replace aiofiles with async pathlib where possible
This commit is contained in:
@@ -8,9 +8,9 @@ _logs_listener = logs.configure()
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import aiofiles
|
||||
import rich
|
||||
from aiohttp import ClientSession, TCPConnector
|
||||
from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn
|
||||
@@ -78,12 +78,6 @@ def get_async_run() -> Callable[[Coroutine[Any, Any, T]], T]:
|
||||
return asyncio.run
|
||||
|
||||
|
||||
async def read_config(file: str, /) -> dict[str, Any]:
|
||||
async with aiofiles.open(file, "rb") as f:
|
||||
content = await f.read()
|
||||
return tomllib.loads(utils.bytes_decode(content))
|
||||
|
||||
|
||||
def get_summary_table(
|
||||
*, before: Mapping[ProxyType, int], after: Mapping[ProxyType, int]
|
||||
) -> Table:
|
||||
@@ -102,8 +96,12 @@ def get_summary_table(
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
cfg = await read_config("config.toml")
|
||||
if cfg["debug"]:
|
||||
config = tomllib.loads(
|
||||
utils.bytes_decode(
|
||||
await asyncio.to_thread(Path("config.toml").read_bytes)
|
||||
)
|
||||
)
|
||||
if config["debug"]:
|
||||
logging.root.setLevel(logging.DEBUG)
|
||||
should_save = False
|
||||
try:
|
||||
@@ -114,7 +112,7 @@ async def main() -> None:
|
||||
raise_for_status=True,
|
||||
fallback_charset_resolver=http.fallback_charset_resolver,
|
||||
) as session:
|
||||
settings = await Settings.from_mapping(cfg, session=session)
|
||||
settings = await Settings.from_mapping(config, session=session)
|
||||
storage = ProxyStorage(protocols=settings.sources)
|
||||
with Progress(
|
||||
TextColumn("[yellow]{task.fields[module]}"),
|
||||
|
||||
@@ -27,8 +27,7 @@ async def _read_etag() -> str | None:
|
||||
await asyncio.to_thread(
|
||||
fs.add_permission, GEODB_ETAG_PATH, stat.S_IRUSR
|
||||
)
|
||||
async with aiofiles.open(GEODB_ETAG_PATH, "rb") as etag_file:
|
||||
content = await etag_file.read()
|
||||
content = await asyncio.to_thread(GEODB_ETAG_PATH.read_bytes)
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
return bytes_decode(content)
|
||||
@@ -42,10 +41,7 @@ async def _save_etag(etag: str, /) -> None:
|
||||
await asyncio.to_thread(
|
||||
fs.add_permission, GEODB_ETAG_PATH, stat.S_IWUSR, missing_ok=True
|
||||
)
|
||||
async with aiofiles.open(
|
||||
GEODB_ETAG_PATH, "w", encoding="utf-8"
|
||||
) as etag_file:
|
||||
await etag_file.write(etag)
|
||||
await asyncio.to_thread(GEODB_ETAG_PATH.write_text, etag, encoding="utf-8")
|
||||
|
||||
|
||||
async def _save_geodb(
|
||||
|
||||
@@ -3,9 +3,9 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import itertools
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import aiofiles
|
||||
from aiohttp import ClientResponseError, ClientTimeout
|
||||
from aiohttp_socks import ProxyType
|
||||
|
||||
@@ -42,8 +42,7 @@ async def scrape_one(
|
||||
content = await response.read()
|
||||
text = get_response_text(response=response, content=content)
|
||||
else:
|
||||
async with aiofiles.open(source, "rb") as f:
|
||||
content = await f.read()
|
||||
content = await asyncio.to_thread(Path(source).read_bytes)
|
||||
text = bytes_decode(content)
|
||||
except ClientResponseError as e:
|
||||
_logger.warning(
|
||||
|
||||
Reference in New Issue
Block a user