diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py index 75299a9..2c13bf4 100644 --- a/proxy_scraper_checker/__main__.py +++ b/proxy_scraper_checker/__main__.py @@ -8,9 +8,9 @@ _logs_listener = logs.configure() import asyncio import logging import sys +from pathlib import Path from typing import TYPE_CHECKING -import aiofiles import rich from aiohttp import ClientSession, TCPConnector from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn @@ -78,12 +78,6 @@ def get_async_run() -> Callable[[Coroutine[Any, Any, T]], T]: return asyncio.run -async def read_config(file: str, /) -> dict[str, Any]: - async with aiofiles.open(file, "rb") as f: - content = await f.read() - return tomllib.loads(utils.bytes_decode(content)) - - def get_summary_table( *, before: Mapping[ProxyType, int], after: Mapping[ProxyType, int] ) -> Table: @@ -102,8 +96,12 @@ def get_summary_table( async def main() -> None: - cfg = await read_config("config.toml") - if cfg["debug"]: + config = tomllib.loads( + utils.bytes_decode( + await asyncio.to_thread(Path("config.toml").read_bytes) + ) + ) + if config["debug"]: logging.root.setLevel(logging.DEBUG) should_save = False try: @@ -114,7 +112,7 @@ async def main() -> None: raise_for_status=True, fallback_charset_resolver=http.fallback_charset_resolver, ) as session: - settings = await Settings.from_mapping(cfg, session=session) + settings = await Settings.from_mapping(config, session=session) storage = ProxyStorage(protocols=settings.sources) with Progress( TextColumn("[yellow]{task.fields[module]}"), diff --git a/proxy_scraper_checker/geodb.py b/proxy_scraper_checker/geodb.py index 5bcfaf4..c99a4a5 100644 --- a/proxy_scraper_checker/geodb.py +++ b/proxy_scraper_checker/geodb.py @@ -27,8 +27,7 @@ async def _read_etag() -> str | None: await asyncio.to_thread( fs.add_permission, GEODB_ETAG_PATH, stat.S_IRUSR ) - async with aiofiles.open(GEODB_ETAG_PATH, "rb") as etag_file: - content = await etag_file.read() + content = await asyncio.to_thread(GEODB_ETAG_PATH.read_bytes) except FileNotFoundError: return None return bytes_decode(content) @@ -42,10 +41,7 @@ async def _save_etag(etag: str, /) -> None: await asyncio.to_thread( fs.add_permission, GEODB_ETAG_PATH, stat.S_IWUSR, missing_ok=True ) - async with aiofiles.open( - GEODB_ETAG_PATH, "w", encoding="utf-8" - ) as etag_file: - await etag_file.write(etag) + await asyncio.to_thread(GEODB_ETAG_PATH.write_text, etag, encoding="utf-8") async def _save_geodb( diff --git a/proxy_scraper_checker/scraper.py b/proxy_scraper_checker/scraper.py index 7fd04b0..314143e 100644 --- a/proxy_scraper_checker/scraper.py +++ b/proxy_scraper_checker/scraper.py @@ -3,9 +3,9 @@ from __future__ import annotations import asyncio import itertools import logging +from pathlib import Path from typing import TYPE_CHECKING -import aiofiles from aiohttp import ClientResponseError, ClientTimeout from aiohttp_socks import ProxyType @@ -42,8 +42,7 @@ async def scrape_one( content = await response.read() text = get_response_text(response=response, content=content) else: - async with aiofiles.open(source, "rb") as f: - content = await f.read() + content = await asyncio.to_thread(Path(source).read_bytes) text = bytes_decode(content) except ClientResponseError as e: _logger.warning(