diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py index 93cf384..53d271b 100644 --- a/proxy_scraper_checker/__main__.py +++ b/proxy_scraper_checker/__main__.py @@ -172,7 +172,9 @@ async def main() -> None: ) ) - await output.save_proxies(storage=storage, settings=settings) + await asyncio.to_thread( + output.save_proxies, storage=storage, settings=settings + ) logger.info( "Thank you for using https://github.com/monosans/proxy-scraper-checker" diff --git a/proxy_scraper_checker/fs.py b/proxy_scraper_checker/fs.py index e5affc3..ec556ab 100644 --- a/proxy_scraper_checker/fs.py +++ b/proxy_scraper_checker/fs.py @@ -5,8 +5,6 @@ from typing import TYPE_CHECKING import platformdirs -from .utils import asyncify - if TYPE_CHECKING: from pathlib import Path @@ -33,9 +31,6 @@ def add_permission( raise -async_add_permission = asyncify(add_permission) - - def create_or_fix_dir(path: Path, /, *, permission: int) -> None: try: path.mkdir(parents=True) @@ -44,6 +39,3 @@ def create_or_fix_dir(path: Path, /, *, permission: int) -> None: msg = f"{path} is not a directory" raise ValueError(msg) from None add_permission(path, permission) - - -async_create_or_fix_dir = asyncify(create_or_fix_dir) diff --git a/proxy_scraper_checker/geodb.py b/proxy_scraper_checker/geodb.py index b577a0a..c76ec5f 100644 --- a/proxy_scraper_checker/geodb.py +++ b/proxy_scraper_checker/geodb.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import logging import stat from typing import TYPE_CHECKING @@ -8,11 +9,9 @@ import aiofiles from aiohttp import hdrs from . import fs -from .utils import IS_DOCKER, asyncify, bytes_decode +from .utils import IS_DOCKER, bytes_decode if TYPE_CHECKING: - import asyncio - from aiohttp import ClientResponse, ClientSession from rich.progress import Progress, TaskID @@ -25,7 +24,9 @@ GEODB_ETAG_PATH = GEODB_PATH.with_suffix(".mmdb.etag") async def _read_etag() -> str | None: try: - await fs.async_add_permission(GEODB_ETAG_PATH, stat.S_IRUSR) + await asyncio.to_thread( + fs.add_permission, GEODB_ETAG_PATH, stat.S_IRUSR + ) async with aiofiles.open(GEODB_ETAG_PATH, "rb") as etag_file: content = await etag_file.read() except FileNotFoundError: @@ -33,13 +34,13 @@ async def _read_etag() -> str | None: return bytes_decode(content) -def _remove_etag() -> asyncio.Future[None]: - return asyncify(GEODB_ETAG_PATH.unlink)(missing_ok=True) +async def _remove_etag() -> None: + return await asyncio.to_thread(GEODB_ETAG_PATH.unlink, missing_ok=True) async def _save_etag(etag: str, /) -> None: - await fs.async_add_permission( - GEODB_ETAG_PATH, stat.S_IWUSR, missing_ok=True + await asyncio.to_thread( + fs.add_permission, GEODB_ETAG_PATH, stat.S_IWUSR, missing_ok=True ) async with aiofiles.open( GEODB_ETAG_PATH, "w", encoding="utf-8" @@ -50,7 +51,9 @@ async def _save_etag(etag: str, /) -> None: async def _save_geodb( *, progress: Progress, response: ClientResponse, task: TaskID ) -> None: - await fs.async_add_permission(GEODB_PATH, stat.S_IWUSR, missing_ok=True) + await asyncio.to_thread( + fs.add_permission, GEODB_PATH, stat.S_IWUSR, missing_ok=True + ) async with aiofiles.open(GEODB_PATH, "wb") as geodb: async for chunk in response.content.iter_any(): await geodb.write(chunk) @@ -60,7 +63,7 @@ async def _save_geodb( async def download_geodb(*, progress: Progress, session: ClientSession) -> None: headers = ( {hdrs.IF_NONE_MATCH: current_etag} - if await asyncify(GEODB_PATH.exists)() + if await asyncio.to_thread(GEODB_PATH.exists) and (current_etag := await _read_etag()) else None ) diff --git a/proxy_scraper_checker/output.py b/proxy_scraper_checker/output.py index 29360a3..6fb9a06 100644 --- a/proxy_scraper_checker/output.py +++ b/proxy_scraper_checker/output.py @@ -11,7 +11,7 @@ import maxminddb from . import fs, sort from .geodb import GEODB_PATH from .null_context import NullContext -from .utils import IS_DOCKER, asyncify +from .utils import IS_DOCKER if TYPE_CHECKING: from collections.abc import Sequence @@ -34,7 +34,6 @@ def _create_proxy_list_str( ) -@asyncify def save_proxies(*, settings: Settings, storage: ProxyStorage) -> None: if settings.output_json: if settings.enable_geolocation: diff --git a/proxy_scraper_checker/settings.py b/proxy_scraper_checker/settings.py index dbde3a4..8152168 100644 --- a/proxy_scraper_checker/settings.py +++ b/proxy_scraper_checker/settings.py @@ -298,8 +298,10 @@ class Settings: else Path(cfg["output"]["path"]) ) - output_path_future = fs.async_create_or_fix_dir( - output_path, permission=stat.S_IXUSR | stat.S_IWUSR + output_path_future = asyncio.to_thread( + fs.create_or_fix_dir, + output_path, + permission=stat.S_IXUSR | stat.S_IWUSR, ) check_website_type, real_ip = await _get_check_website_type_and_real_ip( @@ -312,7 +314,8 @@ class Settings: ) if enable_geolocation: - await fs.async_create_or_fix_dir( + await asyncio.to_thread( + fs.create_or_fix_dir, fs.CACHE_PATH, permission=stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR, ) diff --git a/proxy_scraper_checker/utils.py b/proxy_scraper_checker/utils.py index 94def8b..8b8ded1 100644 --- a/proxy_scraper_checker/utils.py +++ b/proxy_scraper_checker/utils.py @@ -1,7 +1,5 @@ from __future__ import annotations -import asyncio -import functools import os from typing import TYPE_CHECKING from urllib.parse import urlparse @@ -9,8 +7,6 @@ from urllib.parse import urlparse import charset_normalizer if TYPE_CHECKING: - from typing import Callable - from typing_extensions import ParamSpec, TypeVar T = TypeVar("T") @@ -26,12 +22,3 @@ def is_http_url(value: str, /) -> bool: def bytes_decode(value: bytes, /) -> str: return str(charset_normalizer.from_bytes(value)[0]) - - -def asyncify(f: Callable[P, T], /) -> Callable[P, asyncio.Future[T]]: - def wrapper(*args: P.args, **kwargs: P.kwargs) -> asyncio.Future[T]: - return asyncio.get_running_loop().run_in_executor( - None, functools.partial(f, *args, **kwargs) - ) - - return functools.update_wrapper(wrapper, f)