diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py index d01cca6..9c77aad 100644 --- a/proxy_scraper_checker/__main__.py +++ b/proxy_scraper_checker/__main__.py @@ -6,7 +6,6 @@ import sys from typing import TYPE_CHECKING, Dict, Mapping import aiofiles -import aiofiles.os import rich.traceback from aiohttp import ClientSession, TCPConnector from aiohttp_socks import ProxyType @@ -15,7 +14,7 @@ from rich.logging import RichHandler from rich.progress import BarColumn, MofNCompleteColumn, Progress, TextColumn from rich.table import Table -from . import cache, checker, geodb, http, output, scraper, sort, utils +from . import checker, geodb, http, output, scraper, sort, utils from .settings import Settings from .storage import ProxyStorage from .typing_compat import Any @@ -122,7 +121,6 @@ async def main() -> None: ) await ( asyncio.gather( - cache.create(), geodb.download_geodb(progress=progress, session=session), scrape, ) diff --git a/proxy_scraper_checker/cache.py b/proxy_scraper_checker/cache.py index 0ac81b2..87799ab 100644 --- a/proxy_scraper_checker/cache.py +++ b/proxy_scraper_checker/cache.py @@ -1,14 +1,5 @@ from __future__ import annotations -import asyncio - -import aiofiles.os import platformdirs DIR = platformdirs.user_cache_dir("proxy_scraper_checker") -READY_EVENT = asyncio.Event() - - -async def create() -> None: - await aiofiles.os.makedirs(DIR, exist_ok=True) - READY_EVENT.set() diff --git a/proxy_scraper_checker/geodb.py b/proxy_scraper_checker/geodb.py index 78f92f8..6cc5ca9 100644 --- a/proxy_scraper_checker/geodb.py +++ b/proxy_scraper_checker/geodb.py @@ -59,7 +59,6 @@ async def download_geodb(*, progress: Progress, session: ClientSession) -> None: GEODB_PATH, ) return - await cache.READY_EVENT.wait() await _save_geodb( progress=progress, response=response, diff --git a/proxy_scraper_checker/output.py b/proxy_scraper_checker/output.py index ff731b5..2310fac 100644 --- a/proxy_scraper_checker/output.py +++ b/proxy_scraper_checker/output.py @@ -5,8 +5,6 @@ import logging from shutil import rmtree from typing import Sequence, Union -import aiofiles -import aiofiles.os import aiofiles.ospath import maxminddb @@ -40,7 +38,6 @@ def save_proxies(*, settings: Settings, storage: ProxyStorage) -> None: if settings.enable_geolocation else NullContext() ) - settings.output_path.mkdir(parents=True, exist_ok=True) with mmdb as mmdb_reader: proxy_dicts = [ { diff --git a/proxy_scraper_checker/scraper.py b/proxy_scraper_checker/scraper.py index 6a53883..d7219d1 100644 --- a/proxy_scraper_checker/scraper.py +++ b/proxy_scraper_checker/scraper.py @@ -5,8 +5,6 @@ import itertools import logging import aiofiles -import aiofiles.os -import aiofiles.ospath from aiohttp import ClientResponseError, ClientSession, ClientTimeout from aiohttp_socks import ProxyType from rich.progress import Progress, TaskID diff --git a/proxy_scraper_checker/settings.py b/proxy_scraper_checker/settings.py index fe03b7f..fa3ffa4 100644 --- a/proxy_scraper_checker/settings.py +++ b/proxy_scraper_checker/settings.py @@ -31,7 +31,7 @@ from .http import get_response_text from .null_context import NullContext from .parsers import parse_ipv4 from .typing_compat import Any, Literal, Self -from .utils import IS_DOCKER, check_access +from .utils import IS_DOCKER, create_or_check_dir if TYPE_CHECKING: from .proxy import Proxy @@ -273,8 +273,8 @@ class Settings: ) _, _, (check_website_type, real_ip) = await asyncio.gather( - check_access(output_path, mode=os.W_OK | os.X_OK), - check_access(cache.DIR, mode=os.R_OK | os.W_OK | os.X_OK), + create_or_check_dir(output_path, mode=os.W_OK | os.X_OK), + create_or_check_dir(cache.DIR, mode=os.R_OK | os.W_OK | os.X_OK), _get_check_website_type_and_real_ip( check_website=cfg["check_website"], session=session ), diff --git a/proxy_scraper_checker/utils.py b/proxy_scraper_checker/utils.py index 2d7d454..7941695 100644 --- a/proxy_scraper_checker/utils.py +++ b/proxy_scraper_checker/utils.py @@ -1,4 +1,5 @@ from __future__ import annotations +import asyncio import os from pathlib import Path @@ -6,6 +7,7 @@ from typing import Union from urllib.parse import urlparse import aiofiles.os +import aiofiles.ospath import charset_normalizer IS_DOCKER = os.getenv("IS_DOCKER") == "1" @@ -20,7 +22,14 @@ def bytes_decode(value: bytes, /) -> str: return str(charset_normalizer.from_bytes(value)[0]) -async def check_access(path: Union[Path, str], /, *, mode: int) -> None: - if not await aiofiles.os.access(path, mode): - msg = f"{path} is not accessible" - raise ValueError(msg) +async def create_or_check_dir(path: Union[Path, str], /, *, mode: int) -> None: + try: + await aiofiles.os.makedirs(path) + except FileExistsError: + access_task = asyncio.create_task(aiofiles.os.access(path, mode)) + if not await aiofiles.ospath.isdir(path): + msg = f"{path} is not a directory" + raise ValueError(msg) from None + if not await access_task: + msg = f"{path} is not accessible" + raise ValueError(msg) from None