From bbcd62dc4e428093ca133967ba10b307f4378e4f Mon Sep 17 00:00:00 2001 From: monosans Date: Thu, 27 Feb 2025 11:12:52 +0300 Subject: [PATCH] refactor --- proxy_scraper_checker/checker.py | 24 ++++++++++++++---------- proxy_scraper_checker/counter.py | 15 --------------- proxy_scraper_checker/geodb.py | 10 +++++----- proxy_scraper_checker/incrementor.py | 14 ++++++++++++++ proxy_scraper_checker/scraper.py | 20 ++++++++++++-------- 5 files changed, 45 insertions(+), 38 deletions(-) delete mode 100644 proxy_scraper_checker/counter.py create mode 100644 proxy_scraper_checker/incrementor.py diff --git a/proxy_scraper_checker/checker.py b/proxy_scraper_checker/checker.py index cdaaa90..9e856b7 100644 --- a/proxy_scraper_checker/checker.py +++ b/proxy_scraper_checker/checker.py @@ -5,7 +5,7 @@ import logging from typing import TYPE_CHECKING from proxy_scraper_checker import sort -from proxy_scraper_checker.counter import IncrInt +from proxy_scraper_checker.incrementor import Incrementor if TYPE_CHECKING: from collections.abc import Mapping @@ -22,12 +22,12 @@ _logger = logging.getLogger(__name__) async def check_one( *, - counter: IncrInt, + incrementor: Incrementor, progress: Progress, + progress_task: TaskID, proxy: Proxy, settings: Settings, storage: ProxyStorage, - task: TaskID, ) -> None: try: await proxy.check(settings=settings) @@ -41,8 +41,12 @@ async def check_one( ) storage.remove(proxy) else: - counter.incr() - progress.update(task_id=task, advance=1, successful_count=counter.value) + incrementor.increment() + progress.update( + task_id=progress_task, + advance=1, + successful_count=incrementor.get_value(), + ) async def check_all( @@ -52,8 +56,8 @@ async def check_all( progress: Progress, proxies_count: Mapping[ProxyType, int], ) -> None: - counters = { - proto: IncrInt() + incrementors = { + proto: Incrementor() for proto in sort.PROTOCOL_ORDER if proto in storage.enabled_protocols } @@ -65,17 +69,17 @@ async def check_all( protocol=proto.name, successful_count=0, ) - for proto in counters + for proto in incrementors } await asyncio.gather( *( check_one( - counter=counters[proxy.protocol], + incrementor=incrementors[proxy.protocol], progress=progress, + progress_task=progress_tasks[proxy.protocol], proxy=proxy, settings=settings, storage=storage, - task=progress_tasks[proxy.protocol], ) for proxy in storage ) diff --git a/proxy_scraper_checker/counter.py b/proxy_scraper_checker/counter.py deleted file mode 100644 index cce35d9..0000000 --- a/proxy_scraper_checker/counter.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - - -class IncrInt: - __slots__ = ("_v",) - - def __init__(self) -> None: - self._v = 0 - - @property - def value(self) -> int: - return self._v - - def incr(self) -> None: - self._v += 1 diff --git a/proxy_scraper_checker/geodb.py b/proxy_scraper_checker/geodb.py index 70e2de8..d244512 100644 --- a/proxy_scraper_checker/geodb.py +++ b/proxy_scraper_checker/geodb.py @@ -41,17 +41,17 @@ async def _save_etag(etag: str, /) -> None: async def _save_geodb( - *, progress: Progress, response: ClientResponse, task: TaskID + *, progress: Progress, progress_task: TaskID, response: ClientResponse ) -> None: await fs.add_permission(GEODB_PATH, stat.S_IWUSR, missing_ok=True) geodb = await asyncio.to_thread(GEODB_PATH.open, "wb") try: async for chunk in response.content.iter_any(): await asyncio.to_thread(geodb.write, chunk) - progress.advance(task_id=task, advance=len(chunk)) + progress.advance(task_id=progress_task, advance=len(chunk)) finally: await asyncio.to_thread(geodb.close) - progress.update(task_id=task, successful_count="\N{CHECK MARK}") + progress.update(task_id=progress_task, successful_count="\N{CHECK MARK}") async def download_geodb(*, progress: Progress, session: ClientSession) -> None: @@ -71,14 +71,14 @@ async def download_geodb(*, progress: Progress, session: ClientSession) -> None: return await _save_geodb( progress=progress, - response=response, - task=progress.add_task( + progress_task=progress.add_task( description="", total=response.content_length, module="Downloader", protocol="GeoDB", successful_count="\N{HORIZONTAL ELLIPSIS}", ), + response=response, ) if await asyncio.to_thread(is_docker): diff --git a/proxy_scraper_checker/incrementor.py b/proxy_scraper_checker/incrementor.py new file mode 100644 index 0000000..44e31ab --- /dev/null +++ b/proxy_scraper_checker/incrementor.py @@ -0,0 +1,14 @@ +from __future__ import annotations + + +class Incrementor: + __slots__ = ("_value",) + + def __init__(self) -> None: + self._value = 0 + + def get_value(self) -> int: + return self._value + + def increment(self) -> None: + self._value += 1 diff --git a/proxy_scraper_checker/scraper.py b/proxy_scraper_checker/scraper.py index 2714986..3e7b02a 100644 --- a/proxy_scraper_checker/scraper.py +++ b/proxy_scraper_checker/scraper.py @@ -8,8 +8,8 @@ from typing import TYPE_CHECKING from aiohttp import ClientResponseError, ClientTimeout from aiohttp_socks import ProxyType -from proxy_scraper_checker.counter import IncrInt from proxy_scraper_checker.http import get_response_text +from proxy_scraper_checker.incrementor import Incrementor from proxy_scraper_checker.parsers import PROXY_REGEX from proxy_scraper_checker.proxy import Proxy from proxy_scraper_checker.utils import bytes_decode, is_http_url @@ -26,14 +26,14 @@ _logger = logging.getLogger(__name__) async def scrape_one( *, - counter: IncrInt, + incrementor: Incrementor, progress: Progress, + progress_task: TaskID, proto: ProxyType, session: ClientSession, settings: Settings, source: str, storage: ProxyStorage, - task: TaskID, timeout: ClientTimeout, ) -> None: try: @@ -59,7 +59,7 @@ async def scrape_one( e, ) else: - counter.incr() + incrementor.increment() proxies = tuple(PROXY_REGEX.finditer(text)) if not proxies: _logger.warning("%s | No proxies found", source) @@ -87,7 +87,11 @@ async def scrape_one( password=proxy.group("password"), ) ) - progress.update(task_id=task, advance=1, successful_count=counter.value) + progress.update( + task_id=progress_task, + advance=1, + successful_count=incrementor.get_value(), + ) async def scrape_all( @@ -97,7 +101,7 @@ async def scrape_all( settings: Settings, storage: ProxyStorage, ) -> None: - counters = {proto: IncrInt() for proto in settings.sources} + incrementors = {proto: Incrementor() for proto in settings.sources} progress_tasks = { proto: progress.add_task( description="", @@ -112,14 +116,14 @@ async def scrape_all( await asyncio.gather( *( scrape_one( - counter=counters[proto], + incrementor=incrementors[proto], progress=progress, + progress_task=progress_tasks[proto], proto=proto, session=session, settings=settings, source=source, storage=storage, - task=progress_tasks[proto], timeout=timeout, ) for proto, sources in settings.sources.items()