From ebfcc92df9a3808b7d817889b1f3f64cccb33a9a Mon Sep 17 00:00:00 2001 From: monosans Date: Fri, 22 Nov 2024 13:49:07 +0300 Subject: [PATCH] feat: show amount of succesfull proxy checks in real time --- proxy_scraper_checker/__main__.py | 5 +++-- proxy_scraper_checker/checker.py | 20 +++++++++++++++----- proxy_scraper_checker/counter.py | 19 +++++++++++++++++++ proxy_scraper_checker/geodb.py | 5 +++-- proxy_scraper_checker/scraper.py | 13 +++++++++++-- 5 files changed, 51 insertions(+), 11 deletions(-) create mode 100644 proxy_scraper_checker/counter.py diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py index e4bf2f8..84cf597 100644 --- a/proxy_scraper_checker/__main__.py +++ b/proxy_scraper_checker/__main__.py @@ -109,10 +109,11 @@ async def main() -> None: settings = await Settings.from_mapping(cfg, session=session) storage = ProxyStorage(protocols=settings.sources) with Progress( - TextColumn("[yellow]{task.fields[col1]}"), + TextColumn("[yellow]{task.fields[module]}"), TextColumn("[red]::"), - TextColumn("[green]{task.fields[col2]}"), + TextColumn("[green]{task.fields[protocol]}"), BarColumn(), + TextColumn("[cyan]{task.fields[successful_count]}"), MofNCompleteColumn(), transient=True, ) as progress: diff --git a/proxy_scraper_checker/checker.py b/proxy_scraper_checker/checker.py index 3067ec8..d7c77e6 100644 --- a/proxy_scraper_checker/checker.py +++ b/proxy_scraper_checker/checker.py @@ -5,6 +5,7 @@ import logging from typing import TYPE_CHECKING from . import sort +from .counter import IncrInt if TYPE_CHECKING: from collections.abc import Mapping @@ -21,6 +22,7 @@ _logger = logging.getLogger(__name__) async def check_one( *, + counter: IncrInt, progress: Progress, proxy: Proxy, settings: Settings, @@ -38,7 +40,9 @@ async def check_one( "%s.%s: %s", e.__class__.__module__, e.__class__.__qualname__, e ) storage.remove(proxy) - progress.advance(task_id=task, advance=1) + else: + counter.incr() + progress.update(task_id=task, advance=1, successful_count=counter.value) async def check_all( @@ -48,19 +52,25 @@ async def check_all( progress: Progress, proxies_count: Mapping[ProxyType, int], ) -> None: + counters = { + proto: IncrInt() + for proto in sort.PROTOCOL_ORDER + if proto in storage.enabled_protocols + } progress_tasks = { proto: progress.add_task( description="", total=proxies_count[proto], - col1="Checker", - col2=proto.name, + module="Checker", + protocol=proto.name, + successful_count=0, ) - for proto in sort.PROTOCOL_ORDER - if proto in storage.enabled_protocols + for proto in counters } await asyncio.gather( *( check_one( + counter=counters[proxy.protocol], progress=progress, proxy=proxy, settings=settings, diff --git a/proxy_scraper_checker/counter.py b/proxy_scraper_checker/counter.py new file mode 100644 index 0000000..55553b5 --- /dev/null +++ b/proxy_scraper_checker/counter.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from threading import Lock + + +class IncrInt: + __slots__ = ("_lock", "_v") + + def __init__(self) -> None: + self._lock = Lock() + self._v = 0 + + @property + def value(self) -> int: + return self._v + + def incr(self) -> None: + with self._lock: + self._v += 1 diff --git a/proxy_scraper_checker/geodb.py b/proxy_scraper_checker/geodb.py index 3eefc98..eeec7db 100644 --- a/proxy_scraper_checker/geodb.py +++ b/proxy_scraper_checker/geodb.py @@ -81,8 +81,9 @@ async def download_geodb(*, progress: Progress, session: ClientSession) -> None: task=progress.add_task( description="", total=response.content_length, - col1="Downloader", - col2="GeoDB", + module="Downloader", + protocol="GeoDB", + successful_count="", ), ) diff --git a/proxy_scraper_checker/scraper.py b/proxy_scraper_checker/scraper.py index 3264353..1533103 100644 --- a/proxy_scraper_checker/scraper.py +++ b/proxy_scraper_checker/scraper.py @@ -9,6 +9,7 @@ import aiofiles from aiohttp import ClientResponseError, ClientTimeout from aiohttp_socks import ProxyType +from .counter import IncrInt from .http import get_response_text from .parsers import PROXY_REGEX from .proxy import Proxy @@ -26,6 +27,7 @@ _logger = logging.getLogger(__name__) async def scrape_one( *, + counter: IncrInt, progress: Progress, proto: ProxyType, session: ClientSession, @@ -56,6 +58,7 @@ async def scrape_one( e, ) else: + counter.incr() proxies = PROXY_REGEX.finditer(text) try: proxy = next(proxies) @@ -78,7 +81,7 @@ async def scrape_one( password=proxy.group("password"), ) ) - progress.advance(task_id=task, advance=1) + progress.update(task_id=task, advance=1, successful_count=counter.value) async def scrape_all( @@ -88,9 +91,14 @@ async def scrape_all( settings: Settings, storage: ProxyStorage, ) -> None: + counters = {proto: IncrInt() for proto in settings.sources} progress_tasks = { proto: progress.add_task( - description="", total=len(sources), col1="Scraper", col2=proto.name + description="", + total=len(sources), + module="Scraper", + protocol=proto.name, + successful_count=0, ) for proto, sources in settings.sources.items() } @@ -98,6 +106,7 @@ async def scrape_all( await asyncio.gather( *( scrape_one( + counter=counters[proto], progress=progress, proto=proto, session=session,