proxy-scraper-checker/proxy_scraper_checker/checker.py

from __future__ import annotations

import asyncio
import logging
from typing import TYPE_CHECKING

from proxy_scraper_checker import sort
from proxy_scraper_checker.counter import IncrInt

if TYPE_CHECKING:
    from collections.abc import Mapping

    from aiohttp_socks import ProxyType
    from rich.progress import Progress, TaskID

    from proxy_scraper_checker.proxy import Proxy
    from proxy_scraper_checker.settings import Settings
    from proxy_scraper_checker.storage import ProxyStorage

_logger = logging.getLogger(__name__)


async def check_one(
    *,
    counter: IncrInt,
    progress: Progress,
    proxy: Proxy,
    settings: Settings,
    storage: ProxyStorage,
    task: TaskID,
) -> None:
    try:
        await proxy.check(settings=settings)
    except Exception as e:
        # Too many open files
        if isinstance(e, OSError) and e.errno == 24:  # noqa: PLR2004
            _logger.error("Please, set max_connections to lower value")

        _logger.debug(
            "%s.%s: %s", e.__class__.__module__, e.__class__.__qualname__, e
        )
        storage.remove(proxy)
    else:
        counter.incr()
    progress.update(task_id=task, advance=1, successful_count=counter.value)


async def check_all(
    *,
    settings: Settings,
    storage: ProxyStorage,
    progress: Progress,
    proxies_count: Mapping[ProxyType, int],
) -> None:
    counters = {
        proto: IncrInt()
        for proto in sort.PROTOCOL_ORDER
        if proto in storage.enabled_protocols
    }
    progress_tasks = {
        proto: progress.add_task(
            description="",
            total=proxies_count[proto],
            module="Checker",
            protocol=proto.name,
            successful_count=0,
        )
        for proto in counters
    }
    await asyncio.gather(
        *(
            check_one(
                counter=counters[proxy.protocol],
                progress=progress,
                proxy=proxy,
                settings=settings,
                storage=storage,
                task=progress_tasks[proxy.protocol],
            )
            for proxy in storage
        )
    )