87 lines
2.2 KiB
Python
87 lines
2.2 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from typing import TYPE_CHECKING
|
|
|
|
from proxy_scraper_checker import sort
|
|
from proxy_scraper_checker.incrementor import Incrementor
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Mapping
|
|
|
|
from aiohttp_socks import ProxyType
|
|
from rich.progress import Progress, TaskID
|
|
|
|
from proxy_scraper_checker.proxy import Proxy
|
|
from proxy_scraper_checker.settings import Settings
|
|
from proxy_scraper_checker.storage import ProxyStorage
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def check_one(
|
|
*,
|
|
incrementor: Incrementor,
|
|
progress: Progress,
|
|
progress_task: TaskID,
|
|
proxy: Proxy,
|
|
settings: Settings,
|
|
storage: ProxyStorage,
|
|
) -> None:
|
|
try:
|
|
await proxy.check(settings=settings)
|
|
except Exception as e:
|
|
# Too many open files
|
|
if isinstance(e, OSError) and e.errno == 24: # noqa: PLR2004
|
|
_logger.error("Please, set max_connections to lower value")
|
|
|
|
_logger.debug(
|
|
"%s.%s: %s", e.__class__.__module__, e.__class__.__qualname__, e
|
|
)
|
|
storage.remove(proxy)
|
|
else:
|
|
incrementor.increment()
|
|
progress.update(
|
|
task_id=progress_task,
|
|
advance=1,
|
|
successful_count=incrementor.get_value(),
|
|
)
|
|
|
|
|
|
async def check_all(
|
|
*,
|
|
settings: Settings,
|
|
storage: ProxyStorage,
|
|
progress: Progress,
|
|
proxies_count: Mapping[ProxyType, int],
|
|
) -> None:
|
|
incrementors = {
|
|
proto: Incrementor()
|
|
for proto in sort.PROTOCOL_ORDER
|
|
if proto in storage.enabled_protocols
|
|
}
|
|
progress_tasks = {
|
|
proto: progress.add_task(
|
|
description="",
|
|
total=proxies_count[proto],
|
|
module="Checker",
|
|
protocol=proto.name,
|
|
successful_count=0,
|
|
)
|
|
for proto in incrementors
|
|
}
|
|
await asyncio.gather(
|
|
*(
|
|
check_one(
|
|
incrementor=incrementors[proxy.protocol],
|
|
progress=progress,
|
|
progress_task=progress_tasks[proxy.protocol],
|
|
proxy=proxy,
|
|
settings=settings,
|
|
storage=storage,
|
|
)
|
|
for proxy in storage
|
|
)
|
|
)
|