From 082e12b764a5ddd815711e124d72b4effb1188fd Mon Sep 17 00:00:00 2001 From: monosans Date: Sun, 19 Feb 2023 21:44:15 +0300 Subject: [PATCH] Many changes (see full commit message) Add ability to set MaxConnections to 0, at which this parameter will be set to the maximum available value for the OS Add validation for Timeout and SourceTimeout options Improve existing validators Improve error and warning output Misc refactoring --- config.ini | 1 + proxy_scraper_checker/constants.py | 2 +- proxy_scraper_checker/null_context.py | 13 ++ proxy_scraper_checker/proxy.py | 41 +++--- .../proxy_scraper_checker.py | 117 ++++++++---------- proxy_scraper_checker/validators.py | 77 ++++++++++++ 6 files changed, 164 insertions(+), 87 deletions(-) create mode 100644 proxy_scraper_checker/null_context.py create mode 100644 proxy_scraper_checker/validators.py diff --git a/config.ini b/config.ini index 1978dec..7d77a7e 100644 --- a/config.ini +++ b/config.ini @@ -13,6 +13,7 @@ SourceTimeout = 15 ; The limit on *nix can be seen with the command `ulimit -Hn`. ; Don't be in a hurry to set high values. ; Make sure you have enough RAM first, gradually increasing the default value. +; If set to 0, the maximum value available for your OS will be used. MaxConnections = 512 ; URL to which to send a request to check the proxy. diff --git a/proxy_scraper_checker/constants.py b/proxy_scraper_checker/constants.py index 5f77f26..41464c8 100644 --- a/proxy_scraper_checker/constants.py +++ b/proxy_scraper_checker/constants.py @@ -1,5 +1,5 @@ from __future__ import annotations USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; rv:108.0) Gecko/20100101 Firefox/108.0" + "Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/109.0" ) diff --git a/proxy_scraper_checker/null_context.py b/proxy_scraper_checker/null_context.py new file mode 100644 index 0000000..9571296 --- /dev/null +++ b/proxy_scraper_checker/null_context.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import Any + + +class AsyncNullContext: + __slots__ = () + + async def __aenter__(self) -> None: + pass + + async def __aexit__(self, *_: Any) -> None: + pass diff --git a/proxy_scraper_checker/proxy.py b/proxy_scraper_checker/proxy.py index 6af6dee..99f6987 100644 --- a/proxy_scraper_checker/proxy.py +++ b/proxy_scraper_checker/proxy.py @@ -1,52 +1,53 @@ from __future__ import annotations import asyncio +from dataclasses import dataclass from time import perf_counter +from typing import Union from aiohttp import ClientSession, ClientTimeout from aiohttp.abc import AbstractCookieJar from aiohttp_socks import ProxyConnector, ProxyType from .constants import USER_AGENT +from .null_context import AsyncNullContext + +DEFAULT_CHECK_WEBSITE = "http://ip-api.com/json/?fields=8217" +HEADERS = {"User-Agent": USER_AGENT} +@dataclass(repr=False, unsafe_hash=True) class Proxy: __slots__ = ("geolocation", "host", "is_anonymous", "port", "timeout") - def __init__(self, *, host: str, port: int) -> None: - self.host = host - self.port = port - - @property - def default_check_website(self) -> str: - return "http://ip-api.com/json/?fields=8217" + host: str + port: int async def check( self, *, website: str, - sem: asyncio.Semaphore, + sem: Union[asyncio.Semaphore, AsyncNullContext], cookie_jar: AbstractCookieJar, proto: ProxyType, timeout: ClientTimeout, ) -> None: - check_website = ( - self.default_check_website if website == "default" else website - ) + if website == "default": + website = DEFAULT_CHECK_WEBSITE async with sem: start = perf_counter() async with self.get_connector(proto) as connector, ClientSession( connector=connector, cookie_jar=cookie_jar, timeout=timeout, - headers={"User-Agent": USER_AGENT}, + headers=HEADERS, ) as session, session.get( - check_website, raise_for_status=True + website, raise_for_status=True ) as response: - if website == "default": + if website == DEFAULT_CHECK_WEBSITE: await response.read() self.timeout = perf_counter() - start - if website == "default": + if website == DEFAULT_CHECK_WEBSITE: data = await response.json() self.is_anonymous = self.host != data["query"] self.geolocation = "|{}|{}|{}".format( @@ -56,15 +57,7 @@ class Proxy: def get_connector(self, proto: ProxyType) -> ProxyConnector: return ProxyConnector(proxy_type=proto, host=self.host, port=self.port) - def as_str(self, include_geolocation: bool) -> str: + def as_str(self, *, include_geolocation: bool) -> str: if include_geolocation: return f"{self.host}:{self.port}{self.geolocation}" return f"{self.host}:{self.port}" - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Proxy): - return NotImplemented - return self.host == other.host and self.port == other.port - - def __hash__(self) -> int: - return hash((self.host, self.port)) diff --git a/proxy_scraper_checker/proxy_scraper_checker.py b/proxy_scraper_checker/proxy_scraper_checker.py index 83cbbbb..bb2205d 100644 --- a/proxy_scraper_checker/proxy_scraper_checker.py +++ b/proxy_scraper_checker/proxy_scraper_checker.py @@ -3,13 +3,13 @@ from __future__ import annotations import asyncio import logging import re -import sys from configparser import ConfigParser from pathlib import Path from random import shuffle from typing import ( Callable, Dict, + Iterable, List, Optional, Set, @@ -18,7 +18,6 @@ from typing import ( TypeVar, Union, ) -from urllib.parse import urlparse from aiohttp import ClientSession, ClientTimeout, DummyCookieJar from aiohttp_socks import ProxyType @@ -32,9 +31,10 @@ from rich.progress import ( ) from rich.table import Table -from . import sort +from . import sort, validators from .constants import USER_AGENT from .folder import Folder +from .null_context import AsyncNullContext from .proxy import Proxy logger = logging.getLogger(__name__) @@ -44,27 +44,6 @@ TProxyScraperChecker = TypeVar( ) -def validate_max_connections(value: int) -> int: - if sys.platform != "win32": - import resource - - soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) - if soft_limit < hard_limit: - resource.setrlimit( - resource.RLIMIT_NOFILE, (hard_limit, hard_limit) - ) - elif value > 512 and isinstance( - asyncio.get_event_loop_policy(), asyncio.WindowsSelectorEventLoopPolicy - ): - logger.warning( - "MaxConnections value is too high. " - + "Windows supports a maximum of 512. " - + "The config value will be ignored and 512 will be used." - ) - return 512 - return value - - class ProxyScraperChecker: """HTTP, SOCKS4, SOCKS5 proxies scraper and checker.""" @@ -93,7 +72,7 @@ class ProxyScraperChecker: check_website: str, sort_by_speed: bool, save_path: Path, - folders: Tuple[Folder, ...], + folders: Iterable[Folder], sources: Dict[ProxyType, Optional[str]], console: Optional[Console] = None, ) -> None: @@ -120,15 +99,24 @@ class ProxyScraperChecker: be saved. Leave empty to save the proxies to the current directory. """ - self.folders = folders + validators.timeout(timeout) + self.timeout = ClientTimeout(total=timeout, sock_connect=float("inf")) + + validators.source_timeout(source_timeout) + self.source_timeout = source_timeout + + max_conn = validators.max_connections(max_connections) + self.sem: Union[asyncio.Semaphore, AsyncNullContext] = ( + asyncio.Semaphore(max_conn) if max_conn else AsyncNullContext() + ) + self.check_website = check_website + self.sort_by_speed = sort_by_speed + self.path = save_path + self.folders = folders if self.check_website != "default": - parsed_url = urlparse(check_website) - if not parsed_url.scheme or not parsed_url.netloc: - logger.error("Invalid CheckWebsite URL: %s", check_website) - sys.exit(1) - + validators.check_website(check_website) logger.info( "CheckWebsite is not 'default', " + "so it will not be possible to determine " @@ -138,11 +126,21 @@ class ProxyScraperChecker: folder.is_enabled = ( not folder.for_anonymous and not folder.for_geolocation ) - elif not any(folder for folder in self.folders if folder.is_enabled): - logger.error("All folders are disabled in the config") - sys.exit(1) + else: + validators.folders(self.folders) - self.path = save_path + self.sources = { + proto: frozenset(filter(None, sources.splitlines())) + for proto, sources in sources.items() + if sources + } + validators.sources(self.sources) + self.proxies: Dict[ProxyType, Set[Proxy]] = { + proto: set() for proto in self.sources + } + + self.console = console or Console() + self.cookie_jar = DummyCookieJar() self.regex = re.compile( r"(?:^|\D)?(" + r"(?:[1-9]|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])" # 1-255 @@ -155,23 +153,6 @@ class ProxyScraperChecker: + r"(?:\D|$)" ) - self.sort_by_speed = sort_by_speed - self.timeout = ClientTimeout(total=timeout, sock_connect=float("inf")) - self.source_timeout = source_timeout - self.sources = { - proto: frozenset(filter(None, sources.splitlines())) - for proto, sources in sources.items() - if sources - } - self.proxies: Dict[ProxyType, Set[Proxy]] = { - proto: set() for proto in self.sources - } - self.cookie_jar = DummyCookieJar() - self.console = console or Console() - - max_connections = validate_max_connections(max_connections) - self.sem = asyncio.Semaphore(max_connections) - @classmethod def from_configparser( cls: Type[TProxyScraperChecker], @@ -259,13 +240,24 @@ class ProxyScraperChecker: status = response.status text = await response.text() except Exception as e: - logger.error( - "%s | %s.%s | %s", - source, - e.__class__.__module__, - e.__class__.__qualname__, - e, + e_str = str(e) + args: Tuple[object, ...] = ( + ( + "%s | %s.%s (%s)", + source, + e.__class__.__module__, + e.__class__.__qualname__, + e_str, + ) + if e_str + else ( + "%s | %s.%s", + source, + e.__class__.__module__, + e.__class__.__qualname__, + ) ) + logger.error(*args) else: proxies = tuple(self.regex.finditer(text)) if proxies: @@ -275,11 +267,12 @@ class ProxyScraperChecker: ) self.proxies[proto].add(proxy_obj) else: - logger.warning( - "%s | No proxies found | HTTP status code %d", - source, - status, + args = ( + ("%s | No proxies found", source) + if status == 200 + else ("%s | HTTP status code %d", source, status) ) + logger.warning(*args) progress.update(task, advance=1) async def check_proxy( diff --git a/proxy_scraper_checker/validators.py b/proxy_scraper_checker/validators.py new file mode 100644 index 0000000..2705cec --- /dev/null +++ b/proxy_scraper_checker/validators.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import asyncio +import logging +import sys +from typing import Any, Iterable, Optional +from urllib.parse import urlparse + +from .folder import Folder + +logger = logging.getLogger(__name__) + + +def timeout(value: float) -> None: + if value <= 0: + msg = "Timeout must be positive" + raise ValueError(msg) + + +def source_timeout(value: float) -> None: + if value <= 0: + msg = "SourceTimeout must be positive" + raise ValueError(msg) + + +def max_connections(value: int) -> Optional[int]: + if value < 0: + msg = "MaxConnections must be non-negative" + raise ValueError(msg) + max_supported = _get_supported_max_connections() + if not value: + logger.info("Using %d as MaxConnections value", max_supported or 0) + return max_supported + if not max_supported or value <= max_supported: + return value + logger.warning( + "MaxConnections value is too high. " + + "Your OS supports a maximum of %d. " + + "The config value will be ignored and %d will be used.", + max_supported, + max_supported, + ) + return max_supported + + +def _get_supported_max_connections() -> Optional[int]: + if sys.platform == "win32": + if isinstance( + asyncio.get_event_loop_policy(), + asyncio.WindowsSelectorEventLoopPolicy, + ): + return 512 + return None + import resource + + soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft_limit < hard_limit: + resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit)) + return hard_limit + + +def check_website(value: str) -> None: + parsed_url = urlparse(value) + if not parsed_url.scheme or not parsed_url.netloc: + msg = f"invalid CheckWebsite URL: {value}" + raise ValueError(msg) + + +def folders(value: Iterable[Folder]) -> None: + if not any(folder for folder in value if folder.is_enabled): + msg = "all folders are disabled in the config" + raise ValueError(msg) + + +def sources(value: Any) -> None: + if not value: + raise ValueError("proxy sources list is empty")