diff --git a/config.ini b/config.ini index 1978dec..7d77a7e 100644 --- a/config.ini +++ b/config.ini @@ -13,6 +13,7 @@ SourceTimeout = 15 ; The limit on *nix can be seen with the command `ulimit -Hn`. ; Don't be in a hurry to set high values. ; Make sure you have enough RAM first, gradually increasing the default value. +; If set to 0, the maximum value available for your OS will be used. MaxConnections = 512 ; URL to which to send a request to check the proxy. diff --git a/proxy_scraper_checker/constants.py b/proxy_scraper_checker/constants.py index 5f77f26..41464c8 100644 --- a/proxy_scraper_checker/constants.py +++ b/proxy_scraper_checker/constants.py @@ -1,5 +1,5 @@ from __future__ import annotations USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; rv:108.0) Gecko/20100101 Firefox/108.0" + "Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/109.0" ) diff --git a/proxy_scraper_checker/null_context.py b/proxy_scraper_checker/null_context.py new file mode 100644 index 0000000..9571296 --- /dev/null +++ b/proxy_scraper_checker/null_context.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import Any + + +class AsyncNullContext: + __slots__ = () + + async def __aenter__(self) -> None: + pass + + async def __aexit__(self, *_: Any) -> None: + pass diff --git a/proxy_scraper_checker/proxy.py b/proxy_scraper_checker/proxy.py index 6af6dee..99f6987 100644 --- a/proxy_scraper_checker/proxy.py +++ b/proxy_scraper_checker/proxy.py @@ -1,52 +1,53 @@ from __future__ import annotations import asyncio +from dataclasses import dataclass from time import perf_counter +from typing import Union from aiohttp import ClientSession, ClientTimeout from aiohttp.abc import AbstractCookieJar from aiohttp_socks import ProxyConnector, ProxyType from .constants import USER_AGENT +from .null_context import AsyncNullContext + +DEFAULT_CHECK_WEBSITE = "http://ip-api.com/json/?fields=8217" +HEADERS = {"User-Agent": USER_AGENT} +@dataclass(repr=False, unsafe_hash=True) class Proxy: __slots__ = ("geolocation", "host", "is_anonymous", "port", "timeout") - def __init__(self, *, host: str, port: int) -> None: - self.host = host - self.port = port - - @property - def default_check_website(self) -> str: - return "http://ip-api.com/json/?fields=8217" + host: str + port: int async def check( self, *, website: str, - sem: asyncio.Semaphore, + sem: Union[asyncio.Semaphore, AsyncNullContext], cookie_jar: AbstractCookieJar, proto: ProxyType, timeout: ClientTimeout, ) -> None: - check_website = ( - self.default_check_website if website == "default" else website - ) + if website == "default": + website = DEFAULT_CHECK_WEBSITE async with sem: start = perf_counter() async with self.get_connector(proto) as connector, ClientSession( connector=connector, cookie_jar=cookie_jar, timeout=timeout, - headers={"User-Agent": USER_AGENT}, + headers=HEADERS, ) as session, session.get( - check_website, raise_for_status=True + website, raise_for_status=True ) as response: - if website == "default": + if website == DEFAULT_CHECK_WEBSITE: await response.read() self.timeout = perf_counter() - start - if website == "default": + if website == DEFAULT_CHECK_WEBSITE: data = await response.json() self.is_anonymous = self.host != data["query"] self.geolocation = "|{}|{}|{}".format( @@ -56,15 +57,7 @@ class Proxy: def get_connector(self, proto: ProxyType) -> ProxyConnector: return ProxyConnector(proxy_type=proto, host=self.host, port=self.port) - def as_str(self, include_geolocation: bool) -> str: + def as_str(self, *, include_geolocation: bool) -> str: if include_geolocation: return f"{self.host}:{self.port}{self.geolocation}" return f"{self.host}:{self.port}" - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Proxy): - return NotImplemented - return self.host == other.host and self.port == other.port - - def __hash__(self) -> int: - return hash((self.host, self.port)) diff --git a/proxy_scraper_checker/proxy_scraper_checker.py b/proxy_scraper_checker/proxy_scraper_checker.py index 83cbbbb..bb2205d 100644 --- a/proxy_scraper_checker/proxy_scraper_checker.py +++ b/proxy_scraper_checker/proxy_scraper_checker.py @@ -3,13 +3,13 @@ from __future__ import annotations import asyncio import logging import re -import sys from configparser import ConfigParser from pathlib import Path from random import shuffle from typing import ( Callable, Dict, + Iterable, List, Optional, Set, @@ -18,7 +18,6 @@ from typing import ( TypeVar, Union, ) -from urllib.parse import urlparse from aiohttp import ClientSession, ClientTimeout, DummyCookieJar from aiohttp_socks import ProxyType @@ -32,9 +31,10 @@ from rich.progress import ( ) from rich.table import Table -from . import sort +from . import sort, validators from .constants import USER_AGENT from .folder import Folder +from .null_context import AsyncNullContext from .proxy import Proxy logger = logging.getLogger(__name__) @@ -44,27 +44,6 @@ TProxyScraperChecker = TypeVar( ) -def validate_max_connections(value: int) -> int: - if sys.platform != "win32": - import resource - - soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) - if soft_limit < hard_limit: - resource.setrlimit( - resource.RLIMIT_NOFILE, (hard_limit, hard_limit) - ) - elif value > 512 and isinstance( - asyncio.get_event_loop_policy(), asyncio.WindowsSelectorEventLoopPolicy - ): - logger.warning( - "MaxConnections value is too high. " - + "Windows supports a maximum of 512. " - + "The config value will be ignored and 512 will be used." - ) - return 512 - return value - - class ProxyScraperChecker: """HTTP, SOCKS4, SOCKS5 proxies scraper and checker.""" @@ -93,7 +72,7 @@ class ProxyScraperChecker: check_website: str, sort_by_speed: bool, save_path: Path, - folders: Tuple[Folder, ...], + folders: Iterable[Folder], sources: Dict[ProxyType, Optional[str]], console: Optional[Console] = None, ) -> None: @@ -120,15 +99,24 @@ class ProxyScraperChecker: be saved. Leave empty to save the proxies to the current directory. """ - self.folders = folders + validators.timeout(timeout) + self.timeout = ClientTimeout(total=timeout, sock_connect=float("inf")) + + validators.source_timeout(source_timeout) + self.source_timeout = source_timeout + + max_conn = validators.max_connections(max_connections) + self.sem: Union[asyncio.Semaphore, AsyncNullContext] = ( + asyncio.Semaphore(max_conn) if max_conn else AsyncNullContext() + ) + self.check_website = check_website + self.sort_by_speed = sort_by_speed + self.path = save_path + self.folders = folders if self.check_website != "default": - parsed_url = urlparse(check_website) - if not parsed_url.scheme or not parsed_url.netloc: - logger.error("Invalid CheckWebsite URL: %s", check_website) - sys.exit(1) - + validators.check_website(check_website) logger.info( "CheckWebsite is not 'default', " + "so it will not be possible to determine " @@ -138,11 +126,21 @@ class ProxyScraperChecker: folder.is_enabled = ( not folder.for_anonymous and not folder.for_geolocation ) - elif not any(folder for folder in self.folders if folder.is_enabled): - logger.error("All folders are disabled in the config") - sys.exit(1) + else: + validators.folders(self.folders) - self.path = save_path + self.sources = { + proto: frozenset(filter(None, sources.splitlines())) + for proto, sources in sources.items() + if sources + } + validators.sources(self.sources) + self.proxies: Dict[ProxyType, Set[Proxy]] = { + proto: set() for proto in self.sources + } + + self.console = console or Console() + self.cookie_jar = DummyCookieJar() self.regex = re.compile( r"(?:^|\D)?(" + r"(?:[1-9]|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])" # 1-255 @@ -155,23 +153,6 @@ class ProxyScraperChecker: + r"(?:\D|$)" ) - self.sort_by_speed = sort_by_speed - self.timeout = ClientTimeout(total=timeout, sock_connect=float("inf")) - self.source_timeout = source_timeout - self.sources = { - proto: frozenset(filter(None, sources.splitlines())) - for proto, sources in sources.items() - if sources - } - self.proxies: Dict[ProxyType, Set[Proxy]] = { - proto: set() for proto in self.sources - } - self.cookie_jar = DummyCookieJar() - self.console = console or Console() - - max_connections = validate_max_connections(max_connections) - self.sem = asyncio.Semaphore(max_connections) - @classmethod def from_configparser( cls: Type[TProxyScraperChecker], @@ -259,13 +240,24 @@ class ProxyScraperChecker: status = response.status text = await response.text() except Exception as e: - logger.error( - "%s | %s.%s | %s", - source, - e.__class__.__module__, - e.__class__.__qualname__, - e, + e_str = str(e) + args: Tuple[object, ...] = ( + ( + "%s | %s.%s (%s)", + source, + e.__class__.__module__, + e.__class__.__qualname__, + e_str, + ) + if e_str + else ( + "%s | %s.%s", + source, + e.__class__.__module__, + e.__class__.__qualname__, + ) ) + logger.error(*args) else: proxies = tuple(self.regex.finditer(text)) if proxies: @@ -275,11 +267,12 @@ class ProxyScraperChecker: ) self.proxies[proto].add(proxy_obj) else: - logger.warning( - "%s | No proxies found | HTTP status code %d", - source, - status, + args = ( + ("%s | No proxies found", source) + if status == 200 + else ("%s | HTTP status code %d", source, status) ) + logger.warning(*args) progress.update(task, advance=1) async def check_proxy( diff --git a/proxy_scraper_checker/validators.py b/proxy_scraper_checker/validators.py new file mode 100644 index 0000000..2705cec --- /dev/null +++ b/proxy_scraper_checker/validators.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +import asyncio +import logging +import sys +from typing import Any, Iterable, Optional +from urllib.parse import urlparse + +from .folder import Folder + +logger = logging.getLogger(__name__) + + +def timeout(value: float) -> None: + if value <= 0: + msg = "Timeout must be positive" + raise ValueError(msg) + + +def source_timeout(value: float) -> None: + if value <= 0: + msg = "SourceTimeout must be positive" + raise ValueError(msg) + + +def max_connections(value: int) -> Optional[int]: + if value < 0: + msg = "MaxConnections must be non-negative" + raise ValueError(msg) + max_supported = _get_supported_max_connections() + if not value: + logger.info("Using %d as MaxConnections value", max_supported or 0) + return max_supported + if not max_supported or value <= max_supported: + return value + logger.warning( + "MaxConnections value is too high. " + + "Your OS supports a maximum of %d. " + + "The config value will be ignored and %d will be used.", + max_supported, + max_supported, + ) + return max_supported + + +def _get_supported_max_connections() -> Optional[int]: + if sys.platform == "win32": + if isinstance( + asyncio.get_event_loop_policy(), + asyncio.WindowsSelectorEventLoopPolicy, + ): + return 512 + return None + import resource + + soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE) + if soft_limit < hard_limit: + resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit)) + return hard_limit + + +def check_website(value: str) -> None: + parsed_url = urlparse(value) + if not parsed_url.scheme or not parsed_url.netloc: + msg = f"invalid CheckWebsite URL: {value}" + raise ValueError(msg) + + +def folders(value: Iterable[Folder]) -> None: + if not any(folder for folder in value if folder.is_enabled): + msg = "all folders are disabled in the config" + raise ValueError(msg) + + +def sources(value: Any) -> None: + if not value: + raise ValueError("proxy sources list is empty")