haeaeae/proxy-scraper-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Many changes (see full commit message)

Browse Source 
Add ability to set MaxConnections to 0, at which this parameter will be set to the maximum available value for the OS
Add validation for Timeout and SourceTimeout options
Improve existing validators
Improve error and warning output
Misc refactoring
This commit is contained in:
monosans
2023-02-19 22:18:30 +03:00
parent 635e993d56
commit 082e12b764
6 changed files with 164 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.ini
+1
View File
@@ -13,6 +13,7 @@ SourceTimeout = 15
; The limit on *nix can be seen with the command `ulimit -Hn`.
; Don't be in a hurry to set high values.
; Make sure you have enough RAM first, gradually increasing the default value.
; If set to 0, the maximum value available for your OS will be used.
MaxConnections = 512
; URL to which to send a request to check the proxy.
proxy_scraper_checker/constants.py
+1 -1
View File
@@ -1,5 +1,5 @@
from __future__ import annotations
USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; rv:108.0) Gecko/20100101 Firefox/108.0"
"Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/109.0"
)
proxy_scraper_checker/null_context.py
+13
View File
@@ -0,0 +1,13 @@
from __future__ import annotations
from typing import Any
class AsyncNullContext:
__slots__ = ()
async def __aenter__(self) -> None:
pass
async def __aexit__(self, *_: Any) -> None:
pass
proxy_scraper_checker/proxy.py
+17 -24
View File
@@ -1,52 +1,53 @@
from __future__ import annotations
import asyncio
from dataclasses import dataclass
from time import perf_counter
from typing import Union
from aiohttp import ClientSession, ClientTimeout
from aiohttp.abc import AbstractCookieJar
from aiohttp_socks import ProxyConnector, ProxyType
from .constants import USER_AGENT
from .null_context import AsyncNullContext
DEFAULT_CHECK_WEBSITE = "http://ip-api.com/json/?fields=8217"
HEADERS = {"User-Agent": USER_AGENT}
@dataclass(repr=False, unsafe_hash=True)
class Proxy:
__slots__ = ("geolocation", "host", "is_anonymous", "port", "timeout")
def __init__(self, *, host: str, port: int) -> None:
self.host = host
self.port = port
@property
def default_check_website(self) -> str:
return "http://ip-api.com/json/?fields=8217"
host: str
port: int
async def check(
self,
*,
website: str,
sem: asyncio.Semaphore,
sem: Union[asyncio.Semaphore, AsyncNullContext],
cookie_jar: AbstractCookieJar,
proto: ProxyType,
timeout: ClientTimeout,
) -> None:
check_website = (
self.default_check_website if website == "default" else website
)
if website == "default":
website = DEFAULT_CHECK_WEBSITE
async with sem:
start = perf_counter()
async with self.get_connector(proto) as connector, ClientSession(
connector=connector,
cookie_jar=cookie_jar,
timeout=timeout,
headers={"User-Agent": USER_AGENT},
headers=HEADERS,
) as session, session.get(
check_website, raise_for_status=True
website, raise_for_status=True
) as response:
if website == "default":
if website == DEFAULT_CHECK_WEBSITE:
await response.read()
self.timeout = perf_counter() - start
if website == "default":
if website == DEFAULT_CHECK_WEBSITE:
data = await response.json()
self.is_anonymous = self.host != data["query"]
self.geolocation = "|{}|{}|{}".format(
@@ -56,15 +57,7 @@ class Proxy:
def get_connector(self, proto: ProxyType) -> ProxyConnector:
return ProxyConnector(proxy_type=proto, host=self.host, port=self.port)
def as_str(self, include_geolocation: bool) -> str:
def as_str(self, *, include_geolocation: bool) -> str:
if include_geolocation:
return f"{self.host}:{self.port}{self.geolocation}"
return f"{self.host}:{self.port}"
def __eq__(self, other: object) -> bool:
if not isinstance(other, Proxy):
return NotImplemented
return self.host == other.host and self.port == other.port
def __hash__(self) -> int:
return hash((self.host, self.port))
proxy_scraper_checker/proxy_scraper_checker.py
+55 -62
View File
@@ -3,13 +3,13 @@ from __future__ import annotations
import asyncio
import logging
import re
import sys
from configparser import ConfigParser
from pathlib import Path
from random import shuffle
from typing import (
Callable,
Dict,
Iterable,
List,
Optional,
Set,
@@ -18,7 +18,6 @@ from typing import (
TypeVar,
Union,
)
from urllib.parse import urlparse
from aiohttp import ClientSession, ClientTimeout, DummyCookieJar
from aiohttp_socks import ProxyType
@@ -32,9 +31,10 @@ from rich.progress import (
)
from rich.table import Table
from . import sort
from . import sort, validators
from .constants import USER_AGENT
from .folder import Folder
from .null_context import AsyncNullContext
from .proxy import Proxy
logger = logging.getLogger(__name__)
@@ -44,27 +44,6 @@ TProxyScraperChecker = TypeVar(
)
def validate_max_connections(value: int) -> int:
if sys.platform != "win32":
import resource
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft_limit < hard_limit:
resource.setrlimit(
resource.RLIMIT_NOFILE, (hard_limit, hard_limit)
)
elif value > 512 and isinstance(
asyncio.get_event_loop_policy(), asyncio.WindowsSelectorEventLoopPolicy
):
logger.warning(
"MaxConnections value is too high. "
+ "Windows supports a maximum of 512. "
+ "The config value will be ignored and 512 will be used."
)
return 512
return value
class ProxyScraperChecker:
"""HTTP, SOCKS4, SOCKS5 proxies scraper and checker."""
@@ -93,7 +72,7 @@ class ProxyScraperChecker:
check_website: str,
sort_by_speed: bool,
save_path: Path,
folders: Tuple[Folder, ...],
folders: Iterable[Folder],
sources: Dict[ProxyType, Optional[str]],
console: Optional[Console] = None,
) -> None:
@@ -120,15 +99,24 @@ class ProxyScraperChecker:
be saved. Leave empty to save the proxies to the current
directory.
"""
self.folders = folders
validators.timeout(timeout)
self.timeout = ClientTimeout(total=timeout, sock_connect=float("inf"))
validators.source_timeout(source_timeout)
self.source_timeout = source_timeout
max_conn = validators.max_connections(max_connections)
self.sem: Union[asyncio.Semaphore, AsyncNullContext] = (
asyncio.Semaphore(max_conn) if max_conn else AsyncNullContext()
)
self.check_website = check_website
self.sort_by_speed = sort_by_speed
self.path = save_path
self.folders = folders
if self.check_website != "default":
parsed_url = urlparse(check_website)
if not parsed_url.scheme or not parsed_url.netloc:
logger.error("Invalid CheckWebsite URL: %s", check_website)
sys.exit(1)
validators.check_website(check_website)
logger.info(
"CheckWebsite is not 'default', "
+ "so it will not be possible to determine "
@@ -138,11 +126,21 @@ class ProxyScraperChecker:
folder.is_enabled = (
not folder.for_anonymous and not folder.for_geolocation
)
elif not any(folder for folder in self.folders if folder.is_enabled):
logger.error("All folders are disabled in the config")
sys.exit(1)
else:
validators.folders(self.folders)
self.path = save_path
self.sources = {
proto: frozenset(filter(None, sources.splitlines()))
for proto, sources in sources.items()
if sources
}
validators.sources(self.sources)
self.proxies: Dict[ProxyType, Set[Proxy]] = {
proto: set() for proto in self.sources
}
self.console = console or Console()
self.cookie_jar = DummyCookieJar()
self.regex = re.compile(
r"(?:^|\D)?("
+ r"(?:[1-9]|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])" # 1-255
@@ -155,23 +153,6 @@ class ProxyScraperChecker:
+ r"(?:\D|$)"
)
self.sort_by_speed = sort_by_speed
self.timeout = ClientTimeout(total=timeout, sock_connect=float("inf"))
self.source_timeout = source_timeout
self.sources = {
proto: frozenset(filter(None, sources.splitlines()))
for proto, sources in sources.items()
if sources
}
self.proxies: Dict[ProxyType, Set[Proxy]] = {
proto: set() for proto in self.sources
}
self.cookie_jar = DummyCookieJar()
self.console = console or Console()
max_connections = validate_max_connections(max_connections)
self.sem = asyncio.Semaphore(max_connections)
@classmethod
def from_configparser(
cls: Type[TProxyScraperChecker],
@@ -259,13 +240,24 @@ class ProxyScraperChecker:
status = response.status
text = await response.text()
except Exception as e:
logger.error(
"%s | %s.%s | %s",
source,
e.__class__.__module__,
e.__class__.__qualname__,
e,
e_str = str(e)
args: Tuple[object, ...] = (
(
"%s | %s.%s (%s)",
source,
e.__class__.__module__,
e.__class__.__qualname__,
e_str,
)
if e_str
else (
"%s | %s.%s",
source,
e.__class__.__module__,
e.__class__.__qualname__,
)
)
logger.error(*args)
else:
proxies = tuple(self.regex.finditer(text))
if proxies:
@@ -275,11 +267,12 @@ class ProxyScraperChecker:
)
self.proxies[proto].add(proxy_obj)
else:
logger.warning(
"%s | No proxies found | HTTP status code %d",
source,
status,
args = (
("%s | No proxies found", source)
if status == 200
else ("%s | HTTP status code %d", source, status)
)
logger.warning(*args)
progress.update(task, advance=1)
async def check_proxy(
proxy_scraper_checker/validators.py
+77
View File
@@ -0,0 +1,77 @@
from __future__ import annotations
import asyncio
import logging
import sys
from typing import Any, Iterable, Optional
from urllib.parse import urlparse
from .folder import Folder
logger = logging.getLogger(__name__)
def timeout(value: float) -> None:
if value <= 0:
msg = "Timeout must be positive"
raise ValueError(msg)
def source_timeout(value: float) -> None:
if value <= 0:
msg = "SourceTimeout must be positive"
raise ValueError(msg)
def max_connections(value: int) -> Optional[int]:
if value < 0:
msg = "MaxConnections must be non-negative"
raise ValueError(msg)
max_supported = _get_supported_max_connections()
if not value:
logger.info("Using %d as MaxConnections value", max_supported or 0)
return max_supported
if not max_supported or value <= max_supported:
return value
logger.warning(
"MaxConnections value is too high. "
+ "Your OS supports a maximum of %d. "
+ "The config value will be ignored and %d will be used.",
max_supported,
max_supported,
)
return max_supported
def _get_supported_max_connections() -> Optional[int]:
if sys.platform == "win32":
if isinstance(
asyncio.get_event_loop_policy(),
asyncio.WindowsSelectorEventLoopPolicy,
):
return 512
return None
import resource
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft_limit < hard_limit:
resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit))
return hard_limit
def check_website(value: str) -> None:
parsed_url = urlparse(value)
if not parsed_url.scheme or not parsed_url.netloc:
msg = f"invalid CheckWebsite URL: {value}"
raise ValueError(msg)
def folders(value: Iterable[Folder]) -> None:
if not any(folder for folder in value if folder.is_enabled):
msg = "all folders are disabled in the config"
raise ValueError(msg)
def sources(value: Any) -> None:
if not value:
raise ValueError("proxy sources list is empty")