diff --git a/README.md b/README.md index 16028df..7233a94 100644 --- a/README.md +++ b/README.md @@ -9,8 +9,6 @@ HTTP, SOCKS4, SOCKS5 proxies scraper and checker. - Supports determining the geolocation of the proxy exit node. - Can determine if a proxy is anonymous. -For a version that uses Python's built-in `logging` instead of [rich](https://github.com/willmcgugan/rich), see the [simple-output](https://github.com/monosans/proxy-scraper-checker/tree/simple-output) branch. - You can get proxies obtained using this script in [monosans/proxy-list](https://github.com/monosans/proxy-list). ## Installation and usage @@ -24,8 +22,8 @@ You can get proxies obtained using this script in [monosans/proxy-list](https:// - On Unix-like OS run `start.sh` - Manual: 1. `cd` into the unpacked folder - 1. Install dependencies with the command `python -m pip install -U -r requirements.txt` - 1. Run with the command `python main.py` + 1. Install dependencies with the command `python -m pip install -U --no-cache-dir --disable-pip-version-check pip setuptools wheel; python -m pip install -U --no-cache-dir --disable-pip-version-check -r requirements.txt` + 1. Run with the command `python -m proxy_scraper_checker` ## Folders description diff --git a/proxy_scraper_checker/__init__.py b/proxy_scraper_checker/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py new file mode 100644 index 0000000..5d0396a --- /dev/null +++ b/proxy_scraper_checker/__main__.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import asyncio +import logging +import sys + +import rich.traceback +from rich.console import Console +from rich.logging import RichHandler + + +def install_uvloop() -> None: + if sys.implementation.name == "cpython" and sys.platform in { + "darwin", + "linux", + }: + try: + import uvloop + except ImportError: + pass + else: + uvloop.install() + + +def setup_logging(console: Console) -> None: + rich.traceback.install(console=console) + logging.basicConfig( + level=logging.INFO, + format="%(message)s", + handlers=[ + RichHandler(console=console, show_path=False, rich_tracebacks=True) + ], + ) + + +if __name__ == "__main__": + install_uvloop() + + console = Console() + setup_logging(console) + + from .proxy_scraper_checker import ProxyScraperChecker + + psc = ProxyScraperChecker.from_ini("config.ini", console=console) + asyncio.run(psc.run()) diff --git a/proxy_scraper_checker/folder.py b/proxy_scraper_checker/folder.py new file mode 100644 index 0000000..9512ec7 --- /dev/null +++ b/proxy_scraper_checker/folder.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from pathlib import Path +from shutil import rmtree + + +class Folder: + __slots__ = ("for_anonymous", "for_geolocation", "path") + + def __init__(self, *, path: Path, folder_name: str) -> None: + self.path = path / folder_name + self.for_anonymous = "anon" in folder_name + self.for_geolocation = "geo" in folder_name + + def remove(self) -> None: + try: + rmtree(self.path) + except FileNotFoundError: + pass + + def create(self) -> None: + self.path.mkdir(parents=True, exist_ok=True) diff --git a/proxy_scraper_checker/proxy.py b/proxy_scraper_checker/proxy.py new file mode 100644 index 0000000..a117719 --- /dev/null +++ b/proxy_scraper_checker/proxy.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import asyncio +from time import perf_counter + +from aiohttp import ClientSession +from aiohttp_socks import ProxyConnector + + +class Proxy: + __slots__ = ( + "geolocation", + "ip", + "is_anonymous", + "socket_address", + "timeout", + ) + + def __init__(self, *, socket_address: str, ip: str) -> None: + """ + Args: + socket_address: ip:port + """ + self.socket_address = socket_address + self.ip = ip + + async def check( + self, *, sem: asyncio.Semaphore, proto: str, timeout: float + ) -> None: + async with sem: + proxy_url = f"{proto}://{self.socket_address}" + start = perf_counter() + async with ProxyConnector.from_url(proxy_url) as connector: + async with ClientSession(connector=connector) as session: + async with session.get( + "http://ip-api.com/json/?fields=8217", + timeout=timeout, + raise_for_status=True, + ) as response: + data = await response.json() + self.timeout = perf_counter() - start + self.is_anonymous = self.ip != data["query"] + self.geolocation = "|{}|{}|{}".format( + data["country"], data["regionName"], data["city"] + ) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Proxy): + return NotImplemented + return self.socket_address == other.socket_address + + def __hash__(self) -> int: + return hash(self.socket_address) diff --git a/main.py b/proxy_scraper_checker/proxy_scraper_checker.py old mode 100755 new mode 100644 similarity index 65% rename from main.py rename to proxy_scraper_checker/proxy_scraper_checker.py index 3c49b58..74dbfa9 --- a/main.py +++ b/proxy_scraper_checker/proxy_scraper_checker.py @@ -1,100 +1,43 @@ -#!/usr/bin/env python3 from __future__ import annotations import asyncio +import logging import re -import sys from configparser import ConfigParser from pathlib import Path from random import shuffle -from shutil import rmtree -from time import perf_counter -from typing import Callable, Dict, List, Optional, Set, Tuple, Union +from typing import ( + Callable, + Dict, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, +) from aiohttp import ClientSession -from aiohttp_socks import ProxyConnector from rich.console import Console from rich.progress import ( BarColumn, MofNCompleteColumn, Progress, TaskID, - TaskProgressColumn, TextColumn, ) from rich.table import Table +from . import sort +from .folder import Folder +from .proxy import Proxy -class Proxy: - __slots__ = ( - "geolocation", - "ip", - "is_anonymous", - "socket_address", - "timeout", - ) +logger = logging.getLogger(__name__) - def __init__(self, *, socket_address: str, ip: str) -> None: - """ - Args: - socket_address: ip:port - """ - self.socket_address = socket_address - self.ip = ip - - async def check( - self, *, sem: asyncio.Semaphore, proto: str, timeout: float - ) -> None: - async with sem: - proxy_url = f"{proto}://{self.socket_address}" - start = perf_counter() - async with ProxyConnector.from_url(proxy_url) as connector: - async with ClientSession(connector=connector) as session: - async with session.get( - "http://ip-api.com/json/?fields=8217", - timeout=timeout, - raise_for_status=True, - ) as response: - data = await response.json() - self.timeout = perf_counter() - start - self.is_anonymous = self.ip != data["query"] - self.geolocation = "|{}|{}|{}".format( - data["country"], data["regionName"], data["city"] - ) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Proxy): - return NotImplemented - return self.socket_address == other.socket_address - - def __hash__(self) -> int: - return hash(self.socket_address) - - -class Folder: - __slots__ = ("for_anonymous", "for_geolocation", "path") - - def __init__(self, *, path: Path, folder_name: str) -> None: - self.path = path / folder_name - self.for_anonymous = "anon" in folder_name - self.for_geolocation = "geo" in folder_name - - def remove(self) -> None: - try: - rmtree(self.path) - except FileNotFoundError: - pass - - def create(self) -> None: - self.path.mkdir(parents=True, exist_ok=True) - - -def timeout_sort_key(proxy: Proxy) -> float: - return proxy.timeout - - -def natural_sort_key(proxy: Proxy) -> Tuple[int, ...]: - return tuple(map(int, proxy.socket_address.replace(":", ".").split("."))) +TProxyScraperChecker = TypeVar( + "TProxyScraperChecker", bound="ProxyScraperChecker" +) class ProxyScraperChecker: @@ -197,6 +140,45 @@ class ProxyScraperChecker: self.console = console or Console() self.sem = asyncio.Semaphore(max_connections) + @classmethod + def from_ini( + cls: Type[TProxyScraperChecker], + file_name: str, + *, + console: Optional[Console] = None, + ) -> TProxyScraperChecker: + cfg = ConfigParser(interpolation=None) + cfg.read(file_name, encoding="utf-8") + general = cfg["General"] + folders = cfg["Folders"] + http = cfg["HTTP"] + socks4 = cfg["SOCKS4"] + socks5 = cfg["SOCKS5"] + return cls( + timeout=general.getfloat("Timeout", 10), + max_connections=general.getint("MaxConnections", 900), + sort_by_speed=general.getboolean("SortBySpeed", True), + save_path=general.get("SavePath", ""), + proxies=folders.getboolean("proxies", True), + proxies_anonymous=folders.getboolean("proxies_anonymous", True), + proxies_geolocation=folders.getboolean( + "proxies_geolocation", True + ), + proxies_geolocation_anonymous=folders.getboolean( + "proxies_geolocation_anonymous", True + ), + http_sources=http.get("Sources") + if http.getboolean("Enabled", True) + else None, + socks4_sources=socks4.get("Sources") + if socks4.getboolean("Enabled", True) + else None, + socks5_sources=socks5.get("Sources") + if socks5.getboolean("Enabled", True) + else None, + console=console, + ) + async def fetch_source( self, *, @@ -218,11 +200,11 @@ class ProxyScraperChecker: status = response.status text = await response.text() except Exception as e: - msg = f"{source} | Error" + msg = source exc_str = str(e) if exc_str and exc_str != source: msg += f": {exc_str}" - self.console.print(msg) + logger.error(msg) else: proxies = tuple(self.regex.finditer(text)) if proxies: @@ -235,7 +217,7 @@ class ProxyScraperChecker: msg = f"{source} | No proxies found" if status != 200: msg += f" | Status code {status}" - self.console.print(msg) + logger.warning(msg) progress.update(task, advance=1) async def check_proxy( @@ -247,9 +229,7 @@ class ProxyScraperChecker: except Exception as e: # Too many open files if isinstance(e, OSError) and e.errno == 24: - self.console.print( - "[red]Please, set MAX_CONNECTIONS to lower value." - ) + logging.error("Please, set MAX_CONNECTIONS to lower value.") self.proxies[proto].remove(proxy) progress.update(task, advance=1) @@ -306,7 +286,7 @@ class ProxyScraperChecker: def save_proxies(self) -> None: """Delete old proxies and save new ones.""" - sorted_proxies = self.sorted_proxies.items() + sorted_proxies = self.get_sorted_proxies().items() for folder in self.all_folders: folder.remove() for folder in self.enabled_folders: @@ -322,11 +302,35 @@ class ProxyScraperChecker: file = folder.path / f"{proto}.txt" file.write_text(text, encoding="utf-8") - async def main(self) -> None: - with self._progress as progress: + async def run(self) -> None: + with self._get_progress_bar() as progress: await self.fetch_all_sources(progress) await self.check_all_proxies(progress) + table = self._get_results_table() + self.console.print(table) + + self.save_proxies() + logger.info( + "Proxy folders have been created in the %s folder." + + "\nThank you for using proxy-scraper-checker :)", + self.path.resolve(), + ) + + def get_sorted_proxies(self) -> Dict[str, List[Proxy]]: + key: Union[ + Callable[[Proxy], float], Callable[[Proxy], Tuple[int, ...]] + ] = ( + sort.timeout_sort_key + if self.sort_by_speed + else sort.natural_sort_key + ) + return { + proto: sorted(proxies, key=key) + for proto, proxies in self.proxies.items() + } + + def _get_results_table(self) -> Table: table = Table() table.add_column("Protocol", style="cyan") table.add_column("Working", style="magenta") @@ -338,76 +342,12 @@ class ProxyScraperChecker: table.add_row( proto.upper(), f"{working} ({percentage:.1f}%)", str(total) ) - self.console.print(table) + return table - self.save_proxies() - self.console.print( - "[green]Proxy folders have been created in the " - + f"{self.path.resolve()} folder." - + "\nThank you for using proxy-scraper-checker :)" - ) - - @property - def sorted_proxies(self) -> Dict[str, List[Proxy]]: - key: Union[ - Callable[[Proxy], float], Callable[[Proxy], Tuple[int, ...]] - ] = (timeout_sort_key if self.sort_by_speed else natural_sort_key) - return { - proto: sorted(proxies, key=key) - for proto, proxies in self.proxies.items() - } - - @property - def _progress(self) -> Progress: + def _get_progress_bar(self) -> Progress: return Progress( TextColumn("[progress.description]{task.description}"), BarColumn(), - TaskProgressColumn(), MofNCompleteColumn(), console=self.console, ) - - -async def main() -> None: - cfg = ConfigParser(interpolation=None) - cfg.read("config.ini", encoding="utf-8") - general = cfg["General"] - folders = cfg["Folders"] - http = cfg["HTTP"] - socks4 = cfg["SOCKS4"] - socks5 = cfg["SOCKS5"] - await ProxyScraperChecker( - timeout=general.getfloat("Timeout", 10), - max_connections=general.getint("MaxConnections", 900), - sort_by_speed=general.getboolean("SortBySpeed", True), - save_path=general.get("SavePath", ""), - proxies=folders.getboolean("proxies", True), - proxies_anonymous=folders.getboolean("proxies_anonymous", True), - proxies_geolocation=folders.getboolean("proxies_geolocation", True), - proxies_geolocation_anonymous=folders.getboolean( - "proxies_geolocation_anonymous", True - ), - http_sources=http.get("Sources") - if http.getboolean("Enabled", True) - else None, - socks4_sources=socks4.get("Sources") - if socks4.getboolean("Enabled", True) - else None, - socks5_sources=socks5.get("Sources") - if socks5.getboolean("Enabled", True) - else None, - ).main() - - -if __name__ == "__main__": - if sys.implementation.name == "cpython" and sys.platform in { - "darwin", - "linux", - }: - try: - import uvloop - except ImportError: - pass - else: - uvloop.install() - asyncio.run(main()) diff --git a/proxy_scraper_checker/sort.py b/proxy_scraper_checker/sort.py new file mode 100644 index 0000000..d542d8f --- /dev/null +++ b/proxy_scraper_checker/sort.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +from typing import Tuple + +from .proxy import Proxy + + +def timeout_sort_key(proxy: Proxy) -> float: + return proxy.timeout + + +def natural_sort_key(proxy: Proxy) -> Tuple[int, ...]: + return tuple(map(int, proxy.socket_address.replace(":", ".").split("."))) diff --git a/screenshot.png b/screenshot.png index c4580a9..31e2cee 100644 Binary files a/screenshot.png and b/screenshot.png differ diff --git a/start.cmd b/start.cmd index b5f75c6..baf290f 100755 --- a/start.cmd +++ b/start.cmd @@ -1,3 +1,4 @@ -python -m pip install -U -r requirements.txt -python main.py +python -m pip install -U --no-cache-dir --disable-pip-version-check pip setuptools wheel +python -m pip install -U --no-cache-dir --disable-pip-version-check -r requirements.txt +python -m proxy_scraper_checker Pause diff --git a/start.sh b/start.sh index e3fb8c7..41780cf 100755 --- a/start.sh +++ b/start.sh @@ -1,3 +1,2 @@ #!/bin/sh -python -m pip install -U -r requirements.txt -python main.py +python -m pip install -U --no-cache-dir --disable-pip-version-check pip setuptools wheel && python -m pip install -U --no-cache-dir --disable-pip-version-check -r requirements.txt && python -m proxy_scraper_checker