From 8d16ec6fff5c4ce8d323e0d5acf5eba5df4b588f Mon Sep 17 00:00:00 2001 From: monosans Date: Sat, 25 Feb 2023 19:31:34 +0300 Subject: [PATCH] Add pre-commit --- .pre-commit-config.yaml | 44 ++++++++++ proxy_scraper_checker/__main__.py | 9 +- proxy_scraper_checker/constants.py | 4 +- proxy_scraper_checker/proxy.py | 4 +- .../proxy_scraper_checker.py | 87 +++++++------------ proxy_scraper_checker/validators.py | 14 +-- pyproject.toml | 78 +++++++++++++++++ start.cmd | 0 8 files changed, 169 insertions(+), 71 deletions(-) create mode 100644 .pre-commit-config.yaml create mode 100644 pyproject.toml mode change 100755 => 100644 start.cmd diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6c32d7b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,44 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-case-conflict + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-symlinks + - id: check-toml + - id: check-xml + - id: destroyed-symlinks + - id: end-of-file-fixer + - id: mixed-line-ending + args: [--fix=lf] + - id: trailing-whitespace + + - repo: https://github.com/monosans/pre-commit-prettier + rev: v2.8.4 + hooks: + - id: prettier + + - repo: https://github.com/psf/black + rev: 23.1.0 + hooks: + - id: black + + - repo: https://github.com/charliermarsh/ruff-pre-commit + rev: v0.0.252 + hooks: + - id: ruff + args: [--fix] + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.0.1 + hooks: + - id: mypy + args: [] + additional_dependencies: + - aiodns>=3.0,<4 + - aiohttp-socks>=0.7,<0.9 + - aiohttp>=3.8,<4 + - rich>=12.3,<14 + - uvloop>=0.16,<0.18; implementation_name == 'cpython' and (sys_platform == 'darwin' or sys_platform == 'linux') diff --git a/proxy_scraper_checker/__main__.py b/proxy_scraper_checker/__main__.py index 96de7e3..51fc632 100644 --- a/proxy_scraper_checker/__main__.py +++ b/proxy_scraper_checker/__main__.py @@ -15,10 +15,7 @@ from .proxy_scraper_checker import ProxyScraperChecker def set_event_loop_policy() -> None: if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) - elif sys.implementation.name == "cpython" and sys.platform in { - "darwin", - "linux", - }: + elif sys.implementation.name == "cpython" and sys.platform in {"darwin", "linux"}: try: import uvloop except ImportError: @@ -54,7 +51,9 @@ async def main() -> None: cfg = get_config("config.ini") console = Console() - configure_logging(console, debug=cfg["General"].getboolean("Debug", False)) + configure_logging( + console, debug=cfg["General"].getboolean("Debug", False) # noqa: FBT003 + ) psc = ProxyScraperChecker.from_configparser(cfg, console=console) await psc.run() diff --git a/proxy_scraper_checker/constants.py b/proxy_scraper_checker/constants.py index 41464c8..cb087c0 100644 --- a/proxy_scraper_checker/constants.py +++ b/proxy_scraper_checker/constants.py @@ -1,5 +1,3 @@ from __future__ import annotations -USER_AGENT = ( - "Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/109.0" -) +USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; rv:109.0) Gecko/20100101 Firefox/109.0" diff --git a/proxy_scraper_checker/proxy.py b/proxy_scraper_checker/proxy.py index 99f6987..cfd9844 100644 --- a/proxy_scraper_checker/proxy.py +++ b/proxy_scraper_checker/proxy.py @@ -41,9 +41,7 @@ class Proxy: cookie_jar=cookie_jar, timeout=timeout, headers=HEADERS, - ) as session, session.get( - website, raise_for_status=True - ) as response: + ) as session, session.get(website, raise_for_status=True) as response: if website == DEFAULT_CHECK_WEBSITE: await response.read() self.timeout = perf_counter() - start diff --git a/proxy_scraper_checker/proxy_scraper_checker.py b/proxy_scraper_checker/proxy_scraper_checker.py index f035418..e23fb0b 100644 --- a/proxy_scraper_checker/proxy_scraper_checker.py +++ b/proxy_scraper_checker/proxy_scraper_checker.py @@ -22,13 +22,7 @@ from typing import ( from aiohttp import ClientSession, ClientTimeout, DummyCookieJar from aiohttp_socks import ProxyType from rich.console import Console -from rich.progress import ( - BarColumn, - MofNCompleteColumn, - Progress, - TaskID, - TextColumn, -) +from rich.progress import BarColumn, MofNCompleteColumn, Progress, TaskID, TextColumn from rich.table import Table from . import sort, validators @@ -39,9 +33,7 @@ from .proxy import Proxy logger = logging.getLogger(__name__) -TProxyScraperChecker = TypeVar( - "TProxyScraperChecker", bound="ProxyScraperChecker" -) +TProxyScraperChecker = TypeVar("TProxyScraperChecker", bound="ProxyScraperChecker") class ProxyScraperChecker: @@ -121,8 +113,8 @@ class ProxyScraperChecker: validators.check_website(check_website) logger.info( "CheckWebsite is not 'default', " - + "so it will not be possible to determine " - + "the anonymity and geolocation of the proxies" + "so it will not be possible to determine " + "the anonymity and geolocation of the proxies" ) for folder in self.folders: folder.is_enabled = ( @@ -145,12 +137,12 @@ class ProxyScraperChecker: self.cookie_jar = DummyCookieJar() self.regex = re.compile( r"(?:^|\D)?(" - + r"(?:[1-9]|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])" # 1-255 + r"(?:[1-9]|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])" # 1-255 + r"\.(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])" * 3 # 0-255 + r"):" + ( r"(\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}" - + r"|65[0-4]\d{2}|655[0-2]\d|6553[0-5])" + r"|65[0-4]\d{2}|655[0-2]\d|6553[0-5])" ) # 0-65535 + r"(?:\D|$)" ) @@ -173,31 +165,35 @@ class ProxyScraperChecker: source_timeout=general.getfloat("SourceTimeout", 15), max_connections=general.getint("MaxConnections", 512), check_website=general.get("CheckWebsite", "default"), - sort_by_speed=general.getboolean("SortBySpeed", True), + sort_by_speed=general.getboolean("SortBySpeed", True), # noqa: FBT003 save_path=save_path, folders=( Folder( path=save_path / "proxies", - is_enabled=folders.getboolean("proxies", True), + is_enabled=folders.getboolean("proxies", True), # noqa: FBT003 for_anonymous=False, for_geolocation=False, ), Folder( path=save_path / "proxies_anonymous", - is_enabled=folders.getboolean("proxies_anonymous", True), + is_enabled=folders.getboolean( + "proxies_anonymous", True # noqa: FBT003 + ), for_anonymous=True, for_geolocation=False, ), Folder( path=save_path / "proxies_geolocation", - is_enabled=folders.getboolean("proxies_geolocation", True), + is_enabled=folders.getboolean( + "proxies_geolocation", True # noqa: FBT003 + ), for_anonymous=False, for_geolocation=True, ), Folder( path=save_path / "proxies_geolocation_anonymous", is_enabled=folders.getboolean( - "proxies_geolocation_anonymous", True + "proxies_geolocation_anonymous", True # noqa: FBT003 ), for_anonymous=True, for_geolocation=True, @@ -206,17 +202,17 @@ class ProxyScraperChecker: sources={ ProxyType.HTTP: ( http.get("Sources") - if http.getboolean("Enabled", True) + if http.getboolean("Enabled", True) # noqa: FBT003 else None ), ProxyType.SOCKS4: ( socks4.get("Sources") - if socks4.getboolean("Enabled", True) + if socks4.getboolean("Enabled", True) # noqa: FBT003 else None ), ProxyType.SOCKS5: ( socks5.get("Sources") - if socks5.getboolean("Enabled", True) + if socks5.getboolean("Enabled", True) # noqa: FBT003 else None ), }, @@ -264,26 +260,19 @@ class ProxyScraperChecker: proxies = tuple(self.regex.finditer(text)) if proxies: for proxy in proxies: - proxy_obj = Proxy( - host=proxy.group(1), port=int(proxy.group(2)) - ) + proxy_obj = Proxy(host=proxy.group(1), port=int(proxy.group(2))) self.proxies[proto].add(proxy_obj) else: args = ( ("%s | No proxies found", source) - if status == 200 + if status == 200 # noqa: PLR2004 else ("%s | HTTP status code %d", source, status) ) logger.warning(*args) progress.update(task, advance=1) async def check_proxy( - self, - *, - proxy: Proxy, - proto: ProxyType, - progress: Progress, - task: TaskID, + self, *, proxy: Proxy, proto: ProxyType, progress: Progress, task: TaskID ) -> None: """Check if proxy is alive.""" try: @@ -296,14 +285,11 @@ class ProxyScraperChecker: ) except Exception as e: # Too many open files - if isinstance(e, OSError) and e.errno == 24: + if isinstance(e, OSError) and e.errno == 24: # noqa: PLR2004 logger.error("Please, set MaxConnections to lower value.") logger.debug( - "%s.%s | %s", - e.__class__.__module__, - e.__class__.__qualname__, - e, + "%s.%s | %s", e.__class__.__module__, e.__class__.__qualname__, e ) self.proxies[proto].remove(proxy) progress.update(task, advance=1) @@ -311,8 +297,7 @@ class ProxyScraperChecker: async def fetch_all_sources(self, progress: Progress) -> None: tasks = { proto: progress.add_task( - f"[yellow]Scraper [red]:: [green]{proto.name}", - total=len(sources), + f"[yellow]Scraper [red]:: [green]{proto.name}", total=len(sources) ) for proto, sources in self.sources.items() } @@ -341,8 +326,7 @@ class ProxyScraperChecker: async def check_all_proxies(self, progress: Progress) -> None: tasks = { proto: progress.add_task( - f"[yellow]Checker [red]:: [green]{proto.name}", - total=len(proxies), + f"[yellow]Checker [red]:: [green]{proto.name}", total=len(proxies) ) for proto, proxies in self.proxies.items() } @@ -384,22 +368,19 @@ class ProxyScraperChecker: self.save_proxies() logger.info( - "Proxy folders have been created in the %s folder." - + "\nThank you for using proxy-scraper-checker :)", + ( + "Proxy folders have been created in the %s folder." + "\nThank you for using proxy-scraper-checker :)" + ), self.path.resolve(), ) def get_sorted_proxies(self) -> Dict[ProxyType, List[Proxy]]: - key: Union[ - Callable[[Proxy], float], Callable[[Proxy], Tuple[int, ...]] - ] = ( - sort.timeout_sort_key - if self.sort_by_speed - else sort.natural_sort_key + key: Union[Callable[[Proxy], float], Callable[[Proxy], Tuple[int, ...]]] = ( + sort.timeout_sort_key if self.sort_by_speed else sort.natural_sort_key ) return { - proto: sorted(proxies, key=key) - for proto, proxies in self.proxies.items() + proto: sorted(proxies, key=key) for proto, proxies in self.proxies.items() } def _get_results_table(self) -> Table: @@ -411,9 +392,7 @@ class ProxyScraperChecker: working = len(proxies) total = self.proxies_count[proto] percentage = working / total if total else 0 - table.add_row( - proto.name, f"{working} ({percentage:.1%})", str(total) - ) + table.add_row(proto.name, f"{working} ({percentage:.1%})", str(total)) return table def _get_progress_bar(self) -> Progress: diff --git a/proxy_scraper_checker/validators.py b/proxy_scraper_checker/validators.py index 2705cec..3fdbad3 100644 --- a/proxy_scraper_checker/validators.py +++ b/proxy_scraper_checker/validators.py @@ -34,9 +34,11 @@ def max_connections(value: int) -> Optional[int]: if not max_supported or value <= max_supported: return value logger.warning( - "MaxConnections value is too high. " - + "Your OS supports a maximum of %d. " - + "The config value will be ignored and %d will be used.", + ( + "MaxConnections value is too high. " + "Your OS supports a maximum of %d. " + "The config value will be ignored and %d will be used." + ), max_supported, max_supported, ) @@ -46,8 +48,7 @@ def max_connections(value: int) -> Optional[int]: def _get_supported_max_connections() -> Optional[int]: if sys.platform == "win32": if isinstance( - asyncio.get_event_loop_policy(), - asyncio.WindowsSelectorEventLoopPolicy, + asyncio.get_event_loop_policy(), asyncio.WindowsSelectorEventLoopPolicy ): return 512 return None @@ -74,4 +75,5 @@ def folders(value: Iterable[Folder]) -> None: def sources(value: Any) -> None: if not value: - raise ValueError("proxy sources list is empty") + msg = "proxy sources list is empty" + raise ValueError(msg) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4ebe9d7 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,78 @@ +[tool.black] +target-version = ["py37"] +skip-magic-trailing-comma = true +preview = true + +[tool.mypy] +ignore_missing_imports = true +python_version = "3.7" +disallow_subclassing_any = false +disallow_untyped_calls = false +disallow_untyped_decorators = false +warn_unreachable = true +local_partial_types = true +enable_error_code = [ + "redundant-self", + "redundant-expr", + "truthy-bool", + "truthy-iterable", + "ignore-without-code", + "unused-awaitable", +] +strict = true + +[tool.ruff] +target-version = "py37" +select = ["ALL"] +ignore = [ + "ANN", + "B008", + "BLE001", + "C901", + "COM", + "D100", + "D101", + "D102", + "D103", + "D104", + "D105", + "D106", + "D107", + "D203", + "D205", + "D212", + "D213", + "D400", + "D407", + "D415", + "D417", + "DJ008", + "ERA001", + "PD901", + "PLR0911", + "PLR0912", + "PLR0913", + "PLR0915", + "PT012", + "RUF001", + "RUF002", + "RUF003", + "S110", + "S112", + "SIM105", + "TCH001", + "TCH002", + "TCH003", + "TID252", + "TRY400", +] + +[tool.ruff.flake8-unused-arguments] +ignore-variadic-names = true + +[tool.ruff.isort] +combine-as-imports = true +required-imports = ["from __future__ import annotations"] + +[tool.ruff.pyupgrade] +keep-runtime-typing = true diff --git a/start.cmd b/start.cmd old mode 100755 new mode 100644