From 1e56188aa48af615a6a778e414a2e4fb2b53a1bd Mon Sep 17 00:00:00 2001
From: monosans <hsyqixco@protonmail.com>
Date: Fri, 10 Jan 2025 12:49:36 +0300
Subject: [PATCH] add proxies_per_source_limit  setting

---
 config.toml                       | 6 ++++++
 proxy_scraper_checker/scraper.py  | 9 +++++++--
 proxy_scraper_checker/settings.py | 4 ++++
 3 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/config.toml b/config.toml
index fd324ca..cf25776 100644
--- a/config.toml
+++ b/config.toml
@@ -6,6 +6,12 @@ timeout = 5
 # The number of seconds to wait for the proxies to be downloaded from the source.
 source_timeout = 15
 
+# Maximum number of proxies from a single source.
+# If the source has more proxies than this value,
+# then the source has a huge number of low-quality proxies and can be ignored.
+# Set to 0 to disable this limit.
+proxies_per_source_limit = 100000
+
 # Maximum concurrent connections.
 # Don't be in a hurry to set high values.
 # Make sure you have enough RAM first, gradually increasing the default value.
diff --git a/proxy_scraper_checker/scraper.py b/proxy_scraper_checker/scraper.py
index 3363dd9..2714986 100644
--- a/proxy_scraper_checker/scraper.py
+++ b/proxy_scraper_checker/scraper.py
@@ -30,6 +30,7 @@ async def scrape_one(
     progress: Progress,
     proto: ProxyType,
     session: ClientSession,
+    settings: Settings,
     source: str,
     storage: ProxyStorage,
     task: TaskID,
@@ -59,10 +60,13 @@ async def scrape_one(
         )
     else:
         counter.incr()
-        proxies = PROXY_REGEX.findall(text)
+        proxies = tuple(PROXY_REGEX.finditer(text))
         if not proxies:
             _logger.warning("%s | No proxies found", source)
-        elif len(proxies) <= 100_000:  # noqa: PLR2004
+        elif (
+            settings.proxies_per_source_limit
+            and len(proxies) > settings.proxies_per_source_limit
+        ):
             _logger.warning(
                 "%s has too many proxies (%d), skipping", source, len(proxies)
             )
@@ -112,6 +116,7 @@ async def scrape_all(
                 progress=progress,
                 proto=proto,
                 session=session,
+                settings=settings,
                 source=source,
                 storage=storage,
                 task=progress_tasks[proto],
diff --git a/proxy_scraper_checker/settings.py b/proxy_scraper_checker/settings.py
index f91be74..d663c7d 100644
--- a/proxy_scraper_checker/settings.py
+++ b/proxy_scraper_checker/settings.py
@@ -197,6 +197,9 @@ class Settings:
     )
     output_path: Path = attrs.field(converter=Path)
     output_txt: bool = attrs.field(validator=attrs.validators.instance_of(bool))
+    proxies_per_source_limit: int = attrs.field(
+        validator=attrs.validators.ge(0)
+    )
     real_ip: str | None = attrs.field(
         validator=attrs.validators.optional(attrs.validators.instance_of(str))
     )
@@ -321,6 +324,7 @@ class Settings:
             output_json=cfg["output"]["json"],
             output_path=output_path,
             output_txt=cfg["output"]["txt"],
+            proxies_per_source_limit=cfg["proxies_per_source_limit"],
             real_ip=real_ip,
             semaphore=cfg["max_connections"],
             sort_by_speed=cfg["sort_by_speed"],