diff --git a/README.md b/README.md index bc51d4e..5d4ff37 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Collect, test, and organize HTTP/SOCKS4/SOCKS5 proxies from multiple sources wit ## ✨ Key Features - **🔥 Blazing Performance** - Rust-powered async engine with configurable concurrency -- **🌍 Rich Metadata** - ASN, geolocation, and response time data via offline MaxMind databases +- **🌍 Rich Metadata** - ASN, country/city geolocation, and response time data via offline MaxMind databases - **🎯 Smart Parsing** - Advanced regex engine extracts proxies from any format (`protocol://user:pass@host:port`) - **🔐 Auth Support** - Handles username/password authentication seamlessly - **📊 Interactive TUI** - Real-time progress monitoring with beautiful terminal interface @@ -26,11 +26,11 @@ Get pre-checked proxies from [monosans/proxy-list](https://github.com/monosans/p ## ⚠️ SAFETY WARNING ⚠️ -This tool makes many network requests and can impact your IP-address reputation. Consider using a VPN for safer operation. +This tool makes thousands of concurrent network requests to test proxies, which may trigger rate limiting or security flags from your ISP or network provider. Consider using a VPN to protect your IP reputation. ## 🚀 Quick Start -> All configuration options are documented in `config.toml` - edit it to customize sources, timeouts, and output preferences. +> All configuration options are documented in `config.toml` with detailed comments explaining each setting.
💻 Binary Installation @@ -52,7 +52,7 @@ This tool makes many network requests and can impact your IP-address reputation.
🐳 Docker Installation -> **Note:** Docker version uses a simplified log-based interface (no TUI). +> **Note:** Docker version outputs logs to stdout instead of the interactive TUI (terminal user interface). 1. **Install** [Docker Compose](https://docs.docker.com/compose/install/) @@ -80,7 +80,7 @@ This tool makes many network requests and can impact your IP-address reputation. > **Important:** Download Termux from [F-Droid](https://f-droid.org/en/packages/com.termux/), not Google Play ([why?](https://github.com/termux/termux-app#google-play-store-experimental-branch)). -1. **Auto-install** with one command: +1. **Install** with one command: ```bash bash <(curl -fsSL 'https://raw.githubusercontent.com/monosans/proxy-scraper-checker/main/termux.sh') diff --git a/config.toml b/config.toml index 9411e2f..1ed1ab0 100644 --- a/config.toml +++ b/config.toml @@ -1,19 +1,19 @@ -# Enable debug logging (shows detailed checking process) -# Warning: Produces very verbose output +# Enable debug logging to see detailed checking process for each proxy +# Warning: Extremely verbose - logs every connection attempt and failure debug = false [scraping] # Maximum proxies to collect per source (0 = unlimited) -# Helps skip unreliable sources with too many proxies +# When this limit is reached, the source is skipped with a warning to prevent memory issues max_proxies_per_source = 100000 # Request timeout for fetching proxy sources (seconds) -# Higher values may find more sources but take longer +# Higher values allow slower sources to complete but increase total scraping time timeout = 60.0 connect_timeout = 5.0 -# HTTP(S),SOCKS4 or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable. +# HTTP(S), SOCKS4, or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable. proxy = "" # User-Agent header for scraping requests @@ -25,19 +25,19 @@ user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTM # httpbin-compatible: Returns JSON with IP info for ASN/geo data # plain-text: Returns just IP address for basic connectivity # Examples: -# "https://httpbin.org/ip" - JSON with "origin" key. Full featured checking. -# "https://api.ipify.org" - Simple IP return. Full featured checking. -# "https://google.com" - Basic connect/read check -# "" - Skip checking (scrape only) +# "https://httpbin.org/ip" - JSON with "origin" key. Full-featured checking. +# "https://api.ipify.org" - Simple IP return. Full-featured checking. +# "https://google.com" - Basic connect/read check only +# "" - Skip checking entirely (scrape only) check_url = "https://api.ipify.org" -# Concurrent proxy checks (adjust based on RAM/network capacity) -# Start low and increase gradually if system handles it well +# Number of proxies to check simultaneously +# Higher values = faster checking but more RAM/network usage. Lower if you experience crashes or timeouts. max_concurrent_checks = 1024 # Proxy response timeout (seconds) -# Higher = more working proxies found, slower checking -# Lower = faster checking, may miss slower proxies +# Higher = finds more slow proxies but takes longer +# Lower = faster checking but discards slow proxies timeout = 60.0 connect_timeout = 5.0 @@ -46,10 +46,11 @@ user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTM [output] -# Output directory (Docker ignores this setting) +# Output directory (Docker overrides this to /home/app/.local/share/proxy_scraper_checker via volume mount) path = "./out" -# Sort by response time (true) or IP address (false) +# Sort proxies by response time (true) or by IP address (false) +# Speed sorting: fastest proxies first. IP sorting: natural order by protocol, then IP, then port. sort_by_speed = true @@ -66,8 +67,8 @@ enabled = true # Uses offline MaxMind database include_asn = true -# Add geolocation (country/city) info to JSON output -# Uses offline MaxMind database +# Add geolocation (country/city) data to JSON output +# Uses offline MaxMind GeoLite2 City database include_geolocation = true