docs: update README and config.toml for clarity and detail improvements
This commit is contained in:
@@ -11,7 +11,7 @@ Collect, test, and organize HTTP/SOCKS4/SOCKS5 proxies from multiple sources wit
|
||||
## ✨ Key Features
|
||||
|
||||
- **🔥 Blazing Performance** - Rust-powered async engine with configurable concurrency
|
||||
- **🌍 Rich Metadata** - ASN, geolocation, and response time data via offline MaxMind databases
|
||||
- **🌍 Rich Metadata** - ASN, country/city geolocation, and response time data via offline MaxMind databases
|
||||
- **🎯 Smart Parsing** - Advanced regex engine extracts proxies from any format (`protocol://user:pass@host:port`)
|
||||
- **🔐 Auth Support** - Handles username/password authentication seamlessly
|
||||
- **📊 Interactive TUI** - Real-time progress monitoring with beautiful terminal interface
|
||||
@@ -26,11 +26,11 @@ Get pre-checked proxies from [monosans/proxy-list](https://github.com/monosans/p
|
||||
|
||||
## ⚠️ SAFETY WARNING ⚠️
|
||||
|
||||
This tool makes many network requests and can impact your IP-address reputation. Consider using a VPN for safer operation.
|
||||
This tool makes thousands of concurrent network requests to test proxies, which may trigger rate limiting or security flags from your ISP or network provider. Consider using a VPN to protect your IP reputation.
|
||||
|
||||
## 🚀 Quick Start
|
||||
|
||||
> All configuration options are documented in `config.toml` - edit it to customize sources, timeouts, and output preferences.
|
||||
> All configuration options are documented in `config.toml` with detailed comments explaining each setting.
|
||||
|
||||
<details>
|
||||
<summary>💻 Binary Installation</summary>
|
||||
@@ -52,7 +52,7 @@ This tool makes many network requests and can impact your IP-address reputation.
|
||||
<details>
|
||||
<summary>🐳 Docker Installation</summary>
|
||||
|
||||
> **Note:** Docker version uses a simplified log-based interface (no TUI).
|
||||
> **Note:** Docker version outputs logs to stdout instead of the interactive TUI (terminal user interface).
|
||||
|
||||
1. **Install** [Docker Compose](https://docs.docker.com/compose/install/)
|
||||
|
||||
@@ -80,7 +80,7 @@ This tool makes many network requests and can impact your IP-address reputation.
|
||||
|
||||
> **Important:** Download Termux from [F-Droid](https://f-droid.org/en/packages/com.termux/), not Google Play ([why?](https://github.com/termux/termux-app#google-play-store-experimental-branch)).
|
||||
|
||||
1. **Auto-install** with one command:
|
||||
1. **Install** with one command:
|
||||
|
||||
```bash
|
||||
bash <(curl -fsSL 'https://raw.githubusercontent.com/monosans/proxy-scraper-checker/main/termux.sh')
|
||||
|
||||
+18
-17
@@ -1,19 +1,19 @@
|
||||
# Enable debug logging (shows detailed checking process)
|
||||
# Warning: Produces very verbose output
|
||||
# Enable debug logging to see detailed checking process for each proxy
|
||||
# Warning: Extremely verbose - logs every connection attempt and failure
|
||||
debug = false
|
||||
|
||||
|
||||
[scraping]
|
||||
# Maximum proxies to collect per source (0 = unlimited)
|
||||
# Helps skip unreliable sources with too many proxies
|
||||
# When this limit is reached, the source is skipped with a warning to prevent memory issues
|
||||
max_proxies_per_source = 100000
|
||||
|
||||
# Request timeout for fetching proxy sources (seconds)
|
||||
# Higher values may find more sources but take longer
|
||||
# Higher values allow slower sources to complete but increase total scraping time
|
||||
timeout = 60.0
|
||||
connect_timeout = 5.0
|
||||
|
||||
# HTTP(S),SOCKS4 or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable.
|
||||
# HTTP(S), SOCKS4, or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable.
|
||||
proxy = ""
|
||||
|
||||
# User-Agent header for scraping requests
|
||||
@@ -25,19 +25,19 @@ user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTM
|
||||
# httpbin-compatible: Returns JSON with IP info for ASN/geo data
|
||||
# plain-text: Returns just IP address for basic connectivity
|
||||
# Examples:
|
||||
# "https://httpbin.org/ip" - JSON with "origin" key. Full featured checking.
|
||||
# "https://api.ipify.org" - Simple IP return. Full featured checking.
|
||||
# "https://google.com" - Basic connect/read check
|
||||
# "" - Skip checking (scrape only)
|
||||
# "https://httpbin.org/ip" - JSON with "origin" key. Full-featured checking.
|
||||
# "https://api.ipify.org" - Simple IP return. Full-featured checking.
|
||||
# "https://google.com" - Basic connect/read check only
|
||||
# "" - Skip checking entirely (scrape only)
|
||||
check_url = "https://api.ipify.org"
|
||||
|
||||
# Concurrent proxy checks (adjust based on RAM/network capacity)
|
||||
# Start low and increase gradually if system handles it well
|
||||
# Number of proxies to check simultaneously
|
||||
# Higher values = faster checking but more RAM/network usage. Lower if you experience crashes or timeouts.
|
||||
max_concurrent_checks = 1024
|
||||
|
||||
# Proxy response timeout (seconds)
|
||||
# Higher = more working proxies found, slower checking
|
||||
# Lower = faster checking, may miss slower proxies
|
||||
# Higher = finds more slow proxies but takes longer
|
||||
# Lower = faster checking but discards slow proxies
|
||||
timeout = 60.0
|
||||
connect_timeout = 5.0
|
||||
|
||||
@@ -46,10 +46,11 @@ user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTM
|
||||
|
||||
|
||||
[output]
|
||||
# Output directory (Docker ignores this setting)
|
||||
# Output directory (Docker overrides this to /home/app/.local/share/proxy_scraper_checker via volume mount)
|
||||
path = "./out"
|
||||
|
||||
# Sort by response time (true) or IP address (false)
|
||||
# Sort proxies by response time (true) or by IP address (false)
|
||||
# Speed sorting: fastest proxies first. IP sorting: natural order by protocol, then IP, then port.
|
||||
sort_by_speed = true
|
||||
|
||||
|
||||
@@ -66,8 +67,8 @@ enabled = true
|
||||
# Uses offline MaxMind database
|
||||
include_asn = true
|
||||
|
||||
# Add geolocation (country/city) info to JSON output
|
||||
# Uses offline MaxMind database
|
||||
# Add geolocation (country/city) data to JSON output
|
||||
# Uses offline MaxMind GeoLite2 City database
|
||||
include_geolocation = true
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user