diff --git a/config.toml b/config.toml index c4cb3a9..5421f55 100644 --- a/config.toml +++ b/config.toml @@ -7,8 +7,9 @@ debug = false [output] # Output directory (Docker overrides this to /app/out via volume mount) path = "./out" + # Sort proxies by response time (true) or by IP address (false) -# Speed sorting: fastest proxies first. IP sorting: natural order by protocol, then IP, then port. +# Speed sorting: fastest proxies first. IP sorting: natural order by protocol, IP and port. sort_by_speed = true # Plain text output (.txt files) @@ -18,11 +19,13 @@ enabled = true # JSON output with metadata (.json files) [output.json] enabled = true + # Add ASN (network provider) info to JSON output -# Uses offline MaxMind database +# Uses offline database include_asn = true -# Add geolocation (country/city) data to JSON output -# Uses offline MaxMind GeoLite2 City database + +# Add geolocation data to JSON output +# Uses offline database include_geolocation = true [checking] @@ -35,14 +38,17 @@ include_geolocation = true # "https://google.com" - Basic connect/read check only # "" - Skip checking entirely (scrape only) check_url = "https://api.ipify.org" + # Number of proxies to check simultaneously # Higher values = faster checking but more RAM/network usage. Lower if you experience crashes or timeouts. max_concurrent_checks = 512 + # Proxy response timeout (seconds) # Higher = finds more slow proxies but takes longer # Lower = faster checking but discards slow proxies timeout = 10.0 connect_timeout = 5.0 + # User-Agent header for proxy check requests user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36" @@ -50,19 +56,19 @@ user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTM # Maximum proxies to collect per source (0 = unlimited) # When this limit is reached, the source is skipped with a warning to prevent memory issues max_proxies_per_source = 100000 + # Request timeout for fetching proxy sources (seconds) # Higher values allow slower sources to complete but increase total scraping time timeout = 10.0 connect_timeout = 5.0 + # HTTP(S), SOCKS4, or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable. proxy = "" + # User-Agent header for scraping requests user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36" # Proxy sources configuration -# Add local files: ["./my_proxies.txt"] or URLs -# Sources are fetched in parallel for speed - [scraping.http] enabled = true urls = [ @@ -72,7 +78,7 @@ urls = [ # "C:/Users/user/Desktop/my_http_proxies.txt", # "file:///home/user/my_http_proxies.txt", - # Advanced URL configuration examples (with basic auth or custom headers): + # Advanced configuration examples (with basic auth or custom headers): # HTTP Basic Auth example: # { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } }, # Custom headers example: @@ -96,7 +102,7 @@ urls = [ # "C:/Users/user/Desktop/my_socks4_proxies.txt", # "file:///home/user/my_socks4_proxies.txt", - # Advanced URL configuration examples (with basic auth or custom headers): + # Advanced configuration examples (with basic auth or custom headers): # HTTP Basic Auth example: # { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } }, # Custom headers example: @@ -118,7 +124,7 @@ urls = [ # "C:/Users/user/Desktop/my_socks5_proxies.txt", # "file:///home/user/my_socks5_proxies.txt", - # Advanced URL configuration examples (with basic auth or custom headers): + # Advanced configuration examples (with basic auth or custom headers): # HTTP Basic Auth example: # { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } }, # Custom headers example: