147 lines
6.1 KiB
TOML
147 lines
6.1 KiB
TOML
# Enable debug logging to see detailed checking process for each proxy
|
|
# Warning: Extremely verbose - logs every connection attempt and failure
|
|
debug = false
|
|
|
|
|
|
[scraping]
|
|
# Maximum proxies to collect per source (0 = unlimited)
|
|
# When this limit is reached, the source is skipped with a warning to prevent memory issues
|
|
max_proxies_per_source = 100000
|
|
|
|
# Request timeout for fetching proxy sources (seconds)
|
|
# Higher values allow slower sources to complete but increase total scraping time
|
|
timeout = 60.0
|
|
connect_timeout = 5.0
|
|
|
|
# HTTP(S), SOCKS4, or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable.
|
|
proxy = ""
|
|
|
|
# User-Agent header for scraping requests
|
|
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"
|
|
|
|
|
|
[checking]
|
|
# URL for checking proxy functionality
|
|
# httpbin-compatible: Returns JSON with IP info for ASN/geo data
|
|
# plain-text: Returns just IP address for basic connectivity
|
|
# Examples:
|
|
# "https://httpbin.org/ip" - JSON with "origin" key. Full-featured checking.
|
|
# "https://api.ipify.org" - Simple IP return. Full-featured checking.
|
|
# "https://google.com" - Basic connect/read check only
|
|
# "" - Skip checking entirely (scrape only)
|
|
check_url = "https://api.ipify.org"
|
|
|
|
# Number of proxies to check simultaneously
|
|
# Higher values = faster checking but more RAM/network usage. Lower if you experience crashes or timeouts.
|
|
max_concurrent_checks = 1024
|
|
|
|
# Proxy response timeout (seconds)
|
|
# Higher = finds more slow proxies but takes longer
|
|
# Lower = faster checking but discards slow proxies
|
|
timeout = 60.0
|
|
connect_timeout = 5.0
|
|
|
|
# User-Agent header for proxy check requests
|
|
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36"
|
|
|
|
|
|
[output]
|
|
# Output directory (Docker overrides this to /home/app/.local/share/proxy_scraper_checker via volume mount)
|
|
path = "./out"
|
|
|
|
# Sort proxies by response time (true) or by IP address (false)
|
|
# Speed sorting: fastest proxies first. IP sorting: natural order by protocol, then IP, then port.
|
|
sort_by_speed = true
|
|
|
|
|
|
# Plain text output (.txt files)
|
|
[output.txt]
|
|
enabled = true
|
|
|
|
|
|
# JSON output with metadata (.json files)
|
|
[output.json]
|
|
enabled = true
|
|
|
|
# Add ASN (network provider) info to JSON output
|
|
# Uses offline MaxMind database
|
|
include_asn = true
|
|
|
|
# Add geolocation (country/city) data to JSON output
|
|
# Uses offline MaxMind GeoLite2 City database
|
|
include_geolocation = true
|
|
|
|
|
|
# Proxy sources configuration
|
|
# Add local files: ["./my_proxies.txt"] or URLs
|
|
# Sources are fetched in parallel for speed
|
|
|
|
[scraping.http]
|
|
enabled = true
|
|
urls = [
|
|
# Local file examples:
|
|
# "./my_http_proxies.txt",
|
|
# "/home/user/my_http_proxies.txt",
|
|
# "C:/Users/user/Desktop/my_http_proxies.txt",
|
|
# "file:///home/user/my_http_proxies.txt",
|
|
|
|
# Advanced URL configuration examples (with basic auth or custom headers):
|
|
# HTTP Basic Auth example:
|
|
# { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } },
|
|
# Custom headers example:
|
|
# { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },
|
|
|
|
"https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=http",
|
|
"https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=https",
|
|
"https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/http/data.txt",
|
|
"https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/https/data.txt",
|
|
"https://raw.githubusercontent.com/roosterkid/openproxylist/refs/heads/main/HTTPS_RAW.txt",
|
|
"https://raw.githubusercontent.com/sunny9577/proxy-scraper/refs/heads/master/generated/http_proxies.txt",
|
|
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/http.txt",
|
|
]
|
|
|
|
[scraping.socks4]
|
|
enabled = true
|
|
urls = [
|
|
# Local file examples:
|
|
# "./my_socks4_proxies.txt",
|
|
# "/home/user/my_socks4_proxies.txt",
|
|
# "C:/Users/user/Desktop/my_socks4_proxies.txt",
|
|
# "file:///home/user/my_socks4_proxies.txt",
|
|
|
|
# Advanced URL configuration examples (with basic auth or custom headers):
|
|
# HTTP Basic Auth example:
|
|
# { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } },
|
|
# Custom headers example:
|
|
# { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },
|
|
|
|
"https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=socks4",
|
|
"https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/socks4/data.txt",
|
|
"https://raw.githubusercontent.com/roosterkid/openproxylist/refs/heads/main/SOCKS4_RAW.txt",
|
|
"https://raw.githubusercontent.com/sunny9577/proxy-scraper/refs/heads/master/generated/socks4_proxies.txt",
|
|
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/socks4.txt",
|
|
]
|
|
|
|
[scraping.socks5]
|
|
enabled = true
|
|
urls = [
|
|
# Local file examples:
|
|
# "./my_socks5_proxies.txt",
|
|
# "/home/user/my_socks5_proxies.txt",
|
|
# "C:/Users/user/Desktop/my_socks5_proxies.txt",
|
|
# "file:///home/user/my_socks5_proxies.txt",
|
|
|
|
# Advanced URL configuration examples (with basic auth or custom headers):
|
|
# HTTP Basic Auth example:
|
|
# { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } },
|
|
# Custom headers example:
|
|
# { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },
|
|
|
|
"https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=socks5",
|
|
"https://raw.githubusercontent.com/hookzof/socks5_list/refs/heads/master/proxy.txt",
|
|
"https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/socks5/data.txt",
|
|
"https://raw.githubusercontent.com/roosterkid/openproxylist/refs/heads/main/SOCKS5_RAW.txt",
|
|
"https://raw.githubusercontent.com/sunny9577/proxy-scraper/refs/heads/master/generated/socks5_proxies.txt",
|
|
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/socks5.txt",
|
|
]
|