haeaeae/proxy-scraper-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add config schema

Browse Source
This commit is contained in:
monosans
2026-02-27 11:04:05 +03:00
parent c35a33d582
commit 5c7e5a3e34
2 changed files with 152 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config-schema.json
+117
View File
@@ -0,0 +1,117 @@
{
"$id": "https://github.com/monosans/proxy-scraper-checker/blob/main/config.toml",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "proxy-scraper-checker config",
"type": "object",
"$defs": {
"protocol_config": {
"type": "object",
"properties": {
"enabled": { "type": "boolean" },
"urls": {
"type": "array",
"items": {
"anyOf": [
{ "type": "string" },
{
"type": "object",
"properties": {
"url": { "type": "string" },
"basic_auth": {
"type": "object",
"properties": {
"username": { "type": "string" },
"password": { "type": "string" }
},
"required": ["username", "password"],
"additionalProperties": false
},
"headers": {
"type": "object",
"patternProperties": { ".*": { "type": "string" } },
"additionalProperties": false
}
},
"required": ["url"],
"additionalProperties": false,
"x-tombi-array-values-order-by": "url"
}
]
},
"x-tombi-array-values-order": "ascending"
}
},
"required": ["enabled", "urls"],
"additionalProperties": false
}
},
"properties": {
"debug": { "type": "boolean" },
"output": {
"type": "object",
"properties": {
"path": { "type": "string" },
"sort_by_speed": { "type": "boolean" },
"txt": {
"type": "object",
"properties": { "enabled": { "type": "boolean" } },
"required": ["enabled"],
"additionalProperties": false
},
"json": {
"type": "object",
"properties": {
"enabled": { "type": "boolean" },
"include_asn": { "type": "boolean" },
"include_geolocation": { "type": "boolean" }
},
"required": ["enabled", "include_asn", "include_geolocation"],
"additionalProperties": false
}
},
"required": ["path", "sort_by_speed", "txt", "json"],
"additionalProperties": false
},
"checking": {
"type": "object",
"properties": {
"check_url": { "type": "string" },
"max_concurrent_checks": { "type": "integer", "minimum": 1 },
"timeout": { "type": "number", "minimum": 0 },
"connect_timeout": { "type": "number", "minimum": 0 },
"user_agent": { "type": "string" }
},
"required": [
"check_url",
"max_concurrent_checks",
"timeout",
"connect_timeout",
"user_agent"
],
"additionalProperties": false
},
"scraping": {
"type": "object",
"properties": {
"max_proxies_per_source": { "type": "integer", "minimum": 0 },
"timeout": { "type": "number", "minimum": 0 },
"connect_timeout": { "type": "number", "minimum": 0 },
"proxy": { "type": "string" },
"user_agent": { "type": "string" },
"http": { "$ref": "#/$defs/protocol_config" },
"socks4": { "$ref": "#/$defs/protocol_config" },
"socks5": { "$ref": "#/$defs/protocol_config" }
},
"required": [
"max_proxies_per_source",
"timeout",
"connect_timeout",
"proxy",
"user_agent"
],
"additionalProperties": false
}
},
"required": ["debug", "output", "checking", "scraping"],
"additionalProperties": false
}
config.toml
+35 -48
View File
@@ -1,24 +1,29 @@
#:schema ./config-schema.json
# Enable debug logging to see detailed checking process for each proxy
# Warning: Extremely verbose - logs every connection attempt and failure
debug = false
[output]
# Output directory (Docker overrides this to /app/out via volume mount)
path = "./out"
# Sort proxies by response time (true) or by IP address (false)
# Speed sorting: fastest proxies first. IP sorting: natural order by protocol, then IP, then port.
sort_by_speed = true
[scraping]
# Maximum proxies to collect per source (0 = unlimited)
# When this limit is reached, the source is skipped with a warning to prevent memory issues
max_proxies_per_source = 100000
# Request timeout for fetching proxy sources (seconds)
# Higher values allow slower sources to complete but increase total scraping time
timeout = 10.0
connect_timeout = 5.0
# HTTP(S), SOCKS4, or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable.
proxy = ""
# User-Agent header for scraping requests
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36"
# Plain text output (.txt files)
[output.txt]
enabled = true
# JSON output with metadata (.json files)
[output.json]
enabled = true
# Add ASN (network provider) info to JSON output
# Uses offline MaxMind database
include_asn = true
# Add geolocation (country/city) data to JSON output
# Uses offline MaxMind GeoLite2 City database
include_geolocation = true
[checking]
# URL for checking proxy functionality
@@ -30,47 +35,29 @@ user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTM
# "https://google.com" - Basic connect/read check only
# "" - Skip checking entirely (scrape only)
check_url = "https://api.ipify.org"
# Number of proxies to check simultaneously
# Higher values = faster checking but more RAM/network usage. Lower if you experience crashes or timeouts.
max_concurrent_checks = 512
# Proxy response timeout (seconds)
# Higher = finds more slow proxies but takes longer
# Lower = faster checking but discards slow proxies
timeout = 10.0
connect_timeout = 5.0
# User-Agent header for proxy check requests
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36"
[output]
# Output directory (Docker overrides this to /app/out via volume mount)
path = "./out"
# Sort proxies by response time (true) or by IP address (false)
# Speed sorting: fastest proxies first. IP sorting: natural order by protocol, then IP, then port.
sort_by_speed = true
# Plain text output (.txt files)
[output.txt]
enabled = true
# JSON output with metadata (.json files)
[output.json]
enabled = true
# Add ASN (network provider) info to JSON output
# Uses offline MaxMind database
include_asn = true
# Add geolocation (country/city) data to JSON output
# Uses offline MaxMind GeoLite2 City database
include_geolocation = true
[scraping]
# Maximum proxies to collect per source (0 = unlimited)
# When this limit is reached, the source is skipped with a warning to prevent memory issues
max_proxies_per_source = 100000
# Request timeout for fetching proxy sources (seconds)
# Higher values allow slower sources to complete but increase total scraping time
timeout = 10.0
connect_timeout = 5.0
# HTTP(S), SOCKS4, or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable.
proxy = ""
# User-Agent header for scraping requests
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36"
# Proxy sources configuration
# Add local files: ["./my_proxies.txt"] or URLs
@@ -93,11 +80,11 @@ urls = [
"https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=http",
"https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=https",
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/http.txt",
"https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/http/data.txt",
"https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/https/data.txt",
"https://raw.githubusercontent.com/roosterkid/openproxylist/refs/heads/main/HTTPS_RAW.txt",
"https://raw.githubusercontent.com/sunny9577/proxy-scraper/refs/heads/master/generated/http_proxies.txt",
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/http.txt",
]
[scraping.socks4]
@@ -116,10 +103,10 @@ urls = [
# { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },
"https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=socks4",
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/socks4.txt",
"https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/socks4/data.txt",
"https://raw.githubusercontent.com/roosterkid/openproxylist/refs/heads/main/SOCKS4_RAW.txt",
"https://raw.githubusercontent.com/sunny9577/proxy-scraper/refs/heads/master/generated/socks4_proxies.txt",
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/socks4.txt",
]
[scraping.socks5]
@@ -138,9 +125,9 @@ urls = [
# { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },
"https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=socks5",
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/socks5.txt",
"https://raw.githubusercontent.com/hookzof/socks5_list/refs/heads/master/proxy.txt",
"https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/socks5/data.txt",
"https://raw.githubusercontent.com/roosterkid/openproxylist/refs/heads/main/SOCKS5_RAW.txt",
"https://raw.githubusercontent.com/sunny9577/proxy-scraper/refs/heads/master/generated/socks5_proxies.txt",
"https://raw.githubusercontent.com/TheSpeedX/PROXY-List/refs/heads/master/socks5.txt",
]