From ea3a34ff39f414cb062f3ee7a6615aaee4fd3d9a Mon Sep 17 00:00:00 2001 From: monosans Date: Tue, 5 Aug 2025 09:56:11 +0300 Subject: [PATCH] fix: update dependencies and enhance configuration for proxy handling --- Cargo.lock | 13 ++++----- Cargo.toml | 4 +-- config.toml | 42 +++++++++++++++++++++++++---- src/checker.rs | 2 +- src/config.rs | 39 ++++++++++++++++++++++----- src/http.rs | 37 ++++++++++++++++++++----- src/proxy.rs | 4 +-- src/raw_config.rs | 69 ++++++++++++++++++++++++++++++++++++++++------- src/scraper.rs | 32 +++++++++++++--------- src/utils.rs | 8 ------ 10 files changed, 192 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 294427c..8566fa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2060,9 +2060,9 @@ dependencies = [ [[package]] name = "toml" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41ae868b5a0f67631c14589f7e250c1ea2c574ee5ba21c6c8dd4b1485705a5a1" +checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8" dependencies = [ "indexmap", "serde", @@ -2084,9 +2084,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30" +checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10" dependencies = [ "winnow", ] @@ -2284,6 +2284,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -2857,9 +2858,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +checksum = "bdbb9122ea75b11bf96e7492afb723e8a7fbe12c67417aa95e7e3d18144d37cd" dependencies = [ "yoke", "zerofrom", diff --git a/Cargo.toml b/Cargo.toml index 8ea2bec..2261fde 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,14 +22,14 @@ serde = "=1.0.219" serde_json = "=1.0.142" tokio = { version = "=1.47.1", features = ["full"] } tokio-util = "=0.7.16" -toml = "=0.9.4" +toml = "=0.9.5" tracing = "=0.1.41" tracing-log = "=0.2.0" tracing-subscriber = { version = "=0.3.19", features = ["parking_lot"] } tui-logger = { version = "=0.17.3", features = [ "tracing-support", ], optional = true } -url = "=2.5.4" +url = { version = "=2.5.4", features = ["serde"] } [features] tui = ["dep:crossterm", "dep:futures", "dep:ratatui", "dep:tui-logger"] diff --git a/config.toml b/config.toml index 4f4b372..ce5213a 100644 --- a/config.toml +++ b/config.toml @@ -13,6 +13,10 @@ max_proxies_per_source = 100000 timeout = 60.0 connect_timeout = 5.0 +# HTTP(S),SOCKS4 or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable. +proxy = "" + +# User-Agent header for scraping requests user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36" @@ -37,6 +41,7 @@ max_concurrent_checks = 1024 timeout = 60.0 connect_timeout = 5.0 +# User-Agent header for proxy check requests user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36" @@ -71,10 +76,19 @@ include_geolocation = true # Sources are fetched in parallel for speed [scraping.http] -enabled = true +enabled = false urls = [ - # Local file example: + # Local file examples: # "./my_http_proxies.txt", + # "/home/user/my_http_proxies.txt", + # "C:/Users/user/Desktop/my_http_proxies.txt", + # "file:///home/user/my_http_proxies.txt", + + # Advanced URL configuration examples (with basic auth or custom headers): + # HTTP Basic Auth example: + # { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } }, + # Custom headers example: + # { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } }, "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=http", "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=https", @@ -86,10 +100,19 @@ urls = [ ] [scraping.socks4] -enabled = true +enabled = false urls = [ - # Local file example: + # Local file examples: # "./my_socks4_proxies.txt", + # "/home/user/my_socks4_proxies.txt", + # "C:/Users/user/Desktop/my_socks4_proxies.txt", + # "file:///home/user/my_socks4_proxies.txt", + + # Advanced URL configuration examples (with basic auth or custom headers): + # HTTP Basic Auth example: + # { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } }, + # Custom headers example: + # { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } }, "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=socks4", "https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/socks4/data.txt", @@ -101,8 +124,17 @@ urls = [ [scraping.socks5] enabled = true urls = [ - # Local file example: + # Local file examples: # "./my_socks5_proxies.txt", + # "/home/user/my_socks5_proxies.txt", + # "C:/Users/user/Desktop/my_socks5_proxies.txt", + # "file:///home/user/my_socks5_proxies.txt", + + # Advanced URL configuration examples (with basic auth or custom headers): + # HTTP Basic Auth example: + { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } }, + # Custom headers example: + { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } }, "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=socks5", "https://raw.githubusercontent.com/hookzof/socks5_list/refs/heads/master/proxy.txt", diff --git a/src/checker.rs b/src/checker.rs index f8fe6da..32bdb33 100644 --- a/src/checker.rs +++ b/src/checker.rs @@ -12,7 +12,7 @@ pub async fn check_all( token: tokio_util::sync::CancellationToken, #[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender, ) -> color_eyre::Result> { - if config.checking.check_url.is_empty() { + if config.checking.check_url.is_none() { return Ok(proxies); } diff --git a/src/config.rs b/src/config.rs index 048aacb..14ab183 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,12 +1,12 @@ use std::{ - collections::{HashMap, HashSet, hash_map}, + collections::{HashMap, hash_map}, path::{Path, PathBuf}, sync::Arc, }; use color_eyre::eyre::{OptionExt as _, WrapErr as _}; -use crate::{proxy::ProxyType, raw_config, utils::is_docker}; +use crate::{http::BasicAuth, proxy::ProxyType, raw_config, utils::is_docker}; pub const APP_DIRECTORY_NAME: &str = "proxy_scraper_checker"; @@ -15,16 +15,24 @@ pub struct HttpbinResponse { pub origin: String, } +#[derive(Clone)] +pub struct Source { + pub url: String, + pub basic_auth: Option, + pub headers: Option>, +} + pub struct ScrapingConfig { pub max_proxies_per_source: usize, pub timeout: tokio::time::Duration, pub connect_timeout: tokio::time::Duration, + pub proxy: Option, pub user_agent: String, - pub sources: HashMap>, + pub sources: HashMap>, } pub struct CheckingConfig { - pub check_url: String, + pub check_url: Option, pub max_concurrent_checks: usize, pub timeout: tokio::time::Duration, pub connect_timeout: tokio::time::Duration, @@ -87,7 +95,7 @@ impl Config { pub fn enabled_protocols( &self, - ) -> hash_map::Keys<'_, ProxyType, HashSet> { + ) -> hash_map::Keys<'_, ProxyType, Vec> { self.scraping.sources.keys() } @@ -129,6 +137,7 @@ impl Config { connect_timeout: tokio::time::Duration::from_secs_f64( raw_config.scraping.connect_timeout, ), + proxy: raw_config.scraping.proxy, user_agent: raw_config.scraping.user_agent, sources: [ (ProxyType::Http, raw_config.scraping.http), @@ -137,7 +146,12 @@ impl Config { ] .into_iter() .filter_map(|(proxy_type, section)| { - section.enabled.then_some((proxy_type, section.urls)) + section.enabled.then(move || { + ( + proxy_type, + section.urls.into_iter().map(Into::into).collect(), + ) + }) }) .collect(), }, @@ -169,6 +183,19 @@ impl Config { } } +impl From for Source { + fn from(sc: raw_config::SourceConfig) -> Self { + match sc { + raw_config::SourceConfig::Simple(url) => { + Self { url, basic_auth: None, headers: None } + } + raw_config::SourceConfig::Detailed { url, basic_auth, headers } => { + Self { url, basic_auth, headers } + } + } + } +} + pub async fn load_config() -> color_eyre::Result> { let raw_config_path = raw_config::get_config_path(); let raw_config = raw_config::read_config(Path::new(&raw_config_path)) diff --git a/src/http.rs b/src/http.rs index 87aba08..347f371 100644 --- a/src/http.rs +++ b/src/http.rs @@ -1,4 +1,7 @@ -use std::time::{Duration, SystemTime}; +use std::{ + collections::HashMap, + time::{Duration, SystemTime}, +}; use color_eyre::Result; @@ -17,6 +20,12 @@ static RETRY_STATUSES: &[reqwest::StatusCode] = &[ reqwest::StatusCode::GATEWAY_TIMEOUT, ]; +#[derive(Clone, serde::Deserialize)] +pub struct BasicAuth { + pub username: String, + pub password: Option, +} + fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option { if let Some(val) = headers.get("retry-after-ms") && let Ok(s) = val.to_str() @@ -63,12 +72,23 @@ fn calculate_retry_timeout( pub async fn fetch_text( http_client: reqwest::Client, - url: &str, + url: url::Url, + basic_auth: Option, + headers: Option>, ) -> Result { let mut attempt: u32 = 0; loop { - let resp = http_client.get(url).send().await; - match resp { + let mut request = http_client.get(url.clone()); + if let Some(auth) = &basic_auth { + request = + request.basic_auth(&auth.username, auth.password.as_ref()); + } + if let Some(headers) = &headers { + for (k, v) in headers { + request = request.header(k, v); + } + } + match request.send().await { Ok(resp) => { let status = resp.status(); if status.is_client_error() || status.is_server_error() { @@ -124,10 +144,13 @@ pub async fn fetch_text( pub fn create_reqwest_client( config: &Config, ) -> reqwest::Result { - reqwest::Client::builder() + let mut builder = reqwest::ClientBuilder::new() .user_agent(&config.scraping.user_agent) .timeout(config.scraping.timeout) .connect_timeout(config.scraping.connect_timeout) - .use_rustls_tls() - .build() + .use_rustls_tls(); + if let Some(proxy) = config.scraping.proxy.clone() { + builder = builder.proxy(reqwest::Proxy::all(proxy)?); + } + builder.build() } diff --git a/src/proxy.rs b/src/proxy.rs index f139396..4618deb 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -83,7 +83,7 @@ impl Proxy { } pub async fn check(&mut self, config: &Config) -> color_eyre::Result<()> { - let client = reqwest::Client::builder() + let client = reqwest::ClientBuilder::new() .user_agent(&config.checking.user_agent) .proxy(self.try_into()?) .timeout(config.checking.timeout) @@ -93,7 +93,7 @@ impl Proxy { .wrap_err("failed to create reqwest::Client")?; let start = tokio::time::Instant::now(); let response = client - .get(&config.checking.check_url) + .get(config.checking.check_url.clone().unwrap()) .send() .await? .error_for_status()?; diff --git a/src/raw_config.rs b/src/raw_config.rs index 1719a50..c5d643d 100644 --- a/src/raw_config.rs +++ b/src/raw_config.rs @@ -1,5 +1,5 @@ use std::{ - collections::HashSet, + collections::HashMap, env, num::NonZero, path::{Path, PathBuf}, @@ -8,7 +8,7 @@ use std::{ use color_eyre::eyre::WrapErr as _; use serde::{Deserialize, Deserializer}; -use crate::utils::is_http_url; +use crate::http::BasicAuth; fn validate_positive_f64<'de, D: Deserializer<'de>>( deserializer: D, @@ -21,23 +21,72 @@ fn validate_positive_f64<'de, D: Deserializer<'de>>( } } -fn validate_http_url<'de, D: Deserializer<'de>>( +fn validate_url_generic<'de, D>( deserializer: D, -) -> Result { + allowed_schemes: &[&str], +) -> Result, D::Error> +where + D: Deserializer<'de>, +{ let s = String::deserialize(deserializer)?; - if s.is_empty() || is_http_url(&s) { - Ok(s) + if s.trim().is_empty() { + return Ok(None); + } + if let Ok(u) = url::Url::parse(&s) + && allowed_schemes.contains(&u.scheme()) + && u.host_str().is_some() + { + Ok(Some(u)) } else { + let type_label = if let Some((last, rest)) = allowed_schemes + .iter() + .map(|scheme| format!("'{scheme}'")) + .collect::>() + .split_last() + { + if rest.is_empty() { + last.clone() + } else { + format!("{} or {}", rest.join(", "), last) + } + } else { + String::new() + }; Err(serde::de::Error::custom(format!( - "'{s}' is not a valid 'http' or 'https' url" + "'{s}' is not a valid {type_label} url" ))) } } +fn validate_proxy_url<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + validate_url_generic(deserializer, &["http", "https", "socks4", "socks5"]) +} + +fn validate_http_url<'de, D: Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + validate_url_generic(deserializer, &["http", "https"]) +} + +#[derive(Deserialize)] +#[serde(untagged)] +pub enum SourceConfig { + Simple(String), + Detailed { + url: String, + #[serde(default)] + basic_auth: Option, + #[serde(default)] + headers: Option>, + }, +} + #[derive(Deserialize)] pub struct ScrapingProtocolConfig { pub enabled: bool, - pub urls: HashSet, + pub urls: Vec, } #[derive(Deserialize)] @@ -47,6 +96,8 @@ pub struct ScrapingConfig { pub timeout: f64, #[serde(deserialize_with = "validate_positive_f64")] pub connect_timeout: f64, + #[serde(deserialize_with = "validate_proxy_url")] + pub proxy: Option, pub user_agent: String, pub http: ScrapingProtocolConfig, @@ -57,7 +108,7 @@ pub struct ScrapingConfig { #[derive(Deserialize)] pub struct CheckingConfig { #[serde(deserialize_with = "validate_http_url")] - pub check_url: String, + pub check_url: Option, pub max_concurrent_checks: NonZero, #[serde(deserialize_with = "validate_positive_f64")] pub timeout: f64, diff --git a/src/scraper.rs b/src/scraper.rs index 402b0e4..3adc6c3 100644 --- a/src/scraper.rs +++ b/src/scraper.rs @@ -5,7 +5,7 @@ use color_eyre::eyre::{OptionExt as _, WrapErr as _}; #[cfg(feature = "tui")] use crate::event::{AppEvent, Event}; use crate::{ - config::Config, + config::{Config, Source}, http, parsers::PROXY_REGEX, proxy::{Proxy, ProxyType}, @@ -17,20 +17,28 @@ async fn scrape_one( http_client: reqwest::Client, proto: ProxyType, proxies: Arc>>, - source: &str, + source: Source, #[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender, ) -> color_eyre::Result<()> { - let text_result = if let Ok(u) = url::Url::parse(source) { + let text_result = if let Ok(u) = url::Url::parse(&source.url) { match u.scheme() { - "http" | "https" => http::fetch_text(http_client, source).await, + "http" | "https" => { + http::fetch_text( + http_client, + u, + source.basic_auth, + source.headers, + ) + .await + } _ => match u.to_file_path() { Ok(path) => tokio::fs::read_to_string(path).await, - Err(()) => tokio::fs::read_to_string(source).await, + Err(()) => tokio::fs::read_to_string(&source.url).await, } .map_err(Into::into), } } else { - tokio::fs::read_to_string(source).await.map_err(Into::into) + tokio::fs::read_to_string(&source.url).await.map_err(Into::into) }; #[cfg(feature = "tui")] @@ -39,7 +47,7 @@ async fn scrape_one( let text = match text_result { Ok(text) => text, Err(e) => { - tracing::warn!("{} | {}", source, pretty_error(&e)); + tracing::warn!("{} | {}", source.url, pretty_error(&e)); return Ok(()); } }; @@ -48,7 +56,7 @@ async fn scrape_one( PROXY_REGEX.captures_iter(&text).collect::, _>>()?; if matches.is_empty() { - tracing::warn!("{source} | No proxies found"); + tracing::warn!("{} | No proxies found", source.url); return Ok(()); } @@ -57,7 +65,7 @@ async fn scrape_one( { tracing::warn!( "{} | Too many proxies ({}) - skipped", - source, + source.url, matches.len(), ); return Ok(()); @@ -115,13 +123,13 @@ pub async fn scrape_all( let proxies = Arc::new(tokio::sync::Mutex::new(HashSet::new())); let mut join_set = tokio::task::JoinSet::new(); - for (proto, sources) in config.scraping.sources.clone() { + for (proto, sources) in &config.scraping.sources { #[cfg(feature = "tui")] drop(tx.send(Event::App(AppEvent::SourcesTotal( proto.clone(), sources.len(), )))); - for source in sources { + for source in sources.iter().cloned() { let config = Arc::clone(&config); let http_client = http_client.clone(); let proto = proto.clone(); @@ -137,7 +145,7 @@ pub async fn scrape_all( http_client, proto, proxies, - &source, + source, #[cfg(feature = "tui")] tx, ) => res, diff --git a/src/utils.rs b/src/utils.rs index a190ff2..4090fdb 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -16,14 +16,6 @@ pub async fn is_docker() -> bool { } } -pub fn is_http_url(value: &str) -> bool { - url::Url::parse(value).is_ok_and(|parsed_url| { - let scheme = parsed_url.scheme(); - (scheme == "http" || scheme == "https") - && parsed_url.host_str().is_some() - }) -} - pub fn pretty_error(e: &color_eyre::Report) -> String { e.chain().map(ToString::to_string).collect::>().join(" \u{2192} ") }