fix: update dependencies and enhance configuration for proxy handling

2025-08-05 09:56:11 +03:00
parent e6ef3a1cbe
commit ea3a34ff39
10 changed files with 192 additions and 58 deletions
@@ -2060,9 +2060,9 @@ dependencies = [

 [[package]]
 name = "toml"
-version = "0.9.4"
+version = "0.9.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41ae868b5a0f67631c14589f7e250c1ea2c574ee5ba21c6c8dd4b1485705a5a1"
+checksum = "75129e1dc5000bfbaa9fee9d1b21f974f9fbad9daec557a521ee6e080825f6e8"
 dependencies = [
 "indexmap",
 "serde",
@@ -2084,9 +2084,9 @@ dependencies = [

 [[package]]
 name = "toml_parser"
-version = "1.0.1"
+version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "97200572db069e74c512a14117b296ba0a80a30123fbbb5aa1f4a348f639ca30"
+checksum = "b551886f449aa90d4fe2bdaa9f4a2577ad2dde302c61ecf262d80b116db95c10"
 dependencies = [
 "winnow",
 ]
@@ -2284,6 +2284,7 @@ dependencies = [
 "form_urlencoded",
 "idna",
 "percent-encoding",
+ "serde",
 ]

 [[package]]
@@ -2857,9 +2858,9 @@ dependencies = [

 [[package]]
 name = "zerovec"
-version = "0.11.2"
+version = "0.11.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428"
+checksum = "bdbb9122ea75b11bf96e7492afb723e8a7fbe12c67417aa95e7e3d18144d37cd"
 dependencies = [
 "yoke",
 "zerofrom",
@@ -22,14 +22,14 @@ serde = "=1.0.219"
 serde_json = "=1.0.142"
 tokio = { version = "=1.47.1", features = ["full"] }
 tokio-util = "=0.7.16"
-toml = "=0.9.4"
+toml = "=0.9.5"
 tracing = "=0.1.41"
 tracing-log = "=0.2.0"
 tracing-subscriber = { version = "=0.3.19", features = ["parking_lot"] }
 tui-logger = { version = "=0.17.3", features = [
    "tracing-support",
 ], optional = true }
-url = "=2.5.4"
+url = { version = "=2.5.4", features = ["serde"] }

 [features]
 tui = ["dep:crossterm", "dep:futures", "dep:ratatui", "dep:tui-logger"]
@@ -13,6 +13,10 @@ max_proxies_per_source = 100000
 timeout = 60.0
 connect_timeout = 5.0

+# HTTP(S),SOCKS4 or SOCKS5 proxy used for fetching sources (e.g., "socks5://user:pass@host:port"). Leave empty to disable.
+proxy = ""
+
+# User-Agent header for scraping requests
 user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"


@@ -37,6 +41,7 @@ max_concurrent_checks = 1024
 timeout = 60.0
 connect_timeout = 5.0

+# User-Agent header for proxy check requests
 user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/138.0.0.0 Safari/537.36"


@@ -71,10 +76,19 @@ include_geolocation = true
 # Sources are fetched in parallel for speed

 [scraping.http]
-enabled = true
+enabled = false
 urls = [
-    # Local file example:
+    # Local file examples:
    # "./my_http_proxies.txt",
+    # "/home/user/my_http_proxies.txt",
+    # "C:/Users/user/Desktop/my_http_proxies.txt",
+    # "file:///home/user/my_http_proxies.txt",
+
+    # Advanced URL configuration examples (with basic auth or custom headers):
+    # HTTP Basic Auth example:
+    # { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } },
+    # Custom headers example:
+    # { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },

    "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=http",
    "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=https",
@@ -86,10 +100,19 @@ urls = [
 ]

 [scraping.socks4]
-enabled = true
+enabled = false
 urls = [
-    # Local file example:
+    # Local file examples:
    # "./my_socks4_proxies.txt",
+    # "/home/user/my_socks4_proxies.txt",
+    # "C:/Users/user/Desktop/my_socks4_proxies.txt",
+    # "file:///home/user/my_socks4_proxies.txt",
+
+    # Advanced URL configuration examples (with basic auth or custom headers):
+    # HTTP Basic Auth example:
+    # { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } },
+    # Custom headers example:
+    # { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },

    "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=socks4",
    "https://raw.githubusercontent.com/proxifly/free-proxy-list/refs/heads/main/proxies/protocols/socks4/data.txt",
@@ -101,8 +124,17 @@ urls = [
 [scraping.socks5]
 enabled = true
 urls = [
-    # Local file example:
+    # Local file examples:
    # "./my_socks5_proxies.txt",
+    # "/home/user/my_socks5_proxies.txt",
+    # "C:/Users/user/Desktop/my_socks5_proxies.txt",
+    # "file:///home/user/my_socks5_proxies.txt",
+
+    # Advanced URL configuration examples (with basic auth or custom headers):
+    # HTTP Basic Auth example:
+    { url = "https://some.api/endpoint", basic_auth = { username = "user", password = "password123" } },
+    # Custom headers example:
+    { url = "https://some.api/endpoint", headers = { Authorization = "Bearer YOUR_API_KEY" } },

    "https://api.proxyscrape.com/v3/free-proxy-list/get?request=getproxies&protocol=socks5",
    "https://raw.githubusercontent.com/hookzof/socks5_list/refs/heads/master/proxy.txt",
@@ -12,7 +12,7 @@ pub async fn check_all(
    token: tokio_util::sync::CancellationToken,
    #[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
 ) -> color_eyre::Result<Vec<Proxy>> {
-    if config.checking.check_url.is_empty() {
+    if config.checking.check_url.is_none() {
        return Ok(proxies);
    }

@@ -1,12 +1,12 @@
 use std::{
-    collections::{HashMap, HashSet, hash_map},
+    collections::{HashMap, hash_map},
    path::{Path, PathBuf},
    sync::Arc,
 };

 use color_eyre::eyre::{OptionExt as _, WrapErr as _};

-use crate::{proxy::ProxyType, raw_config, utils::is_docker};
+use crate::{http::BasicAuth, proxy::ProxyType, raw_config, utils::is_docker};

 pub const APP_DIRECTORY_NAME: &str = "proxy_scraper_checker";

@@ -15,16 +15,24 @@ pub struct HttpbinResponse {
    pub origin: String,
 }

+#[derive(Clone)]
+pub struct Source {
+    pub url: String,
+    pub basic_auth: Option<BasicAuth>,
+    pub headers: Option<HashMap<String, String>>,
+}
+
 pub struct ScrapingConfig {
    pub max_proxies_per_source: usize,
    pub timeout: tokio::time::Duration,
    pub connect_timeout: tokio::time::Duration,
+    pub proxy: Option<url::Url>,
    pub user_agent: String,
-    pub sources: HashMap<ProxyType, HashSet<String>>,
+    pub sources: HashMap<ProxyType, Vec<Source>>,
 }

 pub struct CheckingConfig {
-    pub check_url: String,
+    pub check_url: Option<url::Url>,
    pub max_concurrent_checks: usize,
    pub timeout: tokio::time::Duration,
    pub connect_timeout: tokio::time::Duration,
@@ -87,7 +95,7 @@ impl Config {

    pub fn enabled_protocols(
        &self,
-    ) -> hash_map::Keys<'_, ProxyType, HashSet<String>> {
+    ) -> hash_map::Keys<'_, ProxyType, Vec<Source>> {
        self.scraping.sources.keys()
    }

@@ -129,6 +137,7 @@ impl Config {
                connect_timeout: tokio::time::Duration::from_secs_f64(
                    raw_config.scraping.connect_timeout,
                ),
+                proxy: raw_config.scraping.proxy,
                user_agent: raw_config.scraping.user_agent,
                sources: [
                    (ProxyType::Http, raw_config.scraping.http),
@@ -137,7 +146,12 @@ impl Config {
                ]
                .into_iter()
                .filter_map(|(proxy_type, section)| {
-                    section.enabled.then_some((proxy_type, section.urls))
+                    section.enabled.then(move || {
+                        (
+                            proxy_type,
+                            section.urls.into_iter().map(Into::into).collect(),
+                        )
+                    })
                })
                .collect(),
            },
@@ -169,6 +183,19 @@ impl Config {
    }
 }

+impl From<raw_config::SourceConfig> for Source {
+    fn from(sc: raw_config::SourceConfig) -> Self {
+        match sc {
+            raw_config::SourceConfig::Simple(url) => {
+                Self { url, basic_auth: None, headers: None }
+            }
+            raw_config::SourceConfig::Detailed { url, basic_auth, headers } => {
+                Self { url, basic_auth, headers }
+            }
+        }
+    }
+}
+
 pub async fn load_config() -> color_eyre::Result<Arc<Config>> {
    let raw_config_path = raw_config::get_config_path();
    let raw_config = raw_config::read_config(Path::new(&raw_config_path))
@@ -1,4 +1,7 @@
-use std::time::{Duration, SystemTime};
+use std::{
+    collections::HashMap,
+    time::{Duration, SystemTime},
+};

 use color_eyre::Result;

@@ -17,6 +20,12 @@ static RETRY_STATUSES: &[reqwest::StatusCode] = &[
    reqwest::StatusCode::GATEWAY_TIMEOUT,
 ];

+#[derive(Clone, serde::Deserialize)]
+pub struct BasicAuth {
+    pub username: String,
+    pub password: Option<String>,
+}
+
 fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option<Duration> {
    if let Some(val) = headers.get("retry-after-ms")
        && let Ok(s) = val.to_str()
@@ -63,12 +72,23 @@ fn calculate_retry_timeout(

 pub async fn fetch_text(
    http_client: reqwest::Client,
-    url: &str,
+    url: url::Url,
+    basic_auth: Option<BasicAuth>,
+    headers: Option<HashMap<String, String>>,
 ) -> Result<String> {
    let mut attempt: u32 = 0;
    loop {
-        let resp = http_client.get(url).send().await;
-        match resp {
+        let mut request = http_client.get(url.clone());
+        if let Some(auth) = &basic_auth {
+            request =
+                request.basic_auth(&auth.username, auth.password.as_ref());
+        }
+        if let Some(headers) = &headers {
+            for (k, v) in headers {
+                request = request.header(k, v);
+            }
+        }
+        match request.send().await {
            Ok(resp) => {
                let status = resp.status();
                if status.is_client_error() || status.is_server_error() {
@@ -124,10 +144,13 @@ pub async fn fetch_text(
 pub fn create_reqwest_client(
    config: &Config,
 ) -> reqwest::Result<reqwest::Client> {
-    reqwest::Client::builder()
+    let mut builder = reqwest::ClientBuilder::new()
        .user_agent(&config.scraping.user_agent)
        .timeout(config.scraping.timeout)
        .connect_timeout(config.scraping.connect_timeout)
-        .use_rustls_tls()
-        .build()
+        .use_rustls_tls();
+    if let Some(proxy) = config.scraping.proxy.clone() {
+        builder = builder.proxy(reqwest::Proxy::all(proxy)?);
+    }
+    builder.build()
 }
@@ -83,7 +83,7 @@ impl Proxy {
    }

    pub async fn check(&mut self, config: &Config) -> color_eyre::Result<()> {
-        let client = reqwest::Client::builder()
+        let client = reqwest::ClientBuilder::new()
            .user_agent(&config.checking.user_agent)
            .proxy(self.try_into()?)
            .timeout(config.checking.timeout)
@@ -93,7 +93,7 @@ impl Proxy {
            .wrap_err("failed to create reqwest::Client")?;
        let start = tokio::time::Instant::now();
        let response = client
-            .get(&config.checking.check_url)
+            .get(config.checking.check_url.clone().unwrap())
            .send()
            .await?
            .error_for_status()?;
@@ -1,5 +1,5 @@
 use std::{
-    collections::HashSet,
+    collections::HashMap,
    env,
    num::NonZero,
    path::{Path, PathBuf},
@@ -8,7 +8,7 @@ use std::{
 use color_eyre::eyre::WrapErr as _;
 use serde::{Deserialize, Deserializer};

-use crate::utils::is_http_url;
+use crate::http::BasicAuth;

 fn validate_positive_f64<'de, D: Deserializer<'de>>(
    deserializer: D,
@@ -21,23 +21,72 @@ fn validate_positive_f64<'de, D: Deserializer<'de>>(
    }
 }

-fn validate_http_url<'de, D: Deserializer<'de>>(
+fn validate_url_generic<'de, D>(
    deserializer: D,
-) -> Result<String, D::Error> {
+    allowed_schemes: &[&str],
+) -> Result<Option<url::Url>, D::Error>
+where
+    D: Deserializer<'de>,
+{
    let s = String::deserialize(deserializer)?;
-    if s.is_empty() || is_http_url(&s) {
-        Ok(s)
+    if s.trim().is_empty() {
+        return Ok(None);
+    }
+    if let Ok(u) = url::Url::parse(&s)
+        && allowed_schemes.contains(&u.scheme())
+        && u.host_str().is_some()
+    {
+        Ok(Some(u))
    } else {
+        let type_label = if let Some((last, rest)) = allowed_schemes
+            .iter()
+            .map(|scheme| format!("'{scheme}'"))
+            .collect::<Vec<_>>()
+            .split_last()
+        {
+            if rest.is_empty() {
+                last.clone()
+            } else {
+                format!("{} or {}", rest.join(", "), last)
+            }
+        } else {
+            String::new()
+        };
        Err(serde::de::Error::custom(format!(
-            "'{s}' is not a valid 'http' or 'https' url"
+            "'{s}' is not a valid {type_label} url"
        )))
    }
 }

+fn validate_proxy_url<'de, D: Deserializer<'de>>(
+    deserializer: D,
+) -> Result<Option<url::Url>, D::Error> {
+    validate_url_generic(deserializer, &["http", "https", "socks4", "socks5"])
+}
+
+fn validate_http_url<'de, D: Deserializer<'de>>(
+    deserializer: D,
+) -> Result<Option<url::Url>, D::Error> {
+    validate_url_generic(deserializer, &["http", "https"])
+}
+
+#[derive(Deserialize)]
+#[serde(untagged)]
+pub enum SourceConfig {
+    Simple(String),
+    Detailed {
+        url: String,
+        #[serde(default)]
+        basic_auth: Option<BasicAuth>,
+        #[serde(default)]
+        headers: Option<HashMap<String, String>>,
+    },
+}
+
 #[derive(Deserialize)]
 pub struct ScrapingProtocolConfig {
    pub enabled: bool,
-    pub urls: HashSet<String>,
+    pub urls: Vec<SourceConfig>,
 }

 #[derive(Deserialize)]
@@ -47,6 +96,8 @@ pub struct ScrapingConfig {
    pub timeout: f64,
    #[serde(deserialize_with = "validate_positive_f64")]
    pub connect_timeout: f64,
+    #[serde(deserialize_with = "validate_proxy_url")]
+    pub proxy: Option<url::Url>,
    pub user_agent: String,

    pub http: ScrapingProtocolConfig,
@@ -57,7 +108,7 @@ pub struct ScrapingConfig {
 #[derive(Deserialize)]
 pub struct CheckingConfig {
    #[serde(deserialize_with = "validate_http_url")]
-    pub check_url: String,
+    pub check_url: Option<url::Url>,
    pub max_concurrent_checks: NonZero<usize>,
    #[serde(deserialize_with = "validate_positive_f64")]
    pub timeout: f64,
@@ -5,7 +5,7 @@ use color_eyre::eyre::{OptionExt as _, WrapErr as _};
 #[cfg(feature = "tui")]
 use crate::event::{AppEvent, Event};
 use crate::{
-    config::Config,
+    config::{Config, Source},
    http,
    parsers::PROXY_REGEX,
    proxy::{Proxy, ProxyType},
@@ -17,20 +17,28 @@ async fn scrape_one(
    http_client: reqwest::Client,
    proto: ProxyType,
    proxies: Arc<tokio::sync::Mutex<HashSet<Proxy>>>,
-    source: &str,
+    source: Source,
    #[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
 ) -> color_eyre::Result<()> {
-    let text_result = if let Ok(u) = url::Url::parse(source) {
+    let text_result = if let Ok(u) = url::Url::parse(&source.url) {
        match u.scheme() {
-            "http" | "https" => http::fetch_text(http_client, source).await,
+            "http" | "https" => {
+                http::fetch_text(
+                    http_client,
+                    u,
+                    source.basic_auth,
+                    source.headers,
+                )
+                .await
+            }
            _ => match u.to_file_path() {
                Ok(path) => tokio::fs::read_to_string(path).await,
-                Err(()) => tokio::fs::read_to_string(source).await,
+                Err(()) => tokio::fs::read_to_string(&source.url).await,
            }
            .map_err(Into::into),
        }
    } else {
-        tokio::fs::read_to_string(source).await.map_err(Into::into)
+        tokio::fs::read_to_string(&source.url).await.map_err(Into::into)
    };

    #[cfg(feature = "tui")]
@@ -39,7 +47,7 @@ async fn scrape_one(
    let text = match text_result {
        Ok(text) => text,
        Err(e) => {
-            tracing::warn!("{} | {}", source, pretty_error(&e));
+            tracing::warn!("{} | {}", source.url, pretty_error(&e));
            return Ok(());
        }
    };
@@ -48,7 +56,7 @@ async fn scrape_one(
        PROXY_REGEX.captures_iter(&text).collect::<Result<Vec<_>, _>>()?;

    if matches.is_empty() {
-        tracing::warn!("{source} | No proxies found");
+        tracing::warn!("{} | No proxies found", source.url);
        return Ok(());
    }

@@ -57,7 +65,7 @@ async fn scrape_one(
    {
        tracing::warn!(
            "{} | Too many proxies ({}) - skipped",
-            source,
+            source.url,
            matches.len(),
        );
        return Ok(());
@@ -115,13 +123,13 @@ pub async fn scrape_all(
    let proxies = Arc::new(tokio::sync::Mutex::new(HashSet::new()));

    let mut join_set = tokio::task::JoinSet::new();
-    for (proto, sources) in config.scraping.sources.clone() {
+    for (proto, sources) in &config.scraping.sources {
        #[cfg(feature = "tui")]
        drop(tx.send(Event::App(AppEvent::SourcesTotal(
            proto.clone(),
            sources.len(),
        ))));
-        for source in sources {
+        for source in sources.iter().cloned() {
            let config = Arc::clone(&config);
            let http_client = http_client.clone();
            let proto = proto.clone();
@@ -137,7 +145,7 @@ pub async fn scrape_all(
                        http_client,
                        proto,
                        proxies,
-                        &source,
+                        source,
                        #[cfg(feature = "tui")]
                        tx,
                    ) => res,
@@ -16,14 +16,6 @@ pub async fn is_docker() -> bool {
    }
 }

-pub fn is_http_url(value: &str) -> bool {
-    url::Url::parse(value).is_ok_and(|parsed_url| {
-        let scheme = parsed_url.scheme();
-        (scheme == "http" || scheme == "https")
-            && parsed_url.host_str().is_some()
-    })
-}
-
 pub fn pretty_error(e: &color_eyre::Report) -> String {
    e.chain().map(ToString::to_string).collect::<Vec<_>>().join(" \u{2192} ")
 }