diff --git a/config.toml b/config.toml index 7650d9e..69c3853 100644 --- a/config.toml +++ b/config.toml @@ -11,6 +11,7 @@ max_proxies_per_source = 100000 # Request timeout for fetching proxy sources (seconds) # Higher values may find more sources but take longer timeout = 60.0 +connect_timeout = 5.0 [checking] @@ -26,12 +27,13 @@ check_url = "https://api.ipify.org" # Concurrent proxy checks (adjust based on RAM/network capacity) # Start low and increase gradually if system handles it well -max_concurrent_checks = 4096 +max_concurrent_checks = 1024 # Proxy response timeout (seconds) # Higher = more working proxies found, slower checking # Lower = faster checking, may miss slower proxies -timeout = 5.0 +timeout = 60.0 +connect_timeout = 5.0 [output] diff --git a/src/config.rs b/src/config.rs index f6149f8..55e1758 100644 --- a/src/config.rs +++ b/src/config.rs @@ -18,6 +18,7 @@ pub struct HttpbinResponse { pub struct ScrapingConfig { pub max_proxies_per_source: usize, pub timeout: tokio::time::Duration, + pub connect_timeout: tokio::time::Duration, pub sources: HashMap>, } @@ -25,6 +26,7 @@ pub struct CheckingConfig { pub check_url: String, pub max_concurrent_checks: usize, pub timeout: tokio::time::Duration, + pub connect_timeout: tokio::time::Duration, } pub struct TxtOutputConfig { @@ -122,6 +124,9 @@ impl Config { timeout: tokio::time::Duration::from_secs_f64( raw_config.scraping.timeout, ), + connect_timeout: tokio::time::Duration::from_secs_f64( + raw_config.scraping.connect_timeout, + ), sources: [ (ProxyType::Http, raw_config.scraping.http), (ProxyType::Socks4, raw_config.scraping.socks4), @@ -139,6 +144,9 @@ impl Config { timeout: tokio::time::Duration::from_secs_f64( raw_config.checking.timeout, ), + connect_timeout: tokio::time::Duration::from_secs_f64( + raw_config.checking.connect_timeout, + ), }, output: OutputConfig { path: output_path, diff --git a/src/http.rs b/src/http.rs index a06032a..c832a09 100644 --- a/src/http.rs +++ b/src/http.rs @@ -2,11 +2,11 @@ use std::time::{Duration, SystemTime}; use color_eyre::Result; +use crate::config::Config; + pub const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \ AppleWebKit/537.36 (KHTML, like Gecko) \ Chrome/138.0.0.0 Safari/537.36"; -const TIMEOUT: Duration = tokio::time::Duration::from_secs(60); -const CONNECT_TIMEOUT: Duration = tokio::time::Duration::from_secs(5); const DEFAULT_MAX_RETRIES: u32 = 2; const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(500); @@ -68,11 +68,10 @@ fn calculate_retry_timeout( pub async fn fetch_text( http_client: reqwest::Client, url: &str, - timeout: Duration, ) -> Result { let mut attempt: u32 = 0; loop { - let resp = http_client.get(url).timeout(timeout).send().await; + let resp = http_client.get(url).send().await; match resp { Ok(resp) => { let status = resp.status(); @@ -126,11 +125,13 @@ pub async fn fetch_text( } } -pub fn create_reqwest_client() -> reqwest::Result { +pub fn create_reqwest_client( + config: &Config, +) -> reqwest::Result { reqwest::Client::builder() .user_agent(USER_AGENT) - .timeout(TIMEOUT) - .connect_timeout(CONNECT_TIMEOUT) + .timeout(config.scraping.timeout) + .connect_timeout(config.scraping.connect_timeout) .use_rustls_tls() .build() } diff --git a/src/main.rs b/src/main.rs index e506ff7..ce24729 100644 --- a/src/main.rs +++ b/src/main.rs @@ -151,7 +151,7 @@ async fn main_task( event::Event, >, ) -> color_eyre::Result<()> { - let http_client = http::create_reqwest_client() + let http_client = http::create_reqwest_client(&config) .wrap_err("failed to create reqwest HTTP client")?; let ((), mut proxies) = tokio::try_join!( diff --git a/src/proxy.rs b/src/proxy.rs index 6df832c..5adad04 100644 --- a/src/proxy.rs +++ b/src/proxy.rs @@ -88,6 +88,7 @@ impl Proxy { .user_agent(USER_AGENT) .proxy(self.try_into()?) .timeout(config.checking.timeout) + .connect_timeout(config.checking.connect_timeout) .pool_max_idle_per_host(0) .tcp_keepalive(None) .use_rustls_tls() diff --git a/src/raw_config.rs b/src/raw_config.rs index 930b394..2cbfa25 100644 --- a/src/raw_config.rs +++ b/src/raw_config.rs @@ -45,6 +45,8 @@ pub struct ScrapingConfig { pub max_proxies_per_source: usize, #[serde(deserialize_with = "validate_positive_f64")] pub timeout: f64, + #[serde(deserialize_with = "validate_positive_f64")] + pub connect_timeout: f64, pub http: ScrapingProtocolConfig, pub socks4: ScrapingProtocolConfig, @@ -58,6 +60,8 @@ pub struct CheckingConfig { pub max_concurrent_checks: NonZero, #[serde(deserialize_with = "validate_positive_f64")] pub timeout: f64, + #[serde(deserialize_with = "validate_positive_f64")] + pub connect_timeout: f64, } #[derive(Deserialize)] diff --git a/src/scraper.rs b/src/scraper.rs index e86a48b..d563bf7 100644 --- a/src/scraper.rs +++ b/src/scraper.rs @@ -21,7 +21,7 @@ async fn scrape_one( #[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender, ) -> color_eyre::Result<()> { let text_result = if is_http_url(source) { - http::fetch_text(http_client, source, config.scraping.timeout).await + http::fetch_text(http_client, source).await } else { tokio::fs::read_to_string( source.strip_prefix("file://").unwrap_or(source),