refactor: optimize proxy matching logic and improve warning messages

This commit is contained in:
monosans
2025-08-13 17:24:27 +03:00
parent ba5289f3b6
commit 867705f787
+15 -13
View File
@@ -2,7 +2,6 @@ use std::sync::Arc;
use color_eyre::eyre::{OptionExt as _, WrapErr as _};
use foldhash::HashSetExt as _;
use itertools::Itertools as _;
#[cfg(feature = "tui")]
use crate::event::{AppEvent, Event};
@@ -55,24 +54,27 @@ async fn scrape_one(
}
};
let matches: Vec<_> = PROXY_REGEX.captures_iter(&text).try_collect()?;
let limit = config.scraping.max_proxies_per_source;
let matches_iter = PROXY_REGEX.captures_iter(&text);
let mut matches = Vec::with_capacity(matches_iter.size_hint().0);
for (i, maybe_capture) in matches_iter.enumerate() {
if limit != 0 && i >= limit {
tracing::warn!(
"{} | Too many proxies (> {}) - skipped",
source.url,
limit
);
return Ok(());
}
matches.push(maybe_capture?);
}
if matches.is_empty() {
tracing::warn!("{} | No proxies found", source.url);
return Ok(());
}
if config.scraping.max_proxies_per_source != 0
&& matches.len() > config.scraping.max_proxies_per_source
{
tracing::warn!(
"{} | Too many proxies ({}) - skipped",
source.url,
matches.len(),
);
return Ok(());
}
#[cfg(feature = "tui")]
let mut seen_protocols = HashSet::new();
let mut proxies = proxies.lock();