haeaeae/proxy-scraper-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: update dependencies and enhance configuration for proxy handling

Browse Source
This commit is contained in:
monosans
2025-08-05 09:56:11 +03:00
parent e6ef3a1cbe
commit ea3a34ff39
10 changed files with 192 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/checker.rs
+1 -1
View File
@@ -12,7 +12,7 @@ pub async fn check_all(
token: tokio_util::sync::CancellationToken,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<Vec<Proxy>> {
if config.checking.check_url.is_empty() {
if config.checking.check_url.is_none() {
return Ok(proxies);
}
src/config.rs
+33 -6
View File
@@ -1,12 +1,12 @@
use std::{
collections::{HashMap, HashSet, hash_map},
collections::{HashMap, hash_map},
path::{Path, PathBuf},
sync::Arc,
};
use color_eyre::eyre::{OptionExt as _, WrapErr as _};
use crate::{proxy::ProxyType, raw_config, utils::is_docker};
use crate::{http::BasicAuth, proxy::ProxyType, raw_config, utils::is_docker};
pub const APP_DIRECTORY_NAME: &str = "proxy_scraper_checker";
@@ -15,16 +15,24 @@ pub struct HttpbinResponse {
pub origin: String,
}
#[derive(Clone)]
pub struct Source {
pub url: String,
pub basic_auth: Option<BasicAuth>,
pub headers: Option<HashMap<String, String>>,
}
pub struct ScrapingConfig {
pub max_proxies_per_source: usize,
pub timeout: tokio::time::Duration,
pub connect_timeout: tokio::time::Duration,
pub proxy: Option<url::Url>,
pub user_agent: String,
pub sources: HashMap<ProxyType, HashSet<String>>,
pub sources: HashMap<ProxyType, Vec<Source>>,
}
pub struct CheckingConfig {
pub check_url: String,
pub check_url: Option<url::Url>,
pub max_concurrent_checks: usize,
pub timeout: tokio::time::Duration,
pub connect_timeout: tokio::time::Duration,
@@ -87,7 +95,7 @@ impl Config {
pub fn enabled_protocols(
&self,
) -> hash_map::Keys<'_, ProxyType, HashSet<String>> {
) -> hash_map::Keys<'_, ProxyType, Vec<Source>> {
self.scraping.sources.keys()
}
@@ -129,6 +137,7 @@ impl Config {
connect_timeout: tokio::time::Duration::from_secs_f64(
raw_config.scraping.connect_timeout,
),
proxy: raw_config.scraping.proxy,
user_agent: raw_config.scraping.user_agent,
sources: [
(ProxyType::Http, raw_config.scraping.http),
@@ -137,7 +146,12 @@ impl Config {
]
.into_iter()
.filter_map(|(proxy_type, section)| {
section.enabled.then_some((proxy_type, section.urls))
section.enabled.then(move || {
(
proxy_type,
section.urls.into_iter().map(Into::into).collect(),
)
})
})
.collect(),
},
@@ -169,6 +183,19 @@ impl Config {
}
}
impl From<raw_config::SourceConfig> for Source {
fn from(sc: raw_config::SourceConfig) -> Self {
match sc {
raw_config::SourceConfig::Simple(url) => {
Self { url, basic_auth: None, headers: None }
}
raw_config::SourceConfig::Detailed { url, basic_auth, headers } => {
Self { url, basic_auth, headers }
}
}
}
}
pub async fn load_config() -> color_eyre::Result<Arc<Config>> {
let raw_config_path = raw_config::get_config_path();
let raw_config = raw_config::read_config(Path::new(&raw_config_path))
src/http.rs
+30 -7
View File
@@ -1,4 +1,7 @@
use std::time::{Duration, SystemTime};
use std::{
collections::HashMap,
time::{Duration, SystemTime},
};
use color_eyre::Result;
@@ -17,6 +20,12 @@ static RETRY_STATUSES: &[reqwest::StatusCode] = &[
reqwest::StatusCode::GATEWAY_TIMEOUT,
];
#[derive(Clone, serde::Deserialize)]
pub struct BasicAuth {
pub username: String,
pub password: Option<String>,
}
fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option<Duration> {
if let Some(val) = headers.get("retry-after-ms")
&& let Ok(s) = val.to_str()
@@ -63,12 +72,23 @@ fn calculate_retry_timeout(
pub async fn fetch_text(
http_client: reqwest::Client,
url: &str,
url: url::Url,
basic_auth: Option<BasicAuth>,
headers: Option<HashMap<String, String>>,
) -> Result<String> {
let mut attempt: u32 = 0;
loop {
let resp = http_client.get(url).send().await;
match resp {
let mut request = http_client.get(url.clone());
if let Some(auth) = &basic_auth {
request =
request.basic_auth(&auth.username, auth.password.as_ref());
}
if let Some(headers) = &headers {
for (k, v) in headers {
request = request.header(k, v);
}
}
match request.send().await {
Ok(resp) => {
let status = resp.status();
if status.is_client_error() || status.is_server_error() {
@@ -124,10 +144,13 @@ pub async fn fetch_text(
pub fn create_reqwest_client(
config: &Config,
) -> reqwest::Result<reqwest::Client> {
reqwest::Client::builder()
let mut builder = reqwest::ClientBuilder::new()
.user_agent(&config.scraping.user_agent)
.timeout(config.scraping.timeout)
.connect_timeout(config.scraping.connect_timeout)
.use_rustls_tls()
.build()
.use_rustls_tls();
if let Some(proxy) = config.scraping.proxy.clone() {
builder = builder.proxy(reqwest::Proxy::all(proxy)?);
}
builder.build()
}
src/proxy.rs
+2 -2
View File
@@ -83,7 +83,7 @@ impl Proxy {
}
pub async fn check(&mut self, config: &Config) -> color_eyre::Result<()> {
let client = reqwest::Client::builder()
let client = reqwest::ClientBuilder::new()
.user_agent(&config.checking.user_agent)
.proxy(self.try_into()?)
.timeout(config.checking.timeout)
@@ -93,7 +93,7 @@ impl Proxy {
.wrap_err("failed to create reqwest::Client")?;
let start = tokio::time::Instant::now();
let response = client
.get(&config.checking.check_url)
.get(config.checking.check_url.clone().unwrap())
.send()
.await?
.error_for_status()?;
src/raw_config.rs
+60 -9
View File
@@ -1,5 +1,5 @@
use std::{
collections::HashSet,
collections::HashMap,
env,
num::NonZero,
path::{Path, PathBuf},
@@ -8,7 +8,7 @@ use std::{
use color_eyre::eyre::WrapErr as _;
use serde::{Deserialize, Deserializer};
use crate::utils::is_http_url;
use crate::http::BasicAuth;
fn validate_positive_f64<'de, D: Deserializer<'de>>(
deserializer: D,
@@ -21,23 +21,72 @@ fn validate_positive_f64<'de, D: Deserializer<'de>>(
}
}
fn validate_http_url<'de, D: Deserializer<'de>>(
fn validate_url_generic<'de, D>(
deserializer: D,
) -> Result<String, D::Error> {
allowed_schemes: &[&str],
) -> Result<Option<url::Url>, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
if s.is_empty() || is_http_url(&s) {
Ok(s)
if s.trim().is_empty() {
return Ok(None);
}
if let Ok(u) = url::Url::parse(&s)
&& allowed_schemes.contains(&u.scheme())
&& u.host_str().is_some()
{
Ok(Some(u))
} else {
let type_label = if let Some((last, rest)) = allowed_schemes
.iter()
.map(|scheme| format!("'{scheme}'"))
.collect::<Vec<_>>()
.split_last()
{
if rest.is_empty() {
last.clone()
} else {
format!("{} or {}", rest.join(", "), last)
}
} else {
String::new()
};
Err(serde::de::Error::custom(format!(
"'{s}' is not a valid 'http' or 'https' url"
"'{s}' is not a valid {type_label} url"
)))
}
}
fn validate_proxy_url<'de, D: Deserializer<'de>>(
deserializer: D,
) -> Result<Option<url::Url>, D::Error> {
validate_url_generic(deserializer, &["http", "https", "socks4", "socks5"])
}
fn validate_http_url<'de, D: Deserializer<'de>>(
deserializer: D,
) -> Result<Option<url::Url>, D::Error> {
validate_url_generic(deserializer, &["http", "https"])
}
#[derive(Deserialize)]
#[serde(untagged)]
pub enum SourceConfig {
Simple(String),
Detailed {
url: String,
#[serde(default)]
basic_auth: Option<BasicAuth>,
#[serde(default)]
headers: Option<HashMap<String, String>>,
},
}
#[derive(Deserialize)]
pub struct ScrapingProtocolConfig {
pub enabled: bool,
pub urls: HashSet<String>,
pub urls: Vec<SourceConfig>,
}
#[derive(Deserialize)]
@@ -47,6 +96,8 @@ pub struct ScrapingConfig {
pub timeout: f64,
#[serde(deserialize_with = "validate_positive_f64")]
pub connect_timeout: f64,
#[serde(deserialize_with = "validate_proxy_url")]
pub proxy: Option<url::Url>,
pub user_agent: String,
pub http: ScrapingProtocolConfig,
@@ -57,7 +108,7 @@ pub struct ScrapingConfig {
#[derive(Deserialize)]
pub struct CheckingConfig {
#[serde(deserialize_with = "validate_http_url")]
pub check_url: String,
pub check_url: Option<url::Url>,
pub max_concurrent_checks: NonZero<usize>,
#[serde(deserialize_with = "validate_positive_f64")]
pub timeout: f64,
src/scraper.rs
+20 -12
View File
@@ -5,7 +5,7 @@ use color_eyre::eyre::{OptionExt as _, WrapErr as _};
#[cfg(feature = "tui")]
use crate::event::{AppEvent, Event};
use crate::{
config::Config,
config::{Config, Source},
http,
parsers::PROXY_REGEX,
proxy::{Proxy, ProxyType},
@@ -17,20 +17,28 @@ async fn scrape_one(
http_client: reqwest::Client,
proto: ProxyType,
proxies: Arc<tokio::sync::Mutex<HashSet<Proxy>>>,
source: &str,
source: Source,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<()> {
let text_result = if let Ok(u) = url::Url::parse(source) {
let text_result = if let Ok(u) = url::Url::parse(&source.url) {
match u.scheme() {
"http" | "https" => http::fetch_text(http_client, source).await,
"http" | "https" => {
http::fetch_text(
http_client,
u,
source.basic_auth,
source.headers,
)
.await
}
_ => match u.to_file_path() {
Ok(path) => tokio::fs::read_to_string(path).await,
Err(()) => tokio::fs::read_to_string(source).await,
Err(()) => tokio::fs::read_to_string(&source.url).await,
}
.map_err(Into::into),
}
} else {
tokio::fs::read_to_string(source).await.map_err(Into::into)
tokio::fs::read_to_string(&source.url).await.map_err(Into::into)
};
#[cfg(feature = "tui")]
@@ -39,7 +47,7 @@ async fn scrape_one(
let text = match text_result {
Ok(text) => text,
Err(e) => {
tracing::warn!("{} | {}", source, pretty_error(&e));
tracing::warn!("{} | {}", source.url, pretty_error(&e));
return Ok(());
}
};
@@ -48,7 +56,7 @@ async fn scrape_one(
PROXY_REGEX.captures_iter(&text).collect::<Result<Vec<_>, _>>()?;
if matches.is_empty() {
tracing::warn!("{source} | No proxies found");
tracing::warn!("{} | No proxies found", source.url);
return Ok(());
}
@@ -57,7 +65,7 @@ async fn scrape_one(
{
tracing::warn!(
"{} | Too many proxies ({}) - skipped",
source,
source.url,
matches.len(),
);
return Ok(());
@@ -115,13 +123,13 @@ pub async fn scrape_all(
let proxies = Arc::new(tokio::sync::Mutex::new(HashSet::new()));
let mut join_set = tokio::task::JoinSet::new();
for (proto, sources) in config.scraping.sources.clone() {
for (proto, sources) in &config.scraping.sources {
#[cfg(feature = "tui")]
drop(tx.send(Event::App(AppEvent::SourcesTotal(
proto.clone(),
sources.len(),
))));
for source in sources {
for source in sources.iter().cloned() {
let config = Arc::clone(&config);
let http_client = http_client.clone();
let proto = proto.clone();
@@ -137,7 +145,7 @@ pub async fn scrape_all(
http_client,
proto,
proxies,
&source,
source,
#[cfg(feature = "tui")]
tx,
) => res,
src/utils.rs
-8
View File
@@ -16,14 +16,6 @@ pub async fn is_docker() -> bool {
}
}
pub fn is_http_url(value: &str) -> bool {
url::Url::parse(value).is_ok_and(|parsed_url| {
let scheme = parsed_url.scheme();
(scheme == "http" || scheme == "https")
&& parsed_url.host_str().is_some()
})
}
pub fn pretty_error(e: &color_eyre::Report) -> String {
e.chain().map(ToString::to_string).collect::<Vec<_>>().join(" \u{2192} ")
}