diff --git a/README.md b/README.md index 6b34b09..24a6b93 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ HTTP, SOCKS4, SOCKS5 proxies scraper and checker. - Written in Rust. - Can determine if the proxy is anonymous. -- Supports determining the geolocation of the proxy exit node. +- Supports determining the geolocation and ASN of the proxy exit node. - Can sort proxies by speed. - Uses regex to find proxies of format `protocol://username:password@host:port` on a web page or in a local file, allowing proxies to be extracted even from json without code changes. - Supports proxies with authentication. diff --git a/config.toml b/config.toml index d3a3303..36f7a71 100644 --- a/config.toml +++ b/config.toml @@ -15,10 +15,10 @@ timeout = 5.0 [checking] # Website used to check proxy functionality -# Use httpbin-compatible services to get geolocation and anonymity info +# Use httpbin-compatible services to get ASN, geolocation and anonymity info # Examples: -# "https://httpbin.org/ip" (or any other httpbin-compatible service) - Supports getting geolocation and anonymity info -# "https://checkip.amazonaws.com" (or any other plain-ip service) - Supports getting geolocation and anonymity info +# "https://httpbin.org/ip" (or any other httpbin-compatible service) - Supports getting ASN, geolocation and anonymity info +# "https://checkip.amazonaws.com" (or any other plain-ip service) - Supports getting ASN, geolocation and anonymity info # "https://google.com" - Checks general connectivity # "" - Disable checking entirely check_url = "https://checkip.amazonaws.com" @@ -57,6 +57,10 @@ enabled = true [output.json] enabled = true +# Include ASN (Autonomous System Number) data for each proxy +# Uses offline database for privacy and speed +include_asn = true + # Include geolocation data for each proxy # Uses offline database for privacy and speed include_geolocation = true diff --git a/src/config.rs b/src/config.rs index 19caf29..6fb3c79 100644 --- a/src/config.rs +++ b/src/config.rs @@ -36,6 +36,7 @@ pub struct TxtOutputConfig { pub struct JsonOutputConfig { pub enabled: bool, + pub include_asn: bool, pub include_geolocation: bool, } @@ -75,6 +76,10 @@ async fn get_output_path( } impl Config { + pub const fn asn_enabled(&self) -> bool { + self.output.json.enabled && self.output.json.include_asn + } + pub const fn geolocation_enabled(&self) -> bool { self.output.json.enabled && self.output.json.include_geolocation } @@ -145,6 +150,7 @@ impl Config { txt: TxtOutputConfig { enabled: raw_config.output.txt.enabled }, json: JsonOutputConfig { enabled: raw_config.output.json.enabled, + include_asn: raw_config.output.json.include_asn, include_geolocation: raw_config .output .json diff --git a/src/event.rs b/src/event.rs index c710b4d..42269d0 100644 --- a/src/event.rs +++ b/src/event.rs @@ -1,10 +1,12 @@ #[cfg(feature = "tui")] +use crate::ipdb; +#[cfg(feature = "tui")] use crate::proxy::ProxyType; #[cfg(feature = "tui")] pub enum AppEvent { - GeoDbTotal(Option), - GeoDbDownloaded(usize), + IpDbTotal(ipdb::DbType, Option), + IpDbDownloaded(ipdb::DbType, usize), SourcesTotal(ProxyType, usize), SourceScraped(ProxyType), diff --git a/src/geodb.rs b/src/geodb.rs deleted file mode 100644 index ef37bd8..0000000 --- a/src/geodb.rs +++ /dev/null @@ -1,166 +0,0 @@ -use std::{io, path::PathBuf}; - -use color_eyre::eyre::{WrapErr as _, eyre}; -use tokio::io::AsyncWriteExt as _; - -#[cfg(feature = "tui")] -use crate::event::{AppEvent, Event}; -use crate::{fs::get_cache_path, utils::is_docker}; - -const GEODB_URL: &str = "https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb"; - -pub async fn get_geodb_path() -> color_eyre::Result { - let mut cache_path = - get_cache_path().await.wrap_err("failed to get cache path")?; - cache_path.push("geolocation_database.mmdb"); - Ok(cache_path) -} - -async fn get_geodb_etag_path() -> color_eyre::Result { - let mut geodb_path = get_geodb_path() - .await - .wrap_err("failed to get geolocation database path")?; - geodb_path.set_extension("mmdb.etag"); - Ok(geodb_path) -} - -async fn read_etag() -> color_eyre::Result> -{ - let etag_path = get_geodb_etag_path() - .await - .wrap_err("failed to get geolocation database ETag path")?; - match tokio::fs::read_to_string(&etag_path).await { - Ok(text) => Ok(text.parse().ok()), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), - Err(e) => Err(e).wrap_err_with(move || { - format!("failed to read file {} to string", etag_path.display()) - }), - } -} - -async fn remove_etag() -> color_eyre::Result<()> { - let etag_path = get_geodb_etag_path() - .await - .wrap_err("failed to get geolocation database ETag path")?; - match tokio::fs::remove_file(&etag_path).await { - Ok(()) => Ok(()), - Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), - Err(e) => Err(e).wrap_err_with(move || { - format!("failed to remove {}", etag_path.display()) - }), - } -} - -async fn save_etag( - etag: reqwest::header::HeaderValue, -) -> color_eyre::Result<()> { - let etag_file = get_geodb_etag_path() - .await - .wrap_err("failed to get geolocation database ETag path")?; - tokio::fs::write(&etag_file, etag).await.wrap_err_with(move || { - format!("failed to write to file {}", etag_file.display()) - }) -} - -async fn save_geodb( - mut response: reqwest::Response, - #[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender, -) -> color_eyre::Result<()> { - #[cfg(feature = "tui")] - tx.send(Event::App(AppEvent::GeoDbTotal(response.content_length())))?; - let geodb_file = get_geodb_path() - .await - .wrap_err("failed to get geolocation database path")?; - let mut file = - tokio::fs::File::create(&geodb_file).await.wrap_err_with(|| { - format!("failed to create file {}", geodb_file.display()) - })?; - while let Some(chunk) = response - .chunk() - .await - .wrap_err("failed to read geolocation database response chunk")? - { - file.write_all(&chunk).await.wrap_err_with(|| { - format!("failed to write to file {}", geodb_file.display()) - })?; - #[cfg(feature = "tui")] - tx.send(Event::App(AppEvent::GeoDbDownloaded(chunk.len())))?; - } - Ok(()) -} - -pub async fn download_geodb( - http_client: reqwest::Client, - #[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender, -) -> color_eyre::Result<()> { - let geodb_file = get_geodb_path() - .await - .wrap_err("failed to get geolocation database path")?; - - let mut headers = reqwest::header::HeaderMap::new(); - if tokio::fs::metadata(&geodb_file).await.is_ok_and(|m| m.is_file()) { - if let Some(etag) = read_etag().await.wrap_err("failed to read ETag")? { - headers.insert(reqwest::header::IF_NONE_MATCH, etag); - } - } - - let response = http_client - .get(GEODB_URL) - .headers(headers) - .send() - .await - .wrap_err("failed to send geolocation database download request")? - .error_for_status() - .wrap_err( - "got error HTTP status code when downloading geolocation database", - )?; - - if response.status() == reqwest::StatusCode::NOT_MODIFIED { - log::info!( - "Latest geolocation database is already cached at {}", - geodb_file.display() - ); - return Ok(()); - } - - if response.status() != reqwest::StatusCode::OK { - return Err(eyre!( - "HTTP status error ({}) for url ({})", - response.status(), - response.url() - )); - } - - let etag = response.headers().get(reqwest::header::ETAG).cloned(); - - save_geodb( - response, - #[cfg(feature = "tui")] - tx.clone(), - ) - .await - .wrap_err("failed to save geolocation database")?; - - if is_docker().await { - log::info!( - "Downloaded geolocation database to Docker volume ({} in \ - container)", - geodb_file.display() - ); - } else { - log::info!( - "Downloaded geolocation database to {}", - geodb_file.display() - ); - } - - if let Some(etag_value) = etag { - save_etag(etag_value) - .await - .wrap_err("failed to save geolocation database ETag") - } else { - remove_etag() - .await - .wrap_err("failed to remove geolocation database ETag") - } -} diff --git a/src/ipdb.rs b/src/ipdb.rs new file mode 100644 index 0000000..83d676a --- /dev/null +++ b/src/ipdb.rs @@ -0,0 +1,203 @@ +use std::{io, path::PathBuf}; + +use color_eyre::eyre::{WrapErr as _, eyre}; +use tokio::io::AsyncWriteExt as _; + +#[cfg(feature = "tui")] +use crate::event::{AppEvent, Event}; +use crate::{fs::get_cache_path, utils::is_docker}; + +#[derive(Clone)] +pub enum DbType { + Asn, + Geo, +} + +impl DbType { + const fn url(&self) -> &'static str { + match self { + Self::Asn => { + "https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-ASN.mmdb" + } + Self::Geo => { + "https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb" + } + } + } + + const fn name(&self) -> &'static str { + match self { + Self::Asn => "ASN", + Self::Geo => "geolocation", + } + } + + pub async fn db_path(&self) -> color_eyre::Result { + let mut cache_path = + get_cache_path().await.wrap_err("failed to get cache path")?; + match self { + Self::Asn => cache_path.push("asn_database.mmdb"), + Self::Geo => cache_path.push("geolocation_database.mmdb"), + } + Ok(cache_path) + } + + async fn etag_path(&self) -> color_eyre::Result { + let mut db_path = self.db_path().await.wrap_err_with(move || { + format!("failed to get {} database path", self.name()) + })?; + db_path.set_extension("mmdb.etag"); + Ok(db_path) + } + + async fn read_etag( + &self, + ) -> color_eyre::Result> { + let path = self.etag_path().await?; + match tokio::fs::read_to_string(&path).await { + Ok(text) => Ok(text.parse().ok()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(e).wrap_err_with(move || { + format!("failed to read file {} to string", path.display()) + }), + } + } + + async fn remove_etag(&self) -> color_eyre::Result<()> { + let path = self.etag_path().await?; + match tokio::fs::remove_file(&path).await { + Ok(()) => Ok(()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e).wrap_err_with(move || { + format!("failed to remove {}", path.display()) + }), + } + } + + async fn save_etag( + &self, + etag: reqwest::header::HeaderValue, + ) -> color_eyre::Result<()> { + let path = self.etag_path().await?; + tokio::fs::write(&path, etag).await.wrap_err_with(move || { + format!("failed to write to file {}", path.display()) + }) + } + + async fn save_db( + &self, + mut response: reqwest::Response, + #[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender, + ) -> color_eyre::Result<()> { + #[cfg(feature = "tui")] + tx.send(Event::App(AppEvent::IpDbTotal( + self.clone(), + response.content_length(), + )))?; + + let db_path = self.db_path().await?; + let mut file = + tokio::fs::File::create(&db_path).await.wrap_err_with(|| { + format!("failed to create file {}", db_path.display()) + })?; + while let Some(chunk) = response.chunk().await.wrap_err_with(|| { + format!("failed to read {} database response chunk", self.name()) + })? { + file.write_all(&chunk).await.wrap_err_with(|| { + format!("failed to write to file {}", db_path.display()) + })?; + #[cfg(feature = "tui")] + tx.send(Event::App(AppEvent::IpDbDownloaded( + self.clone(), + chunk.len(), + )))?; + } + Ok(()) + } + + pub async fn download_db( + self, + http_client: reqwest::Client, + #[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender, + ) -> color_eyre::Result<()> { + let db_path = self.db_path().await?; + let mut headers = reqwest::header::HeaderMap::new(); + if tokio::fs::metadata(&db_path).await.is_ok_and(|m| m.is_file()) { + if let Some(etag) = + self.read_etag().await.wrap_err("failed to read ETag")? + { + headers.insert(reqwest::header::IF_NONE_MATCH, etag); + } + } + + let response = http_client + .get(self.url()) + .headers(headers) + .send() + .await + .wrap_err_with(|| { + format!( + "failed to send {} database download request", + self.name() + ) + })? + .error_for_status() + .wrap_err_with(|| { + format!( + "got error HTTP status code when downloading {} database", + self.name() + ) + })?; + + if response.status() == reqwest::StatusCode::NOT_MODIFIED { + log::info!( + "Latest {} database is already cached at {}", + self.name(), + db_path.display() + ); + return Ok(()); + } + + if response.status() != reqwest::StatusCode::OK { + return Err(eyre!( + "HTTP status error ({}) for url ({})", + response.status(), + response.url() + )); + } + + let etag = response.headers().get(reqwest::header::ETAG).cloned(); + + self.save_db( + response, + #[cfg(feature = "tui")] + tx.clone(), + ) + .await + .wrap_err_with(|| format!("failed to save {} database", self.name()))?; + + if is_docker().await { + log::info!( + "Downloaded {} database to Docker volume ({} in container)", + self.name(), + db_path.display() + ); + } else { + log::info!( + "Downloaded {} database to {}", + self.name(), + db_path.display() + ); + } + + if let Some(etag_value) = etag { + self.save_etag(etag_value).await.wrap_err_with(|| { + format!("failed to save {} database ETag", self.name()) + }) + } else { + self.remove_etag().await.wrap_err_with(|| { + format!("failed to remove {} database ETag", self.name()) + }) + } + } +} diff --git a/src/main.rs b/src/main.rs index 9686b27..1ec6bc4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -38,7 +38,7 @@ mod checker; mod config; mod event; mod fs; -mod geodb; +mod ipdb; mod output; mod parsers; mod proxy; @@ -88,19 +88,37 @@ async fn main() -> color_eyre::Result<()> { let http_client = create_reqwest_client() .wrap_err("failed to create reqwest HTTP client")?; - let maybe_geodb_task = config.geolocation_enabled().then(|| { + let mut output_dependencies_tasks = tokio::task::JoinSet::new(); + + if config.asn_enabled() { let http_client = http_client.clone(); #[cfg(feature = "tui")] let tx = tx.clone(); - tokio::spawn(async move { - geodb::download_geodb( - http_client, - #[cfg(feature = "tui")] - tx, - ) - .await - }) - }); + output_dependencies_tasks.spawn(async move { + ipdb::DbType::Asn + .download_db( + http_client, + #[cfg(feature = "tui")] + tx, + ) + .await + }); + } + + if config.geolocation_enabled() { + let http_client = http_client.clone(); + #[cfg(feature = "tui")] + let tx = tx.clone(); + output_dependencies_tasks.spawn(async move { + ipdb::DbType::Geo + .download_db( + http_client, + #[cfg(feature = "tui")] + tx, + ) + .await + }); + } let proxies = scraper::scrape_all( Arc::clone(&config), @@ -113,11 +131,8 @@ async fn main() -> color_eyre::Result<()> { drop(http_client); - if let Some(geodb_task) = maybe_geodb_task { - geodb_task - .await - .wrap_err("failed to join geolocation database download task")? - .wrap_err("failed to download geolocation database")?; + while let Some(task) = output_dependencies_tasks.join_next().await { + task.wrap_err("failed to join output dependencies task")??; } let proxies = if config.checking.check_url.is_empty() { diff --git a/src/output.rs b/src/output.rs index 6f83cc2..5eaf7f3 100644 --- a/src/output.rs +++ b/src/output.rs @@ -9,7 +9,7 @@ use color_eyre::eyre::WrapErr as _; use crate::{ config::Config, - geodb::get_geodb_path, + ipdb, proxy::{Proxy, ProxyType}, utils::is_docker, }; @@ -35,6 +35,7 @@ struct ProxyJson<'a> { port: u16, timeout: Option, exit_ip: Option, + asn: Option>, geolocation: Option>, } @@ -64,13 +65,25 @@ pub async fn save_proxies( } if config.output.json.enabled { - let maybe_mmdb = if config.output.json.include_geolocation { - let geodb_path = get_geodb_path() - .await - .wrap_err("failed to get geolocation database path")?; + let maybe_asn_db = if config.output.json.include_asn { + let path = ipdb::DbType::Asn.db_path().await?; Some( tokio::task::spawn_blocking(move || { - maxminddb::Reader::open_mmap(geodb_path) + maxminddb::Reader::open_mmap(path) + }) + .await + .wrap_err("failed to spawn tokio blocking task")? + .wrap_err("failed to open ASN database")?, + ) + } else { + None + }; + + let maybe_geo_db = if config.output.json.include_geolocation { + let path = ipdb::DbType::Geo.db_path().await?; + Some( + tokio::task::spawn_blocking(move || { + maxminddb::Reader::open_mmap(path) }) .await .wrap_err("failed to spawn tokio blocking task")? @@ -82,22 +95,6 @@ pub async fn save_proxies( let mut proxy_dicts = Vec::with_capacity(proxies.len()); for proxy in &proxies { - let geolocation = if let Some(mmdb) = &maybe_mmdb { - if let Some(exit_ip) = proxy.exit_ip.clone() { - let exit_ip_addr: IpAddr = exit_ip.parse().wrap_err( - "failed to parse proxy's exit ip as IpAddr", - )?; - mmdb.lookup::(exit_ip_addr) - .wrap_err_with(move || { - format!("failed to lookup {exit_ip_addr} in geolocation database") - })? - } else { - None - } - } else { - None - }; - proxy_dicts.push(ProxyJson { protocol: proxy.protocol.clone(), username: proxy.username.clone(), @@ -108,7 +105,44 @@ pub async fn save_proxies( .timeout .map(|d| (d.as_secs_f64() * 100.0).round() / 100.0_f64), exit_ip: proxy.exit_ip.clone(), - geolocation, + asn: if let Some(asn_db) = &maybe_asn_db { + if let Some(exit_ip) = proxy.exit_ip.clone() { + let exit_ip_addr: IpAddr = exit_ip.parse().wrap_err( + "failed to parse proxy's exit ip as IpAddr", + )?; + asn_db + .lookup::(exit_ip_addr) + .wrap_err_with(move || { + format!( + "failed to lookup {exit_ip_addr} in ASN \ + database" + ) + })? + } else { + None + } + } else { + None + }, + geolocation: if let Some(geo_db) = &maybe_geo_db { + if let Some(exit_ip) = proxy.exit_ip.clone() { + let exit_ip_addr: IpAddr = exit_ip.parse().wrap_err( + "failed to parse proxy's exit ip as IpAddr", + )?; + geo_db + .lookup::(exit_ip_addr) + .wrap_err_with(move || { + format!( + "failed to lookup {exit_ip_addr} in \ + geolocation database" + ) + })? + } else { + None + } + } else { + None + }, }); } @@ -196,9 +230,11 @@ pub async fn save_proxies( } } - let path = config.output.path.canonicalize().wrap_err_with(move || { - format!("failed to canonicalize {}", config.output.path.display()) - })?; + let path = config + .output + .path + .canonicalize() + .unwrap_or_else(move |_| config.output.path.clone()); if is_docker().await { log::info!( "Proxies have been saved to ./out ({} in container)", diff --git a/src/raw_config.rs b/src/raw_config.rs index 1960a28..b27a2d2 100644 --- a/src/raw_config.rs +++ b/src/raw_config.rs @@ -64,6 +64,7 @@ pub struct TxtOutputConfig { #[derive(Deserialize)] pub struct JsonOutputConfig { pub enabled: bool, + pub include_asn: bool, pub include_geolocation: bool, } diff --git a/src/ui/tui.rs b/src/ui/tui.rs index e421bd8..0759e01 100644 --- a/src/ui/tui.rs +++ b/src/ui/tui.rs @@ -22,6 +22,7 @@ use tui_logger::{TuiLoggerWidget, TuiWidgetEvent, TuiWidgetState}; use crate::{ event::{AppEvent, Event}, + ipdb, proxy::ProxyType, utils::is_docker, }; @@ -98,8 +99,11 @@ pub enum AppMode { pub struct AppState { pub mode: AppMode, - pub geodb_total: u64, - pub geodb_downloaded: usize, + pub asn_db_total: u64, + pub asn_db_downloaded: usize, + + pub geo_db_total: u64, + pub geo_db_downloaded: usize, pub sources_total: HashMap, pub sources_scraped: HashMap, @@ -164,7 +168,7 @@ fn draw(f: &mut Frame, state: &AppState, logger_state: &TuiWidgetState) { .constraints([ // Logs Constraint::Fill(1), - // Geolocation database download + // IP database download Constraint::Length(3), // Scraping and checking Constraint::Length(1 + (3 * 3) + 1), @@ -187,17 +191,35 @@ fn draw(f: &mut Frame, state: &AppState, logger_state: &TuiWidgetState) { outer_layout[0], ); + let ipdb_layout = Layout::default() + .direction(Direction::Horizontal) + .constraints([Constraint::Fill(1); 2]) + .split(outer_layout[1]); + f.render_widget( + Gauge::default() + .block(Block::bordered().title("ASN database download")) + .ratio({ + if state.asn_db_total == 0 { + 1.0 + } else { + (state.asn_db_downloaded as f64) + / (state.asn_db_total as f64) + } + }), + ipdb_layout[0], + ); f.render_widget( Gauge::default() .block(Block::bordered().title("Geolocation database download")) .ratio({ - if state.geodb_total == 0 { + if state.geo_db_total == 0 { 1.0 } else { - (state.geodb_downloaded as f64) / (state.geodb_total as f64) + (state.geo_db_downloaded as f64) + / (state.geo_db_total as f64) } }), - outer_layout[1], + ipdb_layout[1], ); let proxies_layout = Layout::default() @@ -337,12 +359,19 @@ async fn handle_event( } Event::App(app_event) => { match app_event { - AppEvent::GeoDbTotal(bytes) => { - state.geodb_total = bytes.unwrap_or_default(); + AppEvent::IpDbTotal(ipdb::DbType::Asn, bytes) => { + state.asn_db_total = bytes.unwrap_or_default(); } - AppEvent::GeoDbDownloaded(bytes) => { - state.geodb_downloaded = - state.geodb_downloaded.saturating_add(bytes); + AppEvent::IpDbTotal(ipdb::DbType::Geo, bytes) => { + state.geo_db_total = bytes.unwrap_or_default(); + } + AppEvent::IpDbDownloaded(ipdb::DbType::Asn, bytes) => { + state.asn_db_downloaded = + state.asn_db_downloaded.saturating_add(bytes); + } + AppEvent::IpDbDownloaded(ipdb::DbType::Geo, bytes) => { + state.geo_db_downloaded = + state.geo_db_downloaded.saturating_add(bytes); } AppEvent::SourcesTotal(proxy_type, amount) => { state.sources_total.insert(proxy_type, amount);