add asn field to json output

This commit is contained in:
monosans
2025-05-24 02:31:23 +03:00
parent 9d78bfc423
commit 5b935043a0
10 changed files with 355 additions and 225 deletions
+1 -1
View File
@@ -8,7 +8,7 @@ HTTP, SOCKS4, SOCKS5 proxies scraper and checker.
- Written in Rust.
- Can determine if the proxy is anonymous.
- Supports determining the geolocation of the proxy exit node.
- Supports determining the geolocation and ASN of the proxy exit node.
- Can sort proxies by speed.
- Uses regex to find proxies of format `protocol://username:password@host:port` on a web page or in a local file, allowing proxies to be extracted even from json without code changes.
- Supports proxies with authentication.
+7 -3
View File
@@ -15,10 +15,10 @@ timeout = 5.0
[checking]
# Website used to check proxy functionality
# Use httpbin-compatible services to get geolocation and anonymity info
# Use httpbin-compatible services to get ASN, geolocation and anonymity info
# Examples:
# "https://httpbin.org/ip" (or any other httpbin-compatible service) - Supports getting geolocation and anonymity info
# "https://checkip.amazonaws.com" (or any other plain-ip service) - Supports getting geolocation and anonymity info
# "https://httpbin.org/ip" (or any other httpbin-compatible service) - Supports getting ASN, geolocation and anonymity info
# "https://checkip.amazonaws.com" (or any other plain-ip service) - Supports getting ASN, geolocation and anonymity info
# "https://google.com" - Checks general connectivity
# "" - Disable checking entirely
check_url = "https://checkip.amazonaws.com"
@@ -57,6 +57,10 @@ enabled = true
[output.json]
enabled = true
# Include ASN (Autonomous System Number) data for each proxy
# Uses offline database for privacy and speed
include_asn = true
# Include geolocation data for each proxy
# Uses offline database for privacy and speed
include_geolocation = true
+6
View File
@@ -36,6 +36,7 @@ pub struct TxtOutputConfig {
pub struct JsonOutputConfig {
pub enabled: bool,
pub include_asn: bool,
pub include_geolocation: bool,
}
@@ -75,6 +76,10 @@ async fn get_output_path(
}
impl Config {
pub const fn asn_enabled(&self) -> bool {
self.output.json.enabled && self.output.json.include_asn
}
pub const fn geolocation_enabled(&self) -> bool {
self.output.json.enabled && self.output.json.include_geolocation
}
@@ -145,6 +150,7 @@ impl Config {
txt: TxtOutputConfig { enabled: raw_config.output.txt.enabled },
json: JsonOutputConfig {
enabled: raw_config.output.json.enabled,
include_asn: raw_config.output.json.include_asn,
include_geolocation: raw_config
.output
.json
+4 -2
View File
@@ -1,10 +1,12 @@
#[cfg(feature = "tui")]
use crate::ipdb;
#[cfg(feature = "tui")]
use crate::proxy::ProxyType;
#[cfg(feature = "tui")]
pub enum AppEvent {
GeoDbTotal(Option<u64>),
GeoDbDownloaded(usize),
IpDbTotal(ipdb::DbType, Option<u64>),
IpDbDownloaded(ipdb::DbType, usize),
SourcesTotal(ProxyType, usize),
SourceScraped(ProxyType),
-166
View File
@@ -1,166 +0,0 @@
use std::{io, path::PathBuf};
use color_eyre::eyre::{WrapErr as _, eyre};
use tokio::io::AsyncWriteExt as _;
#[cfg(feature = "tui")]
use crate::event::{AppEvent, Event};
use crate::{fs::get_cache_path, utils::is_docker};
const GEODB_URL: &str = "https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb";
pub async fn get_geodb_path() -> color_eyre::Result<PathBuf> {
let mut cache_path =
get_cache_path().await.wrap_err("failed to get cache path")?;
cache_path.push("geolocation_database.mmdb");
Ok(cache_path)
}
async fn get_geodb_etag_path() -> color_eyre::Result<PathBuf> {
let mut geodb_path = get_geodb_path()
.await
.wrap_err("failed to get geolocation database path")?;
geodb_path.set_extension("mmdb.etag");
Ok(geodb_path)
}
async fn read_etag() -> color_eyre::Result<Option<reqwest::header::HeaderValue>>
{
let etag_path = get_geodb_etag_path()
.await
.wrap_err("failed to get geolocation database ETag path")?;
match tokio::fs::read_to_string(&etag_path).await {
Ok(text) => Ok(text.parse().ok()),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e).wrap_err_with(move || {
format!("failed to read file {} to string", etag_path.display())
}),
}
}
async fn remove_etag() -> color_eyre::Result<()> {
let etag_path = get_geodb_etag_path()
.await
.wrap_err("failed to get geolocation database ETag path")?;
match tokio::fs::remove_file(&etag_path).await {
Ok(()) => Ok(()),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e).wrap_err_with(move || {
format!("failed to remove {}", etag_path.display())
}),
}
}
async fn save_etag(
etag: reqwest::header::HeaderValue,
) -> color_eyre::Result<()> {
let etag_file = get_geodb_etag_path()
.await
.wrap_err("failed to get geolocation database ETag path")?;
tokio::fs::write(&etag_file, etag).await.wrap_err_with(move || {
format!("failed to write to file {}", etag_file.display())
})
}
async fn save_geodb(
mut response: reqwest::Response,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<()> {
#[cfg(feature = "tui")]
tx.send(Event::App(AppEvent::GeoDbTotal(response.content_length())))?;
let geodb_file = get_geodb_path()
.await
.wrap_err("failed to get geolocation database path")?;
let mut file =
tokio::fs::File::create(&geodb_file).await.wrap_err_with(|| {
format!("failed to create file {}", geodb_file.display())
})?;
while let Some(chunk) = response
.chunk()
.await
.wrap_err("failed to read geolocation database response chunk")?
{
file.write_all(&chunk).await.wrap_err_with(|| {
format!("failed to write to file {}", geodb_file.display())
})?;
#[cfg(feature = "tui")]
tx.send(Event::App(AppEvent::GeoDbDownloaded(chunk.len())))?;
}
Ok(())
}
pub async fn download_geodb(
http_client: reqwest::Client,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<()> {
let geodb_file = get_geodb_path()
.await
.wrap_err("failed to get geolocation database path")?;
let mut headers = reqwest::header::HeaderMap::new();
if tokio::fs::metadata(&geodb_file).await.is_ok_and(|m| m.is_file()) {
if let Some(etag) = read_etag().await.wrap_err("failed to read ETag")? {
headers.insert(reqwest::header::IF_NONE_MATCH, etag);
}
}
let response = http_client
.get(GEODB_URL)
.headers(headers)
.send()
.await
.wrap_err("failed to send geolocation database download request")?
.error_for_status()
.wrap_err(
"got error HTTP status code when downloading geolocation database",
)?;
if response.status() == reqwest::StatusCode::NOT_MODIFIED {
log::info!(
"Latest geolocation database is already cached at {}",
geodb_file.display()
);
return Ok(());
}
if response.status() != reqwest::StatusCode::OK {
return Err(eyre!(
"HTTP status error ({}) for url ({})",
response.status(),
response.url()
));
}
let etag = response.headers().get(reqwest::header::ETAG).cloned();
save_geodb(
response,
#[cfg(feature = "tui")]
tx.clone(),
)
.await
.wrap_err("failed to save geolocation database")?;
if is_docker().await {
log::info!(
"Downloaded geolocation database to Docker volume ({} in \
container)",
geodb_file.display()
);
} else {
log::info!(
"Downloaded geolocation database to {}",
geodb_file.display()
);
}
if let Some(etag_value) = etag {
save_etag(etag_value)
.await
.wrap_err("failed to save geolocation database ETag")
} else {
remove_etag()
.await
.wrap_err("failed to remove geolocation database ETag")
}
}
+203
View File
@@ -0,0 +1,203 @@
use std::{io, path::PathBuf};
use color_eyre::eyre::{WrapErr as _, eyre};
use tokio::io::AsyncWriteExt as _;
#[cfg(feature = "tui")]
use crate::event::{AppEvent, Event};
use crate::{fs::get_cache_path, utils::is_docker};
#[derive(Clone)]
pub enum DbType {
Asn,
Geo,
}
impl DbType {
const fn url(&self) -> &'static str {
match self {
Self::Asn => {
"https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-ASN.mmdb"
}
Self::Geo => {
"https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb"
}
}
}
const fn name(&self) -> &'static str {
match self {
Self::Asn => "ASN",
Self::Geo => "geolocation",
}
}
pub async fn db_path(&self) -> color_eyre::Result<PathBuf> {
let mut cache_path =
get_cache_path().await.wrap_err("failed to get cache path")?;
match self {
Self::Asn => cache_path.push("asn_database.mmdb"),
Self::Geo => cache_path.push("geolocation_database.mmdb"),
}
Ok(cache_path)
}
async fn etag_path(&self) -> color_eyre::Result<PathBuf> {
let mut db_path = self.db_path().await.wrap_err_with(move || {
format!("failed to get {} database path", self.name())
})?;
db_path.set_extension("mmdb.etag");
Ok(db_path)
}
async fn read_etag(
&self,
) -> color_eyre::Result<Option<reqwest::header::HeaderValue>> {
let path = self.etag_path().await?;
match tokio::fs::read_to_string(&path).await {
Ok(text) => Ok(text.parse().ok()),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
Err(e) => Err(e).wrap_err_with(move || {
format!("failed to read file {} to string", path.display())
}),
}
}
async fn remove_etag(&self) -> color_eyre::Result<()> {
let path = self.etag_path().await?;
match tokio::fs::remove_file(&path).await {
Ok(()) => Ok(()),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e).wrap_err_with(move || {
format!("failed to remove {}", path.display())
}),
}
}
async fn save_etag(
&self,
etag: reqwest::header::HeaderValue,
) -> color_eyre::Result<()> {
let path = self.etag_path().await?;
tokio::fs::write(&path, etag).await.wrap_err_with(move || {
format!("failed to write to file {}", path.display())
})
}
async fn save_db(
&self,
mut response: reqwest::Response,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<()> {
#[cfg(feature = "tui")]
tx.send(Event::App(AppEvent::IpDbTotal(
self.clone(),
response.content_length(),
)))?;
let db_path = self.db_path().await?;
let mut file =
tokio::fs::File::create(&db_path).await.wrap_err_with(|| {
format!("failed to create file {}", db_path.display())
})?;
while let Some(chunk) = response.chunk().await.wrap_err_with(|| {
format!("failed to read {} database response chunk", self.name())
})? {
file.write_all(&chunk).await.wrap_err_with(|| {
format!("failed to write to file {}", db_path.display())
})?;
#[cfg(feature = "tui")]
tx.send(Event::App(AppEvent::IpDbDownloaded(
self.clone(),
chunk.len(),
)))?;
}
Ok(())
}
pub async fn download_db(
self,
http_client: reqwest::Client,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<()> {
let db_path = self.db_path().await?;
let mut headers = reqwest::header::HeaderMap::new();
if tokio::fs::metadata(&db_path).await.is_ok_and(|m| m.is_file()) {
if let Some(etag) =
self.read_etag().await.wrap_err("failed to read ETag")?
{
headers.insert(reqwest::header::IF_NONE_MATCH, etag);
}
}
let response = http_client
.get(self.url())
.headers(headers)
.send()
.await
.wrap_err_with(|| {
format!(
"failed to send {} database download request",
self.name()
)
})?
.error_for_status()
.wrap_err_with(|| {
format!(
"got error HTTP status code when downloading {} database",
self.name()
)
})?;
if response.status() == reqwest::StatusCode::NOT_MODIFIED {
log::info!(
"Latest {} database is already cached at {}",
self.name(),
db_path.display()
);
return Ok(());
}
if response.status() != reqwest::StatusCode::OK {
return Err(eyre!(
"HTTP status error ({}) for url ({})",
response.status(),
response.url()
));
}
let etag = response.headers().get(reqwest::header::ETAG).cloned();
self.save_db(
response,
#[cfg(feature = "tui")]
tx.clone(),
)
.await
.wrap_err_with(|| format!("failed to save {} database", self.name()))?;
if is_docker().await {
log::info!(
"Downloaded {} database to Docker volume ({} in container)",
self.name(),
db_path.display()
);
} else {
log::info!(
"Downloaded {} database to {}",
self.name(),
db_path.display()
);
}
if let Some(etag_value) = etag {
self.save_etag(etag_value).await.wrap_err_with(|| {
format!("failed to save {} database ETag", self.name())
})
} else {
self.remove_etag().await.wrap_err_with(|| {
format!("failed to remove {} database ETag", self.name())
})
}
}
}
+31 -16
View File
@@ -38,7 +38,7 @@ mod checker;
mod config;
mod event;
mod fs;
mod geodb;
mod ipdb;
mod output;
mod parsers;
mod proxy;
@@ -88,19 +88,37 @@ async fn main() -> color_eyre::Result<()> {
let http_client = create_reqwest_client()
.wrap_err("failed to create reqwest HTTP client")?;
let maybe_geodb_task = config.geolocation_enabled().then(|| {
let mut output_dependencies_tasks = tokio::task::JoinSet::new();
if config.asn_enabled() {
let http_client = http_client.clone();
#[cfg(feature = "tui")]
let tx = tx.clone();
tokio::spawn(async move {
geodb::download_geodb(
http_client,
#[cfg(feature = "tui")]
tx,
)
.await
})
});
output_dependencies_tasks.spawn(async move {
ipdb::DbType::Asn
.download_db(
http_client,
#[cfg(feature = "tui")]
tx,
)
.await
});
}
if config.geolocation_enabled() {
let http_client = http_client.clone();
#[cfg(feature = "tui")]
let tx = tx.clone();
output_dependencies_tasks.spawn(async move {
ipdb::DbType::Geo
.download_db(
http_client,
#[cfg(feature = "tui")]
tx,
)
.await
});
}
let proxies = scraper::scrape_all(
Arc::clone(&config),
@@ -113,11 +131,8 @@ async fn main() -> color_eyre::Result<()> {
drop(http_client);
if let Some(geodb_task) = maybe_geodb_task {
geodb_task
.await
.wrap_err("failed to join geolocation database download task")?
.wrap_err("failed to download geolocation database")?;
while let Some(task) = output_dependencies_tasks.join_next().await {
task.wrap_err("failed to join output dependencies task")??;
}
let proxies = if config.checking.check_url.is_empty() {
+62 -26
View File
@@ -9,7 +9,7 @@ use color_eyre::eyre::WrapErr as _;
use crate::{
config::Config,
geodb::get_geodb_path,
ipdb,
proxy::{Proxy, ProxyType},
utils::is_docker,
};
@@ -35,6 +35,7 @@ struct ProxyJson<'a> {
port: u16,
timeout: Option<f64>,
exit_ip: Option<String>,
asn: Option<maxminddb::geoip2::Asn<'a>>,
geolocation: Option<maxminddb::geoip2::City<'a>>,
}
@@ -64,13 +65,25 @@ pub async fn save_proxies(
}
if config.output.json.enabled {
let maybe_mmdb = if config.output.json.include_geolocation {
let geodb_path = get_geodb_path()
.await
.wrap_err("failed to get geolocation database path")?;
let maybe_asn_db = if config.output.json.include_asn {
let path = ipdb::DbType::Asn.db_path().await?;
Some(
tokio::task::spawn_blocking(move || {
maxminddb::Reader::open_mmap(geodb_path)
maxminddb::Reader::open_mmap(path)
})
.await
.wrap_err("failed to spawn tokio blocking task")?
.wrap_err("failed to open ASN database")?,
)
} else {
None
};
let maybe_geo_db = if config.output.json.include_geolocation {
let path = ipdb::DbType::Geo.db_path().await?;
Some(
tokio::task::spawn_blocking(move || {
maxminddb::Reader::open_mmap(path)
})
.await
.wrap_err("failed to spawn tokio blocking task")?
@@ -82,22 +95,6 @@ pub async fn save_proxies(
let mut proxy_dicts = Vec::with_capacity(proxies.len());
for proxy in &proxies {
let geolocation = if let Some(mmdb) = &maybe_mmdb {
if let Some(exit_ip) = proxy.exit_ip.clone() {
let exit_ip_addr: IpAddr = exit_ip.parse().wrap_err(
"failed to parse proxy's exit ip as IpAddr",
)?;
mmdb.lookup::<maxminddb::geoip2::City>(exit_ip_addr)
.wrap_err_with(move || {
format!("failed to lookup {exit_ip_addr} in geolocation database")
})?
} else {
None
}
} else {
None
};
proxy_dicts.push(ProxyJson {
protocol: proxy.protocol.clone(),
username: proxy.username.clone(),
@@ -108,7 +105,44 @@ pub async fn save_proxies(
.timeout
.map(|d| (d.as_secs_f64() * 100.0).round() / 100.0_f64),
exit_ip: proxy.exit_ip.clone(),
geolocation,
asn: if let Some(asn_db) = &maybe_asn_db {
if let Some(exit_ip) = proxy.exit_ip.clone() {
let exit_ip_addr: IpAddr = exit_ip.parse().wrap_err(
"failed to parse proxy's exit ip as IpAddr",
)?;
asn_db
.lookup::<maxminddb::geoip2::Asn>(exit_ip_addr)
.wrap_err_with(move || {
format!(
"failed to lookup {exit_ip_addr} in ASN \
database"
)
})?
} else {
None
}
} else {
None
},
geolocation: if let Some(geo_db) = &maybe_geo_db {
if let Some(exit_ip) = proxy.exit_ip.clone() {
let exit_ip_addr: IpAddr = exit_ip.parse().wrap_err(
"failed to parse proxy's exit ip as IpAddr",
)?;
geo_db
.lookup::<maxminddb::geoip2::City>(exit_ip_addr)
.wrap_err_with(move || {
format!(
"failed to lookup {exit_ip_addr} in \
geolocation database"
)
})?
} else {
None
}
} else {
None
},
});
}
@@ -196,9 +230,11 @@ pub async fn save_proxies(
}
}
let path = config.output.path.canonicalize().wrap_err_with(move || {
format!("failed to canonicalize {}", config.output.path.display())
})?;
let path = config
.output
.path
.canonicalize()
.unwrap_or_else(move |_| config.output.path.clone());
if is_docker().await {
log::info!(
"Proxies have been saved to ./out ({} in container)",
+1
View File
@@ -64,6 +64,7 @@ pub struct TxtOutputConfig {
#[derive(Deserialize)]
pub struct JsonOutputConfig {
pub enabled: bool,
pub include_asn: bool,
pub include_geolocation: bool,
}
+40 -11
View File
@@ -22,6 +22,7 @@ use tui_logger::{TuiLoggerWidget, TuiWidgetEvent, TuiWidgetState};
use crate::{
event::{AppEvent, Event},
ipdb,
proxy::ProxyType,
utils::is_docker,
};
@@ -98,8 +99,11 @@ pub enum AppMode {
pub struct AppState {
pub mode: AppMode,
pub geodb_total: u64,
pub geodb_downloaded: usize,
pub asn_db_total: u64,
pub asn_db_downloaded: usize,
pub geo_db_total: u64,
pub geo_db_downloaded: usize,
pub sources_total: HashMap<ProxyType, usize>,
pub sources_scraped: HashMap<ProxyType, usize>,
@@ -164,7 +168,7 @@ fn draw(f: &mut Frame, state: &AppState, logger_state: &TuiWidgetState) {
.constraints([
// Logs
Constraint::Fill(1),
// Geolocation database download
// IP database download
Constraint::Length(3),
// Scraping and checking
Constraint::Length(1 + (3 * 3) + 1),
@@ -187,17 +191,35 @@ fn draw(f: &mut Frame, state: &AppState, logger_state: &TuiWidgetState) {
outer_layout[0],
);
let ipdb_layout = Layout::default()
.direction(Direction::Horizontal)
.constraints([Constraint::Fill(1); 2])
.split(outer_layout[1]);
f.render_widget(
Gauge::default()
.block(Block::bordered().title("ASN database download"))
.ratio({
if state.asn_db_total == 0 {
1.0
} else {
(state.asn_db_downloaded as f64)
/ (state.asn_db_total as f64)
}
}),
ipdb_layout[0],
);
f.render_widget(
Gauge::default()
.block(Block::bordered().title("Geolocation database download"))
.ratio({
if state.geodb_total == 0 {
if state.geo_db_total == 0 {
1.0
} else {
(state.geodb_downloaded as f64) / (state.geodb_total as f64)
(state.geo_db_downloaded as f64)
/ (state.geo_db_total as f64)
}
}),
outer_layout[1],
ipdb_layout[1],
);
let proxies_layout = Layout::default()
@@ -337,12 +359,19 @@ async fn handle_event(
}
Event::App(app_event) => {
match app_event {
AppEvent::GeoDbTotal(bytes) => {
state.geodb_total = bytes.unwrap_or_default();
AppEvent::IpDbTotal(ipdb::DbType::Asn, bytes) => {
state.asn_db_total = bytes.unwrap_or_default();
}
AppEvent::GeoDbDownloaded(bytes) => {
state.geodb_downloaded =
state.geodb_downloaded.saturating_add(bytes);
AppEvent::IpDbTotal(ipdb::DbType::Geo, bytes) => {
state.geo_db_total = bytes.unwrap_or_default();
}
AppEvent::IpDbDownloaded(ipdb::DbType::Asn, bytes) => {
state.asn_db_downloaded =
state.asn_db_downloaded.saturating_add(bytes);
}
AppEvent::IpDbDownloaded(ipdb::DbType::Geo, bytes) => {
state.geo_db_downloaded =
state.geo_db_downloaded.saturating_add(bytes);
}
AppEvent::SourcesTotal(proxy_type, amount) => {
state.sources_total.insert(proxy_type, amount);