add asn field to json output
This commit is contained in:
@@ -8,7 +8,7 @@ HTTP, SOCKS4, SOCKS5 proxies scraper and checker.
|
||||
|
||||
- Written in Rust.
|
||||
- Can determine if the proxy is anonymous.
|
||||
- Supports determining the geolocation of the proxy exit node.
|
||||
- Supports determining the geolocation and ASN of the proxy exit node.
|
||||
- Can sort proxies by speed.
|
||||
- Uses regex to find proxies of format `protocol://username:password@host:port` on a web page or in a local file, allowing proxies to be extracted even from json without code changes.
|
||||
- Supports proxies with authentication.
|
||||
|
||||
+7
-3
@@ -15,10 +15,10 @@ timeout = 5.0
|
||||
|
||||
[checking]
|
||||
# Website used to check proxy functionality
|
||||
# Use httpbin-compatible services to get geolocation and anonymity info
|
||||
# Use httpbin-compatible services to get ASN, geolocation and anonymity info
|
||||
# Examples:
|
||||
# "https://httpbin.org/ip" (or any other httpbin-compatible service) - Supports getting geolocation and anonymity info
|
||||
# "https://checkip.amazonaws.com" (or any other plain-ip service) - Supports getting geolocation and anonymity info
|
||||
# "https://httpbin.org/ip" (or any other httpbin-compatible service) - Supports getting ASN, geolocation and anonymity info
|
||||
# "https://checkip.amazonaws.com" (or any other plain-ip service) - Supports getting ASN, geolocation and anonymity info
|
||||
# "https://google.com" - Checks general connectivity
|
||||
# "" - Disable checking entirely
|
||||
check_url = "https://checkip.amazonaws.com"
|
||||
@@ -57,6 +57,10 @@ enabled = true
|
||||
[output.json]
|
||||
enabled = true
|
||||
|
||||
# Include ASN (Autonomous System Number) data for each proxy
|
||||
# Uses offline database for privacy and speed
|
||||
include_asn = true
|
||||
|
||||
# Include geolocation data for each proxy
|
||||
# Uses offline database for privacy and speed
|
||||
include_geolocation = true
|
||||
|
||||
@@ -36,6 +36,7 @@ pub struct TxtOutputConfig {
|
||||
|
||||
pub struct JsonOutputConfig {
|
||||
pub enabled: bool,
|
||||
pub include_asn: bool,
|
||||
pub include_geolocation: bool,
|
||||
}
|
||||
|
||||
@@ -75,6 +76,10 @@ async fn get_output_path(
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub const fn asn_enabled(&self) -> bool {
|
||||
self.output.json.enabled && self.output.json.include_asn
|
||||
}
|
||||
|
||||
pub const fn geolocation_enabled(&self) -> bool {
|
||||
self.output.json.enabled && self.output.json.include_geolocation
|
||||
}
|
||||
@@ -145,6 +150,7 @@ impl Config {
|
||||
txt: TxtOutputConfig { enabled: raw_config.output.txt.enabled },
|
||||
json: JsonOutputConfig {
|
||||
enabled: raw_config.output.json.enabled,
|
||||
include_asn: raw_config.output.json.include_asn,
|
||||
include_geolocation: raw_config
|
||||
.output
|
||||
.json
|
||||
|
||||
+4
-2
@@ -1,10 +1,12 @@
|
||||
#[cfg(feature = "tui")]
|
||||
use crate::ipdb;
|
||||
#[cfg(feature = "tui")]
|
||||
use crate::proxy::ProxyType;
|
||||
|
||||
#[cfg(feature = "tui")]
|
||||
pub enum AppEvent {
|
||||
GeoDbTotal(Option<u64>),
|
||||
GeoDbDownloaded(usize),
|
||||
IpDbTotal(ipdb::DbType, Option<u64>),
|
||||
IpDbDownloaded(ipdb::DbType, usize),
|
||||
|
||||
SourcesTotal(ProxyType, usize),
|
||||
SourceScraped(ProxyType),
|
||||
|
||||
-166
@@ -1,166 +0,0 @@
|
||||
use std::{io, path::PathBuf};
|
||||
|
||||
use color_eyre::eyre::{WrapErr as _, eyre};
|
||||
use tokio::io::AsyncWriteExt as _;
|
||||
|
||||
#[cfg(feature = "tui")]
|
||||
use crate::event::{AppEvent, Event};
|
||||
use crate::{fs::get_cache_path, utils::is_docker};
|
||||
|
||||
const GEODB_URL: &str = "https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb";
|
||||
|
||||
pub async fn get_geodb_path() -> color_eyre::Result<PathBuf> {
|
||||
let mut cache_path =
|
||||
get_cache_path().await.wrap_err("failed to get cache path")?;
|
||||
cache_path.push("geolocation_database.mmdb");
|
||||
Ok(cache_path)
|
||||
}
|
||||
|
||||
async fn get_geodb_etag_path() -> color_eyre::Result<PathBuf> {
|
||||
let mut geodb_path = get_geodb_path()
|
||||
.await
|
||||
.wrap_err("failed to get geolocation database path")?;
|
||||
geodb_path.set_extension("mmdb.etag");
|
||||
Ok(geodb_path)
|
||||
}
|
||||
|
||||
async fn read_etag() -> color_eyre::Result<Option<reqwest::header::HeaderValue>>
|
||||
{
|
||||
let etag_path = get_geodb_etag_path()
|
||||
.await
|
||||
.wrap_err("failed to get geolocation database ETag path")?;
|
||||
match tokio::fs::read_to_string(&etag_path).await {
|
||||
Ok(text) => Ok(text.parse().ok()),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(e) => Err(e).wrap_err_with(move || {
|
||||
format!("failed to read file {} to string", etag_path.display())
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_etag() -> color_eyre::Result<()> {
|
||||
let etag_path = get_geodb_etag_path()
|
||||
.await
|
||||
.wrap_err("failed to get geolocation database ETag path")?;
|
||||
match tokio::fs::remove_file(&etag_path).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(e) => Err(e).wrap_err_with(move || {
|
||||
format!("failed to remove {}", etag_path.display())
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn save_etag(
|
||||
etag: reqwest::header::HeaderValue,
|
||||
) -> color_eyre::Result<()> {
|
||||
let etag_file = get_geodb_etag_path()
|
||||
.await
|
||||
.wrap_err("failed to get geolocation database ETag path")?;
|
||||
tokio::fs::write(&etag_file, etag).await.wrap_err_with(move || {
|
||||
format!("failed to write to file {}", etag_file.display())
|
||||
})
|
||||
}
|
||||
|
||||
async fn save_geodb(
|
||||
mut response: reqwest::Response,
|
||||
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> color_eyre::Result<()> {
|
||||
#[cfg(feature = "tui")]
|
||||
tx.send(Event::App(AppEvent::GeoDbTotal(response.content_length())))?;
|
||||
let geodb_file = get_geodb_path()
|
||||
.await
|
||||
.wrap_err("failed to get geolocation database path")?;
|
||||
let mut file =
|
||||
tokio::fs::File::create(&geodb_file).await.wrap_err_with(|| {
|
||||
format!("failed to create file {}", geodb_file.display())
|
||||
})?;
|
||||
while let Some(chunk) = response
|
||||
.chunk()
|
||||
.await
|
||||
.wrap_err("failed to read geolocation database response chunk")?
|
||||
{
|
||||
file.write_all(&chunk).await.wrap_err_with(|| {
|
||||
format!("failed to write to file {}", geodb_file.display())
|
||||
})?;
|
||||
#[cfg(feature = "tui")]
|
||||
tx.send(Event::App(AppEvent::GeoDbDownloaded(chunk.len())))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn download_geodb(
|
||||
http_client: reqwest::Client,
|
||||
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> color_eyre::Result<()> {
|
||||
let geodb_file = get_geodb_path()
|
||||
.await
|
||||
.wrap_err("failed to get geolocation database path")?;
|
||||
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
if tokio::fs::metadata(&geodb_file).await.is_ok_and(|m| m.is_file()) {
|
||||
if let Some(etag) = read_etag().await.wrap_err("failed to read ETag")? {
|
||||
headers.insert(reqwest::header::IF_NONE_MATCH, etag);
|
||||
}
|
||||
}
|
||||
|
||||
let response = http_client
|
||||
.get(GEODB_URL)
|
||||
.headers(headers)
|
||||
.send()
|
||||
.await
|
||||
.wrap_err("failed to send geolocation database download request")?
|
||||
.error_for_status()
|
||||
.wrap_err(
|
||||
"got error HTTP status code when downloading geolocation database",
|
||||
)?;
|
||||
|
||||
if response.status() == reqwest::StatusCode::NOT_MODIFIED {
|
||||
log::info!(
|
||||
"Latest geolocation database is already cached at {}",
|
||||
geodb_file.display()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
return Err(eyre!(
|
||||
"HTTP status error ({}) for url ({})",
|
||||
response.status(),
|
||||
response.url()
|
||||
));
|
||||
}
|
||||
|
||||
let etag = response.headers().get(reqwest::header::ETAG).cloned();
|
||||
|
||||
save_geodb(
|
||||
response,
|
||||
#[cfg(feature = "tui")]
|
||||
tx.clone(),
|
||||
)
|
||||
.await
|
||||
.wrap_err("failed to save geolocation database")?;
|
||||
|
||||
if is_docker().await {
|
||||
log::info!(
|
||||
"Downloaded geolocation database to Docker volume ({} in \
|
||||
container)",
|
||||
geodb_file.display()
|
||||
);
|
||||
} else {
|
||||
log::info!(
|
||||
"Downloaded geolocation database to {}",
|
||||
geodb_file.display()
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(etag_value) = etag {
|
||||
save_etag(etag_value)
|
||||
.await
|
||||
.wrap_err("failed to save geolocation database ETag")
|
||||
} else {
|
||||
remove_etag()
|
||||
.await
|
||||
.wrap_err("failed to remove geolocation database ETag")
|
||||
}
|
||||
}
|
||||
+203
@@ -0,0 +1,203 @@
|
||||
use std::{io, path::PathBuf};
|
||||
|
||||
use color_eyre::eyre::{WrapErr as _, eyre};
|
||||
use tokio::io::AsyncWriteExt as _;
|
||||
|
||||
#[cfg(feature = "tui")]
|
||||
use crate::event::{AppEvent, Event};
|
||||
use crate::{fs::get_cache_path, utils::is_docker};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub enum DbType {
|
||||
Asn,
|
||||
Geo,
|
||||
}
|
||||
|
||||
impl DbType {
|
||||
const fn url(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Asn => {
|
||||
"https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-ASN.mmdb"
|
||||
}
|
||||
Self::Geo => {
|
||||
"https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const fn name(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Asn => "ASN",
|
||||
Self::Geo => "geolocation",
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn db_path(&self) -> color_eyre::Result<PathBuf> {
|
||||
let mut cache_path =
|
||||
get_cache_path().await.wrap_err("failed to get cache path")?;
|
||||
match self {
|
||||
Self::Asn => cache_path.push("asn_database.mmdb"),
|
||||
Self::Geo => cache_path.push("geolocation_database.mmdb"),
|
||||
}
|
||||
Ok(cache_path)
|
||||
}
|
||||
|
||||
async fn etag_path(&self) -> color_eyre::Result<PathBuf> {
|
||||
let mut db_path = self.db_path().await.wrap_err_with(move || {
|
||||
format!("failed to get {} database path", self.name())
|
||||
})?;
|
||||
db_path.set_extension("mmdb.etag");
|
||||
Ok(db_path)
|
||||
}
|
||||
|
||||
async fn read_etag(
|
||||
&self,
|
||||
) -> color_eyre::Result<Option<reqwest::header::HeaderValue>> {
|
||||
let path = self.etag_path().await?;
|
||||
match tokio::fs::read_to_string(&path).await {
|
||||
Ok(text) => Ok(text.parse().ok()),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(None),
|
||||
Err(e) => Err(e).wrap_err_with(move || {
|
||||
format!("failed to read file {} to string", path.display())
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_etag(&self) -> color_eyre::Result<()> {
|
||||
let path = self.etag_path().await?;
|
||||
match tokio::fs::remove_file(&path).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(e) => Err(e).wrap_err_with(move || {
|
||||
format!("failed to remove {}", path.display())
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
async fn save_etag(
|
||||
&self,
|
||||
etag: reqwest::header::HeaderValue,
|
||||
) -> color_eyre::Result<()> {
|
||||
let path = self.etag_path().await?;
|
||||
tokio::fs::write(&path, etag).await.wrap_err_with(move || {
|
||||
format!("failed to write to file {}", path.display())
|
||||
})
|
||||
}
|
||||
|
||||
async fn save_db(
|
||||
&self,
|
||||
mut response: reqwest::Response,
|
||||
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> color_eyre::Result<()> {
|
||||
#[cfg(feature = "tui")]
|
||||
tx.send(Event::App(AppEvent::IpDbTotal(
|
||||
self.clone(),
|
||||
response.content_length(),
|
||||
)))?;
|
||||
|
||||
let db_path = self.db_path().await?;
|
||||
let mut file =
|
||||
tokio::fs::File::create(&db_path).await.wrap_err_with(|| {
|
||||
format!("failed to create file {}", db_path.display())
|
||||
})?;
|
||||
while let Some(chunk) = response.chunk().await.wrap_err_with(|| {
|
||||
format!("failed to read {} database response chunk", self.name())
|
||||
})? {
|
||||
file.write_all(&chunk).await.wrap_err_with(|| {
|
||||
format!("failed to write to file {}", db_path.display())
|
||||
})?;
|
||||
#[cfg(feature = "tui")]
|
||||
tx.send(Event::App(AppEvent::IpDbDownloaded(
|
||||
self.clone(),
|
||||
chunk.len(),
|
||||
)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn download_db(
|
||||
self,
|
||||
http_client: reqwest::Client,
|
||||
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> color_eyre::Result<()> {
|
||||
let db_path = self.db_path().await?;
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
if tokio::fs::metadata(&db_path).await.is_ok_and(|m| m.is_file()) {
|
||||
if let Some(etag) =
|
||||
self.read_etag().await.wrap_err("failed to read ETag")?
|
||||
{
|
||||
headers.insert(reqwest::header::IF_NONE_MATCH, etag);
|
||||
}
|
||||
}
|
||||
|
||||
let response = http_client
|
||||
.get(self.url())
|
||||
.headers(headers)
|
||||
.send()
|
||||
.await
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to send {} database download request",
|
||||
self.name()
|
||||
)
|
||||
})?
|
||||
.error_for_status()
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"got error HTTP status code when downloading {} database",
|
||||
self.name()
|
||||
)
|
||||
})?;
|
||||
|
||||
if response.status() == reqwest::StatusCode::NOT_MODIFIED {
|
||||
log::info!(
|
||||
"Latest {} database is already cached at {}",
|
||||
self.name(),
|
||||
db_path.display()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
return Err(eyre!(
|
||||
"HTTP status error ({}) for url ({})",
|
||||
response.status(),
|
||||
response.url()
|
||||
));
|
||||
}
|
||||
|
||||
let etag = response.headers().get(reqwest::header::ETAG).cloned();
|
||||
|
||||
self.save_db(
|
||||
response,
|
||||
#[cfg(feature = "tui")]
|
||||
tx.clone(),
|
||||
)
|
||||
.await
|
||||
.wrap_err_with(|| format!("failed to save {} database", self.name()))?;
|
||||
|
||||
if is_docker().await {
|
||||
log::info!(
|
||||
"Downloaded {} database to Docker volume ({} in container)",
|
||||
self.name(),
|
||||
db_path.display()
|
||||
);
|
||||
} else {
|
||||
log::info!(
|
||||
"Downloaded {} database to {}",
|
||||
self.name(),
|
||||
db_path.display()
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(etag_value) = etag {
|
||||
self.save_etag(etag_value).await.wrap_err_with(|| {
|
||||
format!("failed to save {} database ETag", self.name())
|
||||
})
|
||||
} else {
|
||||
self.remove_etag().await.wrap_err_with(|| {
|
||||
format!("failed to remove {} database ETag", self.name())
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
+31
-16
@@ -38,7 +38,7 @@ mod checker;
|
||||
mod config;
|
||||
mod event;
|
||||
mod fs;
|
||||
mod geodb;
|
||||
mod ipdb;
|
||||
mod output;
|
||||
mod parsers;
|
||||
mod proxy;
|
||||
@@ -88,19 +88,37 @@ async fn main() -> color_eyre::Result<()> {
|
||||
let http_client = create_reqwest_client()
|
||||
.wrap_err("failed to create reqwest HTTP client")?;
|
||||
|
||||
let maybe_geodb_task = config.geolocation_enabled().then(|| {
|
||||
let mut output_dependencies_tasks = tokio::task::JoinSet::new();
|
||||
|
||||
if config.asn_enabled() {
|
||||
let http_client = http_client.clone();
|
||||
#[cfg(feature = "tui")]
|
||||
let tx = tx.clone();
|
||||
tokio::spawn(async move {
|
||||
geodb::download_geodb(
|
||||
http_client,
|
||||
#[cfg(feature = "tui")]
|
||||
tx,
|
||||
)
|
||||
.await
|
||||
})
|
||||
});
|
||||
output_dependencies_tasks.spawn(async move {
|
||||
ipdb::DbType::Asn
|
||||
.download_db(
|
||||
http_client,
|
||||
#[cfg(feature = "tui")]
|
||||
tx,
|
||||
)
|
||||
.await
|
||||
});
|
||||
}
|
||||
|
||||
if config.geolocation_enabled() {
|
||||
let http_client = http_client.clone();
|
||||
#[cfg(feature = "tui")]
|
||||
let tx = tx.clone();
|
||||
output_dependencies_tasks.spawn(async move {
|
||||
ipdb::DbType::Geo
|
||||
.download_db(
|
||||
http_client,
|
||||
#[cfg(feature = "tui")]
|
||||
tx,
|
||||
)
|
||||
.await
|
||||
});
|
||||
}
|
||||
|
||||
let proxies = scraper::scrape_all(
|
||||
Arc::clone(&config),
|
||||
@@ -113,11 +131,8 @@ async fn main() -> color_eyre::Result<()> {
|
||||
|
||||
drop(http_client);
|
||||
|
||||
if let Some(geodb_task) = maybe_geodb_task {
|
||||
geodb_task
|
||||
.await
|
||||
.wrap_err("failed to join geolocation database download task")?
|
||||
.wrap_err("failed to download geolocation database")?;
|
||||
while let Some(task) = output_dependencies_tasks.join_next().await {
|
||||
task.wrap_err("failed to join output dependencies task")??;
|
||||
}
|
||||
|
||||
let proxies = if config.checking.check_url.is_empty() {
|
||||
|
||||
+62
-26
@@ -9,7 +9,7 @@ use color_eyre::eyre::WrapErr as _;
|
||||
|
||||
use crate::{
|
||||
config::Config,
|
||||
geodb::get_geodb_path,
|
||||
ipdb,
|
||||
proxy::{Proxy, ProxyType},
|
||||
utils::is_docker,
|
||||
};
|
||||
@@ -35,6 +35,7 @@ struct ProxyJson<'a> {
|
||||
port: u16,
|
||||
timeout: Option<f64>,
|
||||
exit_ip: Option<String>,
|
||||
asn: Option<maxminddb::geoip2::Asn<'a>>,
|
||||
geolocation: Option<maxminddb::geoip2::City<'a>>,
|
||||
}
|
||||
|
||||
@@ -64,13 +65,25 @@ pub async fn save_proxies(
|
||||
}
|
||||
|
||||
if config.output.json.enabled {
|
||||
let maybe_mmdb = if config.output.json.include_geolocation {
|
||||
let geodb_path = get_geodb_path()
|
||||
.await
|
||||
.wrap_err("failed to get geolocation database path")?;
|
||||
let maybe_asn_db = if config.output.json.include_asn {
|
||||
let path = ipdb::DbType::Asn.db_path().await?;
|
||||
Some(
|
||||
tokio::task::spawn_blocking(move || {
|
||||
maxminddb::Reader::open_mmap(geodb_path)
|
||||
maxminddb::Reader::open_mmap(path)
|
||||
})
|
||||
.await
|
||||
.wrap_err("failed to spawn tokio blocking task")?
|
||||
.wrap_err("failed to open ASN database")?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let maybe_geo_db = if config.output.json.include_geolocation {
|
||||
let path = ipdb::DbType::Geo.db_path().await?;
|
||||
Some(
|
||||
tokio::task::spawn_blocking(move || {
|
||||
maxminddb::Reader::open_mmap(path)
|
||||
})
|
||||
.await
|
||||
.wrap_err("failed to spawn tokio blocking task")?
|
||||
@@ -82,22 +95,6 @@ pub async fn save_proxies(
|
||||
|
||||
let mut proxy_dicts = Vec::with_capacity(proxies.len());
|
||||
for proxy in &proxies {
|
||||
let geolocation = if let Some(mmdb) = &maybe_mmdb {
|
||||
if let Some(exit_ip) = proxy.exit_ip.clone() {
|
||||
let exit_ip_addr: IpAddr = exit_ip.parse().wrap_err(
|
||||
"failed to parse proxy's exit ip as IpAddr",
|
||||
)?;
|
||||
mmdb.lookup::<maxminddb::geoip2::City>(exit_ip_addr)
|
||||
.wrap_err_with(move || {
|
||||
format!("failed to lookup {exit_ip_addr} in geolocation database")
|
||||
})?
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
proxy_dicts.push(ProxyJson {
|
||||
protocol: proxy.protocol.clone(),
|
||||
username: proxy.username.clone(),
|
||||
@@ -108,7 +105,44 @@ pub async fn save_proxies(
|
||||
.timeout
|
||||
.map(|d| (d.as_secs_f64() * 100.0).round() / 100.0_f64),
|
||||
exit_ip: proxy.exit_ip.clone(),
|
||||
geolocation,
|
||||
asn: if let Some(asn_db) = &maybe_asn_db {
|
||||
if let Some(exit_ip) = proxy.exit_ip.clone() {
|
||||
let exit_ip_addr: IpAddr = exit_ip.parse().wrap_err(
|
||||
"failed to parse proxy's exit ip as IpAddr",
|
||||
)?;
|
||||
asn_db
|
||||
.lookup::<maxminddb::geoip2::Asn>(exit_ip_addr)
|
||||
.wrap_err_with(move || {
|
||||
format!(
|
||||
"failed to lookup {exit_ip_addr} in ASN \
|
||||
database"
|
||||
)
|
||||
})?
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
},
|
||||
geolocation: if let Some(geo_db) = &maybe_geo_db {
|
||||
if let Some(exit_ip) = proxy.exit_ip.clone() {
|
||||
let exit_ip_addr: IpAddr = exit_ip.parse().wrap_err(
|
||||
"failed to parse proxy's exit ip as IpAddr",
|
||||
)?;
|
||||
geo_db
|
||||
.lookup::<maxminddb::geoip2::City>(exit_ip_addr)
|
||||
.wrap_err_with(move || {
|
||||
format!(
|
||||
"failed to lookup {exit_ip_addr} in \
|
||||
geolocation database"
|
||||
)
|
||||
})?
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
None
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -196,9 +230,11 @@ pub async fn save_proxies(
|
||||
}
|
||||
}
|
||||
|
||||
let path = config.output.path.canonicalize().wrap_err_with(move || {
|
||||
format!("failed to canonicalize {}", config.output.path.display())
|
||||
})?;
|
||||
let path = config
|
||||
.output
|
||||
.path
|
||||
.canonicalize()
|
||||
.unwrap_or_else(move |_| config.output.path.clone());
|
||||
if is_docker().await {
|
||||
log::info!(
|
||||
"Proxies have been saved to ./out ({} in container)",
|
||||
|
||||
@@ -64,6 +64,7 @@ pub struct TxtOutputConfig {
|
||||
#[derive(Deserialize)]
|
||||
pub struct JsonOutputConfig {
|
||||
pub enabled: bool,
|
||||
pub include_asn: bool,
|
||||
pub include_geolocation: bool,
|
||||
}
|
||||
|
||||
|
||||
+40
-11
@@ -22,6 +22,7 @@ use tui_logger::{TuiLoggerWidget, TuiWidgetEvent, TuiWidgetState};
|
||||
|
||||
use crate::{
|
||||
event::{AppEvent, Event},
|
||||
ipdb,
|
||||
proxy::ProxyType,
|
||||
utils::is_docker,
|
||||
};
|
||||
@@ -98,8 +99,11 @@ pub enum AppMode {
|
||||
pub struct AppState {
|
||||
pub mode: AppMode,
|
||||
|
||||
pub geodb_total: u64,
|
||||
pub geodb_downloaded: usize,
|
||||
pub asn_db_total: u64,
|
||||
pub asn_db_downloaded: usize,
|
||||
|
||||
pub geo_db_total: u64,
|
||||
pub geo_db_downloaded: usize,
|
||||
|
||||
pub sources_total: HashMap<ProxyType, usize>,
|
||||
pub sources_scraped: HashMap<ProxyType, usize>,
|
||||
@@ -164,7 +168,7 @@ fn draw(f: &mut Frame, state: &AppState, logger_state: &TuiWidgetState) {
|
||||
.constraints([
|
||||
// Logs
|
||||
Constraint::Fill(1),
|
||||
// Geolocation database download
|
||||
// IP database download
|
||||
Constraint::Length(3),
|
||||
// Scraping and checking
|
||||
Constraint::Length(1 + (3 * 3) + 1),
|
||||
@@ -187,17 +191,35 @@ fn draw(f: &mut Frame, state: &AppState, logger_state: &TuiWidgetState) {
|
||||
outer_layout[0],
|
||||
);
|
||||
|
||||
let ipdb_layout = Layout::default()
|
||||
.direction(Direction::Horizontal)
|
||||
.constraints([Constraint::Fill(1); 2])
|
||||
.split(outer_layout[1]);
|
||||
f.render_widget(
|
||||
Gauge::default()
|
||||
.block(Block::bordered().title("ASN database download"))
|
||||
.ratio({
|
||||
if state.asn_db_total == 0 {
|
||||
1.0
|
||||
} else {
|
||||
(state.asn_db_downloaded as f64)
|
||||
/ (state.asn_db_total as f64)
|
||||
}
|
||||
}),
|
||||
ipdb_layout[0],
|
||||
);
|
||||
f.render_widget(
|
||||
Gauge::default()
|
||||
.block(Block::bordered().title("Geolocation database download"))
|
||||
.ratio({
|
||||
if state.geodb_total == 0 {
|
||||
if state.geo_db_total == 0 {
|
||||
1.0
|
||||
} else {
|
||||
(state.geodb_downloaded as f64) / (state.geodb_total as f64)
|
||||
(state.geo_db_downloaded as f64)
|
||||
/ (state.geo_db_total as f64)
|
||||
}
|
||||
}),
|
||||
outer_layout[1],
|
||||
ipdb_layout[1],
|
||||
);
|
||||
|
||||
let proxies_layout = Layout::default()
|
||||
@@ -337,12 +359,19 @@ async fn handle_event(
|
||||
}
|
||||
Event::App(app_event) => {
|
||||
match app_event {
|
||||
AppEvent::GeoDbTotal(bytes) => {
|
||||
state.geodb_total = bytes.unwrap_or_default();
|
||||
AppEvent::IpDbTotal(ipdb::DbType::Asn, bytes) => {
|
||||
state.asn_db_total = bytes.unwrap_or_default();
|
||||
}
|
||||
AppEvent::GeoDbDownloaded(bytes) => {
|
||||
state.geodb_downloaded =
|
||||
state.geodb_downloaded.saturating_add(bytes);
|
||||
AppEvent::IpDbTotal(ipdb::DbType::Geo, bytes) => {
|
||||
state.geo_db_total = bytes.unwrap_or_default();
|
||||
}
|
||||
AppEvent::IpDbDownloaded(ipdb::DbType::Asn, bytes) => {
|
||||
state.asn_db_downloaded =
|
||||
state.asn_db_downloaded.saturating_add(bytes);
|
||||
}
|
||||
AppEvent::IpDbDownloaded(ipdb::DbType::Geo, bytes) => {
|
||||
state.geo_db_downloaded =
|
||||
state.geo_db_downloaded.saturating_add(bytes);
|
||||
}
|
||||
AppEvent::SourcesTotal(proxy_type, amount) => {
|
||||
state.sources_total.insert(proxy_type, amount);
|
||||
|
||||
Reference in New Issue
Block a user