haeaeae/proxy-scraper-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

don't send unused events on non-tui

Browse Source
This commit is contained in:
monosans
2025-04-27 21:59:32 +03:00
parent 1f40832955
commit 6b08bf17bb
5 changed files with 77 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/checker.rs
+16 -9
View File
@@ -2,25 +2,25 @@ use std::sync::Arc;
use color_eyre::eyre::WrapErr as _;
use crate::{
config::Config,
event::{AppEvent, Event},
proxy::Proxy,
storage::ProxyStorage,
};
use crate::{config::Config, proxy::Proxy, storage::ProxyStorage};
#[cfg(feature = "tui")]
use crate::event::{AppEvent, Event};
async fn check_one(
config: Arc<Config>,
mut proxy: Proxy,
tx: tokio::sync::mpsc::UnboundedSender<Event>,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<Proxy> {
let check_result = proxy
.check(Arc::clone(&config))
.await
.wrap_err("proxy did not pass checking");
#[cfg(feature = "tui")]
tx.send(Event::App(AppEvent::ProxyChecked(proxy.protocol.clone())))?;
match check_result {
Ok(()) => {
#[cfg(feature = "tui")]
tx.send(Event::App(AppEvent::ProxyWorking(
proxy.protocol.clone(),
)))?;
@@ -45,20 +45,27 @@ async fn check_one(
pub async fn check_all(
config: Arc<Config>,
storage: ProxyStorage,
tx: tokio::sync::mpsc::UnboundedSender<Event>,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<ProxyStorage> {
let semaphore =
Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_checks));
let mut join_set = tokio::task::JoinSet::new();
for proxy in storage {
let config = Arc::clone(&config);
#[cfg(feature = "tui")]
let tx = tx.clone();
let permit = Arc::clone(&semaphore)
.acquire_owned()
.await
.wrap_err("failed to acquire semaphore")?;
join_set.spawn(async move {
let result = check_one(config, proxy, tx).await;
let result = check_one(
config,
proxy,
#[cfg(feature = "tui")]
tx,
)
.await;
drop(permit);
result
});
src/event.rs
+8 -1
View File
@@ -1,15 +1,22 @@
#[cfg(feature = "tui")]
use crate::proxy::ProxyType;
#[cfg_attr(not(feature = "tui"), expect(dead_code))]
pub enum AppEvent {
#[cfg(feature = "tui")]
GeoDbTotal(Option<u64>),
#[cfg(feature = "tui")]
GeoDbDownloaded(usize),
#[cfg(feature = "tui")]
SourcesTotal(ProxyType, usize),
#[cfg(feature = "tui")]
SourceScraped(ProxyType),
#[cfg(feature = "tui")]
TotalProxies(ProxyType, usize),
#[cfg(feature = "tui")]
ProxyChecked(ProxyType),
#[cfg(feature = "tui")]
ProxyWorking(ProxyType),
Done,
src/geodb.rs
+15 -8
View File
@@ -3,11 +3,10 @@ use std::{io, path::PathBuf};
use color_eyre::eyre::WrapErr as _;
use tokio::io::AsyncWriteExt as _;
use crate::{
event::{AppEvent, Event},
fs::get_cache_path,
utils::is_docker,
};
use crate::{fs::get_cache_path, utils::is_docker};
#[cfg(feature = "tui")]
use crate::event::{AppEvent, Event};
const GEODB_URL: &str = "https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb";
@@ -65,8 +64,9 @@ async fn save_etag(
async fn save_geodb(
mut response: reqwest::Response,
tx: tokio::sync::mpsc::UnboundedSender<Event>,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<()> {
#[cfg(feature = "tui")]
tx.send(Event::App(AppEvent::GeoDbTotal(response.content_length())))?;
let geodb_file =
get_geodb_path().await.wrap_err("failed to get GeoDB path")?;
@@ -82,6 +82,7 @@ async fn save_geodb(
file.write_all(&chunk).await.wrap_err_with(|| {
format!("failed to write to file {}", geodb_file.display())
})?;
#[cfg(feature = "tui")]
tx.send(Event::App(AppEvent::GeoDbDownloaded(chunk.len())))?;
}
Ok(())
@@ -89,7 +90,7 @@ async fn save_geodb(
pub async fn download_geodb(
http_client: reqwest::Client,
tx: tokio::sync::mpsc::UnboundedSender<Event>,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<()> {
let geodb_file =
get_geodb_path().await.wrap_err("failed to get GeoDB path")?;
@@ -120,7 +121,13 @@ pub async fn download_geodb(
let etag = response.headers().get(reqwest::header::ETAG).cloned();
save_geodb(response, tx.clone()).await.wrap_err("failed to save GeoDB")?;
save_geodb(
response,
#[cfg(feature = "tui")]
tx.clone(),
)
.await
.wrap_err("failed to save GeoDB")?;
if is_docker().await {
log::info!(
src/main.rs
+18 -6
View File
@@ -87,15 +87,22 @@ async fn main() -> color_eyre::Result<()> {
let maybe_geodb_task = config.enable_geolocation.then(|| {
let http_client = http_client.clone();
#[cfg(feature = "tui")]
let tx = tx.clone();
tokio::spawn(
async move { geodb::download_geodb(http_client, tx).await },
)
tokio::spawn(async move {
geodb::download_geodb(
http_client,
#[cfg(feature = "tui")]
tx,
)
.await
})
});
let mut storage = scraper::scrape_all(
Arc::clone(&config),
http_client.clone(),
#[cfg(feature = "tui")]
tx.clone(),
)
.await?;
@@ -110,9 +117,14 @@ async fn main() -> color_eyre::Result<()> {
}
if !config.check_website.is_empty() {
storage = checker::check_all(Arc::clone(&config), storage, tx.clone())
.await
.wrap_err("failed to check proxies")?;
storage = checker::check_all(
Arc::clone(&config),
storage,
#[cfg(feature = "tui")]
tx.clone(),
)
.await
.wrap_err("failed to check proxies")?;
}
output::save_proxies(config, storage).await?;
src/scraper.rs
+20 -4
View File
@@ -4,13 +4,15 @@ use color_eyre::eyre::WrapErr as _;
use crate::{
config::Config,
event::{AppEvent, Event},
parsers::PROXY_REGEX,
proxy::{Proxy, ProxyType},
storage::ProxyStorage,
utils::is_http_url,
};
#[cfg(feature = "tui")]
use crate::event::{AppEvent, Event};
async fn fetch_text(
config: Arc<Config>,
http_client: reqwest::Client,
@@ -48,11 +50,12 @@ async fn scrape_one(
http_client: reqwest::Client,
proto: ProxyType,
source: &str,
tx: tokio::sync::mpsc::UnboundedSender<Event>,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<HashSet<Proxy>> {
let text_result =
fetch_text(Arc::clone(&config), http_client.clone(), source).await;
#[cfg(feature = "tui")]
tx.send(Event::App(AppEvent::SourceScraped(proto.clone())))?;
let text = match text_result {
@@ -111,10 +114,11 @@ async fn scrape_one(
pub async fn scrape_all(
config: Arc<Config>,
http_client: reqwest::Client,
tx: tokio::sync::mpsc::UnboundedSender<Event>,
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<ProxyStorage> {
let mut join_set = tokio::task::JoinSet::new();
for (proto, sources) in config.sources.clone() {
#[cfg(feature = "tui")]
tx.send(Event::App(AppEvent::SourcesTotal(
proto.clone(),
sources.len(),
@@ -123,9 +127,18 @@ pub async fn scrape_all(
let config = Arc::clone(&config);
let http_client = http_client.clone();
let proto = proto.clone();
#[cfg(feature = "tui")]
let tx = tx.clone();
join_set.spawn(async move {
scrape_one(config, http_client, proto, &source, tx).await
scrape_one(
config,
http_client,
proto,
&source,
#[cfg(feature = "tui")]
tx,
)
.await
});
}
}
@@ -133,11 +146,14 @@ pub async fn scrape_all(
let mut storage =
ProxyStorage::new(config.sources.keys().cloned().collect());
while let Some(res) = join_set.join_next().await {
#[cfg(feature = "tui")]
let mut seen_protocols = HashSet::new();
for proxy in res.wrap_err("failed to join proxy scrape task")?? {
#[cfg(feature = "tui")]
seen_protocols.insert(proxy.protocol.clone());
storage.insert(proxy);
}
#[cfg(feature = "tui")]
for proto in seen_protocols {
let count = storage.iter().filter(|p| p.protocol == proto).count();
tx.send(Event::App(AppEvent::TotalProxies(proto, count)))?;