haeaeae/proxy-scraper-checker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: reformatting

Browse Source
This commit is contained in:
monosans
2025-04-24 13:44:02 +03:00
parent 673ef8cb3e
commit e30d7b1019
10 changed files with 60 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/config.rs
+8 -4
View File
@@ -4,13 +4,17 @@ use std::{
};
use color_eyre::eyre::{OptionExt as _, WrapErr as _};
use serde::Deserialize;
use crate::{
APP_DIRECTORY_NAME, parsers::parse_ipv4, proxy::ProxyType,
raw_config::RawConfig, utils::is_docker,
parsers::parse_ipv4, proxy::ProxyType, raw_config::RawConfig,
utils::is_docker,
};
pub const APP_DIRECTORY_NAME: &str = "proxy_scraper_checker";
pub const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/135.0.0.0 Safari/537.36";
#[derive(Clone)]
pub enum CheckWebsiteType {
Unknown,
@@ -18,7 +22,7 @@ pub enum CheckWebsiteType {
HttpbinIp,
}
#[derive(Deserialize)]
#[derive(serde::Deserialize)]
pub struct HttpbinResponse {
pub origin: String,
}
src/fs.rs
+1 -1
View File
@@ -2,7 +2,7 @@ use std::path::PathBuf;
use color_eyre::eyre::{OptionExt as _, WrapErr as _};
use crate::APP_DIRECTORY_NAME;
use crate::config::APP_DIRECTORY_NAME;
pub async fn get_cache_path() -> color_eyre::Result<PathBuf> {
static CACHE: tokio::sync::OnceCell<PathBuf> =
src/main.rs
+25 -45
View File
@@ -32,6 +32,7 @@
clippy::std_instead_of_core,
clippy::unwrap_used
)]
mod checker;
mod config;
mod event;
@@ -46,21 +47,14 @@ mod storage;
mod ui;
mod utils;
use std::sync::Arc;
use color_eyre::eyre::WrapErr as _;
use ui::UI as _;
pub const APP_DIRECTORY_NAME: &str = "proxy_scraper_checker";
pub const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
AppleWebKit/537.36 (KHTML, like Gecko) \
Chrome/135.0.0.0 Safari/537.36";
const CONFIG_ENV_VAR: &str = "PROXY_SCRAPER_CHECKER_CONFIG";
fn get_config_path() -> String {
std::env::var(CONFIG_ENV_VAR).unwrap_or_else(|_| "config.toml".to_owned())
}
fn create_reqwest_client() -> reqwest::Result<reqwest::Client> {
reqwest::Client::builder()
.user_agent(USER_AGENT)
.user_agent(config::USER_AGENT)
.timeout(tokio::time::Duration::from_secs(60))
.connect_timeout(tokio::time::Duration::from_secs(5))
.use_rustls_tls()
@@ -69,30 +63,25 @@ fn create_reqwest_client() -> reqwest::Result<reqwest::Client> {
#[tokio::main]
async fn main() -> color_eyre::Result<()> {
color_eyre::eyre::Context::wrap_err(
color_eyre::install(),
"failed to install color_eyre hooks",
)?;
color_eyre::install().wrap_err("failed to install color_eyre hooks")?;
let ui_impl = ui::UIImpl::new()?;
let (tx, rx) = tokio::sync::mpsc::unbounded_channel();
let ui_task = tokio::task::spawn(ui_impl.run(tx.clone(), rx));
let http_client = color_eyre::eyre::Context::wrap_err(
create_reqwest_client(),
"failed to create reqwest HTTP client",
)?;
let http_client = create_reqwest_client()
.wrap_err("failed to create reqwest HTTP client")?;
let raw_config_path = get_config_path();
let raw_config = color_eyre::eyre::Context::wrap_err_with(
raw_config::read_config(&raw_config_path).await,
move || format!("failed to read {raw_config_path}"),
)?;
let raw_config_path = raw_config::get_config_path();
let raw_config = raw_config::read_config(&raw_config_path)
.await
.wrap_err_with(move || format!("failed to read {raw_config_path}"))?;
let config = std::sync::Arc::new(color_eyre::eyre::Context::wrap_err(
config::Config::from_raw_config(raw_config, http_client.clone()).await,
"failed to create Config from RawConfig",
)?);
let config = Arc::new(
config::Config::from_raw_config(raw_config, http_client.clone())
.await
.wrap_err("failed to create Config from RawConfig")?,
);
if config.debug {
ui::UIImpl::set_log_level(log::LevelFilter::Debug);
@@ -107,7 +96,7 @@ async fn main() -> color_eyre::Result<()> {
});
let mut storage = scraper::scrape_all(
std::sync::Arc::clone(&config),
Arc::clone(&config),
http_client.clone(),
tx.clone(),
)
@@ -116,25 +105,16 @@ async fn main() -> color_eyre::Result<()> {
drop(http_client);
if let Some(geodb_task) = maybe_geodb_task {
color_eyre::eyre::Context::wrap_err(
color_eyre::eyre::Context::wrap_err(
geodb_task.await,
"failed to join GeoDB download task",
)?,
"failed to download GeoDB",
)?;
geodb_task
.await
.wrap_err("failed to join GeoDB download task")?
.wrap_err("failed to download GeoDB")?;
}
if !config.check_website.is_empty() {
storage = color_eyre::eyre::Context::wrap_err(
checker::check_all(
std::sync::Arc::clone(&config),
storage,
tx.clone(),
)
.await,
"failed to check proxies",
)?;
storage = checker::check_all(Arc::clone(&config), storage, tx.clone())
.await
.wrap_err("failed to check proxies")?;
}
output::save_proxies(config, storage).await?;
src/output.rs
+5 -7
View File
@@ -1,10 +1,10 @@
use std::{
io, iter,
net::{IpAddr, Ipv4Addr},
sync::Arc,
};
use color_eyre::eyre::WrapErr as _;
use serde::Serialize;
use crate::{
config::Config,
@@ -20,15 +20,13 @@ fn sort_by_timeout(proxy: &Proxy) -> tokio::time::Duration {
fn sort_naturally(proxy: &Proxy) -> (ProxyType, Vec<u8>, u16) {
let host_key = proxy.host.parse::<Ipv4Addr>().map_or_else(
move |_| {
std::iter::repeat_n(u8::MAX, 4).chain(proxy.host.bytes()).collect()
},
move |_| iter::repeat_n(u8::MAX, 4).chain(proxy.host.bytes()).collect(),
|ip| ip.octets().to_vec(),
);
(proxy.protocol.clone(), host_key, proxy.port)
}
#[derive(Serialize)]
#[derive(serde::Serialize)]
struct ProxyJson<'a> {
protocol: ProxyType,
username: Option<String>,
@@ -99,7 +97,7 @@ pub async fn save_proxies(
] {
match tokio::fs::remove_file(&path).await {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e).wrap_err_with(|| {
format!("failed to remove file {}", path.display())
}),
@@ -134,7 +132,7 @@ pub async fn save_proxies(
let folder_path = config.output_path.join(folder);
match tokio::fs::remove_dir_all(&folder_path).await {
Ok(()) => Ok(()),
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()),
Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(()),
Err(e) => Err(e).wrap_err_with(|| {
format!(
"failed to remove directory {}",
src/proxy.rs
+5 -7
View File
@@ -1,11 +1,9 @@
use std::sync::Arc;
use std::{fmt, sync::Arc};
use color_eyre::eyre::{OptionExt as _, WrapErr as _, eyre};
use derivative::Derivative;
use crate::{
USER_AGENT,
config::{CheckWebsiteType, Config, HttpbinResponse},
config::{CheckWebsiteType, Config, HttpbinResponse, USER_AGENT},
parsers::parse_ipv4,
};
@@ -32,8 +30,8 @@ impl TryFrom<&str> for ProxyType {
}
}
impl std::fmt::Display for ProxyType {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
impl fmt::Display for ProxyType {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
write!(
f,
"{}",
@@ -46,7 +44,7 @@ impl std::fmt::Display for ProxyType {
}
}
#[derive(Derivative, Eq)]
#[derive(derivative::Derivative, Eq)]
#[derivative(Hash, PartialEq)]
pub struct Proxy {
pub protocol: ProxyType,
src/raw_config.rs
+7 -1
View File
@@ -1,4 +1,4 @@
use std::{collections::HashSet, path::PathBuf};
use std::{collections::HashSet, env, path::PathBuf};
use color_eyre::eyre::WrapErr as _;
use serde::{Deserialize, Deserializer};
@@ -96,6 +96,12 @@ pub struct ProxySection {
pub sources: HashSet<String>,
}
const CONFIG_ENV_VAR: &str = "PROXY_SCRAPER_CHECKER_CONFIG";
pub fn get_config_path() -> String {
env::var(CONFIG_ENV_VAR).unwrap_or_else(|_| "config.toml".to_owned())
}
pub async fn read_config(path: &str) -> color_eyre::Result<RawConfig> {
let raw_config = tokio::fs::read_to_string(path)
.await
src/scraper.rs
+1 -1
View File
@@ -109,7 +109,7 @@ async fn scrape_one(
}
pub async fn scrape_all(
config: std::sync::Arc<Config>,
config: Arc<Config>,
http_client: reqwest::Client,
tx: tokio::sync::mpsc::UnboundedSender<Event>,
) -> color_eyre::Result<ProxyStorage> {
src/storage.rs
+3 -3
View File
@@ -1,4 +1,4 @@
use std::collections::{HashMap, HashSet};
use std::collections::{HashMap, HashSet, hash_set};
use crate::proxy::{Proxy, ProxyType};
@@ -30,13 +30,13 @@ impl ProxyStorage {
groups
}
pub fn iter(&self) -> std::collections::hash_set::Iter<'_, Proxy> {
pub fn iter(&self) -> hash_set::Iter<'_, Proxy> {
self.proxies.iter()
}
}
impl IntoIterator for ProxyStorage {
type IntoIter = std::collections::hash_set::IntoIter<Proxy>;
type IntoIter = hash_set::IntoIter<Proxy>;
type Item = Proxy;
fn into_iter(self) -> Self::IntoIter {
src/ui/mod.rs
+4 -1
View File
@@ -1,10 +1,13 @@
#![expect(clippy::pub_use)]
use std::marker::Sized;
use crate::event::Event;
pub trait UI {
fn new() -> color_eyre::Result<Self>
where
Self: std::marker::Sized;
Self: Sized;
fn set_log_level(log_level: log::LevelFilter);
src/ui/tui.rs
+1
View File
@@ -7,6 +7,7 @@
clippy::missing_asserts_for_indexing,
clippy::wildcard_enum_match_arm
)]
use std::collections::HashMap;
use color_eyre::eyre::WrapErr as _;