reconfigure clippy
This commit is contained in:
@@ -403,7 +403,7 @@ jobs:
|
||||
persist-credentials: false
|
||||
- run: rustup toolchain install beta --profile minimal --component clippy --allow-downgrade
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- run: cargo +beta clippy --all-targets ${{ matrix.features }} -- -Dwarnings -Wclippy::pedantic
|
||||
- run: cargo +beta clippy --all-targets ${{ matrix.features }} -- -Dwarnings
|
||||
pre-commit:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
|
||||
@@ -7,6 +7,8 @@ repos:
|
||||
- id: check-illegal-windows-names
|
||||
- id: check-merge-conflict
|
||||
- id: check-shebang-scripts-are-executable
|
||||
exclude_types:
|
||||
- rust
|
||||
- id: check-symlinks
|
||||
- id: check-toml
|
||||
- id: check-xml
|
||||
|
||||
+6
-7
@@ -1,6 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use color_eyre::eyre::WrapErr;
|
||||
use color_eyre::eyre::WrapErr as _;
|
||||
|
||||
use crate::{
|
||||
config::Config,
|
||||
@@ -15,7 +15,7 @@ async fn check_one(
|
||||
tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> color_eyre::Result<Proxy> {
|
||||
let check_result = proxy
|
||||
.check(config.clone())
|
||||
.check(Arc::clone(&config))
|
||||
.await
|
||||
.wrap_err("proxy did not pass checking");
|
||||
tx.send(Event::App(AppEvent::ProxyChecked(proxy.protocol.clone())))?;
|
||||
@@ -34,7 +34,7 @@ async fn check_one(
|
||||
e.chain()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(" → ")
|
||||
.join(" \u{2192} ")
|
||||
);
|
||||
}
|
||||
Err(e)
|
||||
@@ -42,7 +42,7 @@ async fn check_one(
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn check_all(
|
||||
pub async fn check_all(
|
||||
config: Arc<Config>,
|
||||
storage: ProxyStorage,
|
||||
tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
@@ -51,10 +51,9 @@ pub(crate) async fn check_all(
|
||||
Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_checks));
|
||||
let mut join_set = tokio::task::JoinSet::new();
|
||||
for proxy in storage {
|
||||
let config = config.clone();
|
||||
let config = Arc::clone(&config);
|
||||
let tx = tx.clone();
|
||||
let permit = semaphore
|
||||
.clone()
|
||||
let permit = Arc::clone(&semaphore)
|
||||
.acquire_owned()
|
||||
.await
|
||||
.wrap_err("failed to acquire semaphore")?;
|
||||
|
||||
+29
-27
@@ -3,7 +3,7 @@ use std::{
|
||||
path::PathBuf,
|
||||
};
|
||||
|
||||
use color_eyre::eyre::{OptionExt, WrapErr};
|
||||
use color_eyre::eyre::{OptionExt as _, WrapErr as _};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::{
|
||||
@@ -12,19 +12,19 @@ use crate::{
|
||||
};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(crate) enum CheckWebsiteType {
|
||||
pub enum CheckWebsiteType {
|
||||
Unknown,
|
||||
PlainIp,
|
||||
HttpbinIp,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(crate) struct HttpbinResponse {
|
||||
pub(crate) origin: String,
|
||||
pub struct HttpbinResponse {
|
||||
pub origin: String,
|
||||
}
|
||||
|
||||
impl CheckWebsiteType {
|
||||
pub(crate) async fn guess(
|
||||
pub async fn guess(
|
||||
check_website: &str,
|
||||
http_client: reqwest::Client,
|
||||
) -> Self {
|
||||
@@ -81,29 +81,29 @@ impl CheckWebsiteType {
|
||||
Self::Unknown
|
||||
}
|
||||
|
||||
pub(crate) fn supports_geolocation(&self) -> bool {
|
||||
pub const fn supports_geolocation(&self) -> bool {
|
||||
match self {
|
||||
CheckWebsiteType::Unknown => false,
|
||||
CheckWebsiteType::PlainIp | CheckWebsiteType::HttpbinIp => true,
|
||||
Self::Unknown => false,
|
||||
Self::PlainIp | Self::HttpbinIp => true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::struct_excessive_bools)]
|
||||
pub(crate) struct Config {
|
||||
pub(crate) timeout: tokio::time::Duration,
|
||||
pub(crate) source_timeout: tokio::time::Duration,
|
||||
pub(crate) proxies_per_source_limit: usize,
|
||||
pub(crate) max_concurrent_checks: usize,
|
||||
pub(crate) check_website: String,
|
||||
pub(crate) check_website_type: CheckWebsiteType,
|
||||
pub(crate) sort_by_speed: bool,
|
||||
pub(crate) enable_geolocation: bool,
|
||||
pub(crate) debug: bool,
|
||||
pub(crate) output_path: PathBuf,
|
||||
pub(crate) output_json: bool,
|
||||
pub(crate) output_txt: bool,
|
||||
pub(crate) sources: HashMap<ProxyType, HashSet<String>>,
|
||||
#[expect(clippy::struct_excessive_bools)]
|
||||
pub struct Config {
|
||||
pub timeout: tokio::time::Duration,
|
||||
pub source_timeout: tokio::time::Duration,
|
||||
pub proxies_per_source_limit: usize,
|
||||
pub max_concurrent_checks: usize,
|
||||
pub check_website: String,
|
||||
pub check_website_type: CheckWebsiteType,
|
||||
pub sort_by_speed: bool,
|
||||
pub enable_geolocation: bool,
|
||||
pub debug: bool,
|
||||
pub output_path: PathBuf,
|
||||
pub output_json: bool,
|
||||
pub output_txt: bool,
|
||||
pub sources: HashMap<ProxyType, HashSet<String>>,
|
||||
}
|
||||
|
||||
async fn get_output_path(
|
||||
@@ -126,7 +126,7 @@ async fn get_output_path(
|
||||
}
|
||||
|
||||
impl Config {
|
||||
pub(crate) async fn from_raw_config(
|
||||
pub async fn from_raw_config(
|
||||
raw_config: RawConfig,
|
||||
http_client: reqwest::Client,
|
||||
) -> color_eyre::Result<Self> {
|
||||
@@ -143,8 +143,9 @@ impl Config {
|
||||
|
||||
let max_concurrent_checks =
|
||||
match rlimit::increase_nofile_limit(u64::MAX) {
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
Ok(lim) => {
|
||||
#[expect(clippy::as_conversions)]
|
||||
#[expect(clippy::cast_possible_truncation)]
|
||||
if raw_config.max_concurrent_checks > (lim as usize) {
|
||||
log::warn!(
|
||||
"max_concurrent_checks config value is too high \
|
||||
@@ -181,8 +182,9 @@ impl Config {
|
||||
(ProxyType::Socks5, raw_config.socks5),
|
||||
]
|
||||
.into_iter()
|
||||
.filter(|(_, section)| section.enabled)
|
||||
.map(|(proxy_type, section)| (proxy_type, section.sources))
|
||||
.filter_map(|(proxy_type, section)| {
|
||||
section.enabled.then_some((proxy_type, section.sources))
|
||||
})
|
||||
.collect(),
|
||||
})
|
||||
}
|
||||
|
||||
+2
-2
@@ -1,7 +1,7 @@
|
||||
use crate::proxy::ProxyType;
|
||||
|
||||
#[allow(dead_code)]
|
||||
pub(crate) enum AppEvent {
|
||||
pub enum AppEvent {
|
||||
GeoDbTotal(Option<u64>),
|
||||
GeoDbDownloaded(usize),
|
||||
|
||||
@@ -15,7 +15,7 @@ pub(crate) enum AppEvent {
|
||||
Done,
|
||||
}
|
||||
|
||||
pub(crate) enum Event {
|
||||
pub enum Event {
|
||||
#[cfg(feature = "tui")]
|
||||
Tick,
|
||||
#[cfg(feature = "tui")]
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
use color_eyre::eyre::{OptionExt, WrapErr};
|
||||
use color_eyre::eyre::{OptionExt as _, WrapErr as _};
|
||||
|
||||
use crate::APP_DIRECTORY_NAME;
|
||||
|
||||
pub(crate) async fn get_cache_path() -> color_eyre::Result<PathBuf> {
|
||||
pub async fn get_cache_path() -> color_eyre::Result<PathBuf> {
|
||||
static CACHE: tokio::sync::OnceCell<PathBuf> =
|
||||
tokio::sync::OnceCell::const_new();
|
||||
Ok(CACHE
|
||||
|
||||
+4
-4
@@ -1,7 +1,7 @@
|
||||
use std::{io, path::PathBuf};
|
||||
|
||||
use color_eyre::eyre::WrapErr;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use color_eyre::eyre::WrapErr as _;
|
||||
use tokio::io::AsyncWriteExt as _;
|
||||
|
||||
use crate::{
|
||||
event::{AppEvent, Event},
|
||||
@@ -11,7 +11,7 @@ use crate::{
|
||||
|
||||
const GEODB_URL: &str = "https://raw.githubusercontent.com/P3TERX/GeoLite.mmdb/download/GeoLite2-City.mmdb";
|
||||
|
||||
pub(crate) async fn get_geodb_path() -> color_eyre::Result<PathBuf> {
|
||||
pub async fn get_geodb_path() -> color_eyre::Result<PathBuf> {
|
||||
let mut cache_path =
|
||||
get_cache_path().await.wrap_err("failed to get cache path")?;
|
||||
cache_path.push("geolocation_database.mmdb");
|
||||
@@ -88,7 +88,7 @@ async fn save_geodb(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn download_geodb(
|
||||
pub async fn download_geodb(
|
||||
http_client: reqwest::Client,
|
||||
tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> color_eyre::Result<()> {
|
||||
|
||||
+70
-29
@@ -1,22 +1,57 @@
|
||||
pub(crate) mod checker;
|
||||
pub(crate) mod config;
|
||||
pub(crate) mod event;
|
||||
pub(crate) mod fs;
|
||||
pub(crate) mod geodb;
|
||||
pub(crate) mod output;
|
||||
pub(crate) mod parsers;
|
||||
pub(crate) mod proxy;
|
||||
pub(crate) mod raw_config;
|
||||
pub(crate) mod scraper;
|
||||
pub(crate) mod storage;
|
||||
pub(crate) mod ui;
|
||||
pub(crate) mod utils;
|
||||
use ui::UI;
|
||||
#![warn(
|
||||
clippy::all,
|
||||
clippy::pedantic,
|
||||
clippy::restriction,
|
||||
clippy::nursery,
|
||||
clippy::cargo
|
||||
)]
|
||||
#![allow(
|
||||
clippy::absolute_paths,
|
||||
clippy::allow_attributes_without_reason,
|
||||
clippy::allow_attributes,
|
||||
clippy::arbitrary_source_item_ordering,
|
||||
clippy::blanket_clippy_restriction_lints,
|
||||
clippy::default_numeric_fallback,
|
||||
clippy::else_if_without_else,
|
||||
clippy::float_arithmetic,
|
||||
clippy::implicit_return,
|
||||
clippy::iter_over_hash_type,
|
||||
clippy::min_ident_chars,
|
||||
clippy::missing_docs_in_private_items,
|
||||
clippy::mod_module_files,
|
||||
clippy::multiple_crate_versions,
|
||||
clippy::pattern_type_mismatch,
|
||||
clippy::pub_with_shorthand,
|
||||
clippy::too_many_lines,
|
||||
clippy::question_mark_used,
|
||||
clippy::shadow_reuse,
|
||||
clippy::shadow_unrelated,
|
||||
clippy::single_call_fn,
|
||||
clippy::single_char_lifetime_names,
|
||||
clippy::std_instead_of_alloc,
|
||||
clippy::std_instead_of_core,
|
||||
clippy::unwrap_used
|
||||
)]
|
||||
mod checker;
|
||||
mod config;
|
||||
mod event;
|
||||
mod fs;
|
||||
mod geodb;
|
||||
mod output;
|
||||
mod parsers;
|
||||
mod proxy;
|
||||
mod raw_config;
|
||||
mod scraper;
|
||||
mod storage;
|
||||
mod ui;
|
||||
mod utils;
|
||||
|
||||
pub(crate) const APP_DIRECTORY_NAME: &str = "proxy_scraper_checker";
|
||||
pub(crate) const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; \
|
||||
x64) AppleWebKit/537.36 (KHTML, like \
|
||||
Gecko) Chrome/135.0.0.0 Safari/537.36";
|
||||
use ui::UI as _;
|
||||
|
||||
pub const APP_DIRECTORY_NAME: &str = "proxy_scraper_checker";
|
||||
pub const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \
|
||||
AppleWebKit/537.36 (KHTML, like Gecko) \
|
||||
Chrome/135.0.0.0 Safari/537.36";
|
||||
const CONFIG_ENV_VAR: &str = "PROXY_SCRAPER_CHECKER_CONFIG";
|
||||
|
||||
fn get_config_path() -> String {
|
||||
@@ -63,19 +98,20 @@ async fn main() -> color_eyre::Result<()> {
|
||||
ui::UIImpl::set_log_level(log::LevelFilter::Debug);
|
||||
}
|
||||
|
||||
let maybe_geodb_task = if config.enable_geolocation {
|
||||
let maybe_geodb_task = config.enable_geolocation.then(|| {
|
||||
let http_client = http_client.clone();
|
||||
let tx = tx.clone();
|
||||
Some(tokio::spawn(async move {
|
||||
geodb::download_geodb(http_client, tx).await
|
||||
}))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
tokio::spawn(
|
||||
async move { geodb::download_geodb(http_client, tx).await },
|
||||
)
|
||||
});
|
||||
|
||||
let mut storage =
|
||||
scraper::scrape_all(config.clone(), http_client.clone(), tx.clone())
|
||||
.await?;
|
||||
let mut storage = scraper::scrape_all(
|
||||
std::sync::Arc::clone(&config),
|
||||
http_client.clone(),
|
||||
tx.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
drop(http_client);
|
||||
|
||||
@@ -91,7 +127,12 @@ async fn main() -> color_eyre::Result<()> {
|
||||
|
||||
if !config.check_website.is_empty() {
|
||||
storage = color_eyre::eyre::Context::wrap_err(
|
||||
checker::check_all(config.clone(), storage, tx.clone()).await,
|
||||
checker::check_all(
|
||||
std::sync::Arc::clone(&config),
|
||||
storage,
|
||||
tx.clone(),
|
||||
)
|
||||
.await,
|
||||
"failed to check proxies",
|
||||
)?;
|
||||
}
|
||||
|
||||
+10
-10
@@ -3,7 +3,7 @@ use std::{
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use color_eyre::eyre::WrapErr;
|
||||
use color_eyre::eyre::WrapErr as _;
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::{
|
||||
@@ -19,12 +19,12 @@ fn sort_by_timeout(proxy: &Proxy) -> tokio::time::Duration {
|
||||
}
|
||||
|
||||
fn sort_naturally(proxy: &Proxy) -> (ProxyType, Vec<u8>, u16) {
|
||||
let host_key = match proxy.host.parse::<Ipv4Addr>() {
|
||||
Ok(ip) => ip.octets().to_vec(),
|
||||
Err(_) => {
|
||||
let host_key = proxy.host.parse::<Ipv4Addr>().map_or_else(
|
||||
move |_| {
|
||||
std::iter::repeat_n(u8::MAX, 4).chain(proxy.host.bytes()).collect()
|
||||
}
|
||||
};
|
||||
},
|
||||
|ip| ip.octets().to_vec(),
|
||||
);
|
||||
(proxy.protocol.clone(), host_key, proxy.port)
|
||||
}
|
||||
|
||||
@@ -40,13 +40,13 @@ struct ProxyJson<'a> {
|
||||
geolocation: Option<maxminddb::geoip2::City<'a>>,
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_lines)]
|
||||
pub(crate) async fn save_proxies(
|
||||
#[expect(clippy::too_many_lines)]
|
||||
pub async fn save_proxies(
|
||||
config: Arc<Config>,
|
||||
storage: ProxyStorage,
|
||||
) -> color_eyre::Result<()> {
|
||||
if config.output_json {
|
||||
let mmdb = if config.enable_geolocation {
|
||||
let maybe_mmdb = if config.enable_geolocation {
|
||||
let geodb_path =
|
||||
get_geodb_path().await.wrap_err("failed to get GeoDB path")?;
|
||||
let buffer = tokio::fs::read(&geodb_path).await.wrap_err_with(
|
||||
@@ -63,7 +63,7 @@ pub(crate) async fn save_proxies(
|
||||
let mut proxy_dicts = Vec::with_capacity(sorted_proxies.len());
|
||||
|
||||
for proxy in sorted_proxies {
|
||||
let geolocation = if let Some(mmdb) = &mmdb {
|
||||
let geolocation = if let Some(mmdb) = &maybe_mmdb {
|
||||
if let Some(exit_ip) = proxy.exit_ip.clone() {
|
||||
let exit_ip_addr: IpAddr = exit_ip.parse().wrap_err(
|
||||
"failed to parse proxy's exit ip as IpAddr",
|
||||
|
||||
+4
-6
@@ -1,12 +1,10 @@
|
||||
use std::sync::LazyLock;
|
||||
|
||||
pub(crate) static PROXY_REGEX: LazyLock<fancy_regex::Regex> = LazyLock::new(
|
||||
|| {
|
||||
fancy_regex::Regex::new(
|
||||
pub static PROXY_REGEX: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
|
||||
fancy_regex::Regex::new(
|
||||
r"(?:^|[^\dA-Za-z])(?:(?P<protocol>https?|socks[45]):\/\/)?(?:(?P<username>[^\s:@]+):(?P<password>[^\s:@]+)@)?(?P<host>(?:[\-\.\dA-Za-z]+|(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])(?:\.(?:\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])){3})):(?P<port>\d|[1-9]\d{1,3}|[1-5]\d{4}|6[0-4]\d{3}|65[0-4]\d{2}|655[0-2]\d|6553[0-5])(?=[^\dA-Za-z]|$)"
|
||||
).unwrap()
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
static IPV4_REGEX: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
|
||||
fancy_regex::Regex::new(
|
||||
@@ -14,7 +12,7 @@ static IPV4_REGEX: LazyLock<fancy_regex::Regex> = LazyLock::new(|| {
|
||||
).unwrap()
|
||||
});
|
||||
|
||||
pub(crate) fn parse_ipv4(s: &str) -> Option<String> {
|
||||
pub fn parse_ipv4(s: &str) -> Option<String> {
|
||||
IPV4_REGEX
|
||||
.captures(s)
|
||||
.unwrap()
|
||||
|
||||
+16
-16
@@ -1,6 +1,6 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use color_eyre::eyre::{OptionExt, WrapErr, eyre};
|
||||
use color_eyre::eyre::{OptionExt as _, WrapErr as _, eyre};
|
||||
use derivative::Derivative;
|
||||
|
||||
use crate::{
|
||||
@@ -10,7 +10,7 @@ use crate::{
|
||||
};
|
||||
|
||||
#[derive(serde::Serialize, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
|
||||
pub(crate) enum ProxyType {
|
||||
pub enum ProxyType {
|
||||
#[serde(rename = "http")]
|
||||
Http,
|
||||
#[serde(rename = "socks4")]
|
||||
@@ -24,9 +24,9 @@ impl TryFrom<&str> for ProxyType {
|
||||
|
||||
fn try_from(string: &str) -> color_eyre::Result<Self> {
|
||||
match string {
|
||||
"http" | "https" => Ok(ProxyType::Http),
|
||||
"socks4" => Ok(ProxyType::Socks4),
|
||||
"socks5" => Ok(ProxyType::Socks5),
|
||||
"http" | "https" => Ok(Self::Http),
|
||||
"socks4" => Ok(Self::Socks4),
|
||||
"socks5" => Ok(Self::Socks5),
|
||||
_ => Err(eyre!("Failed to convert {string} to ProxyType")),
|
||||
}
|
||||
}
|
||||
@@ -46,24 +46,24 @@ impl std::fmt::Display for ProxyType {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Derivative, Eq)]
|
||||
#[derive(Derivative, Eq)]
|
||||
#[derivative(Hash, PartialEq)]
|
||||
pub(crate) struct Proxy {
|
||||
pub(crate) protocol: ProxyType,
|
||||
pub(crate) host: String,
|
||||
pub(crate) port: u16,
|
||||
pub(crate) username: Option<String>,
|
||||
pub(crate) password: Option<String>,
|
||||
pub struct Proxy {
|
||||
pub protocol: ProxyType,
|
||||
pub host: String,
|
||||
pub port: u16,
|
||||
pub username: Option<String>,
|
||||
pub password: Option<String>,
|
||||
#[derivative(Hash = "ignore")]
|
||||
#[derivative(PartialEq = "ignore")]
|
||||
pub(crate) timeout: Option<tokio::time::Duration>,
|
||||
pub timeout: Option<tokio::time::Duration>,
|
||||
#[derivative(Hash = "ignore")]
|
||||
#[derivative(PartialEq = "ignore")]
|
||||
pub(crate) exit_ip: Option<String>,
|
||||
pub exit_ip: Option<String>,
|
||||
}
|
||||
|
||||
impl Proxy {
|
||||
pub(crate) async fn check(
|
||||
pub async fn check(
|
||||
&mut self,
|
||||
config: Arc<Config>,
|
||||
) -> color_eyre::Result<()> {
|
||||
@@ -125,7 +125,7 @@ impl Proxy {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) fn as_str(&self, include_protocol: bool) -> String {
|
||||
pub fn as_str(&self, include_protocol: bool) -> String {
|
||||
let mut s = String::new();
|
||||
if include_protocol {
|
||||
s.push_str(&self.protocol.to_string());
|
||||
|
||||
+24
-23
@@ -1,6 +1,6 @@
|
||||
use std::{collections::HashSet, path::PathBuf};
|
||||
|
||||
use color_eyre::eyre::WrapErr;
|
||||
use color_eyre::eyre::WrapErr as _;
|
||||
use serde::{Deserialize, Deserializer};
|
||||
|
||||
use crate::utils::is_http_url;
|
||||
@@ -41,31 +41,32 @@ fn validate_http_url<'de, D: Deserializer<'de>>(
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(crate) struct RawConfig {
|
||||
pub struct RawConfig {
|
||||
#[serde(deserialize_with = "validate_positive_f64")]
|
||||
pub(crate) timeout: f64,
|
||||
pub timeout: f64,
|
||||
#[serde(deserialize_with = "validate_positive_f64")]
|
||||
pub(crate) source_timeout: f64,
|
||||
pub(crate) proxies_per_source_limit: usize,
|
||||
pub source_timeout: f64,
|
||||
pub proxies_per_source_limit: usize,
|
||||
#[serde(deserialize_with = "validate_positive_usize")]
|
||||
pub(crate) max_concurrent_checks: usize,
|
||||
pub max_concurrent_checks: usize,
|
||||
#[serde(deserialize_with = "validate_http_url")]
|
||||
pub(crate) check_website: String,
|
||||
pub(crate) sort_by_speed: bool,
|
||||
pub(crate) enable_geolocation: bool,
|
||||
pub(crate) debug: bool,
|
||||
pub(crate) output: Output,
|
||||
pub(crate) http: ProxySection,
|
||||
pub(crate) socks4: ProxySection,
|
||||
pub(crate) socks5: ProxySection,
|
||||
pub check_website: String,
|
||||
pub sort_by_speed: bool,
|
||||
pub enable_geolocation: bool,
|
||||
pub debug: bool,
|
||||
pub output: Output,
|
||||
pub http: ProxySection,
|
||||
pub socks4: ProxySection,
|
||||
pub socks5: ProxySection,
|
||||
}
|
||||
|
||||
pub(crate) struct Output {
|
||||
pub(crate) path: PathBuf,
|
||||
pub(crate) json: bool,
|
||||
pub(crate) txt: bool,
|
||||
pub struct Output {
|
||||
pub path: PathBuf,
|
||||
pub json: bool,
|
||||
pub txt: bool,
|
||||
}
|
||||
|
||||
#[expect(clippy::missing_trait_methods)]
|
||||
impl<'de> Deserialize<'de> for Output {
|
||||
fn deserialize<D: Deserializer<'de>>(
|
||||
deserializer: D,
|
||||
@@ -85,17 +86,17 @@ impl<'de> Deserialize<'de> for Output {
|
||||
));
|
||||
}
|
||||
|
||||
Ok(Output { path: inner.path, json: inner.json, txt: inner.txt })
|
||||
Ok(Self { path: inner.path, json: inner.json, txt: inner.txt })
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub(crate) struct ProxySection {
|
||||
pub(crate) enabled: bool,
|
||||
pub(crate) sources: HashSet<String>,
|
||||
pub struct ProxySection {
|
||||
pub enabled: bool,
|
||||
pub sources: HashSet<String>,
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config(path: &str) -> color_eyre::Result<RawConfig> {
|
||||
pub async fn read_config(path: &str) -> color_eyre::Result<RawConfig> {
|
||||
let raw_config = tokio::fs::read_to_string(path)
|
||||
.await
|
||||
.wrap_err_with(move || format!("failed to read {path} to string"))?;
|
||||
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
use std::{collections::HashSet, sync::Arc};
|
||||
|
||||
use color_eyre::eyre::WrapErr;
|
||||
use color_eyre::eyre::WrapErr as _;
|
||||
|
||||
use crate::{
|
||||
config::Config,
|
||||
@@ -51,7 +51,7 @@ async fn scrape_one(
|
||||
tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> color_eyre::Result<HashSet<Proxy>> {
|
||||
let text_result =
|
||||
fetch_text(config.clone(), http_client.clone(), source).await;
|
||||
fetch_text(Arc::clone(&config), http_client.clone(), source).await;
|
||||
|
||||
tx.send(Event::App(AppEvent::SourceScraped(proto.clone())))?;
|
||||
|
||||
@@ -65,7 +65,7 @@ async fn scrape_one(
|
||||
e.chain()
|
||||
.map(ToString::to_string)
|
||||
.collect::<Vec<_>>()
|
||||
.join(" → "),
|
||||
.join(" \u{2192} "),
|
||||
);
|
||||
return Ok(proxies);
|
||||
}
|
||||
@@ -108,7 +108,7 @@ async fn scrape_one(
|
||||
Ok(proxies)
|
||||
}
|
||||
|
||||
pub(crate) async fn scrape_all(
|
||||
pub async fn scrape_all(
|
||||
config: std::sync::Arc<Config>,
|
||||
http_client: reqwest::Client,
|
||||
tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
@@ -120,7 +120,7 @@ pub(crate) async fn scrape_all(
|
||||
sources.len(),
|
||||
)))?;
|
||||
for source in sources {
|
||||
let config = config.clone();
|
||||
let config = Arc::clone(&config);
|
||||
let http_client = http_client.clone();
|
||||
let proto = proto.clone();
|
||||
let tx = tx.clone();
|
||||
|
||||
+5
-5
@@ -2,23 +2,23 @@ use std::collections::{HashMap, HashSet};
|
||||
|
||||
use crate::proxy::{Proxy, ProxyType};
|
||||
|
||||
pub(crate) struct ProxyStorage {
|
||||
pub struct ProxyStorage {
|
||||
proxies: HashSet<Proxy>,
|
||||
enabled_protocols: HashSet<ProxyType>,
|
||||
}
|
||||
|
||||
impl ProxyStorage {
|
||||
pub(crate) fn new(protocols: HashSet<ProxyType>) -> Self {
|
||||
pub fn new(protocols: HashSet<ProxyType>) -> Self {
|
||||
Self { proxies: HashSet::new(), enabled_protocols: protocols }
|
||||
}
|
||||
|
||||
pub(crate) fn insert(&mut self, proxy: Proxy) {
|
||||
pub fn insert(&mut self, proxy: Proxy) {
|
||||
if self.enabled_protocols.contains(&proxy.protocol) {
|
||||
self.proxies.insert(proxy);
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn get_grouped(&self) -> HashMap<ProxyType, Vec<&Proxy>> {
|
||||
pub fn get_grouped(&self) -> HashMap<ProxyType, Vec<&Proxy>> {
|
||||
let mut groups: HashMap<ProxyType, Vec<&Proxy>> = self
|
||||
.enabled_protocols
|
||||
.iter()
|
||||
@@ -30,7 +30,7 @@ impl ProxyStorage {
|
||||
groups
|
||||
}
|
||||
|
||||
pub(crate) fn iter(&self) -> std::collections::hash_set::Iter<'_, Proxy> {
|
||||
pub fn iter(&self) -> std::collections::hash_set::Iter<'_, Proxy> {
|
||||
self.proxies.iter()
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -1,6 +1,6 @@
|
||||
use crate::event::{AppEvent, Event};
|
||||
|
||||
pub(crate) struct LoggerUI {}
|
||||
pub struct LoggerUI;
|
||||
|
||||
impl super::UI for LoggerUI {
|
||||
fn new() -> color_eyre::Result<Self> {
|
||||
@@ -18,7 +18,7 @@ impl super::UI for LoggerUI {
|
||||
mut rx: tokio::sync::mpsc::UnboundedReceiver<Event>,
|
||||
) -> color_eyre::Result<()> {
|
||||
while let Some(event) = rx.recv().await {
|
||||
if let Event::App(AppEvent::Done) = event {
|
||||
if matches!(event, Event::App(AppEvent::Done)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
+4
-3
@@ -1,6 +1,7 @@
|
||||
#![expect(clippy::pub_use)]
|
||||
use crate::event::Event;
|
||||
|
||||
pub(crate) trait UI {
|
||||
pub trait UI {
|
||||
fn new() -> color_eyre::Result<Self>
|
||||
where
|
||||
Self: std::marker::Sized;
|
||||
@@ -17,9 +18,9 @@ pub(crate) trait UI {
|
||||
#[cfg(feature = "tui")]
|
||||
mod tui;
|
||||
#[cfg(feature = "tui")]
|
||||
pub(crate) use self::tui::Tui as UIImpl;
|
||||
pub use tui::Tui as UIImpl;
|
||||
|
||||
#[cfg(not(feature = "tui"))]
|
||||
mod logger;
|
||||
#[cfg(not(feature = "tui"))]
|
||||
pub(crate) use self::logger::LoggerUI as UIImpl;
|
||||
pub use logger::LoggerUI as UIImpl;
|
||||
|
||||
+27
-26
@@ -1,10 +1,20 @@
|
||||
#![expect(
|
||||
clippy::arithmetic_side_effects,
|
||||
clippy::as_conversions,
|
||||
clippy::cast_precision_loss,
|
||||
clippy::float_arithmetic,
|
||||
clippy::indexing_slicing,
|
||||
clippy::integer_division_remainder_used,
|
||||
clippy::missing_asserts_for_indexing,
|
||||
clippy::wildcard_enum_match_arm
|
||||
)]
|
||||
use std::collections::HashMap;
|
||||
|
||||
use color_eyre::eyre::WrapErr;
|
||||
use color_eyre::eyre::WrapErr as _;
|
||||
use crossterm::event::{
|
||||
Event as CrosstermEvent, KeyCode, KeyModifiers, MouseEventKind,
|
||||
};
|
||||
use futures::StreamExt;
|
||||
use futures::StreamExt as _;
|
||||
use ratatui::{
|
||||
DefaultTerminal, Frame,
|
||||
layout::{Alignment, Constraint, Direction, Layout},
|
||||
@@ -22,7 +32,7 @@ use crate::{
|
||||
|
||||
const FPS: f64 = 30.0;
|
||||
|
||||
pub(crate) struct Tui {
|
||||
pub struct Tui {
|
||||
terminal: DefaultTerminal,
|
||||
}
|
||||
|
||||
@@ -74,7 +84,7 @@ impl Drop for Tui {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) enum AppMode {
|
||||
pub enum AppMode {
|
||||
#[default]
|
||||
Running,
|
||||
/// Wait for the user confirmation to close the UI
|
||||
@@ -84,23 +94,23 @@ pub(crate) enum AppMode {
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(crate) struct AppState {
|
||||
pub(crate) mode: AppMode,
|
||||
pub struct AppState {
|
||||
pub mode: AppMode,
|
||||
|
||||
pub(crate) geodb_total: u64,
|
||||
pub(crate) geodb_downloaded: usize,
|
||||
pub geodb_total: u64,
|
||||
pub geodb_downloaded: usize,
|
||||
|
||||
pub(crate) sources_total: HashMap<ProxyType, usize>,
|
||||
pub(crate) sources_scraped: HashMap<ProxyType, usize>,
|
||||
pub sources_total: HashMap<ProxyType, usize>,
|
||||
pub sources_scraped: HashMap<ProxyType, usize>,
|
||||
|
||||
pub(crate) proxies_total: HashMap<ProxyType, usize>,
|
||||
pub(crate) proxies_checked: HashMap<ProxyType, usize>,
|
||||
pub(crate) proxies_working: HashMap<ProxyType, usize>,
|
||||
pub proxies_total: HashMap<ProxyType, usize>,
|
||||
pub proxies_checked: HashMap<ProxyType, usize>,
|
||||
pub proxies_working: HashMap<ProxyType, usize>,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
pub(crate) fn new() -> Self {
|
||||
AppState::default()
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -147,9 +157,6 @@ async fn crossterm_event_listener(
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::cast_possible_truncation)]
|
||||
#[allow(clippy::cast_precision_loss)]
|
||||
#[allow(clippy::too_many_lines)]
|
||||
fn draw(f: &mut Frame, state: &AppState, logger_state: &TuiWidgetState) {
|
||||
let outer_block = Block::default()
|
||||
.title("https://github.com/monosans/proxy-scraper-checker")
|
||||
@@ -196,13 +203,7 @@ fn draw(f: &mut Frame, state: &AppState, logger_state: &TuiWidgetState) {
|
||||
|
||||
let proxies_layout = Layout::default()
|
||||
.direction(Direction::Horizontal)
|
||||
.constraints(
|
||||
state
|
||||
.sources_total
|
||||
.keys()
|
||||
.map(|_| Constraint::Fill(1))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
.constraints(state.sources_total.keys().map(|_| Constraint::Fill(1)))
|
||||
.split(outer_layout[2]);
|
||||
|
||||
let mut proxy_types: Vec<_> = state.sources_total.keys().collect();
|
||||
@@ -284,7 +285,7 @@ fn draw(f: &mut Frame, state: &AppState, logger_state: &TuiWidgetState) {
|
||||
Line::from("Up/PageUp/k - scroll logs up"),
|
||||
Line::from("Down/PageDown/j - scroll logs down"),
|
||||
];
|
||||
if let AppMode::Done = state.mode {
|
||||
if matches!(state.mode, AppMode::Done) {
|
||||
lines.push(
|
||||
Line::from("Enter/ESC/q/Ctrl-C - exit")
|
||||
.style(Style::default().fg(Color::Red)),
|
||||
|
||||
+4
-6
@@ -1,4 +1,4 @@
|
||||
pub(crate) async fn is_docker() -> bool {
|
||||
pub async fn is_docker() -> bool {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
static CACHE: tokio::sync::OnceCell<bool> =
|
||||
@@ -16,12 +16,10 @@ pub(crate) async fn is_docker() -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn is_http_url(value: &str) -> bool {
|
||||
if let Ok(parsed_url) = url::Url::parse(value) {
|
||||
pub fn is_http_url(value: &str) -> bool {
|
||||
url::Url::parse(value).is_ok_and(|parsed_url| {
|
||||
let scheme = parsed_url.scheme();
|
||||
(scheme == "http" || scheme == "https")
|
||||
&& parsed_url.host_str().is_some()
|
||||
} else {
|
||||
false
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user