feat: add Dockerfile and Docker Compose configuration for samply service; refactor proxy checking logic
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
# syntax=docker.io/docker/dockerfile:1
|
||||
|
||||
FROM docker.io/rust:1-slim-trixie AS builder
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN rm -f /etc/apt/apt.conf.d/docker-clean \
|
||||
&& echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
apt-get update \
|
||||
&& apt-get install -y --no-install-recommends cmake make
|
||||
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
|
||||
cargo install --locked --git https://github.com/mstange/samply samply
|
||||
|
||||
RUN --mount=source=src,target=src \
|
||||
--mount=source=Cargo.toml,target=Cargo.toml \
|
||||
--mount=source=Cargo.lock,target=Cargo.lock \
|
||||
--mount=type=cache,target=/app/target,sharing=locked \
|
||||
--mount=type=cache,target=/usr/local/cargo/registry,sharing=locked \
|
||||
cargo build --profile profiling --locked \
|
||||
&& cp target/profiling/proxy-scraper-checker .
|
||||
|
||||
|
||||
FROM docker.io/debian:trixie-slim AS final
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN rm -f /etc/apt/apt.conf.d/docker-clean \
|
||||
&& echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache
|
||||
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates
|
||||
|
||||
COPY --from=builder --link /usr/local/cargo/bin/samply .
|
||||
COPY --from=builder --link /app/proxy-scraper-checker .
|
||||
|
||||
CMD ["/app/samply", "record", "--rate", "1750", "--presymbolicate", "--save-only", "-o", "/app/out/samply-record.json", "/app/proxy-scraper-checker"]
|
||||
@@ -0,0 +1,13 @@
|
||||
services:
|
||||
samply:
|
||||
build:
|
||||
dockerfile: bench/samply/Dockerfile
|
||||
args:
|
||||
- "UID=${UID:-1000}"
|
||||
- "GID=${GID:-1000}"
|
||||
pull: true
|
||||
init: true
|
||||
privileged: true
|
||||
volumes:
|
||||
- ./out:/app/out
|
||||
- ./config.toml:/app/config.toml
|
||||
+7
-8
@@ -10,7 +10,7 @@ pub async fn check_all<R: reqwest::dns::Resolve + Clone + 'static>(
|
||||
config: Arc<Config>,
|
||||
dns_resolver: R,
|
||||
proxies: Vec<Proxy>,
|
||||
mut tls_backend: rustls::ClientConfig,
|
||||
tls_backend: rustls::ClientConfig,
|
||||
token: tokio_util::sync::CancellationToken,
|
||||
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> crate::Result<Vec<Proxy>> {
|
||||
@@ -24,20 +24,20 @@ pub async fn check_all<R: reqwest::dns::Resolve + Clone + 'static>(
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let client =
|
||||
crate::proxy::build_check_client(&config, dns_resolver, tls_backend)?;
|
||||
|
||||
#[cfg(not(feature = "tui"))]
|
||||
tracing::info!("Started checking {} proxies", proxies.len());
|
||||
|
||||
let queue = Arc::new(parking_lot::Mutex::new(proxies));
|
||||
let checked_proxies = Arc::new(parking_lot::Mutex::new(Vec::new()));
|
||||
|
||||
tls_backend.alpn_protocols = vec![b"http/1.1".to_vec()];
|
||||
|
||||
let mut join_set = tokio::task::JoinSet::<()>::new();
|
||||
for _ in 0..workers_count {
|
||||
let queue = Arc::clone(&queue);
|
||||
let client = client.clone();
|
||||
let config = Arc::clone(&config);
|
||||
let dns_resolver = dns_resolver.clone();
|
||||
let tls_backend = tls_backend.clone();
|
||||
let checked_proxies = Arc::clone(&checked_proxies);
|
||||
let token = token.clone();
|
||||
#[cfg(feature = "tui")]
|
||||
@@ -50,7 +50,7 @@ pub async fn check_all<R: reqwest::dns::Resolve + Clone + 'static>(
|
||||
let Some(mut proxy) = queue.lock().pop() else {
|
||||
break;
|
||||
};
|
||||
let check_result = proxy.check(&config, dns_resolver.clone(), tls_backend.clone()).await;
|
||||
let check_result = proxy.check(&client, &config).await;
|
||||
#[cfg(feature = "tui")]
|
||||
drop(tx.send(Event::App(AppEvent::ProxyChecked(proxy.protocol))));
|
||||
match check_result {
|
||||
@@ -75,10 +75,9 @@ pub async fn check_all<R: reqwest::dns::Resolve + Clone + 'static>(
|
||||
});
|
||||
}
|
||||
|
||||
drop(client);
|
||||
drop(config);
|
||||
drop(dns_resolver);
|
||||
drop(queue);
|
||||
drop(tls_backend);
|
||||
drop(token);
|
||||
#[cfg(feature = "tui")]
|
||||
drop(tx);
|
||||
|
||||
+15
-15
@@ -66,6 +66,21 @@ pub struct Config {
|
||||
pub output: OutputConfig,
|
||||
}
|
||||
|
||||
impl From<raw_config::SourceConfig> for Source {
|
||||
fn from(sc: raw_config::SourceConfig) -> Self {
|
||||
match sc {
|
||||
raw_config::SourceConfig::Simple(url) => {
|
||||
Self { url, basic_auth: None, headers: None }
|
||||
}
|
||||
raw_config::SourceConfig::Detailed(config) => Self {
|
||||
url: config.url,
|
||||
basic_auth: config.basic_auth,
|
||||
headers: config.headers,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn get_output_path(
|
||||
raw_config: &raw_config::RawConfig,
|
||||
) -> crate::Result<PathBuf> {
|
||||
@@ -183,21 +198,6 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<raw_config::SourceConfig> for Source {
|
||||
fn from(sc: raw_config::SourceConfig) -> Self {
|
||||
match sc {
|
||||
raw_config::SourceConfig::Simple(url) => {
|
||||
Self { url, basic_auth: None, headers: None }
|
||||
}
|
||||
raw_config::SourceConfig::Detailed(config) => Self {
|
||||
url: config.url,
|
||||
basic_auth: config.basic_auth,
|
||||
headers: config.headers,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn load_config() -> crate::Result<Arc<Config>> {
|
||||
let raw_config = {
|
||||
let raw_config_path = raw_config::get_config_path();
|
||||
|
||||
@@ -4,10 +4,10 @@ use color_eyre::eyre::{OptionExt as _, WrapErr as _};
|
||||
|
||||
use crate::config::APP_DIRECTORY_NAME;
|
||||
|
||||
pub async fn get_cache_path() -> crate::Result<PathBuf> {
|
||||
static CACHE: tokio::sync::OnceCell<PathBuf> =
|
||||
tokio::sync::OnceCell::const_new();
|
||||
static CACHE: tokio::sync::OnceCell<PathBuf> =
|
||||
tokio::sync::OnceCell::const_new();
|
||||
|
||||
pub async fn get_cache_path() -> crate::Result<PathBuf> {
|
||||
Ok(CACHE
|
||||
.get_or_try_init(async || -> crate::Result<PathBuf> {
|
||||
let mut path = tokio::task::spawn_blocking(dirs::cache_dir)
|
||||
|
||||
+2
-2
@@ -79,6 +79,8 @@ pub async fn build_rustls_config() -> crate::Result<rustls::ClientConfig> {
|
||||
.with_no_client_auth())
|
||||
}
|
||||
|
||||
pub struct RetryMiddleware;
|
||||
|
||||
fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option<Duration> {
|
||||
if let Some(val) = headers.get("retry-after-ms")
|
||||
&& let Ok(s) = val.to_str()
|
||||
@@ -123,8 +125,6 @@ fn calculate_retry_timeout(
|
||||
Some(base.mul_f64(jitter))
|
||||
}
|
||||
|
||||
pub struct RetryMiddleware;
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl reqwest_middleware::Middleware for RetryMiddleware {
|
||||
async fn handle(
|
||||
|
||||
+88
-88
@@ -14,6 +14,94 @@ pub enum DbType {
|
||||
}
|
||||
|
||||
impl DbType {
|
||||
pub async fn download(
|
||||
self,
|
||||
http_client: reqwest_middleware::ClientWithMiddleware,
|
||||
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> crate::Result<()> {
|
||||
let db_path = self.db_path().await?;
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
#[expect(clippy::collapsible_if)]
|
||||
if tokio::fs::metadata(&db_path).await.is_ok_and(|m| m.is_file()) {
|
||||
if let Some(etag) = self.read_etag().await? {
|
||||
headers.insert(reqwest::header::IF_NONE_MATCH, etag);
|
||||
}
|
||||
}
|
||||
|
||||
let response = http_client
|
||||
.get(self.url())
|
||||
.headers(headers)
|
||||
.timeout(Duration::MAX)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
|
||||
if response.status() == reqwest::StatusCode::NOT_MODIFIED {
|
||||
tracing::info!(
|
||||
"Latest {} database is already cached at {}",
|
||||
self.name(),
|
||||
db_path.display()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
return Err(eyre!(
|
||||
"HTTP status error ({}) for url ({})",
|
||||
response.status(),
|
||||
response.url()
|
||||
));
|
||||
}
|
||||
|
||||
let etag = response.headers().get(reqwest::header::ETAG).cloned();
|
||||
|
||||
self.save_db(
|
||||
response,
|
||||
#[cfg(feature = "tui")]
|
||||
tx,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if is_docker().await {
|
||||
tracing::info!(
|
||||
"Downloaded {} database to Docker volume ({} in container)",
|
||||
self.name(),
|
||||
db_path.display()
|
||||
);
|
||||
} else {
|
||||
tracing::info!(
|
||||
"Downloaded {} database to {}",
|
||||
self.name(),
|
||||
db_path.display()
|
||||
);
|
||||
}
|
||||
drop(db_path);
|
||||
|
||||
if let Some(etag_value) = etag {
|
||||
self.save_etag(etag_value).await
|
||||
} else {
|
||||
self.remove_etag().await
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn open_mmap(
|
||||
self,
|
||||
) -> crate::Result<maxminddb::Reader<maxminddb::Mmap>> {
|
||||
let path = self.db_path().await?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
#[expect(clippy::undocumented_unsafe_blocks)]
|
||||
unsafe { maxminddb::Reader::open_mmap(&path) }.wrap_err_with(
|
||||
move || {
|
||||
compact_str::format_compact!(
|
||||
"failed to open IP database: {}",
|
||||
path.display()
|
||||
)
|
||||
},
|
||||
)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
|
||||
const fn name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Asn => "ASN",
|
||||
@@ -132,92 +220,4 @@ impl DbType {
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn download(
|
||||
self,
|
||||
http_client: reqwest_middleware::ClientWithMiddleware,
|
||||
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> crate::Result<()> {
|
||||
let db_path = self.db_path().await?;
|
||||
let mut headers = reqwest::header::HeaderMap::new();
|
||||
#[expect(clippy::collapsible_if)]
|
||||
if tokio::fs::metadata(&db_path).await.is_ok_and(|m| m.is_file()) {
|
||||
if let Some(etag) = self.read_etag().await? {
|
||||
headers.insert(reqwest::header::IF_NONE_MATCH, etag);
|
||||
}
|
||||
}
|
||||
|
||||
let response = http_client
|
||||
.get(self.url())
|
||||
.headers(headers)
|
||||
.timeout(Duration::MAX)
|
||||
.send()
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
|
||||
if response.status() == reqwest::StatusCode::NOT_MODIFIED {
|
||||
tracing::info!(
|
||||
"Latest {} database is already cached at {}",
|
||||
self.name(),
|
||||
db_path.display()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
return Err(eyre!(
|
||||
"HTTP status error ({}) for url ({})",
|
||||
response.status(),
|
||||
response.url()
|
||||
));
|
||||
}
|
||||
|
||||
let etag = response.headers().get(reqwest::header::ETAG).cloned();
|
||||
|
||||
self.save_db(
|
||||
response,
|
||||
#[cfg(feature = "tui")]
|
||||
tx,
|
||||
)
|
||||
.await?;
|
||||
|
||||
if is_docker().await {
|
||||
tracing::info!(
|
||||
"Downloaded {} database to Docker volume ({} in container)",
|
||||
self.name(),
|
||||
db_path.display()
|
||||
);
|
||||
} else {
|
||||
tracing::info!(
|
||||
"Downloaded {} database to {}",
|
||||
self.name(),
|
||||
db_path.display()
|
||||
);
|
||||
}
|
||||
drop(db_path);
|
||||
|
||||
if let Some(etag_value) = etag {
|
||||
self.save_etag(etag_value).await
|
||||
} else {
|
||||
self.remove_etag().await
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn open_mmap(
|
||||
self,
|
||||
) -> crate::Result<maxminddb::Reader<maxminddb::Mmap>> {
|
||||
let path = self.db_path().await?;
|
||||
tokio::task::spawn_blocking(move || {
|
||||
#[expect(clippy::undocumented_unsafe_blocks)]
|
||||
unsafe { maxminddb::Reader::open_mmap(&path) }.wrap_err_with(
|
||||
move || {
|
||||
compact_str::format_compact!(
|
||||
"failed to open IP database: {}",
|
||||
path.display()
|
||||
)
|
||||
},
|
||||
)
|
||||
})
|
||||
.await?
|
||||
}
|
||||
}
|
||||
|
||||
+52
-52
@@ -63,6 +63,58 @@ fn group_proxies<'a>(
|
||||
groups
|
||||
}
|
||||
|
||||
async fn write_proxy_list_to_file<'a, I>(
|
||||
path: &Path,
|
||||
proxies: I,
|
||||
include_protocol: bool,
|
||||
) -> crate::Result<()>
|
||||
where
|
||||
I: IntoIterator<Item = &'a Proxy>,
|
||||
{
|
||||
let file =
|
||||
tokio::fs::File::create(path).await.wrap_err_with(move || {
|
||||
compact_str::format_compact!(
|
||||
"failed to create file: {}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
let mut writer = tokio::io::BufWriter::new(file);
|
||||
|
||||
let mut first = true;
|
||||
let mut tmp = Vec::new();
|
||||
for proxy in proxies {
|
||||
if first {
|
||||
first = false;
|
||||
} else {
|
||||
writer.write_all(b"\n").await.wrap_err_with(move || {
|
||||
compact_str::format_compact!(
|
||||
"failed to write to file: {}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
proxy.write_to_sink(&mut tmp, include_protocol);
|
||||
writer.write_all(&tmp).await.wrap_err_with(move || {
|
||||
compact_str::format_compact!(
|
||||
"failed to write to file: {}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
tmp.clear();
|
||||
}
|
||||
drop(tmp);
|
||||
|
||||
writer.flush().await.wrap_err_with(move || {
|
||||
compact_str::format_compact!(
|
||||
"failed to write to file: {}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[expect(clippy::too_many_lines)]
|
||||
pub async fn save_proxies(
|
||||
config: Arc<Config>,
|
||||
@@ -199,55 +251,3 @@ pub async fn save_proxies(
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_proxy_list_to_file<'a, I>(
|
||||
path: &Path,
|
||||
proxies: I,
|
||||
include_protocol: bool,
|
||||
) -> crate::Result<()>
|
||||
where
|
||||
I: IntoIterator<Item = &'a Proxy>,
|
||||
{
|
||||
let file =
|
||||
tokio::fs::File::create(path).await.wrap_err_with(move || {
|
||||
compact_str::format_compact!(
|
||||
"failed to create file: {}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
let mut writer = tokio::io::BufWriter::new(file);
|
||||
|
||||
let mut first = true;
|
||||
let mut tmp = Vec::new();
|
||||
for proxy in proxies {
|
||||
if first {
|
||||
first = false;
|
||||
} else {
|
||||
writer.write_all(b"\n").await.wrap_err_with(move || {
|
||||
compact_str::format_compact!(
|
||||
"failed to write to file: {}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
proxy.write_to_sink(&mut tmp, include_protocol);
|
||||
writer.write_all(&tmp).await.wrap_err_with(move || {
|
||||
compact_str::format_compact!(
|
||||
"failed to write to file: {}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
tmp.clear();
|
||||
}
|
||||
drop(tmp);
|
||||
|
||||
writer.flush().await.wrap_err_with(move || {
|
||||
compact_str::format_compact!(
|
||||
"failed to write to file: {}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+83
-65
@@ -47,6 +47,7 @@ impl ProxyType {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "tui")]
|
||||
pub const fn as_str_uppercase(self) -> &'static str {
|
||||
match self {
|
||||
Self::Http => "HTTP",
|
||||
@@ -67,29 +68,36 @@ pub struct Proxy {
|
||||
pub exit_ip: Option<compact_str::CompactString>,
|
||||
}
|
||||
|
||||
impl TryFrom<&mut Proxy> for reqwest::Proxy {
|
||||
type Error = crate::Error;
|
||||
tokio::task_local! {
|
||||
static CHECK_PROXY_URL: reqwest::Url;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn try_from(value: &mut Proxy) -> Result<Self, Self::Error> {
|
||||
let proxy = Self::all(
|
||||
compact_str::format_compact!(
|
||||
"{}://{}:{}",
|
||||
value.protocol.as_str_lowercase(),
|
||||
value.host,
|
||||
value.port
|
||||
)
|
||||
.as_str(),
|
||||
)?;
|
||||
pub fn build_check_client<R: reqwest::dns::Resolve + 'static>(
|
||||
config: &Config,
|
||||
dns_resolver: R,
|
||||
mut tls_backend: rustls::ClientConfig,
|
||||
) -> reqwest::Result<reqwest::Client> {
|
||||
tls_backend.alpn_protocols = vec![b"http/1.1".to_vec()];
|
||||
let builder = reqwest::ClientBuilder::new()
|
||||
.user_agent(config.checking.user_agent.as_bytes())
|
||||
.proxy(reqwest::Proxy::custom(|_| Some(CHECK_PROXY_URL.get())))
|
||||
.timeout(config.checking.timeout)
|
||||
.connect_timeout(config.checking.connect_timeout)
|
||||
.pool_idle_timeout(Duration::ZERO)
|
||||
.pool_max_idle_per_host(0)
|
||||
.http1_only()
|
||||
.tcp_keepalive(None)
|
||||
.tls_backend_preconfigured(tls_backend)
|
||||
.dns_resolver(dns_resolver);
|
||||
|
||||
if let (Some(username), Some(password)) =
|
||||
(value.username.as_ref(), value.password.as_ref())
|
||||
{
|
||||
Ok(proxy.basic_auth(username, password))
|
||||
} else {
|
||||
Ok(proxy)
|
||||
}
|
||||
}
|
||||
#[cfg(any(
|
||||
target_os = "android",
|
||||
target_os = "fuchsia",
|
||||
target_os = "linux"
|
||||
))]
|
||||
let builder = builder.tcp_user_timeout(None);
|
||||
|
||||
builder.build()
|
||||
}
|
||||
|
||||
pub trait ProxySink {
|
||||
@@ -117,52 +125,62 @@ impl ProxySink for Vec<u8> {
|
||||
}
|
||||
}
|
||||
|
||||
impl Proxy {
|
||||
pub const fn is_checked(&self) -> bool {
|
||||
self.timeout.is_some()
|
||||
}
|
||||
impl TryFrom<&mut Proxy> for url::Url {
|
||||
type Error = crate::Error;
|
||||
|
||||
pub async fn check<R: reqwest::dns::Resolve + 'static>(
|
||||
&mut self,
|
||||
config: &Config,
|
||||
dns_resolver: R,
|
||||
tls_backend: rustls::ClientConfig,
|
||||
) -> crate::Result<()> {
|
||||
if let Some(check_url) = config.checking.check_url.clone() {
|
||||
let builder = reqwest::ClientBuilder::new()
|
||||
.user_agent(config.checking.user_agent.as_bytes())
|
||||
.proxy(self.try_into()?)
|
||||
.timeout(config.checking.timeout)
|
||||
.connect_timeout(config.checking.connect_timeout)
|
||||
.pool_idle_timeout(Duration::ZERO)
|
||||
.pool_max_idle_per_host(0)
|
||||
.http1_only()
|
||||
.tcp_keepalive(None)
|
||||
.tls_backend_preconfigured(tls_backend)
|
||||
.dns_resolver(dns_resolver);
|
||||
#[cfg(any(
|
||||
target_os = "android",
|
||||
target_os = "fuchsia",
|
||||
target_os = "linux"
|
||||
))]
|
||||
let builder = builder.tcp_user_timeout(None);
|
||||
let request = {
|
||||
let client = builder.build()?;
|
||||
client.get(check_url)
|
||||
};
|
||||
let start = Instant::now();
|
||||
let response = request.send().await?.error_for_status()?;
|
||||
self.timeout = Some(start.elapsed());
|
||||
self.exit_ip = response.text().await.map_or(None, |text| {
|
||||
if let Ok(httpbin) =
|
||||
serde_json::from_str::<HttpbinResponse>(&text)
|
||||
{
|
||||
parse_ipv4(&httpbin.origin)
|
||||
} else {
|
||||
parse_ipv4(&text)
|
||||
}
|
||||
});
|
||||
#[inline]
|
||||
fn try_from(proxy: &mut Proxy) -> Result<Self, Self::Error> {
|
||||
let mut url = Self::parse("http://0.0.0.0")?;
|
||||
|
||||
url.set_scheme(proxy.protocol.as_str_lowercase())
|
||||
.map_err(|()| eyre!("invalid proxy url scheme"))?;
|
||||
|
||||
if let (Some(username), Some(password)) =
|
||||
(proxy.username.as_deref(), proxy.password.as_deref())
|
||||
{
|
||||
url.set_username(username)
|
||||
.map_err(|()| eyre!("invalid proxy url username"))?;
|
||||
url.set_password(Some(password))
|
||||
.map_err(|()| eyre!("invalid proxy url password"))?;
|
||||
}
|
||||
|
||||
url.set_host(Some(proxy.host.as_str()))?;
|
||||
url.set_port(Some(proxy.port))
|
||||
.map_err(|()| eyre!("invalid proxy url port"))?;
|
||||
|
||||
Ok(url)
|
||||
}
|
||||
}
|
||||
|
||||
impl Proxy {
|
||||
pub async fn check(
|
||||
&mut self,
|
||||
client: &reqwest::Client,
|
||||
config: &Config,
|
||||
) -> crate::Result<()> {
|
||||
let Some(check_url) = config.checking.check_url.clone() else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let proxy_url = self.try_into()?;
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
let response = CHECK_PROXY_URL
|
||||
.scope(proxy_url, async move { client.get(check_url).send().await })
|
||||
.await?
|
||||
.error_for_status()?;
|
||||
|
||||
self.timeout = Some(start.elapsed());
|
||||
self.exit_ip = response.text().await.map_or(None, |text| {
|
||||
if let Ok(httpbin) = serde_json::from_str::<HttpbinResponse>(&text)
|
||||
{
|
||||
parse_ipv4(&httpbin.origin)
|
||||
} else {
|
||||
parse_ipv4(&text)
|
||||
}
|
||||
});
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -9,6 +9,8 @@ use serde::Deserialize as _;
|
||||
|
||||
use crate::{HashMap, http::BasicAuth};
|
||||
|
||||
const CONFIG_ENV: &str = "PROXY_SCRAPER_CHECKER_CONFIG";
|
||||
|
||||
fn validate_positive_f64<'de, D: serde::Deserializer<'de>>(
|
||||
deserializer: D,
|
||||
) -> Result<f64, D::Error> {
|
||||
@@ -185,8 +187,6 @@ impl<'de> serde::Deserialize<'de> for OutputConfig {
|
||||
}
|
||||
}
|
||||
|
||||
const CONFIG_ENV: &str = "PROXY_SCRAPER_CHECKER_CONFIG";
|
||||
|
||||
pub fn get_config_path() -> compact_str::CompactString {
|
||||
env::var(CONFIG_ENV).map_or_else(
|
||||
|_| compact_str::CompactString::const_new("config.toml"),
|
||||
|
||||
+58
-58
@@ -13,6 +13,64 @@ use crate::{
|
||||
utils::pretty_error,
|
||||
};
|
||||
|
||||
pub async fn scrape_all(
|
||||
config: Arc<Config>,
|
||||
http_client: reqwest_middleware::ClientWithMiddleware,
|
||||
token: tokio_util::sync::CancellationToken,
|
||||
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> crate::Result<Vec<Proxy>> {
|
||||
let proxies = Arc::new(parking_lot::Mutex::new(HashSet::new()));
|
||||
|
||||
let mut join_set = tokio::task::JoinSet::new();
|
||||
for (&proto, sources) in &config.scraping.sources {
|
||||
#[cfg(feature = "tui")]
|
||||
drop(tx.send(Event::App(AppEvent::SourcesTotal(proto, sources.len()))));
|
||||
|
||||
for source in sources {
|
||||
let config = Arc::clone(&config);
|
||||
let http_client = http_client.clone();
|
||||
let proxies = Arc::clone(&proxies);
|
||||
let token = token.clone();
|
||||
let source = Arc::clone(source);
|
||||
#[cfg(feature = "tui")]
|
||||
let tx = tx.clone();
|
||||
join_set.spawn(async move {
|
||||
tokio::select! {
|
||||
biased;
|
||||
res = scrape_one(
|
||||
config,
|
||||
http_client,
|
||||
proto,
|
||||
proxies,
|
||||
source,
|
||||
#[cfg(feature = "tui")]
|
||||
tx,
|
||||
) => res,
|
||||
() = token.cancelled() => Ok(()),
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
drop(config);
|
||||
drop(http_client);
|
||||
drop(token);
|
||||
#[cfg(feature = "tui")]
|
||||
drop(tx);
|
||||
|
||||
while let Some(res) = join_set.join_next().await {
|
||||
res??;
|
||||
}
|
||||
|
||||
drop(join_set);
|
||||
|
||||
Ok(Arc::into_inner(proxies)
|
||||
.ok_or_eyre("failed to unwrap Arc")?
|
||||
.into_inner()
|
||||
.into_iter()
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn scrape_one(
|
||||
config: Arc<Config>,
|
||||
http_client: reqwest_middleware::ClientWithMiddleware,
|
||||
@@ -155,61 +213,3 @@ async fn scrape_one(
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn scrape_all(
|
||||
config: Arc<Config>,
|
||||
http_client: reqwest_middleware::ClientWithMiddleware,
|
||||
token: tokio_util::sync::CancellationToken,
|
||||
#[cfg(feature = "tui")] tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
) -> crate::Result<Vec<Proxy>> {
|
||||
let proxies = Arc::new(parking_lot::Mutex::new(HashSet::new()));
|
||||
|
||||
let mut join_set = tokio::task::JoinSet::new();
|
||||
for (&proto, sources) in &config.scraping.sources {
|
||||
#[cfg(feature = "tui")]
|
||||
drop(tx.send(Event::App(AppEvent::SourcesTotal(proto, sources.len()))));
|
||||
|
||||
for source in sources {
|
||||
let config = Arc::clone(&config);
|
||||
let http_client = http_client.clone();
|
||||
let proxies = Arc::clone(&proxies);
|
||||
let token = token.clone();
|
||||
let source = Arc::clone(source);
|
||||
#[cfg(feature = "tui")]
|
||||
let tx = tx.clone();
|
||||
join_set.spawn(async move {
|
||||
tokio::select! {
|
||||
biased;
|
||||
res = scrape_one(
|
||||
config,
|
||||
http_client,
|
||||
proto,
|
||||
proxies,
|
||||
source,
|
||||
#[cfg(feature = "tui")]
|
||||
tx,
|
||||
) => res,
|
||||
() = token.cancelled() => Ok(()),
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
drop(config);
|
||||
drop(http_client);
|
||||
drop(token);
|
||||
#[cfg(feature = "tui")]
|
||||
drop(tx);
|
||||
|
||||
while let Some(res) = join_set.join_next().await {
|
||||
res??;
|
||||
}
|
||||
|
||||
drop(join_set);
|
||||
|
||||
Ok(Arc::into_inner(proxies)
|
||||
.ok_or_eyre("failed to unwrap Arc")?
|
||||
.into_inner()
|
||||
.into_iter()
|
||||
.collect())
|
||||
}
|
||||
|
||||
+25
-25
@@ -34,31 +34,6 @@ impl Drop for RatatuiRestoreGuard {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn run(
|
||||
mut terminal: ratatui::DefaultTerminal,
|
||||
token: tokio_util::sync::CancellationToken,
|
||||
tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
mut rx: tokio::sync::mpsc::UnboundedReceiver<Event>,
|
||||
) -> crate::Result<()> {
|
||||
tokio::spawn(tick_event_listener(tx.clone()));
|
||||
tokio::spawn(crossterm_event_listener(tx));
|
||||
|
||||
let mut app_state = AppState::default();
|
||||
let logger_state = TuiWidgetState::default();
|
||||
|
||||
while !matches!(app_state.mode, AppMode::Quit) {
|
||||
if let Some(event) = rx.recv().await {
|
||||
if handle_event(event, &mut app_state, &token, &logger_state) {
|
||||
terminal
|
||||
.draw(|frame| draw(frame, &app_state, &logger_state))?;
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub enum AppMode {
|
||||
#[default]
|
||||
@@ -94,6 +69,31 @@ pub struct AppState {
|
||||
pub proxies_working: HashMap<ProxyType, usize>,
|
||||
}
|
||||
|
||||
pub async fn run(
|
||||
mut terminal: ratatui::DefaultTerminal,
|
||||
token: tokio_util::sync::CancellationToken,
|
||||
tx: tokio::sync::mpsc::UnboundedSender<Event>,
|
||||
mut rx: tokio::sync::mpsc::UnboundedReceiver<Event>,
|
||||
) -> crate::Result<()> {
|
||||
tokio::spawn(tick_event_listener(tx.clone()));
|
||||
tokio::spawn(crossterm_event_listener(tx));
|
||||
|
||||
let mut app_state = AppState::default();
|
||||
let logger_state = TuiWidgetState::default();
|
||||
|
||||
while !matches!(app_state.mode, AppMode::Quit) {
|
||||
if let Some(event) = rx.recv().await {
|
||||
if handle_event(event, &mut app_state, &token, &logger_state) {
|
||||
terminal
|
||||
.draw(|frame| draw(frame, &app_state, &logger_state))?;
|
||||
}
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn tick_event_listener(tx: tokio::sync::mpsc::UnboundedSender<Event>) {
|
||||
let mut tick = tokio::time::interval(Duration::from_millis(100));
|
||||
loop {
|
||||
|
||||
+18
-18
@@ -1,23 +1,5 @@
|
||||
use std::fmt::{Display, Write as _};
|
||||
|
||||
pub async fn is_docker() -> bool {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
static CACHE: tokio::sync::OnceCell<bool> =
|
||||
tokio::sync::OnceCell::const_new();
|
||||
|
||||
*CACHE
|
||||
.get_or_init(async || {
|
||||
tokio::fs::try_exists("/.dockerenv").await.unwrap_or(false)
|
||||
})
|
||||
.await
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub trait CompactStrJoin: Iterator {
|
||||
fn join(&mut self, sep: &str) -> compact_str::CompactString
|
||||
where
|
||||
@@ -47,3 +29,21 @@ impl<T: Iterator> CompactStrJoin for T {}
|
||||
pub fn pretty_error(e: &crate::Error) -> compact_str::CompactString {
|
||||
e.chain().join(" \u{2192} ")
|
||||
}
|
||||
|
||||
pub async fn is_docker() -> bool {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
static CACHE: tokio::sync::OnceCell<bool> =
|
||||
tokio::sync::OnceCell::const_new();
|
||||
|
||||
*CACHE
|
||||
.get_or_init(async || {
|
||||
tokio::fs::try_exists("/.dockerenv").await.unwrap_or(false)
|
||||
})
|
||||
.await
|
||||
}
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
{
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user