feat: automatically retry scraping errors

Cargo.lock

@@ -762,6 +762,12 @@ version = "1.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
 
+[[package]]
+name = "httpdate"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+
 [[package]]
 name = "hyper"
 version = "1.6.0"
@@ -1339,7 +1345,9 @@ dependencies = [
  "dirs",
  "fancy-regex",
  "futures",
+ "httpdate",
  "maxminddb",
+ "rand 0.9.2",
  "ratatui",
  "reqwest",
  "rlimit",

Cargo.toml

@@ -14,7 +14,9 @@ derivative = "=2.2.0"
 dirs = "=6.0.0"
 fancy-regex = "=0.15.0"
 futures = { version = "=0.3.31", optional = true }
+httpdate = "=1.0.3"
 maxminddb = { version = "=0.26.0", features = ["mmap"] }
+rand = "=0.9.2"
 ratatui = { version = "=0.29.0", optional = true }
 rlimit = "=0.10.2"
 serde = "=1.0.219"

src/scraper.rs

@@ -1,4 +1,8 @@
-use std::{collections::HashSet, sync::Arc};
+use std::{
+    collections::HashSet,
+    sync::Arc,
+    time::{Duration, SystemTime},
+};
 
 use color_eyre::eyre::{OptionExt as _, WrapErr as _};
 
@@ -11,26 +15,132 @@ use crate::{
     utils::{is_http_url, pretty_error},
 };
 
+const DEFAULT_MAX_RETRIES: usize = 2;
+const INITIAL_RETRY_DELAY: Duration = Duration::from_millis(500);
+const MAX_RETRY_DELAY: Duration = Duration::from_secs(8);
+
+static RETRY_STATUSES: &[reqwest::StatusCode] = &[
+    reqwest::StatusCode::REQUEST_TIMEOUT,
+    reqwest::StatusCode::TOO_MANY_REQUESTS,
+    reqwest::StatusCode::INTERNAL_SERVER_ERROR,
+    reqwest::StatusCode::BAD_GATEWAY,
+    reqwest::StatusCode::SERVICE_UNAVAILABLE,
+    reqwest::StatusCode::GATEWAY_TIMEOUT,
+];
+
+fn parse_retry_after(headers: &reqwest::header::HeaderMap) -> Option<Duration> {
+    if let Some(val) = headers.get("retry-after-ms")
+        && let Ok(s) = val.to_str()
+        && let Ok(ms) = s.parse()
+    {
+        return Some(Duration::from_millis(ms));
+    }
+
+    if let Some(val) = headers.get(reqwest::header::RETRY_AFTER)
+        && let Ok(s) = val.to_str()
+    {
+        if let Ok(sec) = s.parse() {
+            return Some(Duration::from_secs(sec));
+        }
+
+        if let Ok(parsed) = httpdate::parse_http_date(s)
+            && let Ok(dur) = parsed.duration_since(SystemTime::now())
+        {
+            return Some(dur);
+        }
+    }
+    None
+}
+
+fn calculate_retry_timeout(
+    headers: Option<&reqwest::header::HeaderMap>,
+    attempt: usize,
+) -> Option<Duration> {
+    if let Some(h) = headers
+        && let Some(after) = parse_retry_after(h)
+    {
+        if after > Duration::from_secs(60) {
+            return None;
+        }
+        return Some(after);
+    }
+
+    let base = INITIAL_RETRY_DELAY
+        .saturating_mul((2_u32).pow(u32::try_from(attempt).unwrap_or(u32::MAX)))
+        .min(MAX_RETRY_DELAY);
+    let jitter = 0.25_f64.mul_add(-rand::random::<f64>(), 1.0);
+    Some(base.mul_f64(jitter))
+}
+
 async fn fetch_text(
     config: &Config,
     http_client: reqwest::Client,
     source: &str,
 ) -> color_eyre::Result<String> {
-    Ok(if is_http_url(source) {
-        http_client
-            .get(source)
-            .timeout(config.scraping.timeout)
-            .send()
-            .await?
-            .error_for_status()?
-            .text()
-            .await?
-    } else {
-        tokio::fs::read_to_string(
-            source.strip_prefix("file://").unwrap_or(source),
-        )
-        .await?
-    })
+    if is_http_url(source) {
+        let mut attempt = 0;
+        loop {
+            let req = http_client
+                .get(source)
+                .timeout(config.scraping.timeout)
+                .send()
+                .await;
+
+            match req {
+                Ok(resp) => {
+                    let status = resp.status();
+                    if status.is_client_error() || status.is_server_error() {
+                        if attempt < DEFAULT_MAX_RETRIES
+                            && RETRY_STATUSES.contains(&status)
+                            && let Some(delay) = calculate_retry_timeout(
+                                Some(resp.headers()),
+                                attempt,
+                            )
+                        {
+                            tracing::info!(
+                                "Request to {} returned status {}. Retrying attempt {}/{} after {:?}",
+                                source,
+                                status,
+                                attempt.saturating_add(1),
+                                DEFAULT_MAX_RETRIES,
+                                delay
+                            );
+                            tokio::time::sleep(delay).await;
+                            attempt = attempt.saturating_add(1);
+                            continue;
+                        }
+                        resp.error_for_status_ref()?;
+                    }
+                    let text = resp.text().await?;
+                    return Ok(text);
+                }
+                Err(err) => {
+                    if attempt < DEFAULT_MAX_RETRIES
+                        && err.is_connect()
+                        && let Some(delay) =
+                            calculate_retry_timeout(None, attempt)
+                    {
+                        tracing::info!(
+                            "Connection error while requesting {}: {}. Retrying attempt {}/{} after {:?}",
+                            source,
+                            err,
+                            attempt.saturating_add(1),
+                            DEFAULT_MAX_RETRIES,
+                            delay
+                        );
+                        tokio::time::sleep(delay).await;
+                        attempt = attempt.saturating_add(1);
+                        continue;
+                    }
+                    return Err(err.into());
+                }
+            }
+        }
+    }
+    Ok(tokio::fs::read_to_string(
+        source.strip_prefix("file://").unwrap_or(source),
+    )
+    .await?)
 }
 
 async fn scrape_one(