From dfce4972e4dbef9705a061c77b87fa333dfa3942 Mon Sep 17 00:00:00 2001
From: rxwatcher <rxwatcher@users.noreply.github.com>
Date: Mon, 20 Jul 2026 13:23:20 +0200
Subject: [PATCH] fix(ebooks): floor rate-limited requeue delays

The ebook-metadata plugin attaches ~1s RetryInfo to ResourceExhausted
errors as request-pacing advice for its internal token bucket. Adopting
that hint verbatim as the queue horizon made rate-limited rows claimable
again immediately, so every backfill run re-claimed the same saturated
tail. Clamp rate-limited requeues to a 15m floor (SILO_EBOOK_RATE_LIMIT_COOLDOWN
to tune); hints above the floor are honored up to the existing 24h cap.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 internal/ebooks/enrichment_queue.go      | 26 +++++++++++++++----
 internal/ebooks/enrichment_queue_test.go | 32 ++++++++++++++++++++++++
 2 files changed, 53 insertions(+), 5 deletions(-)

diff --git a/internal/ebooks/enrichment_queue.go b/internal/ebooks/enrichment_queue.go
index b3aca11e..024b7e48 100644
--- a/internal/ebooks/enrichment_queue.go
+++ b/internal/ebooks/enrichment_queue.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"strings"
 	"time"
 
@@ -13,13 +14,24 @@ import (
 )
 
 const (
-	defaultEnrichmentLease = 10 * time.Minute
-	maxEnrichmentRetry     = 24 * time.Hour
-	transientRetryBase     = 5 * time.Minute
-	skippedRetryHorizon    = 15 * time.Minute
-	claimCandidateWindow   = maxEnrichWorkers
+	defaultEnrichmentLease   = 10 * time.Minute
+	maxEnrichmentRetry       = 24 * time.Hour
+	transientRetryBase       = 5 * time.Minute
+	skippedRetryHorizon      = 15 * time.Minute
+	claimCandidateWindow     = maxEnrichWorkers
+	defaultRateLimitCooldown = 15 * time.Minute
+	rateLimitCooldownEnv     = "SILO_EBOOK_RATE_LIMIT_COOLDOWN"
 )
 
+func rateLimitCooldownFloor() time.Duration {
+	if v := os.Getenv(rateLimitCooldownEnv); v != "" {
+		if parsed, err := time.ParseDuration(v); err == nil && parsed > 0 {
+			return parsed
+		}
+	}
+	return defaultRateLimitCooldown
+}
+
 type EnrichmentOutcome string
 
 const (
@@ -787,6 +799,10 @@ func enrichmentRetryDelay(errorClass EnrichmentErrorClass, attempts int, retryAf
 		if retryAfter <= 0 {
 			retryAfter = transientRetryDelay(attempts)
 		}
+		// A provider's short RetryInfo is request-pacing advice for its own
+		// token bucket, not a queue horizon; requeueing that fast re-claims
+		// the same saturated tail every run.
+		retryAfter = max(retryAfter, rateLimitCooldownFloor())
 		return min(retryAfter, maxEnrichmentRetry)
 	case EnrichmentErrorPermanent:
 		return 30 * 24 * time.Hour
diff --git a/internal/ebooks/enrichment_queue_test.go b/internal/ebooks/enrichment_queue_test.go
index 72f5d4e9..ede1d664 100644
--- a/internal/ebooks/enrichment_queue_test.go
+++ b/internal/ebooks/enrichment_queue_test.go
@@ -490,6 +490,38 @@ func TestEnrichmentRetryPolicy(t *testing.T) {
 		}
 	})
 
+	t.Run("rate limits never requeue below the cooldown floor", func(t *testing.T) {
+		// A provider's 1s RetryInfo is request-pacing advice, not a queue
+		// horizon; adopting it verbatim re-claims the same saturated tail.
+		if got := enrichmentRetryDelay(EnrichmentErrorRateLimited, 1, time.Second); got != 15*time.Minute {
+			t.Fatalf("short-hint rate-limited retry = %s, want 15m floor", got)
+		}
+		if got := enrichmentRetryDelay(EnrichmentErrorRateLimited, 1, 0); got != 15*time.Minute {
+			t.Fatalf("no-hint rate-limited retry = %s, want 15m floor", got)
+		}
+		if got := enrichmentRetryDelay(EnrichmentErrorRateLimited, 20, 0); got != 24*time.Hour {
+			t.Fatalf("no-hint high-attempt rate-limited retry = %s, want capped backoff", got)
+		}
+	})
+
+	t.Run("cooldown floor is env tunable with a tolerant fallback", func(t *testing.T) {
+		t.Setenv("SILO_EBOOK_RATE_LIMIT_COOLDOWN", "30m")
+		if got := enrichmentRetryDelay(EnrichmentErrorRateLimited, 1, time.Second); got != 30*time.Minute {
+			t.Fatalf("tuned floor retry = %s, want 30m", got)
+		}
+		if got := enrichmentRetryDelay(EnrichmentErrorRateLimited, 1, 45*time.Minute); got != 45*time.Minute {
+			t.Fatalf("hint above tuned floor = %s, want 45m", got)
+		}
+		t.Setenv("SILO_EBOOK_RATE_LIMIT_COOLDOWN", "banana")
+		if got := enrichmentRetryDelay(EnrichmentErrorRateLimited, 1, time.Second); got != 15*time.Minute {
+			t.Fatalf("invalid floor retry = %s, want 15m default", got)
+		}
+		t.Setenv("SILO_EBOOK_RATE_LIMIT_COOLDOWN", "-5m")
+		if got := enrichmentRetryDelay(EnrichmentErrorRateLimited, 1, time.Second); got != 15*time.Minute {
+			t.Fatalf("non-positive floor retry = %s, want 15m default", got)
+		}
+	})
+
 	t.Run("permanent failures refresh after 30 days", func(t *testing.T) {
 		if got := enrichmentRetryDelay(EnrichmentErrorPermanent, 1, 0); got != 30*24*time.Hour {
 			t.Fatalf("permanent retry = %s, want 30 days", got)