From 537b3f6476db22ad7edac26951fbe1f1640fbf73 Mon Sep 17 00:00:00 2001 From: rxwatcher Date: Mon, 20 Jul 2026 07:40:13 +0200 Subject: [PATCH] perf(ebooks): allow large backfill worker pools --- internal/ebooks/enrichment.go | 7 ++++--- internal/ebooks/enrichment_test.go | 11 ++++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/internal/ebooks/enrichment.go b/internal/ebooks/enrichment.go index cc6383f7..46a9f569 100644 --- a/internal/ebooks/enrichment.go +++ b/internal/ebooks/enrichment.go @@ -31,8 +31,9 @@ import ( const ( ebookMetadataImageProviderID = "ebook-metadata" - defaultEnrichBatchSize = 50 + defaultEnrichBatchSize = 5000 defaultEnrichWorkers = 4 + maxEnrichWorkers = 5000 defaultEnrichmentItemTimeout = 2 * time.Minute ) @@ -69,8 +70,8 @@ func ebookEnrichWorkers() int { n = parsed } } - if n > defaultEnrichBatchSize { - n = defaultEnrichBatchSize + if n > maxEnrichWorkers { + n = maxEnrichWorkers } return n } diff --git a/internal/ebooks/enrichment_test.go b/internal/ebooks/enrichment_test.go index 60e080ec..28492235 100644 --- a/internal/ebooks/enrichment_test.go +++ b/internal/ebooks/enrichment_test.go @@ -69,9 +69,14 @@ func TestEbookEnrichWorkersFromEnv(t *testing.T) { t.Fatalf("ebookEnrichWorkers() with zero = %d, want default %d", got, defaultEnrichWorkers) } - t.Setenv("SILO_EBOOK_ENRICH_WORKERS", "999") - if got := ebookEnrichWorkers(); got != defaultEnrichBatchSize { - t.Fatalf("ebookEnrichWorkers() capped = %d, want %d", got, defaultEnrichBatchSize) + t.Setenv("SILO_EBOOK_ENRICH_WORKERS", "5000") + if got := ebookEnrichWorkers(); got != 5000 { + t.Fatalf("ebookEnrichWorkers() = %d, want 5000", got) + } + + t.Setenv("SILO_EBOOK_ENRICH_WORKERS", "9999") + if got := ebookEnrichWorkers(); got != maxEnrichWorkers { + t.Fatalf("ebookEnrichWorkers() capped = %d, want %d", got, maxEnrichWorkers) } }