Files
silo-server/internal/taskmanager/tasks/reconcile_artwork_cache_test.go
04c4344f52 feat(metadata): reconcile artwork cache after public S3 provider changes (#349)
* feat(metadata): reconcile artwork cache after public S3 provider changes

Changing the public S3 provider previously broke every cached image
permanently: the DB keeps bucket-relative keys, the image cache pipeline
treats a cached path as its durable dedup marker and never re-enqueues,
and clients eat the 404s straight from S3 so the server never notices.

Add a storage identity fingerprint (s3.public_storage_identity, seeded
via SetIfAbsent at boot) and a reconcile_artwork_cache task whose
startup trigger only fires when the identity changed; manual runs
always sweep, doubling as bucket-data-loss recovery. The task probes a
random sample of cached objects, then either bulk-resets (near-total
miss) or per-row verifies. Missing provider-sourced artwork is reset to
its *_source_path so the existing enqueue loop re-caches it; surfaces
without a re-downloadable source (chapter thumbnails, collection
artwork, library posters, branding refs, embedded book covers) are
cleared so their owning pipelines refill them. Small upload-holding
tables are always per-row verified so bulk mode cannot blind-clear an
upload that survived migration, and transport errors never reset rows.

Users never see broken images during the transition: reset rows serve
the provider's original URL via the existing absolute-URL pass-through
and thumbhashes are preserved. The storage settings page now warns that
uploads cannot be re-downloaded when the identity fields are edited.

Part of #348

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(metadata): harden artwork reconcile per code review

Address the confirmed findings from the PR review:

- Fingerprint the key prefix case-sensitively and slash-trimmed exactly
  as s3client applies it (new exported NormalizeKeyPrefix): a case-only
  prefix edit is a real storage move and must reconcile; a slash-only
  edit is not and must not.
- Certify the storage fingerprint immediately after the artwork sweep
  succeeds and make the 4-object branding check non-fatal (reported in
  the task message), so a transient branding error cannot discard a
  completed catalog sweep and force it to repeat every boot.
- Fail closed on conditional-task preflight errors in the task manager
  (previously fail-open ran the task), and retry transient settings
  reads in ShouldRun since the startup trigger fires once per process.
- Track probe HEAD errors against a separate baseline so a flaky probe
  cannot consume the sweep's error budget.
- Probe before counting: bulk mode skips the per-surface count(*)
  full scans entirely, and probe sampling drops ORDER BY random()
  (plain LIMIT answers "is the cache in this bucket" just as well).
- Verify chapter thumbnails across a whole 500-file batch in one HEAD
  fan-out instead of per file, keeping the worker pool saturated.
- Replace the 10 inline non-provider-scheme ARRAY literals in the
  enqueue query with the shared nonProviderImageSchemesSQL constant.

Part of #348

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

* fix(metadata): guard bulk reset against degraded probes, certify only clean sweeps

Address bot review feedback on the reconcile hardening:

- A probe where more than half the HEAD requests error aborts the run:
  errored requests are excluded from the sample, so a partial outage
  could otherwise present a handful of surviving 404s as a ~100% miss
  rate and bulk-reset the catalog. Bulk mode additionally requires a
  minimum number of successful samples; thinned probes and tiny
  catalogs take the safe per-row verify path.
- Track sweep errors separately from probe/branding errors
  (stats.sweep_errors) and certify the storage fingerprint only when
  the sweep completed with zero of them — skipped rows were never
  verified, so the next startup retries. Applied resets stay durable.
- Give each ObjectExists attempt its own timeout so a stalled HEAD
  fails that attempt instead of pinning the retry loop to the run
  context.
- Report branding assets checked (not just cleared) in stats.Checked.
- Drop the dead settingsRepo/brandingSvc nil guards in cmd/silo and
  sync spec numbers with the implementation constants.

Part of #348

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-07-09 12:00:24 -04:00

190 lines
7.3 KiB
Go

package tasks
import (
"context"
"encoding/json"
"errors"
"strings"
"testing"
"github.com/Silo-Server/silo-server/internal/metadata"
)
type fakeSettingsStore struct {
values map[string]string
getErr error
}
func (f *fakeSettingsStore) Get(_ context.Context, key string) (string, error) {
if f.getErr != nil {
return "", f.getErr
}
return f.values[key], nil
}
func (f *fakeSettingsStore) Set(_ context.Context, key, value string) error {
if f.values == nil {
f.values = map[string]string{}
}
f.values[key] = value
return nil
}
type fakeReconcileRunner struct {
stats metadata.ArtworkReconcileStats
err error
runs int
}
func (f *fakeReconcileRunner) Run(context.Context, func(float64, string)) (metadata.ArtworkReconcileStats, error) {
f.runs++
return f.stats, f.err
}
type fakeBrandingReconciler struct {
checked int
cleared int
err error
}
func (f *fakeBrandingReconciler) ReconcileMissingAssets(context.Context) (int, int, error) {
return f.checked, f.cleared, f.err
}
type fakeProgress struct {
lastMessage string
resultData json.RawMessage
}
func (f *fakeProgress) Report(_ float64, message string) { f.lastMessage = message }
func (f *fakeProgress) SetResultData(data json.RawMessage) { f.resultData = data }
func TestArtworkStorageIdentityNormalizes(t *testing.T) {
// Endpoint and bucket are case-insensitive; whitespace is trimmed.
a := ArtworkStorageIdentity(" https://S3.Example.com ", "Assets", "silo/prod")
b := ArtworkStorageIdentity("https://s3.example.com", "assets", "silo/prod")
if a != b {
t.Fatalf("identity not normalized: %q != %q", a, b)
}
// The key prefix is slash-insensitive (the s3client trims slashes, so
// 'art' and '/art/' are the same storage location)...
if ArtworkStorageIdentity("e", "b", "art") != ArtworkStorageIdentity("e", "b", " /art/ ") {
t.Fatal("slash-only prefix differences must not change the identity")
}
// ...but case-SENSITIVE: S3 object keys are case-sensitive, so a
// case-only prefix edit is a real storage move and must reconcile.
if ArtworkStorageIdentity("e", "b", "Art") == ArtworkStorageIdentity("e", "b", "art") {
t.Fatal("case-only prefix differences are real storage moves and must change the identity")
}
if a == ArtworkStorageIdentity("https://s3.example.com", "assets", "") {
t.Fatal("key prefix must participate in the identity")
}
if a == ArtworkStorageIdentity("https://other.example.com", "assets", "silo/prod") {
t.Fatal("endpoint must participate in the identity")
}
}
func TestReconcileArtworkCacheShouldRun(t *testing.T) {
runner := &fakeReconcileRunner{}
store := &fakeSettingsStore{values: map[string]string{}}
task := NewReconcileArtworkCacheTask(runner, store, nil, "endpoint|bucket|prefix")
// No stored fingerprint: first boot, seeding happens at wiring time; the
// scheduled run must not sweep a catalog it has no baseline for.
if run, err := task.ShouldRun(context.Background()); err != nil || run {
t.Fatalf("ShouldRun with empty fingerprint = %v, %v; want false, nil", run, err)
}
store.values[ArtworkStorageIdentityKey] = "endpoint|bucket|prefix"
if run, err := task.ShouldRun(context.Background()); err != nil || run {
t.Fatalf("ShouldRun with matching fingerprint = %v, %v; want false, nil", run, err)
}
store.values[ArtworkStorageIdentityKey] = "old-endpoint|bucket|prefix"
if run, err := task.ShouldRun(context.Background()); err != nil || !run {
t.Fatalf("ShouldRun with changed fingerprint = %v, %v; want true, nil", run, err)
}
}
func TestReconcileArtworkCacheExecutePersistsFingerprintOnlyOnSuccess(t *testing.T) {
store := &fakeSettingsStore{values: map[string]string{ArtworkStorageIdentityKey: "old"}}
failing := &fakeReconcileRunner{err: errors.New("storage unreachable")}
task := NewReconcileArtworkCacheTask(failing, store, nil, "new")
if err := task.Execute(context.Background(), &fakeProgress{}); err == nil {
t.Fatal("Execute with failing runner returned nil error")
}
if got := store.values[ArtworkStorageIdentityKey]; got != "old" {
t.Fatalf("fingerprint after failed run = %q, want unchanged %q", got, "old")
}
ok := &fakeReconcileRunner{stats: metadata.ArtworkReconcileStats{Mode: "verify", Verified: 3, Requeued: 2, Cleared: 1}}
task = NewReconcileArtworkCacheTask(ok, store, nil, "new")
progress := &fakeProgress{}
if err := task.Execute(context.Background(), progress); err != nil {
t.Fatalf("Execute = %v, want nil", err)
}
if got := store.values[ArtworkStorageIdentityKey]; got != "new" {
t.Fatalf("fingerprint after successful run = %q, want %q", got, "new")
}
if progress.resultData == nil {
t.Fatal("Execute did not record result data")
}
}
func TestReconcileArtworkCacheExecuteDoesNotCertifyOnSweepErrors(t *testing.T) {
// Rows skipped on storage errors were never verified, so the sweep did
// not fully cover the catalog: the fingerprint must stay stale so the
// next startup retries.
store := &fakeSettingsStore{values: map[string]string{ArtworkStorageIdentityKey: "old"}}
runner := &fakeReconcileRunner{stats: metadata.ArtworkReconcileStats{
Mode: "verify", Verified: 10, Errors: 3, SweepErrors: 3,
}}
branding := &fakeBrandingReconciler{checked: 4}
task := NewReconcileArtworkCacheTask(runner, store, branding, "new")
if err := task.Execute(context.Background(), &fakeProgress{}); err == nil {
t.Fatal("Execute with sweep errors returned nil error")
}
if got := store.values[ArtworkStorageIdentityKey]; got != "old" {
t.Fatalf("fingerprint after sweep errors = %q, want unchanged %q", got, "old")
}
}
func TestReconcileArtworkCacheExecuteIncludesBranding(t *testing.T) {
store := &fakeSettingsStore{values: map[string]string{}}
runner := &fakeReconcileRunner{stats: metadata.ArtworkReconcileStats{Mode: "verify", Cleared: 1}}
task := NewReconcileArtworkCacheTask(runner, store, &fakeBrandingReconciler{checked: 4, cleared: 2}, "id")
progress := &fakeProgress{}
if err := task.Execute(context.Background(), progress); err != nil {
t.Fatalf("Execute = %v, want nil", err)
}
var stats metadata.ArtworkReconcileStats
if err := json.Unmarshal(progress.resultData, &stats); err != nil {
t.Fatalf("decode result data: %v", err)
}
if stats.Cleared != 3 {
t.Fatalf("Cleared = %d, want 3 (1 artwork + 2 branding)", stats.Cleared)
}
if stats.Checked != 4 {
t.Fatalf("Checked = %d, want 4 (all probed branding assets, not just cleared ones)", stats.Checked)
}
// A branding failure must NOT discard the completed sweep: the
// fingerprint is certified first and the failure is reported in the
// message instead, so the full catalog sweep never repeats over a
// transient error on a 4-object branding check.
fpStore := &fakeSettingsStore{values: map[string]string{}}
failing := NewReconcileArtworkCacheTask(runner, fpStore,
&fakeBrandingReconciler{err: errors.New("storage unreachable")}, "id")
failingProgress := &fakeProgress{}
if err := failing.Execute(context.Background(), failingProgress); err != nil {
t.Fatalf("Execute with failing branding reconcile = %v, want nil (non-fatal)", err)
}
if got := fpStore.values[ArtworkStorageIdentityKey]; got != "id" {
t.Fatalf("fingerprint after branding failure = %q, want certified %q", got, "id")
}
if !strings.Contains(failingProgress.lastMessage, "branding asset check failed") {
t.Fatalf("completion message %q does not surface the branding failure", failingProgress.lastMessage)
}
}