* feat(metadata): reconcile artwork cache after public S3 provider changes Changing the public S3 provider previously broke every cached image permanently: the DB keeps bucket-relative keys, the image cache pipeline treats a cached path as its durable dedup marker and never re-enqueues, and clients eat the 404s straight from S3 so the server never notices. Add a storage identity fingerprint (s3.public_storage_identity, seeded via SetIfAbsent at boot) and a reconcile_artwork_cache task whose startup trigger only fires when the identity changed; manual runs always sweep, doubling as bucket-data-loss recovery. The task probes a random sample of cached objects, then either bulk-resets (near-total miss) or per-row verifies. Missing provider-sourced artwork is reset to its *_source_path so the existing enqueue loop re-caches it; surfaces without a re-downloadable source (chapter thumbnails, collection artwork, library posters, branding refs, embedded book covers) are cleared so their owning pipelines refill them. Small upload-holding tables are always per-row verified so bulk mode cannot blind-clear an upload that survived migration, and transport errors never reset rows. Users never see broken images during the transition: reset rows serve the provider's original URL via the existing absolute-URL pass-through and thumbhashes are preserved. The storage settings page now warns that uploads cannot be re-downloaded when the identity fields are edited. Part of #348 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(metadata): harden artwork reconcile per code review Address the confirmed findings from the PR review: - Fingerprint the key prefix case-sensitively and slash-trimmed exactly as s3client applies it (new exported NormalizeKeyPrefix): a case-only prefix edit is a real storage move and must reconcile; a slash-only edit is not and must not. - Certify the storage fingerprint immediately after the artwork sweep succeeds and make the 4-object branding check non-fatal (reported in the task message), so a transient branding error cannot discard a completed catalog sweep and force it to repeat every boot. - Fail closed on conditional-task preflight errors in the task manager (previously fail-open ran the task), and retry transient settings reads in ShouldRun since the startup trigger fires once per process. - Track probe HEAD errors against a separate baseline so a flaky probe cannot consume the sweep's error budget. - Probe before counting: bulk mode skips the per-surface count(*) full scans entirely, and probe sampling drops ORDER BY random() (plain LIMIT answers "is the cache in this bucket" just as well). - Verify chapter thumbnails across a whole 500-file batch in one HEAD fan-out instead of per file, keeping the worker pool saturated. - Replace the 10 inline non-provider-scheme ARRAY literals in the enqueue query with the shared nonProviderImageSchemesSQL constant. Part of #348 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * fix(metadata): guard bulk reset against degraded probes, certify only clean sweeps Address bot review feedback on the reconcile hardening: - A probe where more than half the HEAD requests error aborts the run: errored requests are excluded from the sample, so a partial outage could otherwise present a handful of surviving 404s as a ~100% miss rate and bulk-reset the catalog. Bulk mode additionally requires a minimum number of successful samples; thinned probes and tiny catalogs take the safe per-row verify path. - Track sweep errors separately from probe/branding errors (stats.sweep_errors) and certify the storage fingerprint only when the sweep completed with zero of them — skipped rows were never verified, so the next startup retries. Applied resets stay durable. - Give each ObjectExists attempt its own timeout so a stalled HEAD fails that attempt instead of pinning the retry loop to the run context. - Report branding assets checked (not just cleared) in stats.Checked. - Drop the dead settingsRepo/brandingSvc nil guards in cmd/silo and sync spec numbers with the implementation constants. Part of #348 Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
190 lines
7.3 KiB
Go
190 lines
7.3 KiB
Go
package tasks
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/Silo-Server/silo-server/internal/metadata"
|
|
)
|
|
|
|
type fakeSettingsStore struct {
|
|
values map[string]string
|
|
getErr error
|
|
}
|
|
|
|
func (f *fakeSettingsStore) Get(_ context.Context, key string) (string, error) {
|
|
if f.getErr != nil {
|
|
return "", f.getErr
|
|
}
|
|
return f.values[key], nil
|
|
}
|
|
|
|
func (f *fakeSettingsStore) Set(_ context.Context, key, value string) error {
|
|
if f.values == nil {
|
|
f.values = map[string]string{}
|
|
}
|
|
f.values[key] = value
|
|
return nil
|
|
}
|
|
|
|
type fakeReconcileRunner struct {
|
|
stats metadata.ArtworkReconcileStats
|
|
err error
|
|
runs int
|
|
}
|
|
|
|
func (f *fakeReconcileRunner) Run(context.Context, func(float64, string)) (metadata.ArtworkReconcileStats, error) {
|
|
f.runs++
|
|
return f.stats, f.err
|
|
}
|
|
|
|
type fakeBrandingReconciler struct {
|
|
checked int
|
|
cleared int
|
|
err error
|
|
}
|
|
|
|
func (f *fakeBrandingReconciler) ReconcileMissingAssets(context.Context) (int, int, error) {
|
|
return f.checked, f.cleared, f.err
|
|
}
|
|
|
|
type fakeProgress struct {
|
|
lastMessage string
|
|
resultData json.RawMessage
|
|
}
|
|
|
|
func (f *fakeProgress) Report(_ float64, message string) { f.lastMessage = message }
|
|
func (f *fakeProgress) SetResultData(data json.RawMessage) { f.resultData = data }
|
|
|
|
func TestArtworkStorageIdentityNormalizes(t *testing.T) {
|
|
// Endpoint and bucket are case-insensitive; whitespace is trimmed.
|
|
a := ArtworkStorageIdentity(" https://S3.Example.com ", "Assets", "silo/prod")
|
|
b := ArtworkStorageIdentity("https://s3.example.com", "assets", "silo/prod")
|
|
if a != b {
|
|
t.Fatalf("identity not normalized: %q != %q", a, b)
|
|
}
|
|
// The key prefix is slash-insensitive (the s3client trims slashes, so
|
|
// 'art' and '/art/' are the same storage location)...
|
|
if ArtworkStorageIdentity("e", "b", "art") != ArtworkStorageIdentity("e", "b", " /art/ ") {
|
|
t.Fatal("slash-only prefix differences must not change the identity")
|
|
}
|
|
// ...but case-SENSITIVE: S3 object keys are case-sensitive, so a
|
|
// case-only prefix edit is a real storage move and must reconcile.
|
|
if ArtworkStorageIdentity("e", "b", "Art") == ArtworkStorageIdentity("e", "b", "art") {
|
|
t.Fatal("case-only prefix differences are real storage moves and must change the identity")
|
|
}
|
|
if a == ArtworkStorageIdentity("https://s3.example.com", "assets", "") {
|
|
t.Fatal("key prefix must participate in the identity")
|
|
}
|
|
if a == ArtworkStorageIdentity("https://other.example.com", "assets", "silo/prod") {
|
|
t.Fatal("endpoint must participate in the identity")
|
|
}
|
|
}
|
|
|
|
func TestReconcileArtworkCacheShouldRun(t *testing.T) {
|
|
runner := &fakeReconcileRunner{}
|
|
store := &fakeSettingsStore{values: map[string]string{}}
|
|
task := NewReconcileArtworkCacheTask(runner, store, nil, "endpoint|bucket|prefix")
|
|
|
|
// No stored fingerprint: first boot, seeding happens at wiring time; the
|
|
// scheduled run must not sweep a catalog it has no baseline for.
|
|
if run, err := task.ShouldRun(context.Background()); err != nil || run {
|
|
t.Fatalf("ShouldRun with empty fingerprint = %v, %v; want false, nil", run, err)
|
|
}
|
|
|
|
store.values[ArtworkStorageIdentityKey] = "endpoint|bucket|prefix"
|
|
if run, err := task.ShouldRun(context.Background()); err != nil || run {
|
|
t.Fatalf("ShouldRun with matching fingerprint = %v, %v; want false, nil", run, err)
|
|
}
|
|
|
|
store.values[ArtworkStorageIdentityKey] = "old-endpoint|bucket|prefix"
|
|
if run, err := task.ShouldRun(context.Background()); err != nil || !run {
|
|
t.Fatalf("ShouldRun with changed fingerprint = %v, %v; want true, nil", run, err)
|
|
}
|
|
}
|
|
|
|
func TestReconcileArtworkCacheExecutePersistsFingerprintOnlyOnSuccess(t *testing.T) {
|
|
store := &fakeSettingsStore{values: map[string]string{ArtworkStorageIdentityKey: "old"}}
|
|
failing := &fakeReconcileRunner{err: errors.New("storage unreachable")}
|
|
task := NewReconcileArtworkCacheTask(failing, store, nil, "new")
|
|
|
|
if err := task.Execute(context.Background(), &fakeProgress{}); err == nil {
|
|
t.Fatal("Execute with failing runner returned nil error")
|
|
}
|
|
if got := store.values[ArtworkStorageIdentityKey]; got != "old" {
|
|
t.Fatalf("fingerprint after failed run = %q, want unchanged %q", got, "old")
|
|
}
|
|
|
|
ok := &fakeReconcileRunner{stats: metadata.ArtworkReconcileStats{Mode: "verify", Verified: 3, Requeued: 2, Cleared: 1}}
|
|
task = NewReconcileArtworkCacheTask(ok, store, nil, "new")
|
|
progress := &fakeProgress{}
|
|
if err := task.Execute(context.Background(), progress); err != nil {
|
|
t.Fatalf("Execute = %v, want nil", err)
|
|
}
|
|
if got := store.values[ArtworkStorageIdentityKey]; got != "new" {
|
|
t.Fatalf("fingerprint after successful run = %q, want %q", got, "new")
|
|
}
|
|
if progress.resultData == nil {
|
|
t.Fatal("Execute did not record result data")
|
|
}
|
|
}
|
|
|
|
func TestReconcileArtworkCacheExecuteDoesNotCertifyOnSweepErrors(t *testing.T) {
|
|
// Rows skipped on storage errors were never verified, so the sweep did
|
|
// not fully cover the catalog: the fingerprint must stay stale so the
|
|
// next startup retries.
|
|
store := &fakeSettingsStore{values: map[string]string{ArtworkStorageIdentityKey: "old"}}
|
|
runner := &fakeReconcileRunner{stats: metadata.ArtworkReconcileStats{
|
|
Mode: "verify", Verified: 10, Errors: 3, SweepErrors: 3,
|
|
}}
|
|
branding := &fakeBrandingReconciler{checked: 4}
|
|
task := NewReconcileArtworkCacheTask(runner, store, branding, "new")
|
|
if err := task.Execute(context.Background(), &fakeProgress{}); err == nil {
|
|
t.Fatal("Execute with sweep errors returned nil error")
|
|
}
|
|
if got := store.values[ArtworkStorageIdentityKey]; got != "old" {
|
|
t.Fatalf("fingerprint after sweep errors = %q, want unchanged %q", got, "old")
|
|
}
|
|
}
|
|
|
|
func TestReconcileArtworkCacheExecuteIncludesBranding(t *testing.T) {
|
|
store := &fakeSettingsStore{values: map[string]string{}}
|
|
runner := &fakeReconcileRunner{stats: metadata.ArtworkReconcileStats{Mode: "verify", Cleared: 1}}
|
|
task := NewReconcileArtworkCacheTask(runner, store, &fakeBrandingReconciler{checked: 4, cleared: 2}, "id")
|
|
progress := &fakeProgress{}
|
|
if err := task.Execute(context.Background(), progress); err != nil {
|
|
t.Fatalf("Execute = %v, want nil", err)
|
|
}
|
|
var stats metadata.ArtworkReconcileStats
|
|
if err := json.Unmarshal(progress.resultData, &stats); err != nil {
|
|
t.Fatalf("decode result data: %v", err)
|
|
}
|
|
if stats.Cleared != 3 {
|
|
t.Fatalf("Cleared = %d, want 3 (1 artwork + 2 branding)", stats.Cleared)
|
|
}
|
|
if stats.Checked != 4 {
|
|
t.Fatalf("Checked = %d, want 4 (all probed branding assets, not just cleared ones)", stats.Checked)
|
|
}
|
|
|
|
// A branding failure must NOT discard the completed sweep: the
|
|
// fingerprint is certified first and the failure is reported in the
|
|
// message instead, so the full catalog sweep never repeats over a
|
|
// transient error on a 4-object branding check.
|
|
fpStore := &fakeSettingsStore{values: map[string]string{}}
|
|
failing := NewReconcileArtworkCacheTask(runner, fpStore,
|
|
&fakeBrandingReconciler{err: errors.New("storage unreachable")}, "id")
|
|
failingProgress := &fakeProgress{}
|
|
if err := failing.Execute(context.Background(), failingProgress); err != nil {
|
|
t.Fatalf("Execute with failing branding reconcile = %v, want nil (non-fatal)", err)
|
|
}
|
|
if got := fpStore.values[ArtworkStorageIdentityKey]; got != "id" {
|
|
t.Fatalf("fingerprint after branding failure = %q, want certified %q", got, "id")
|
|
}
|
|
if !strings.Contains(failingProgress.lastMessage, "branding asset check failed") {
|
|
t.Fatalf("completion message %q does not surface the branding failure", failingProgress.lastMessage)
|
|
}
|
|
}
|