* feat(metadata): expand provider image cache queue * fix(metadata): harden provider image cache queue Addresses bug-review feedback from Codex/CodeRabbit on the metadata image cache pipeline. All findings validated against the code before fixing; false positives (rows/connection deadlock, PhotoSourcePath merge coupling) were confirmed non-issues and left unchanged. - Honor metadata.cache_images for the background processor. The cache_metadata_images task was registered whenever S3 was configured, so merely enabling object storage downloaded the entire provider-artwork catalog even with caching disabled. Add ImageCacheProcessor.SetEnabled, gate RunOnce/RunUntilIdle on it, and wire it (with hot reload) from cfg.Metadata.CacheImages in main.go. - Guard terminal job updates with lease ownership. EnqueueBatch can repurpose a running row with a new source; MarkSucceeded/MarkFailed keyed on id alone let a stale worker finalize the replacement job and drop the new artwork. Thread locked_by through and add status='running' AND locked_by=$n guards. - Avoid uploading stale jobs onto the live artwork key. Verify the target still references the job's source (CurrentTargetSourcePath) before CacheImage, so a job whose source an admin/refresh already replaced cannot overwrite the deterministic storage object. - COALESCE nullable external IDs in EnqueueExistingProviderArtwork. A NULL tmdb_id/tvdb_id/imdb_id on any candidate failed the scan and aborted the whole cache run; matches the existing item_repo pattern. - Stop re-downloading the catalog every 30 days. Discovery now skips targets whose *_path is already a cached relative path, making the cached row the durable dedup marker instead of the prunable job row. - Decouple catalog sweeps from queue draining. RunOnce no longer runs discovery per batch; RunUntilIdle sweeps only when the queue drains and throttles full sweeps to every 15m, so idle installs stop full-scanning every entity table each minute. - Requeue claimed-but-unstarted jobs on cancellation. Acquire the semaphore before spawning workers and RequeueClaimed any jobs not yet started, instead of leaving them locked until the 15m lease expires. - Skip the backoff sleep after the final upload attempt in putObjectWithRetry (saves ~1.5s on permanent failures). - Add the s3/file/local/upload/generated exclusion to the seasons and episodes backfill in migration 20260617184537 for consistency with the later migration (the bad backfill was inert downstream, but the asymmetry is removed). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
135 lines
4.1 KiB
Go
135 lines
4.1 KiB
Go
package metadata
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/Silo-Server/silo-server/internal/models"
|
|
)
|
|
|
|
type recordingImageCacheJobEnqueuer struct {
|
|
inputs []EnqueueImageCacheJobInput
|
|
}
|
|
|
|
func (r *recordingImageCacheJobEnqueuer) Enqueue(_ context.Context, in EnqueueImageCacheJobInput) error {
|
|
_, err := r.EnqueueBatch(context.Background(), []EnqueueImageCacheJobInput{in})
|
|
return err
|
|
}
|
|
|
|
func (r *recordingImageCacheJobEnqueuer) EnqueueBatch(_ context.Context, inputs []EnqueueImageCacheJobInput) (int, error) {
|
|
r.inputs = append(r.inputs, inputs...)
|
|
return len(inputs), nil
|
|
}
|
|
|
|
func TestPreserveCachedArtworkKeepsCachedPathWhenSourceMatches(t *testing.T) {
|
|
path, thumb, source := preserveCachedArtwork(
|
|
"tvdb://banners/episodes/1.jpg",
|
|
"",
|
|
"tvdb/series/1/seasons/1/episodes/1/still/original.webp",
|
|
"tvdb://banners/episodes/1.jpg",
|
|
"thumb",
|
|
)
|
|
if path != "tvdb/series/1/seasons/1/episodes/1/still/original.webp" {
|
|
t.Fatalf("path = %q", path)
|
|
}
|
|
if thumb != "thumb" {
|
|
t.Fatalf("thumb = %q", thumb)
|
|
}
|
|
if source != "tvdb://banners/episodes/1.jpg" {
|
|
t.Fatalf("source = %q", source)
|
|
}
|
|
}
|
|
|
|
func TestPersistSeasonsAndEpisodesPersistsSourceBeforeEnqueue(t *testing.T) {
|
|
const seriesID = "series-tvdb-123"
|
|
service, _, seasonRepo, episodeRepo := newSeasonEpisodeServiceForTest(seriesID)
|
|
enqueuer := &recordingImageCacheJobEnqueuer{}
|
|
service.SetAutoCacheImages(true)
|
|
service.SetImageCacheJobEnqueuer(enqueuer)
|
|
|
|
series := &models.MediaItem{
|
|
ContentID: seriesID,
|
|
Type: "series",
|
|
TvdbID: "123",
|
|
}
|
|
service.persistSeasonsAndEpisodes(
|
|
context.Background(),
|
|
series,
|
|
map[string]string{"tvdb": "123"},
|
|
"en",
|
|
"en",
|
|
[]SeasonResult{{
|
|
SeasonNumber: 1,
|
|
Title: "Season 1",
|
|
PosterPath: "tvdb://banners/seasons/1.jpg",
|
|
}},
|
|
[]EpisodeResult{{
|
|
ProviderIDs: map[string]string{"tvdb": "ep-1"},
|
|
SeasonNumber: 1,
|
|
EpisodeNumber: 1,
|
|
Title: "Pilot",
|
|
StillPath: "tvdb://banners/episodes/1.jpg",
|
|
StillThumbhash: "provider-thumb",
|
|
}},
|
|
MergeFillEmpty,
|
|
)
|
|
|
|
season := seasonRepo.seasons[seasonKey(seriesID, 1)]
|
|
if season == nil {
|
|
t.Fatal("season was not persisted")
|
|
}
|
|
if season.PosterSourcePath != "tvdb://banners/seasons/1.jpg" {
|
|
t.Fatalf("season source = %q", season.PosterSourcePath)
|
|
}
|
|
episode := episodeRepo.episodes[episodeKey(seriesID, 1, 1)]
|
|
if episode == nil {
|
|
t.Fatal("episode was not persisted")
|
|
}
|
|
if episode.StillSourcePath != "tvdb://banners/episodes/1.jpg" {
|
|
t.Fatalf("episode source = %q", episode.StillSourcePath)
|
|
}
|
|
if len(enqueuer.inputs) != 2 {
|
|
t.Fatalf("queued jobs = %d, want 2", len(enqueuer.inputs))
|
|
}
|
|
if enqueuer.inputs[0].TargetContentID != season.ContentID {
|
|
t.Fatalf("season job target = %q, want %q", enqueuer.inputs[0].TargetContentID, season.ContentID)
|
|
}
|
|
if enqueuer.inputs[1].TargetContentID != episode.ContentID {
|
|
t.Fatalf("episode job target = %q, want %q", enqueuer.inputs[1].TargetContentID, episode.ContentID)
|
|
}
|
|
}
|
|
|
|
func TestPreserveCachedArtworkRecordsNewProviderSource(t *testing.T) {
|
|
path, thumb, source := preserveCachedArtwork(
|
|
"tvdb://banners/episodes/new.jpg",
|
|
"",
|
|
"tvdb/series/1/seasons/1/episodes/1/still/original.webp",
|
|
"tvdb://banners/episodes/old.jpg",
|
|
"old-thumb",
|
|
)
|
|
if path != "tvdb/series/1/seasons/1/episodes/1/still/original.webp" {
|
|
t.Fatalf("cached path should remain visible until worker succeeds, got %q", path)
|
|
}
|
|
if thumb != "old-thumb" {
|
|
t.Fatalf("thumb = %q", thumb)
|
|
}
|
|
if source != "tvdb://banners/episodes/new.jpg" {
|
|
t.Fatalf("source = %q", source)
|
|
}
|
|
}
|
|
|
|
func TestProviderImageSourcePathOnlyRecordsProviderScheme(t *testing.T) {
|
|
if got := providerImageSourcePath("tvdb://banners/episodes/1.jpg"); got != "tvdb://banners/episodes/1.jpg" {
|
|
t.Fatalf("provider source = %q", got)
|
|
}
|
|
if got := providerImageSourcePath("tmdb/series/1/poster/original.webp"); got != "" {
|
|
t.Fatalf("cached path source = %q, want empty", got)
|
|
}
|
|
if got := providerImageSourcePath("file:///media/poster.jpg"); got != "" {
|
|
t.Fatalf("file source = %q, want empty", got)
|
|
}
|
|
if got := providerImageSourcePath("https://image.tmdb.org/t/p/original/a.jpg"); got != "https://image.tmdb.org/t/p/original/a.jpg" {
|
|
t.Fatalf("http source = %q", got)
|
|
}
|
|
}
|