From 14ffc91dfbe31cf90d3c07439883065d156adf0e Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 18 Jun 2026 10:07:58 -0400 Subject: [PATCH] [codex] Expand provider image cache queue (#176) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(metadata): expand provider image cache queue * fix(metadata): harden provider image cache queue Addresses bug-review feedback from Codex/CodeRabbit on the metadata image cache pipeline. All findings validated against the code before fixing; false positives (rows/connection deadlock, PhotoSourcePath merge coupling) were confirmed non-issues and left unchanged. - Honor metadata.cache_images for the background processor. The cache_metadata_images task was registered whenever S3 was configured, so merely enabling object storage downloaded the entire provider-artwork catalog even with caching disabled. Add ImageCacheProcessor.SetEnabled, gate RunOnce/RunUntilIdle on it, and wire it (with hot reload) from cfg.Metadata.CacheImages in main.go. - Guard terminal job updates with lease ownership. EnqueueBatch can repurpose a running row with a new source; MarkSucceeded/MarkFailed keyed on id alone let a stale worker finalize the replacement job and drop the new artwork. Thread locked_by through and add status='running' AND locked_by=$n guards. - Avoid uploading stale jobs onto the live artwork key. Verify the target still references the job's source (CurrentTargetSourcePath) before CacheImage, so a job whose source an admin/refresh already replaced cannot overwrite the deterministic storage object. - COALESCE nullable external IDs in EnqueueExistingProviderArtwork. A NULL tmdb_id/tvdb_id/imdb_id on any candidate failed the scan and aborted the whole cache run; matches the existing item_repo pattern. - Stop re-downloading the catalog every 30 days. Discovery now skips targets whose *_path is already a cached relative path, making the cached row the durable dedup marker instead of the prunable job row. - Decouple catalog sweeps from queue draining. RunOnce no longer runs discovery per batch; RunUntilIdle sweeps only when the queue drains and throttles full sweeps to every 15m, so idle installs stop full-scanning every entity table each minute. - Requeue claimed-but-unstarted jobs on cancellation. Acquire the semaphore before spawning workers and RequeueClaimed any jobs not yet started, instead of leaving them locked until the 15m lease expires. - Skip the backoff sleep after the final upload attempt in putObjectWithRetry (saves ~1.5s on permanent failures). - Add the s3/file/local/upload/generated exclusion to the seasons and episodes backfill in migration 20260617184537 for consistency with the later migration (the bad backfill was inert downstream, but the asymmetry is removed). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.8 --------- Co-authored-by: Claude Opus 4.8 --- cmd/silo/main.go | 27 +- internal/audiobooks/enrichment.go | 83 +- internal/catalog/episode_catalog_source.go | 2 + internal/catalog/episode_repo.go | 51 +- internal/catalog/item_repo.go | 105 ++- internal/catalog/localization_repo.go | 117 ++- internal/catalog/localize.go | 4 + internal/catalog/person_repo.go | 75 +- internal/catalog/season_repo.go | 43 +- internal/catalog/update.go | 78 +- internal/ebooks/enrichment.go | 81 +- internal/imagecache/imagecache.go | 72 +- internal/imagecache/imagecache_test.go | 103 ++- internal/manga/enrichment.go | 86 +- .../metadata/fallback_unification_test.go | 4 +- internal/metadata/image_cache_enqueue_test.go | 134 +++ internal/metadata/image_cache_job_repo.go | 796 ++++++++++++++++++ .../metadata/image_cache_job_repo_test.go | 93 ++ internal/metadata/image_cache_processor.go | 473 +++++++++++ .../metadata/image_cache_processor_test.go | 560 ++++++++++++ internal/metadata/merge.go | 4 + internal/metadata/person_refresh.go | 76 +- internal/metadata/plugin_provider.go | 2 + internal/metadata/service.go | 528 +++++++++--- internal/metadata/types.go | 68 +- internal/models/media.go | 89 +- internal/models/metadata_image_cache_job.go | 27 + .../tasks/cache_metadata_images.go | 77 ++ .../tasks/cache_metadata_images_test.go | 78 ++ internal/taskmanager/triggers/daily.go | 14 +- internal/taskmanager/triggers/interval.go | 15 +- .../taskmanager/triggers/interval_test.go | 25 + internal/taskmanager/triggers/startup.go | 14 +- internal/taskmanager/triggers/weekly.go | 14 +- ...60617184537_metadata_image_cache_queue.sql | 87 ++ ...03000_expand_metadata_image_cache_jobs.sql | 184 ++++ 36 files changed, 3871 insertions(+), 418 deletions(-) create mode 100644 internal/metadata/image_cache_enqueue_test.go create mode 100644 internal/metadata/image_cache_job_repo.go create mode 100644 internal/metadata/image_cache_job_repo_test.go create mode 100644 internal/metadata/image_cache_processor.go create mode 100644 internal/metadata/image_cache_processor_test.go create mode 100644 internal/models/metadata_image_cache_job.go create mode 100644 internal/taskmanager/tasks/cache_metadata_images.go create mode 100644 internal/taskmanager/tasks/cache_metadata_images_test.go create mode 100644 internal/taskmanager/triggers/interval_test.go create mode 100644 migrations/sql/20260617184537_metadata_image_cache_queue.sql create mode 100644 migrations/sql/20260617203000_expand_metadata_image_cache_jobs.sql diff --git a/cmd/silo/main.go b/cmd/silo/main.go index a59dfb1a..a2a68b3c 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -51,7 +51,6 @@ import ( "github.com/Silo-Server/silo-server/internal/config" "github.com/Silo-Server/silo-server/internal/database" "github.com/Silo-Server/silo-server/internal/ebooks" - "github.com/Silo-Server/silo-server/internal/manga" evt "github.com/Silo-Server/silo-server/internal/events" "github.com/Silo-Server/silo-server/internal/historyimport" "github.com/Silo-Server/silo-server/internal/imagecache" @@ -62,6 +61,7 @@ import ( "github.com/Silo-Server/silo-server/internal/logfilter" "github.com/Silo-Server/silo-server/internal/logstream" "github.com/Silo-Server/silo-server/internal/mail" + "github.com/Silo-Server/silo-server/internal/manga" "github.com/Silo-Server/silo-server/internal/markers" "github.com/Silo-Server/silo-server/internal/mdblist" "github.com/Silo-Server/silo-server/internal/metadata" @@ -1042,6 +1042,7 @@ func main() { // Step 4b: Create metadata service and match worker (if needed). var metadataService *metadata.MetadataService + var metadataImageCacheProcessor *metadata.ImageCacheProcessor var personRefreshService *metadata.PersonRefreshService var matchWorker *metadata.MatchWorker var libraryIngestExecutor *libraryingest.Executor @@ -1169,26 +1170,47 @@ func main() { if deps.S3Public != nil { imageCacher := imagecache.New(deps.S3Public) metadataService.SetImageCacher(imageCacher) + imageCacheJobs := metadata.NewImageCacheJobRepository(deps.DB) + metadataService.SetImageCacheJobEnqueuer(imageCacheJobs) + metadataImageCacheProcessor = metadata.NewImageCacheProcessorWithTargets( + imageCacheJobs, + imageCacher, + imageResolver, + metadata.ImageCacheProcessorTargets{ + Items: itemRepo, + Seasons: seasonRepo, + Episodes: episodeRepo, + ItemLocalizations: catalog.NewMediaItemLocalizationRepository(deps.DB), + SeasonLocalizations: catalog.NewSeasonLocalizationRepository(deps.DB), + People: personRepo, + }, + ) metadataService.SetAutoCacheImages(cfg.Metadata.CacheImages) + metadataImageCacheProcessor.SetEnabled(cfg.Metadata.CacheImages) configWatcher.OnChange(func(_, updated *config.Config) { metadataService.SetAutoCacheImages(updated.Metadata.CacheImages) + metadataImageCacheProcessor.SetEnabled(updated.Metadata.CacheImages) }) if deps.Scanner != nil { deps.Scanner.SetImageCacher(imageCacher) } if cfg.Metadata.CacheImages { personRefreshService.SetImageCacher(imageCacher) + personRefreshService.SetImageCacheJobEnqueuer(imageCacheJobs) slog.Info("metadata image caching enabled") } if audiobookEnricher != nil { audiobookEnricher.SetImageCacher(imageCacher) + audiobookEnricher.SetImageCacheJobEnqueuer(imageCacheJobs) audiobookEnricher.SetFFmpegPath(scanner.FFmpegPathFromFFprobe(scanner.FFprobePathFromFFmpeg(cfg.Playback.FFmpegPath))) } if ebookEnricher != nil { ebookEnricher.SetImageCacher(imageCacher) + ebookEnricher.SetImageCacheJobEnqueuer(imageCacheJobs) } if mangaEnricher != nil { mangaEnricher.SetImageCacher(imageCacher) + mangaEnricher.SetImageCacheJobEnqueuer(imageCacheJobs) } } @@ -1721,6 +1743,9 @@ func main() { if refreshWorker != nil && metadataService != nil { taskMgr.Register(tasks.NewRefreshMetadataTask(refreshWorker, metadataService)) } + if metadataImageCacheProcessor != nil { + taskMgr.Register(tasks.NewCacheMetadataImagesTask(metadataImageCacheProcessor)) + } if pluginAutoUpdater != nil { taskMgr.Register(tasks.NewCheckPluginUpdatesTask(pluginAutoUpdater)) } diff --git a/internal/audiobooks/enrichment.go b/internal/audiobooks/enrichment.go index 767e592b..628514c3 100644 --- a/internal/audiobooks/enrichment.go +++ b/internal/audiobooks/enrichment.go @@ -93,17 +93,18 @@ type enrichmentItemRow struct { // Enricher drives the audiobook metadata enrichment sweep. type Enricher struct { - pool *pgxpool.Pool - chainRepo *metadata.ChainRepository - resolver *metadata.PluginResolverAdapter - itemRepo *catalog.ItemRepository - personRepo *catalog.PersonRepository - providerIDs *catalog.ProviderIDRepository - imageCacher audiobookCoverCacher - workLinker literaryWorkLinker - ffmpegPath string - batchSize int - workers int + pool *pgxpool.Pool + chainRepo *metadata.ChainRepository + resolver *metadata.PluginResolverAdapter + itemRepo *catalog.ItemRepository + personRepo *catalog.PersonRepository + providerIDs *catalog.ProviderIDRepository + imageCacher audiobookCoverCacher + imageCacheJobs metadata.ImageCacheJobEnqueuer + workLinker literaryWorkLinker + ffmpegPath string + batchSize int + workers int } type literaryWorkLinker interface { @@ -144,6 +145,13 @@ func (e *Enricher) SetImageCacher(cacher audiobookCoverCacher) { e.imageCacher = cacher } +func (e *Enricher) SetImageCacheJobEnqueuer(enqueuer metadata.ImageCacheJobEnqueuer) { + if e == nil { + return + } + e.imageCacheJobs = enqueuer +} + func (e *Enricher) SetLiteraryWorkLinker(linker literaryWorkLinker) { if e == nil { return @@ -468,12 +476,11 @@ func (e *Enricher) enrichItem(ctx context.Context, item enrichmentItemRow) error return e.stampLastRefreshed(ctx, item.ContentID) } - e.cacheRemotePoster(ctx, item.ContentID, accumulator) - // Phase 3: Persist. if err := e.persist(ctx, item.ContentID, accumulatedIDs, accumulator); err != nil { return fmt.Errorf("persisting enrichment for %s: %w", item.ContentID, err) } + e.enqueueRemoteArtwork(ctx, item.ContentID, accumulator) e.autoLinkLiteraryWork(ctx, item.ContentID) slog.Info("audiobook enrichment: enriched", @@ -577,18 +584,27 @@ func (e *Enricher) persist(ctx context.Context, contentID string, providerIDs ma if result.PosterPath != "" { upd.PosterPath = &result.PosterPath + if isRemoteHTTPImage(result.PosterPath) { + upd.PosterSourcePath = &result.PosterPath + } } if result.PosterThumbhash != "" { upd.PosterThumbhash = &result.PosterThumbhash } if result.BackdropPath != "" { upd.BackdropPath = &result.BackdropPath + if isRemoteHTTPImage(result.BackdropPath) { + upd.BackdropSourcePath = &result.BackdropPath + } } if result.BackdropThumbhash != "" { upd.BackdropThumbhash = &result.BackdropThumbhash } if result.LogoPath != "" { upd.LogoPath = &result.LogoPath + if isRemoteHTTPImage(result.LogoPath) { + upd.LogoSourcePath = &result.LogoPath + } } if result.Overview != "" { upd.Overview = &result.Overview @@ -647,6 +663,47 @@ func (e *Enricher) persist(ctx context.Context, contentID string, providerIDs ma return nil } +func (e *Enricher) enqueueRemoteArtwork(ctx context.Context, contentID string, result *metadata.MetadataResult) { + if e == nil || e.imageCacheJobs == nil || result == nil || contentID == "" { + return + } + inputs := make([]metadata.EnqueueImageCacheJobInput, 0, 3) + add := func(sourcePath string, imageType metadata.ImageType) { + if !isRemoteHTTPImage(sourcePath) { + return + } + inputs = append(inputs, metadata.EnqueueImageCacheJobInput{ + TargetType: metadata.ImageCacheTargetItem, + TargetContentID: contentID, + SeriesID: contentID, + SourcePath: sourcePath, + ProviderID: audiobookMetadataImageProviderID, + ProviderContentID: contentID, + ContentType: "audiobooks", + ImageType: metadata.ImageTypeToString(imageType), + }) + } + add(result.PosterPath, metadata.ImagePoster) + add(result.BackdropPath, metadata.ImageBackdrop) + add(result.LogoPath, metadata.ImageLogo) + if len(inputs) == 0 { + return + } + enqueueCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if _, err := e.imageCacheJobs.EnqueueBatch(enqueueCtx, inputs); err != nil { + slog.Warn("audiobook enrichment: failed to enqueue image cache jobs", + "content_id", contentID, + "count", len(inputs), + "error", err, + ) + } +} + +func isRemoteHTTPImage(path string) bool { + return strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") +} + // updateMetadataAndTimestamps runs UpdateMetadata and also stamps // last_refreshed and matched_at so the item is skipped on future sweeps. func (e *Enricher) updateMetadataAndTimestamps(ctx context.Context, contentID string, upd *catalog.MetadataUpdate) error { diff --git a/internal/catalog/episode_catalog_source.go b/internal/catalog/episode_catalog_source.go index 758fcdc9..0cc313f9 100644 --- a/internal/catalog/episode_catalog_source.go +++ b/internal/catalog/episode_catalog_source.go @@ -32,8 +32,10 @@ const episodeCatalogSelectBody = `( ''::text AS poster_source_path, COALESCE(NULLIF(s.poster_thumbhash, ''), NULLIF(si.poster_thumbhash, ''), NULLIF(e.still_thumbhash, ''), '') AS poster_thumbhash, COALESCE(si.backdrop_path, '') AS backdrop_path, + COALESCE(si.backdrop_source_path, '') AS backdrop_source_path, COALESCE(si.backdrop_thumbhash, '') AS backdrop_thumbhash, COALESCE(si.logo_path, '') AS logo_path, + COALESCE(si.logo_source_path, '') AS logo_source_path, COALESCE(e.metadata_s3_path, '') AS metadata_s3_path, COALESCE(e.metadata_etag, '') AS metadata_etag, NULL::integer AS season_count, diff --git a/internal/catalog/episode_repo.go b/internal/catalog/episode_repo.go index 88658ca8..ac75c1fd 100644 --- a/internal/catalog/episode_repo.go +++ b/internal/catalog/episode_repo.go @@ -33,7 +33,7 @@ const episodeColumns = `content_id, series_id, season_id, season_number, episode title, default_metadata_language, overview, air_date, runtime, rating_imdb, rating_tmdb, imdb_id, tmdb_id, tvdb_id, - still_path, still_thumbhash, + still_path, still_source_path, still_thumbhash, metadata_s3_path, metadata_etag, metadata_source, created_at, updated_at` @@ -113,6 +113,7 @@ func scanEpisode(row pgx.Row) (*models.Episode, error) { var tmdbID *string var tvdbID *string var stillPath *string + var stillSourcePath *string var stillThumbhash *string var metadataS3Path *string var metadataEtag *string @@ -133,6 +134,7 @@ func scanEpisode(row pgx.Row) (*models.Episode, error) { &tmdbID, &tvdbID, &stillPath, + &stillSourcePath, &stillThumbhash, &metadataS3Path, &metadataEtag, @@ -161,6 +163,9 @@ func scanEpisode(row pgx.Row) (*models.Episode, error) { if stillPath != nil { ep.StillPath = *stillPath } + if stillSourcePath != nil { + ep.StillSourcePath = *stillSourcePath + } if stillThumbhash != nil { ep.StillThumbhash = *stillThumbhash } @@ -191,6 +196,7 @@ func scanEpisodes(rows pgx.Rows) ([]*models.Episode, error) { var tmdbID *string var tvdbID *string var stillPath *string + var stillSourcePath *string var stillThumbhash *string var metadataS3Path *string var metadataEtag *string @@ -211,6 +217,7 @@ func scanEpisodes(rows pgx.Rows) ([]*models.Episode, error) { &tmdbID, &tvdbID, &stillPath, + &stillSourcePath, &stillThumbhash, &metadataS3Path, &metadataEtag, @@ -239,6 +246,9 @@ func scanEpisodes(rows pgx.Rows) ([]*models.Episode, error) { if stillPath != nil { ep.StillPath = *stillPath } + if stillSourcePath != nil { + ep.StillSourcePath = *stillSourcePath + } if stillThumbhash != nil { ep.StillThumbhash = *stillThumbhash } @@ -297,15 +307,15 @@ func (r *EpisodeRepository) Upsert(ctx context.Context, ep *models.Episode) erro title, default_metadata_language, overview, air_date, runtime, rating_imdb, rating_tmdb, imdb_id, tmdb_id, tvdb_id, - still_path, still_thumbhash, + still_path, still_source_path, still_thumbhash, metadata_s3_path, metadata_etag, metadata_source ) VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, - $16, $17, - $18, $19, $20 + $16, $17, $18, + $19, $20, $21 ) ON CONFLICT (series_id, season_number, episode_number) DO UPDATE SET season_id = COALESCE(EXCLUDED.season_id, episodes.season_id), @@ -320,6 +330,7 @@ func (r *EpisodeRepository) Upsert(ctx context.Context, ep *models.Episode) erro tmdb_id = COALESCE(NULLIF(EXCLUDED.tmdb_id, ''), episodes.tmdb_id), tvdb_id = COALESCE(NULLIF(EXCLUDED.tvdb_id, ''), episodes.tvdb_id), still_path = EXCLUDED.still_path, + still_source_path = EXCLUDED.still_source_path, still_thumbhash = EXCLUDED.still_thumbhash, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, @@ -345,6 +356,7 @@ func (r *EpisodeRepository) Upsert(ctx context.Context, ep *models.Episode) erro ep.TmdbID, ep.TvdbID, ep.StillPath, + ep.StillSourcePath, ep.StillThumbhash, ep.MetadataS3Path, ep.MetadataEtag, @@ -439,6 +451,7 @@ func (r *EpisodeRepository) BulkUpsert(ctx context.Context, seriesID string, epi epTmdbIDs := make([]string, len(episodes)) epTvdbIDs := make([]string, len(episodes)) stillPaths := make([]string, len(episodes)) + stillSourcePaths := make([]string, len(episodes)) stillThumbs := make([]string, len(episodes)) metaS3Paths := make([]string, len(episodes)) metaEtags := make([]string, len(episodes)) @@ -463,6 +476,7 @@ func (r *EpisodeRepository) BulkUpsert(ctx context.Context, seriesID string, epi epTmdbIDs[i] = ep.TmdbID epTvdbIDs[i] = ep.TvdbID stillPaths[i] = ep.StillPath + stillSourcePaths[i] = ep.StillSourcePath stillThumbs[i] = ep.StillThumbhash metaS3Paths[i] = ep.MetadataS3Path metaEtags[i] = ep.MetadataEtag @@ -475,7 +489,7 @@ func (r *EpisodeRepository) BulkUpsert(ctx context.Context, seriesID string, epi title, default_metadata_language, overview, air_date, runtime, rating_imdb, rating_tmdb, imdb_id, tmdb_id, tvdb_id, - still_path, still_thumbhash, + still_path, still_source_path, still_thumbhash, metadata_s3_path, metadata_etag, metadata_source ) SELECT * FROM UNNEST( @@ -483,8 +497,8 @@ func (r *EpisodeRepository) BulkUpsert(ctx context.Context, seriesID string, epi $6::text[], $7::text[], $8::text[], $9::date[], $10::int[], $11::float8[], $12::float8[], $13::text[], $14::text[], $15::text[], - $16::text[], $17::text[], - $18::text[], $19::text[], $20::text[] + $16::text[], $17::text[], $18::text[], + $19::text[], $20::text[], $21::text[] ) ON CONFLICT (series_id, season_number, episode_number) DO UPDATE SET season_id = COALESCE(EXCLUDED.season_id, episodes.season_id), @@ -499,6 +513,7 @@ func (r *EpisodeRepository) BulkUpsert(ctx context.Context, seriesID string, epi tmdb_id = COALESCE(NULLIF(EXCLUDED.tmdb_id, ''), episodes.tmdb_id), tvdb_id = COALESCE(NULLIF(EXCLUDED.tvdb_id, ''), episodes.tvdb_id), still_path = EXCLUDED.still_path, + still_source_path = EXCLUDED.still_source_path, still_thumbhash = EXCLUDED.still_thumbhash, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, @@ -511,7 +526,7 @@ func (r *EpisodeRepository) BulkUpsert(ctx context.Context, seriesID string, epi titles, defaultMetadataLanguages, overviews, airDates, runtimes, ratingsIMDB, ratingsTMDB, epImdbIDs, epTmdbIDs, epTvdbIDs, - stillPaths, stillThumbs, + stillPaths, stillSourcePaths, stillThumbs, metaS3Paths, metaEtags, metaSources, ) if err != nil { @@ -900,6 +915,10 @@ func (r *EpisodeRepository) UpdateMetadata(ctx context.Context, contentID string addString("tmdb_id", upd.TmdbID) addString("tvdb_id", upd.TvdbID) addString("still_path", upd.StillPath) + if upd.StillPath != nil && upd.StillSourcePath == nil { + setClauses = append(setClauses, "still_source_path = ''") + } + addString("still_source_path", upd.StillSourcePath) addString("still_thumbhash", upd.StillThumbhash) setClauses = append(setClauses, "updated_at = NOW()") @@ -917,3 +936,19 @@ func (r *EpisodeRepository) UpdateMetadata(ctx context.Context, contentID string } return nil } + +func (r *EpisodeRepository) UpdateStillIfSourceMatches(ctx context.Context, contentID, sourcePath, cachedPath, thumbhash string) (bool, error) { + tag, err := r.pool.Exec(ctx, ` + UPDATE episodes + SET still_path = $3, + still_source_path = $2, + still_thumbhash = $4, + updated_at = NOW() + WHERE content_id = $1 + AND still_source_path = $2 + `, contentID, sourcePath, cachedPath, thumbhash) + if err != nil { + return false, fmt.Errorf("updating episode cached still: %w", err) + } + return tag.RowsAffected() > 0, nil +} diff --git a/internal/catalog/item_repo.go b/internal/catalog/item_repo.go index 095352ae..9b9d5ee5 100644 --- a/internal/catalog/item_repo.go +++ b/internal/catalog/item_repo.go @@ -117,7 +117,7 @@ var itemColumnNames = []string{ "content_rating", "runtime", "overview", "tagline", "rating_imdb", "rating_tmdb", "rating_rt_critic", "rating_rt_audience", "imdb_id", "tmdb_id", "tvdb_id", - "poster_path", "poster_source_path", "poster_thumbhash", "backdrop_path", "backdrop_thumbhash", "logo_path", + "poster_path", "poster_source_path", "poster_thumbhash", "backdrop_path", "backdrop_source_path", "backdrop_thumbhash", "logo_path", "logo_source_path", "metadata_s3_path", "metadata_etag", "season_count", "studios", "networks", "countries", "keywords", "original_language", "release_date::text", "first_air_date", "last_air_date", "air_time", "air_timezone", "show_status", @@ -129,14 +129,16 @@ var itemColumnNames = []string{ // scan into plain (non-pointer) string fields on models.MediaItem, so select // lists coalesce them to ”. var nullableStringItemColumns = map[string]bool{ - "poster_path": true, - "poster_source_path": true, - "poster_thumbhash": true, - "backdrop_path": true, - "backdrop_thumbhash": true, - "logo_path": true, - "metadata_s3_path": true, - "metadata_etag": true, + "poster_path": true, + "poster_source_path": true, + "poster_thumbhash": true, + "backdrop_path": true, + "backdrop_source_path": true, + "backdrop_thumbhash": true, + "logo_path": true, + "logo_source_path": true, + "metadata_s3_path": true, + "metadata_etag": true, } // itemColumnExpr renders one select-list entry for col, qualified with alias @@ -219,8 +221,10 @@ func scanItem(row pgx.Row) (*models.MediaItem, error) { &item.PosterSourcePath, &item.PosterThumbhash, &item.BackdropPath, + &item.BackdropSourcePath, &item.BackdropThumbhash, &item.LogoPath, + &item.LogoSourcePath, &item.MetadataS3Path, &item.MetadataEtag, &item.SeasonCount, @@ -283,8 +287,10 @@ func listItemScanDests(item *models.MediaItem) []any { &item.PosterSourcePath, &item.PosterThumbhash, &item.BackdropPath, + &item.BackdropSourcePath, &item.BackdropThumbhash, &item.LogoPath, + &item.LogoSourcePath, &item.MetadataS3Path, &item.MetadataEtag, &item.SeasonCount, @@ -401,7 +407,7 @@ func (r *ItemRepository) upsert(ctx context.Context, execer itemExecer, item *mo content_rating, runtime, overview, tagline, rating_imdb, rating_tmdb, rating_rt_critic, rating_rt_audience, imdb_id, tmdb_id, tvdb_id, - poster_path, poster_source_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, + poster_path, poster_source_path, poster_thumbhash, backdrop_path, backdrop_source_path, backdrop_thumbhash, logo_path, logo_source_path, metadata_s3_path, metadata_etag, season_count, studios, networks, countries, keywords, original_language, release_date, first_air_date, last_air_date, air_time, air_timezone, show_status, @@ -412,12 +418,12 @@ func (r *ItemRepository) upsert(ctx context.Context, execer itemExecer, item *mo $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, - $20, $21, $22, $23, $24, $25, - $26, $27, $28, - $29, $30, $31, $32, $33, $34, $35, $36, $37, $38, - $39, - $40, $41, $42, - $43, $44, $45 + $20, $21, $22, $23, $24, $25, $26, $27, + $28, $29, $30, + $31, $32, $33, $34, $35, $36, $37, $38, $39, $40, + $41, + $42, $43, $44, + $45, $46, $47 ) ON CONFLICT (content_id) DO UPDATE SET type = EXCLUDED.type, @@ -442,8 +448,10 @@ func (r *ItemRepository) upsert(ctx context.Context, execer itemExecer, item *mo poster_source_path = EXCLUDED.poster_source_path, poster_thumbhash = EXCLUDED.poster_thumbhash, backdrop_path = EXCLUDED.backdrop_path, + backdrop_source_path = EXCLUDED.backdrop_source_path, backdrop_thumbhash = EXCLUDED.backdrop_thumbhash, logo_path = EXCLUDED.logo_path, + logo_source_path = EXCLUDED.logo_source_path, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, season_count = EXCLUDED.season_count, @@ -490,8 +498,10 @@ func (r *ItemRepository) upsert(ctx context.Context, execer itemExecer, item *mo item.PosterSourcePath, item.PosterThumbhash, item.BackdropPath, + item.BackdropSourcePath, item.BackdropThumbhash, item.LogoPath, + item.LogoSourcePath, item.MetadataS3Path, item.MetadataEtag, item.SeasonCount, @@ -1335,7 +1345,7 @@ func (r *ItemRepository) ReplacePeople(ctx context.Context, contentID string, pe func (r *ItemRepository) GetPeople(ctx context.Context, contentID string) ([]models.ItemPerson, error) { rows, err := r.pool.Query(ctx, ` SELECT p.id, p.name, p.sort_name, p.bio, p.birth_date, p.death_date, p.birthplace, p.homepage, - p.photo_path, p.photo_thumbhash, p.tmdb_id, p.imdb_id, p.tvdb_id, p.plex_guid, + p.photo_path, p.photo_source_path, p.photo_thumbhash, p.tmdb_id, p.imdb_id, p.tvdb_id, p.plex_guid, p.created_at, p.updated_at, ip.kind, ip.character, ip.sort_order FROM item_people ip @@ -1432,16 +1442,25 @@ func (r *ItemRepository) UpdateMetadata(ctx context.Context, contentID string, u addString("tvdb_id", upd.TvdbID) addIntArray("locked_fields", upd.LockedFields) addString("poster_path", upd.PosterPath) - if upd.PosterPath != nil { + if upd.PosterPath != nil && upd.PosterSourcePath == nil { // An explicit poster override invalidates the provider-origin source // path captured by image caching; outbound embeds must not keep // rendering the replaced provider artwork. - setClauses = append(setClauses, "poster_source_path = NULL") + setClauses = append(setClauses, "poster_source_path = ''") } + addString("poster_source_path", upd.PosterSourcePath) addString("poster_thumbhash", upd.PosterThumbhash) addString("backdrop_path", upd.BackdropPath) + if upd.BackdropPath != nil && upd.BackdropSourcePath == nil { + setClauses = append(setClauses, "backdrop_source_path = ''") + } + addString("backdrop_source_path", upd.BackdropSourcePath) addString("backdrop_thumbhash", upd.BackdropThumbhash) addString("logo_path", upd.LogoPath) + if upd.LogoPath != nil && upd.LogoSourcePath == nil { + setClauses = append(setClauses, "logo_source_path = ''") + } + addString("logo_source_path", upd.LogoSourcePath) setClauses = append(setClauses, "updated_at = NOW()") @@ -1459,6 +1478,54 @@ func (r *ItemRepository) UpdateMetadata(ctx context.Context, contentID string, u return nil } +func (r *ItemRepository) UpdateArtworkIfSourceMatches(ctx context.Context, contentID, imageType, sourcePath, cachedPath, thumbhash string) (bool, error) { + if r == nil || r.pool == nil { + return false, ErrItemNotFound + } + + var query string + var args []any + switch imageType { + case "poster": + query = ` + UPDATE media_items + SET poster_path = $3, + poster_source_path = $2, + poster_thumbhash = NULLIF($4, ''), + updated_at = NOW() + WHERE content_id = $1 + AND poster_source_path = $2` + args = []any{contentID, sourcePath, cachedPath, thumbhash} + case "backdrop": + query = ` + UPDATE media_items + SET backdrop_path = $3, + backdrop_source_path = $2, + backdrop_thumbhash = NULLIF($4, ''), + updated_at = NOW() + WHERE content_id = $1 + AND backdrop_source_path = $2` + args = []any{contentID, sourcePath, cachedPath, thumbhash} + case "logo": + query = ` + UPDATE media_items + SET logo_path = $3, + logo_source_path = $2, + updated_at = NOW() + WHERE content_id = $1 + AND logo_source_path = $2` + args = []any{contentID, sourcePath, cachedPath} + default: + return false, fmt.Errorf("unsupported media item artwork type %q", imageType) + } + + tag, err := r.pool.Exec(ctx, query, args...) + if err != nil { + return false, fmt.Errorf("updating media item cached artwork: %w", err) + } + return tag.RowsAffected() > 0, nil +} + // IncrementRefreshFailure records a failed metadata refresh attempt for an // existing media item. func (r *ItemRepository) IncrementRefreshFailure(ctx context.Context, contentID string) error { diff --git a/internal/catalog/localization_repo.go b/internal/catalog/localization_repo.go index 9e5a3328..cd4e57a3 100644 --- a/internal/catalog/localization_repo.go +++ b/internal/catalog/localization_repo.go @@ -38,10 +38,12 @@ func (r *MediaItemLocalizationRepository) Upsert(ctx context.Context, loc *model _, err := r.pool.Exec(ctx, ` INSERT INTO media_item_localizations ( content_id, language, title, sort_title, overview, tagline, - poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path + poster_path, poster_source_path, poster_thumbhash, + backdrop_path, backdrop_source_path, backdrop_thumbhash, + logo_path, logo_source_path ) VALUES ( $1, $2, $3, $4, $5, $6, - $7, $8, $9, $10, $11 + $7, $8, $9, $10, $11, $12, $13, $14 ) ON CONFLICT (content_id, language) DO UPDATE SET title = EXCLUDED.title, @@ -63,13 +65,18 @@ func (r *MediaItemLocalizationRepository) Upsert(ctx context.Context, loc *model THEN media_item_localizations.tagline_source ELSE 'provider' END, poster_path = EXCLUDED.poster_path, + poster_source_path = EXCLUDED.poster_source_path, poster_thumbhash = EXCLUDED.poster_thumbhash, backdrop_path = EXCLUDED.backdrop_path, + backdrop_source_path = EXCLUDED.backdrop_source_path, backdrop_thumbhash = EXCLUDED.backdrop_thumbhash, logo_path = EXCLUDED.logo_path, + logo_source_path = EXCLUDED.logo_source_path, updated_at = NOW() `, loc.ContentID, loc.Language, loc.Title, loc.SortTitle, loc.Overview, loc.Tagline, - loc.PosterPath, loc.PosterThumbhash, loc.BackdropPath, loc.BackdropThumbhash, loc.LogoPath) + loc.PosterPath, loc.PosterSourcePath, loc.PosterThumbhash, + loc.BackdropPath, loc.BackdropSourcePath, loc.BackdropThumbhash, + loc.LogoPath, loc.LogoSourcePath) if err != nil { return fmt.Errorf("upserting media item localization: %w", err) } @@ -93,11 +100,13 @@ func (r *MediaItemLocalizationRepository) UpsertAITranslation(ctx context.Contex _, err := r.pool.Exec(ctx, ` INSERT INTO media_item_localizations ( content_id, language, title, sort_title, overview, tagline, - poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, + poster_path, poster_source_path, poster_thumbhash, + backdrop_path, backdrop_source_path, backdrop_thumbhash, + logo_path, logo_source_path, overview_source, tagline_source ) VALUES ( $1, $2, '', '', COALESCE($3, ''), COALESCE($4, ''), - '', '', '', '', '', + '', '', '', '', '', '', '', '', CASE WHEN $3::text IS NULL THEN 'provider' ELSE 'ai' END, CASE WHEN $4::text IS NULL THEN 'provider' ELSE 'ai' END ) @@ -137,7 +146,9 @@ func (r *MediaItemLocalizationRepository) UpsertAITranslation(ctx context.Contex func (r *MediaItemLocalizationRepository) Get(ctx context.Context, contentID, language string) (*models.MediaItemLocalization, error) { row := r.pool.QueryRow(ctx, ` SELECT content_id, language, title, sort_title, overview, tagline, - poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, + poster_path, poster_source_path, poster_thumbhash, + backdrop_path, backdrop_source_path, backdrop_thumbhash, + logo_path, logo_source_path, overview_source, tagline_source, created_at, updated_at FROM media_item_localizations @@ -153,7 +164,9 @@ func (r *MediaItemLocalizationRepository) GetByContentIDs(ctx context.Context, c } rows, err := r.pool.Query(ctx, ` SELECT content_id, language, title, sort_title, overview, tagline, - poster_path, poster_thumbhash, backdrop_path, backdrop_thumbhash, logo_path, + poster_path, poster_source_path, poster_thumbhash, + backdrop_path, backdrop_source_path, backdrop_thumbhash, + logo_path, logo_source_path, overview_source, tagline_source, created_at, updated_at FROM media_item_localizations @@ -176,6 +189,57 @@ func (r *MediaItemLocalizationRepository) GetByContentIDs(ctx context.Context, c return result, nil } +func (r *MediaItemLocalizationRepository) UpdateArtworkIfSourceMatches(ctx context.Context, contentID, language, imageType, sourcePath, cachedPath, thumbhash string) (bool, error) { + if r == nil || r.pool == nil { + return false, nil + } + + var query string + var args []any + switch imageType { + case "poster": + query = ` + UPDATE media_item_localizations + SET poster_path = $4, + poster_source_path = $3, + poster_thumbhash = NULLIF($5, ''), + updated_at = NOW() + WHERE content_id = $1 + AND language = $2 + AND poster_source_path = $3` + args = []any{contentID, language, sourcePath, cachedPath, thumbhash} + case "backdrop": + query = ` + UPDATE media_item_localizations + SET backdrop_path = $4, + backdrop_source_path = $3, + backdrop_thumbhash = NULLIF($5, ''), + updated_at = NOW() + WHERE content_id = $1 + AND language = $2 + AND backdrop_source_path = $3` + args = []any{contentID, language, sourcePath, cachedPath, thumbhash} + case "logo": + query = ` + UPDATE media_item_localizations + SET logo_path = $4, + logo_source_path = $3, + updated_at = NOW() + WHERE content_id = $1 + AND language = $2 + AND logo_source_path = $3` + args = []any{contentID, language, sourcePath, cachedPath} + default: + return false, fmt.Errorf("unsupported localized media item artwork type %q", imageType) + } + + tag, err := r.pool.Exec(ctx, query, args...) + if err != nil { + return false, fmt.Errorf("updating localized media item cached artwork: %w", err) + } + return tag.RowsAffected() > 0, nil +} + type SeasonLocalizationRepository struct { pool *pgxpool.Pool } @@ -192,8 +256,8 @@ func (r *SeasonLocalizationRepository) Upsert(ctx context.Context, loc *models.S } _, err := r.pool.Exec(ctx, ` INSERT INTO season_localizations ( - season_content_id, language, title, overview, poster_path, poster_thumbhash - ) VALUES ($1, $2, $3, $4, $5, $6) + season_content_id, language, title, overview, poster_path, poster_source_path, poster_thumbhash + ) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (season_content_id, language) DO UPDATE SET title = EXCLUDED.title, overview = CASE @@ -205,9 +269,10 @@ func (r *SeasonLocalizationRepository) Upsert(ctx context.Context, loc *models.S THEN season_localizations.overview_source ELSE 'provider' END, poster_path = EXCLUDED.poster_path, + poster_source_path = EXCLUDED.poster_source_path, poster_thumbhash = EXCLUDED.poster_thumbhash, updated_at = NOW() - `, loc.SeasonContentID, loc.Language, loc.Title, loc.Overview, loc.PosterPath, loc.PosterThumbhash) + `, loc.SeasonContentID, loc.Language, loc.Title, loc.Overview, loc.PosterPath, loc.PosterSourcePath, loc.PosterThumbhash) if err != nil { return fmt.Errorf("upserting season localization: %w", err) } @@ -222,8 +287,8 @@ func (r *SeasonLocalizationRepository) UpsertAIOverview(ctx context.Context, sea } _, err := r.pool.Exec(ctx, ` INSERT INTO season_localizations ( - season_content_id, language, title, overview, poster_path, poster_thumbhash, overview_source - ) VALUES ($1, $2, '', $3, '', '', 'ai') + season_content_id, language, title, overview, poster_path, poster_source_path, poster_thumbhash, overview_source + ) VALUES ($1, $2, '', $3, '', '', '', 'ai') ON CONFLICT (season_content_id, language) DO UPDATE SET overview = CASE WHEN season_localizations.overview_source = 'manual' @@ -247,7 +312,7 @@ func (r *SeasonLocalizationRepository) UpsertAIOverview(ctx context.Context, sea func (r *SeasonLocalizationRepository) Get(ctx context.Context, seasonContentID, language string) (*models.SeasonLocalization, error) { row := r.pool.QueryRow(ctx, ` - SELECT season_content_id, language, title, overview, poster_path, poster_thumbhash, + SELECT season_content_id, language, title, overview, poster_path, poster_source_path, poster_thumbhash, overview_source, created_at, updated_at FROM season_localizations WHERE season_content_id = $1 AND language = $2 @@ -261,7 +326,7 @@ func (r *SeasonLocalizationRepository) GetBySeasonIDs(ctx context.Context, seaso return result, nil } rows, err := r.pool.Query(ctx, ` - SELECT season_content_id, language, title, overview, poster_path, poster_thumbhash, + SELECT season_content_id, language, title, overview, poster_path, poster_source_path, poster_thumbhash, overview_source, created_at, updated_at FROM season_localizations WHERE language = $1 AND season_content_id = ANY($2) @@ -283,6 +348,26 @@ func (r *SeasonLocalizationRepository) GetBySeasonIDs(ctx context.Context, seaso return result, nil } +func (r *SeasonLocalizationRepository) UpdateArtworkIfSourceMatches(ctx context.Context, seasonContentID, language, sourcePath, cachedPath, thumbhash string) (bool, error) { + if r == nil || r.pool == nil { + return false, nil + } + tag, err := r.pool.Exec(ctx, ` + UPDATE season_localizations + SET poster_path = $4, + poster_source_path = $3, + poster_thumbhash = NULLIF($5, ''), + updated_at = NOW() + WHERE season_content_id = $1 + AND language = $2 + AND poster_source_path = $3 + `, seasonContentID, language, sourcePath, cachedPath, thumbhash) + if err != nil { + return false, fmt.Errorf("updating localized season cached artwork: %w", err) + } + return tag.RowsAffected() > 0, nil +} + type EpisodeLocalizationRepository struct { pool *pgxpool.Pool } @@ -394,10 +479,13 @@ func scanMediaItemLocalization(row pgx.Row) (*models.MediaItemLocalization, erro &loc.Overview, &loc.Tagline, &loc.PosterPath, + &loc.PosterSourcePath, &loc.PosterThumbhash, &loc.BackdropPath, + &loc.BackdropSourcePath, &loc.BackdropThumbhash, &loc.LogoPath, + &loc.LogoSourcePath, &loc.OverviewSource, &loc.TaglineSource, &loc.CreatedAt, @@ -419,6 +507,7 @@ func scanSeasonLocalization(row pgx.Row) (*models.SeasonLocalization, error) { &loc.Title, &loc.Overview, &loc.PosterPath, + &loc.PosterSourcePath, &loc.PosterThumbhash, &loc.OverviewSource, &loc.CreatedAt, diff --git a/internal/catalog/localize.go b/internal/catalog/localize.go index ad04babe..9a5c4380 100644 --- a/internal/catalog/localize.go +++ b/internal/catalog/localize.go @@ -26,14 +26,17 @@ func applyItemLocalization(item *models.MediaItem, loc *models.MediaItemLocaliza } if loc.PosterPath != "" { localized.PosterPath = loc.PosterPath + localized.PosterSourcePath = loc.PosterSourcePath localized.PosterThumbhash = loc.PosterThumbhash } if loc.BackdropPath != "" { localized.BackdropPath = loc.BackdropPath + localized.BackdropSourcePath = loc.BackdropSourcePath localized.BackdropThumbhash = loc.BackdropThumbhash } if loc.LogoPath != "" { localized.LogoPath = loc.LogoPath + localized.LogoSourcePath = loc.LogoSourcePath } return localized } @@ -53,6 +56,7 @@ func applySeasonLocalization(season *models.Season, loc *models.SeasonLocalizati } if loc.PosterPath != "" { localized.PosterPath = loc.PosterPath + localized.PosterSourcePath = loc.PosterSourcePath localized.PosterThumbhash = loc.PosterThumbhash } return localized diff --git a/internal/catalog/person_repo.go b/internal/catalog/person_repo.go index 4b5b2c1e..e1cea749 100644 --- a/internal/catalog/person_repo.go +++ b/internal/catalog/person_repo.go @@ -72,10 +72,10 @@ func (r *PersonRepository) FindOrCreate(ctx context.Context, p models.Person) (i _, err = r.pool.Exec(ctx, ` INSERT INTO people (id, name, sort_name, bio, birth_date, death_date, birthplace, homepage, - photo_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`, + photo_path, photo_source_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15)`, id, p.Name, p.SortName, p.Bio, p.BirthDate, p.DeathDate, p.Birthplace, p.Homepage, - p.PhotoPath, p.PhotoThumbhash, p.TmdbID, p.ImdbID, p.TvdbID, p.PlexGUID, + p.PhotoPath, p.PhotoSourcePath, p.PhotoThumbhash, p.TmdbID, p.ImdbID, p.TvdbID, p.PlexGUID, ) if err != nil { return 0, fmt.Errorf("insert person: %w", err) @@ -119,6 +119,7 @@ func (r *PersonRepository) enrichExisting(ctx context.Context, id int64, p model fillEmpty("tvdb_id", p.TvdbID) fillEmpty("plex_guid", p.PlexGUID) overwriteIfReal("photo_path", p.PhotoPath) + overwriteIfReal("photo_source_path", p.PhotoSourcePath) overwriteIfReal("photo_thumbhash", p.PhotoThumbhash) fillEmpty("bio", p.Bio) fillEmpty("birthplace", p.Birthplace) @@ -288,6 +289,7 @@ func (r *PersonRepository) BatchFindOrCreate(ctx context.Context, people []model eTvdbIDs := make([]string, len(toEnrich)) ePlexGUIDs := make([]string, len(toEnrich)) ePhotoPaths := make([]string, len(toEnrich)) + ePhotoSourcePaths := make([]string, len(toEnrich)) ePhotoThumbs := make([]string, len(toEnrich)) eBios := make([]string, len(toEnrich)) eBirthplaces := make([]string, len(toEnrich)) @@ -299,6 +301,7 @@ func (r *PersonRepository) BatchFindOrCreate(ctx context.Context, people []model eTvdbIDs[i] = e.person.TvdbID ePlexGUIDs[i] = e.person.PlexGUID ePhotoPaths[i] = e.person.PhotoPath + ePhotoSourcePaths[i] = e.person.PhotoSourcePath ePhotoThumbs[i] = e.person.PhotoThumbhash eBios[i] = e.person.Bio eBirthplaces[i] = e.person.Birthplace @@ -315,6 +318,11 @@ func (r *PersonRepository) BatchFindOrCreate(ctx context.Context, people []model WHEN people.photo_path = '' THEN t.photo_path ELSE people.photo_path END, + photo_source_path = CASE + WHEN t.photo_source_path NOT IN ('', '-') THEN t.photo_source_path + WHEN people.photo_source_path = '' THEN t.photo_source_path + ELSE people.photo_source_path + END, photo_thumbhash = CASE WHEN t.photo_thumbhash NOT IN ('', '-') THEN t.photo_thumbhash WHEN people.photo_thumbhash = '' THEN t.photo_thumbhash @@ -325,12 +333,12 @@ func (r *PersonRepository) BatchFindOrCreate(ctx context.Context, people []model homepage = CASE WHEN people.homepage = '' AND t.homepage <> '' THEN t.homepage ELSE people.homepage END, updated_at = NOW() FROM UNNEST($1::bigint[], $2::text[], $3::text[], $4::text[], $5::text[], - $6::text[], $7::text[], $8::text[], $9::text[], $10::text[]) + $6::text[], $7::text[], $8::text[], $9::text[], $10::text[], $11::text[]) AS t(id, tmdb_id, imdb_id, tvdb_id, plex_guid, - photo_path, photo_thumbhash, bio, birthplace, homepage) + photo_path, photo_source_path, photo_thumbhash, bio, birthplace, homepage) WHERE people.id = t.id`, enrichIDs, eTmdbIDs, eImdbIDs, eTvdbIDs, ePlexGUIDs, - ePhotoPaths, ePhotoThumbs, eBios, eBirthplaces, eHomepages, + ePhotoPaths, ePhotoSourcePaths, ePhotoThumbs, eBios, eBirthplaces, eHomepages, ) if err != nil { return nil, fmt.Errorf("batch enrich people: %w", err) @@ -354,6 +362,7 @@ func (r *PersonRepository) BatchFindOrCreate(ctx context.Context, people []model birthplaces := make([]string, len(newIndices)) homepages := make([]string, len(newIndices)) photoPaths := make([]string, len(newIndices)) + photoSourcePaths := make([]string, len(newIndices)) photoThumbs := make([]string, len(newIndices)) nTmdbIDs := make([]string, len(newIndices)) nImdbIDs := make([]string, len(newIndices)) @@ -379,6 +388,7 @@ func (r *PersonRepository) BatchFindOrCreate(ctx context.Context, people []model birthplaces[j] = p.Birthplace homepages[j] = p.Homepage photoPaths[j] = p.PhotoPath + photoSourcePaths[j] = p.PhotoSourcePath photoThumbs[j] = p.PhotoThumbhash nTmdbIDs[j] = p.TmdbID nImdbIDs[j] = p.ImdbID @@ -388,16 +398,16 @@ func (r *PersonRepository) BatchFindOrCreate(ctx context.Context, people []model rows, err := r.pool.Query(ctx, ` INSERT INTO people (id, name, sort_name, bio, birth_date, death_date, birthplace, homepage, - photo_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid) + photo_path, photo_source_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid) SELECT * FROM UNNEST( $1::bigint[], $2::text[], $3::text[], $4::text[], $5::date[], $6::date[], $7::text[], $8::text[], $9::text[], $10::text[], $11::text[], $12::text[], - $13::text[], $14::text[] + $13::text[], $14::text[], $15::text[] ) ON CONFLICT DO NOTHING RETURNING id`, newIDs, names, sortNames, bios, birthDates, deathDates, - birthplaces, homepages, photoPaths, photoThumbs, nTmdbIDs, nImdbIDs, + birthplaces, homepages, photoPaths, photoSourcePaths, photoThumbs, nTmdbIDs, nImdbIDs, nTvdbIDs, nPlexGUIDs, ) if err != nil { @@ -455,10 +465,10 @@ func (r *PersonRepository) Get(ctx context.Context, id int64) (*models.Person, e var p models.Person err := r.pool.QueryRow(ctx, ` SELECT id, name, sort_name, bio, birth_date, death_date, birthplace, homepage, - photo_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid, created_at, updated_at + photo_path, photo_source_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid, created_at, updated_at FROM people WHERE id = $1`, id, ).Scan(&p.ID, &p.Name, &p.SortName, &p.Bio, &p.BirthDate, &p.DeathDate, &p.Birthplace, &p.Homepage, - &p.PhotoPath, &p.PhotoThumbhash, &p.TmdbID, &p.ImdbID, &p.TvdbID, &p.PlexGUID, &p.CreatedAt, &p.UpdatedAt, + &p.PhotoPath, &p.PhotoSourcePath, &p.PhotoThumbhash, &p.TmdbID, &p.ImdbID, &p.TvdbID, &p.PlexGUID, &p.CreatedAt, &p.UpdatedAt, ) if err != nil { return nil, fmt.Errorf("get person %d: %w", id, err) @@ -471,10 +481,10 @@ func (r *PersonRepository) GetByName(ctx context.Context, name string) (*models. var p models.Person err := r.pool.QueryRow(ctx, ` SELECT id, name, sort_name, bio, birth_date, death_date, birthplace, homepage, - photo_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid, created_at, updated_at + photo_path, photo_source_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid, created_at, updated_at FROM people WHERE LOWER(name) = LOWER($1)`, name, ).Scan(&p.ID, &p.Name, &p.SortName, &p.Bio, &p.BirthDate, &p.DeathDate, &p.Birthplace, &p.Homepage, - &p.PhotoPath, &p.PhotoThumbhash, &p.TmdbID, &p.ImdbID, &p.TvdbID, &p.PlexGUID, &p.CreatedAt, &p.UpdatedAt, + &p.PhotoPath, &p.PhotoSourcePath, &p.PhotoThumbhash, &p.TmdbID, &p.ImdbID, &p.TvdbID, &p.PlexGUID, &p.CreatedAt, &p.UpdatedAt, ) if err != nil { return nil, fmt.Errorf("get person by name %q: %w", name, err) @@ -489,7 +499,7 @@ func (r *PersonRepository) Search(ctx context.Context, query string, limit int) } rows, err := r.pool.Query(ctx, ` SELECT id, name, sort_name, bio, birth_date, death_date, birthplace, homepage, - photo_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid, created_at, updated_at + photo_path, photo_source_path, photo_thumbhash, tmdb_id, imdb_id, tvdb_id, plex_guid, created_at, updated_at FROM people WHERE LOWER(name) LIKE '%' || LOWER($1) || '%' ORDER BY name LIMIT $2`, query, limit, ) @@ -502,7 +512,7 @@ func (r *PersonRepository) Search(ctx context.Context, query string, limit int) for rows.Next() { var p models.Person if err := rows.Scan(&p.ID, &p.Name, &p.SortName, &p.Bio, &p.BirthDate, &p.DeathDate, &p.Birthplace, &p.Homepage, - &p.PhotoPath, &p.PhotoThumbhash, &p.TmdbID, &p.ImdbID, &p.TvdbID, &p.PlexGUID, &p.CreatedAt, &p.UpdatedAt, + &p.PhotoPath, &p.PhotoSourcePath, &p.PhotoThumbhash, &p.TmdbID, &p.ImdbID, &p.TvdbID, &p.PlexGUID, &p.CreatedAt, &p.UpdatedAt, ); err != nil { return nil, fmt.Errorf("scan person: %w", err) } @@ -515,16 +525,35 @@ func (r *PersonRepository) Search(ctx context.Context, query string, limit int) func (r *PersonRepository) Update(ctx context.Context, p models.Person) error { _, err := r.pool.Exec(ctx, ` UPDATE people SET name=$2, sort_name=$3, bio=$4, birth_date=$5, death_date=$6, - birthplace=$7, homepage=$8, photo_path=$9, photo_thumbhash=$10, - tmdb_id=$11, imdb_id=$12, tvdb_id=$13, plex_guid=$14, updated_at=now() + birthplace=$7, homepage=$8, photo_path=$9, photo_source_path=$10, photo_thumbhash=$11, + tmdb_id=$12, imdb_id=$13, tvdb_id=$14, plex_guid=$15, updated_at=now() WHERE id = $1`, p.ID, p.Name, p.SortName, p.Bio, p.BirthDate, p.DeathDate, - p.Birthplace, p.Homepage, p.PhotoPath, p.PhotoThumbhash, + p.Birthplace, p.Homepage, p.PhotoPath, p.PhotoSourcePath, p.PhotoThumbhash, p.TmdbID, p.ImdbID, p.TvdbID, p.PlexGUID, ) return err } +func (r *PersonRepository) UpdatePhotoIfSourceMatches(ctx context.Context, personID int64, sourcePath, cachedPath, thumbhash string) (bool, error) { + if r == nil || r.pool == nil { + return false, pgx.ErrNoRows + } + tag, err := r.pool.Exec(ctx, ` + UPDATE people + SET photo_path = $3, + photo_source_path = $2, + photo_thumbhash = NULLIF($4, ''), + updated_at = NOW() + WHERE id = $1 + AND photo_source_path = $2 + `, personID, sourcePath, cachedPath, thumbhash) + if err != nil { + return false, fmt.Errorf("updating cached person photo: %w", err) + } + return tag.RowsAffected() > 0, nil +} + // FindRefreshCandidates returns people with external IDs that are incomplete or stale. func (r *PersonRepository) FindRefreshCandidates( ctx context.Context, @@ -580,7 +609,7 @@ func (r *PersonRepository) FindRefreshCandidates( func (r *PersonRepository) ListForItem(ctx context.Context, contentID string) ([]models.ItemPerson, error) { rows, err := r.pool.Query(ctx, ` SELECT p.id, p.name, p.sort_name, p.bio, p.birth_date, p.death_date, p.birthplace, p.homepage, - p.photo_path, p.photo_thumbhash, p.tmdb_id, p.imdb_id, p.tvdb_id, p.plex_guid, + p.photo_path, p.photo_source_path, p.photo_thumbhash, p.tmdb_id, p.imdb_id, p.tvdb_id, p.plex_guid, p.created_at, p.updated_at, ip.kind, ip.character, ip.sort_order FROM item_people ip @@ -605,7 +634,7 @@ func (r *PersonRepository) ListForItems(ctx context.Context, contentIDs []string rows, err := r.pool.Query(ctx, ` SELECT ip.content_id, p.id, p.name, p.sort_name, p.bio, p.birth_date, p.death_date, p.birthplace, p.homepage, - p.photo_path, p.photo_thumbhash, p.tmdb_id, p.imdb_id, p.tvdb_id, p.plex_guid, + p.photo_path, p.photo_source_path, p.photo_thumbhash, p.tmdb_id, p.imdb_id, p.tvdb_id, p.plex_guid, p.created_at, p.updated_at, ip.kind, ip.character, ip.sort_order FROM item_people ip @@ -625,7 +654,7 @@ func (r *PersonRepository) ListForItems(ctx context.Context, contentIDs []string if err := rows.Scan( &contentID, &p.ID, &p.Name, &p.SortName, &p.Bio, &p.BirthDate, &p.DeathDate, - &p.Birthplace, &p.Homepage, &p.PhotoPath, &p.PhotoThumbhash, + &p.Birthplace, &p.Homepage, &p.PhotoPath, &p.PhotoSourcePath, &p.PhotoThumbhash, &p.TmdbID, &p.ImdbID, &p.TvdbID, &p.PlexGUID, &p.CreatedAt, &p.UpdatedAt, &p.Kind, &p.Character, &p.SortOrder, @@ -641,7 +670,7 @@ func (r *PersonRepository) ListForItems(ctx context.Context, contentIDs []string func (r *PersonRepository) ListByKind(ctx context.Context, contentID string, kind models.PersonKind) ([]models.ItemPerson, error) { rows, err := r.pool.Query(ctx, ` SELECT p.id, p.name, p.sort_name, p.bio, p.birth_date, p.death_date, p.birthplace, p.homepage, - p.photo_path, p.photo_thumbhash, p.tmdb_id, p.imdb_id, p.tvdb_id, p.plex_guid, + p.photo_path, p.photo_source_path, p.photo_thumbhash, p.tmdb_id, p.imdb_id, p.tvdb_id, p.plex_guid, p.created_at, p.updated_at, ip.kind, ip.character, ip.sort_order FROM item_people ip @@ -690,7 +719,7 @@ func scanItemPeople(rows pgx.Rows) ([]models.ItemPerson, error) { var ip models.ItemPerson if err := rows.Scan( &ip.ID, &ip.Name, &ip.SortName, &ip.Bio, &ip.BirthDate, &ip.DeathDate, &ip.Birthplace, &ip.Homepage, - &ip.PhotoPath, &ip.PhotoThumbhash, &ip.TmdbID, &ip.ImdbID, &ip.TvdbID, &ip.PlexGUID, + &ip.PhotoPath, &ip.PhotoSourcePath, &ip.PhotoThumbhash, &ip.TmdbID, &ip.ImdbID, &ip.TvdbID, &ip.PlexGUID, &ip.CreatedAt, &ip.UpdatedAt, &ip.Kind, &ip.Character, &ip.SortOrder, ); err != nil { diff --git a/internal/catalog/season_repo.go b/internal/catalog/season_repo.go index 9f0c3450..1c9ec676 100644 --- a/internal/catalog/season_repo.go +++ b/internal/catalog/season_repo.go @@ -30,7 +30,7 @@ func NewSeasonRepository(pool *pgxpool.Pool) *SeasonRepository { // seasonColumns is the list of columns returned by all SELECT queries on seasons. const seasonColumns = `content_id, series_id, season_number, title, default_metadata_language, overview, - air_date, poster_path, poster_thumbhash, + air_date, poster_path, poster_source_path, poster_thumbhash, metadata_s3_path, metadata_etag, metadata_source, created_at, updated_at` @@ -46,6 +46,7 @@ func scanSeason(row pgx.Row) (*models.Season, error) { &s.Overview, &s.AirDate, &s.PosterPath, + &s.PosterSourcePath, &s.PosterThumbhash, &s.MetadataS3Path, &s.MetadataEtag, @@ -76,6 +77,7 @@ func scanSeasons(rows pgx.Rows) ([]*models.Season, error) { &s.Overview, &s.AirDate, &s.PosterPath, + &s.PosterSourcePath, &s.PosterThumbhash, &s.MetadataS3Path, &s.MetadataEtag, @@ -101,12 +103,12 @@ func (r *SeasonRepository) Upsert(ctx context.Context, s *models.Season) error { query := ` INSERT INTO seasons ( content_id, series_id, season_number, title, default_metadata_language, overview, - air_date, poster_path, poster_thumbhash, + air_date, poster_path, poster_source_path, poster_thumbhash, metadata_s3_path, metadata_etag, metadata_source ) VALUES ( $1, $2, $3, $4, $5, $6, - $7, $8, $9, - $10, $11, $12 + $7, $8, $9, $10, + $11, $12, $13 ) ON CONFLICT (series_id, season_number) DO UPDATE SET title = EXCLUDED.title, @@ -114,6 +116,7 @@ func (r *SeasonRepository) Upsert(ctx context.Context, s *models.Season) error { overview = EXCLUDED.overview, air_date = EXCLUDED.air_date, poster_path = EXCLUDED.poster_path, + poster_source_path = EXCLUDED.poster_source_path, poster_thumbhash = EXCLUDED.poster_thumbhash, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, @@ -131,6 +134,7 @@ func (r *SeasonRepository) Upsert(ctx context.Context, s *models.Season) error { s.Overview, s.AirDate, s.PosterPath, + s.PosterSourcePath, s.PosterThumbhash, s.MetadataS3Path, s.MetadataEtag, @@ -159,6 +163,7 @@ func (r *SeasonRepository) BulkUpsert(ctx context.Context, seasons []*models.Sea overviews := make([]string, len(seasons)) airDates := make([]*time.Time, len(seasons)) posterPaths := make([]string, len(seasons)) + posterSourcePaths := make([]string, len(seasons)) posterThumbs := make([]string, len(seasons)) metaS3Paths := make([]string, len(seasons)) metaEtags := make([]string, len(seasons)) @@ -173,6 +178,7 @@ func (r *SeasonRepository) BulkUpsert(ctx context.Context, seasons []*models.Sea overviews[i] = s.Overview airDates[i] = s.AirDate posterPaths[i] = s.PosterPath + posterSourcePaths[i] = s.PosterSourcePath posterThumbs[i] = s.PosterThumbhash metaS3Paths[i] = s.MetadataS3Path metaEtags[i] = s.MetadataEtag @@ -182,13 +188,13 @@ func (r *SeasonRepository) BulkUpsert(ctx context.Context, seasons []*models.Sea query := ` INSERT INTO seasons ( content_id, series_id, season_number, title, default_metadata_language, overview, - air_date, poster_path, poster_thumbhash, + air_date, poster_path, poster_source_path, poster_thumbhash, metadata_s3_path, metadata_etag, metadata_source ) SELECT * FROM UNNEST( $1::text[], $2::text[], $3::int[], $4::text[], $5::text[], $6::text[], - $7::date[], $8::text[], $9::text[], - $10::text[], $11::text[], $12::text[] + $7::date[], $8::text[], $9::text[], $10::text[], + $11::text[], $12::text[], $13::text[] ) ON CONFLICT (series_id, season_number) DO UPDATE SET title = EXCLUDED.title, @@ -196,6 +202,7 @@ func (r *SeasonRepository) BulkUpsert(ctx context.Context, seasons []*models.Sea overview = EXCLUDED.overview, air_date = EXCLUDED.air_date, poster_path = EXCLUDED.poster_path, + poster_source_path = EXCLUDED.poster_source_path, poster_thumbhash = EXCLUDED.poster_thumbhash, metadata_s3_path = EXCLUDED.metadata_s3_path, metadata_etag = EXCLUDED.metadata_etag, @@ -205,7 +212,7 @@ func (r *SeasonRepository) BulkUpsert(ctx context.Context, seasons []*models.Sea rows, err := r.pool.Query(ctx, query, contentIDs, seriesIDs, seasonNums, titles, defaultMetadataLanguages, overviews, - airDates, posterPaths, posterThumbs, + airDates, posterPaths, posterSourcePaths, posterThumbs, metaS3Paths, metaEtags, metaSources, ) if err != nil { @@ -318,6 +325,10 @@ func (r *SeasonRepository) UpdateMetadata(ctx context.Context, contentID string, addInt("season_number", upd.SeasonNumber) addString("air_date", upd.AirDate) addString("poster_path", upd.PosterPath) + if upd.PosterPath != nil && upd.PosterSourcePath == nil { + setClauses = append(setClauses, "poster_source_path = ''") + } + addString("poster_source_path", upd.PosterSourcePath) addString("poster_thumbhash", upd.PosterThumbhash) setClauses = append(setClauses, "updated_at = NOW()") @@ -335,3 +346,19 @@ func (r *SeasonRepository) UpdateMetadata(ctx context.Context, contentID string, } return nil } + +func (r *SeasonRepository) UpdateArtworkIfSourceMatches(ctx context.Context, contentID, sourcePath, cachedPath, thumbhash string) (bool, error) { + tag, err := r.pool.Exec(ctx, ` + UPDATE seasons + SET poster_path = $3, + poster_source_path = $2, + poster_thumbhash = $4, + updated_at = NOW() + WHERE content_id = $1 + AND poster_source_path = $2 + `, contentID, sourcePath, cachedPath, thumbhash) + if err != nil { + return false, fmt.Errorf("updating season cached artwork: %w", err) + } + return tag.RowsAffected() > 0, nil +} diff --git a/internal/catalog/update.go b/internal/catalog/update.go index b0057327..e703a091 100644 --- a/internal/catalog/update.go +++ b/internal/catalog/update.go @@ -12,43 +12,47 @@ import ( // MetadataUpdate contains the fields that can be updated on a media item, // season, or episode. Nil pointer fields are skipped (not updated). type MetadataUpdate struct { - Title *string - SortTitle *string - OriginalTitle *string - Overview *string - Tagline *string - ContentRating *string - Year *int - Runtime *int - Genres *[]string - Studios *[]string - Networks *[]string - Countries *[]string - ReleaseDate *string - FirstAirDate *string - LastAirDate *string - AirTime *string - AirTimezone *string - AirDate *string - Status *string - ShowStatus *string - RatingIMDB *float64 - RatingTMDB *float64 - RatingRTCritic *int - RatingRTAudience *int - ImdbID *string - TmdbID *string - TvdbID *string - SeasonNumber *int - EpisodeNumber *int - LockedFields *[]int - PosterPath *string - PosterThumbhash *string - BackdropPath *string - BackdropThumbhash *string - LogoPath *string - StillPath *string - StillThumbhash *string + Title *string + SortTitle *string + OriginalTitle *string + Overview *string + Tagline *string + ContentRating *string + Year *int + Runtime *int + Genres *[]string + Studios *[]string + Networks *[]string + Countries *[]string + ReleaseDate *string + FirstAirDate *string + LastAirDate *string + AirTime *string + AirTimezone *string + AirDate *string + Status *string + ShowStatus *string + RatingIMDB *float64 + RatingTMDB *float64 + RatingRTCritic *int + RatingRTAudience *int + ImdbID *string + TmdbID *string + TvdbID *string + SeasonNumber *int + EpisodeNumber *int + LockedFields *[]int + PosterPath *string + PosterSourcePath *string + PosterThumbhash *string + BackdropPath *string + BackdropSourcePath *string + BackdropThumbhash *string + LogoPath *string + LogoSourcePath *string + StillPath *string + StillSourcePath *string + StillThumbhash *string } // UpdateMediaItemMetadata updates specific metadata fields on a media_items row. diff --git a/internal/ebooks/enrichment.go b/internal/ebooks/enrichment.go index 731eb69c..eefcafdd 100644 --- a/internal/ebooks/enrichment.go +++ b/internal/ebooks/enrichment.go @@ -73,16 +73,17 @@ type enrichmentItemRow struct { // Enricher drives the ebook metadata enrichment sweep. type Enricher struct { - pool *pgxpool.Pool - chainRepo *metadata.ChainRepository - resolver *metadata.PluginResolverAdapter - itemRepo *catalog.ItemRepository - personRepo *catalog.PersonRepository - providerIDs *catalog.ProviderIDRepository - imageCacher metadata.ImageCacher - workLinker literaryWorkLinker - batchSize int - workers int + pool *pgxpool.Pool + chainRepo *metadata.ChainRepository + resolver *metadata.PluginResolverAdapter + itemRepo *catalog.ItemRepository + personRepo *catalog.PersonRepository + providerIDs *catalog.ProviderIDRepository + imageCacher metadata.ImageCacher + imageCacheJobs metadata.ImageCacheJobEnqueuer + workLinker literaryWorkLinker + batchSize int + workers int } type literaryWorkLinker interface { @@ -116,6 +117,13 @@ func (e *Enricher) SetImageCacher(cacher metadata.ImageCacher) { e.imageCacher = cacher } +func (e *Enricher) SetImageCacheJobEnqueuer(enqueuer metadata.ImageCacheJobEnqueuer) { + if e == nil { + return + } + e.imageCacheJobs = enqueuer +} + func (e *Enricher) SetLiteraryWorkLinker(linker literaryWorkLinker) { if e == nil { return @@ -342,11 +350,10 @@ func (e *Enricher) enrichWithProviders(ctx context.Context, item enrichmentItemR return e.stampLastRefreshed(ctx, item.ContentID) } - e.cacheRemotePoster(ctx, item.ContentID, accumulator) - if err := e.persist(ctx, item.ContentID, accumulatedIDs, accumulator); err != nil { return fmt.Errorf("persisting enrichment for %s: %w", item.ContentID, err) } + e.enqueueRemoteArtwork(ctx, item.ContentID, accumulator) e.autoLinkLiteraryWork(ctx, item.ContentID) slog.Info("ebook enrichment: enriched", @@ -492,6 +499,47 @@ func (e *Enricher) cacheRemotePoster(ctx context.Context, contentID string, resu } } +func (e *Enricher) enqueueRemoteArtwork(ctx context.Context, contentID string, result *metadata.MetadataResult) { + if e == nil || e.imageCacheJobs == nil || result == nil || contentID == "" { + return + } + inputs := make([]metadata.EnqueueImageCacheJobInput, 0, 3) + add := func(sourcePath string, imageType metadata.ImageType) { + if !isRemoteHTTPImage(sourcePath) { + return + } + inputs = append(inputs, metadata.EnqueueImageCacheJobInput{ + TargetType: metadata.ImageCacheTargetItem, + TargetContentID: contentID, + SeriesID: contentID, + SourcePath: sourcePath, + ProviderID: ebookMetadataImageProviderID, + ProviderContentID: contentID, + ContentType: "ebooks", + ImageType: metadata.ImageTypeToString(imageType), + }) + } + add(result.PosterPath, metadata.ImagePoster) + add(result.BackdropPath, metadata.ImageBackdrop) + add(result.LogoPath, metadata.ImageLogo) + if len(inputs) == 0 { + return + } + enqueueCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if _, err := e.imageCacheJobs.EnqueueBatch(enqueueCtx, inputs); err != nil { + slog.Warn("ebook enrichment: failed to enqueue image cache jobs", + "content_id", contentID, + "count", len(inputs), + "error", err, + ) + } +} + +func isRemoteHTTPImage(path string) bool { + return strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") +} + func isNilImageCacher(cacher metadata.ImageCacher) bool { if cacher == nil { return true @@ -523,18 +571,27 @@ func (e *Enricher) persist(ctx context.Context, contentID string, providerIDs ma if result.PosterPath != "" { upd.PosterPath = &result.PosterPath + if isRemoteHTTPImage(result.PosterPath) { + upd.PosterSourcePath = &result.PosterPath + } } if result.PosterThumbhash != "" { upd.PosterThumbhash = &result.PosterThumbhash } if result.BackdropPath != "" { upd.BackdropPath = &result.BackdropPath + if isRemoteHTTPImage(result.BackdropPath) { + upd.BackdropSourcePath = &result.BackdropPath + } } if result.BackdropThumbhash != "" { upd.BackdropThumbhash = &result.BackdropThumbhash } if result.LogoPath != "" { upd.LogoPath = &result.LogoPath + if isRemoteHTTPImage(result.LogoPath) { + upd.LogoSourcePath = &result.LogoPath + } } if result.Overview != "" { upd.Overview = &result.Overview diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index f4eb3171..1f433f50 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -46,6 +46,7 @@ type CacheRequest struct { ImageType metadata.ImageType SeasonNumber *int EpisodeNumber *int + Language string ImageResolver ImageURLResolver // optional; used when SourceURL is a plugin:// path } @@ -85,6 +86,7 @@ func (c *Cacher) CacheImage(ctx context.Context, req metadata.CacheImageRequest) ImageType: req.ImageType, SeasonNumber: req.SeasonNumber, EpisodeNumber: req.EpisodeNumber, + Language: req.Language, }) if err != nil { return nil, err @@ -156,7 +158,7 @@ func (c *Cacher) CacheBytes(ctx context.Context, data []byte, req CacheRequest) if err != nil { return nil, fmt.Errorf("imagecache: generate variants: %w", err) } - basePath := buildBasePath(req.ProviderID, req.ContentType, req.ContentID, req.ImageType, req.SeasonNumber, req.EpisodeNumber) + basePath := buildBasePath(req.ProviderID, req.ContentType, req.ContentID, req.ImageType, req.Language, req.SeasonNumber, req.EpisodeNumber) bucket := c.s3.Bucket() var wg sync.WaitGroup uploadErrs := make([]error, len(result.Variants)) @@ -165,7 +167,7 @@ func (c *Cacher) CacheBytes(ctx context.Context, data []byte, req CacheRequest) go func(idx int, variant imageutil.Variant) { defer wg.Done() key := basePath + "/" + variant.Key + result.Ext - if err := c.s3.PutObject(ctx, bucket, key, variant.Data); err != nil { + if err := putObjectWithRetry(ctx, c.s3, bucket, key, variant.Data); err != nil { uploadErrs[idx] = fmt.Errorf("imagecache: upload %s: %w", key, err) } }(i, v) @@ -227,7 +229,7 @@ func (c *Cacher) Cache(ctx context.Context, req CacheRequest) (*CacheResult, err return nil, fmt.Errorf("imagecache: generate variants: %w", err) } - basePath := buildBasePath(req.ProviderID, req.ContentType, req.ContentID, req.ImageType, req.SeasonNumber, req.EpisodeNumber) + basePath := buildBasePath(req.ProviderID, req.ContentType, req.ContentID, req.ImageType, req.Language, req.SeasonNumber, req.EpisodeNumber) bucket := c.s3.Bucket() // Upload all variants concurrently. @@ -238,7 +240,7 @@ func (c *Cacher) Cache(ctx context.Context, req CacheRequest) (*CacheResult, err go func(idx int, variant imageutil.Variant) { defer wg.Done() key := basePath + "/" + variant.Key + result.Ext - if err := c.s3.PutObject(ctx, bucket, key, variant.Data); err != nil { + if err := putObjectWithRetry(ctx, c.s3, bucket, key, variant.Data); err != nil { uploadErrs[idx] = fmt.Errorf("imagecache: upload %s: %w", key, err) } }(i, v) @@ -269,21 +271,52 @@ func variantWidths(t metadata.ImageType) []int { return []int{500} case metadata.ImageStill: return []int{500, 300} + case metadata.ImageProfile: + return []int{500, 300} default: return []int{500, 300} } } +func putObjectWithRetry(ctx context.Context, putter ObjectPutter, bucket, key string, data []byte) error { + const maxAttempts = 3 + var lastErr error + for attempt := 0; attempt < maxAttempts; attempt++ { + if err := putter.PutObject(ctx, bucket, key, data); err != nil { + lastErr = err + if attempt == maxAttempts-1 { + // Final attempt failed; return immediately without a pointless backoff. + break + } + timer := time.NewTimer(time.Duration(attempt+1) * 500 * time.Millisecond) + select { + case <-timer.C: + continue + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + } + } + return nil + } + return lastErr +} + // buildBasePath constructs the S3 key prefix for a given image. Season // posters and episode stills nest under their parent series so a single // DeletePrefix on the series prefix cascades to all child images. // -// item-level: {provider}/{type}/{id}/{imageType} -// season: {provider}/{type}/{id}/seasons/{n}/{imageType} -// episode: {provider}/{type}/{id}/seasons/{n}/episodes/{m}/{imageType} -func buildBasePath(providerID, contentType, contentID string, t metadata.ImageType, seasonNumber, episodeNumber *int) string { +// item-level: {provider}/{type}/{id}/{imageType} +// localized item: {provider}/{type}/{id}/localizations/{lang}/{imageType} +// season: {provider}/{type}/{id}/seasons/{n}/{imageType} +// localized season: {provider}/{type}/{id}/localizations/{lang}/seasons/{n}/{imageType} +// episode: {provider}/{type}/{id}/seasons/{n}/episodes/{m}/{imageType} +func buildBasePath(providerID, contentType, contentID string, t metadata.ImageType, language string, seasonNumber, episodeNumber *int) string { imageTypeName := imageTypeName(t) base := fmt.Sprintf("%s/%s/%s", providerID, contentType, contentID) + if lang := normalizeImageLanguage(language); lang != "" { + base = fmt.Sprintf("%s/localizations/%s", base, lang) + } if seasonNumber != nil { base = fmt.Sprintf("%s/seasons/%d", base, *seasonNumber) if episodeNumber != nil { @@ -304,11 +337,34 @@ func imageTypeName(t metadata.ImageType) string { return "logo" case metadata.ImageStill: return "still" + case metadata.ImageProfile: + return "profile" default: return "unknown" } } +func normalizeImageLanguage(language string) string { + language = strings.ToLower(strings.TrimSpace(language)) + if language == "" { + return "" + } + var b strings.Builder + for _, r := range language { + switch { + case r >= 'a' && r <= 'z': + b.WriteRune(r) + case r >= '0' && r <= '9': + b.WriteRune(r) + case r == '-' || r == '_': + b.WriteRune(r) + default: + b.WriteByte('_') + } + } + return strings.Trim(b.String(), "_") +} + // downloadImage fetches the image at the given URL, enforcing size, timeout, // and public-network limits. func (c *Cacher) downloadImage(ctx context.Context, rawURL string) ([]byte, error) { diff --git a/internal/imagecache/imagecache_test.go b/internal/imagecache/imagecache_test.go index 598e6999..694531df 100644 --- a/internal/imagecache/imagecache_test.go +++ b/internal/imagecache/imagecache_test.go @@ -34,10 +34,11 @@ func makeTestJPEG(t *testing.T) []byte { // mockS3 records all PutObject calls for test assertions. type mockS3 struct { - mu sync.Mutex - calls []putCall - bucket string - putErr error // if non-nil, returned for every PutObject call + mu sync.Mutex + calls []putCall + bucket string + putErr error // if non-nil, returned for every PutObject call + failuresBeforeSuccess int } type putCall struct { @@ -47,11 +48,15 @@ type putCall struct { } func (m *mockS3) PutObject(_ context.Context, bucket, key string, data []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + if m.failuresBeforeSuccess > 0 { + m.failuresBeforeSuccess-- + return errors.New("temporary s3 failure") + } if m.putErr != nil { return m.putErr } - m.mu.Lock() - defer m.mu.Unlock() m.calls = append(m.calls, putCall{bucket: bucket, key: key, size: len(data)}) return nil } @@ -226,6 +231,64 @@ func TestCache_Logo(t *testing.T) { } } +func TestCache_LocalizedPosterUsesLanguageScopedPath(t *testing.T) { + jpeg := makeTestJPEG(t) + srv := startImageServer(t, jpeg, http.StatusOK) + + s3 := &mockS3{bucket: "media"} + c := newWithHTTPClient(s3, srv.Client()) + + result, err := c.Cache(context.Background(), CacheRequest{ + SourceURL: srv.URL + "/poster-fr.jpg", + ProviderID: "tmdb", + ContentType: "series", + ContentID: "1396", + ImageType: metadata.ImagePoster, + Language: "fr-CA", + }) + if err != nil { + t.Fatalf("Cache localized poster: %v", err) + } + + wantBase := "tmdb/series/1396/localizations/fr-ca/poster" + if result.BasePath != wantBase { + t.Errorf("BasePath = %q, want %q", result.BasePath, wantBase) + } + if !hasKey(s3.keys(), wantBase+"/original.webp") { + t.Errorf("missing localized original in %v", s3.keys()) + } +} + +func TestCache_ProfileUsesProfileImagePath(t *testing.T) { + jpeg := makeTestJPEG(t) + srv := startImageServer(t, jpeg, http.StatusOK) + + s3 := &mockS3{bucket: "media"} + c := newWithHTTPClient(s3, srv.Client()) + + result, err := c.Cache(context.Background(), CacheRequest{ + SourceURL: srv.URL + "/person.jpg", + ProviderID: "tmdb", + ContentType: "people", + ContentID: "287", + ImageType: metadata.ImageProfile, + }) + if err != nil { + t.Fatalf("Cache profile: %v", err) + } + + wantBase := "tmdb/people/287/profile" + if result.BasePath != wantBase { + t.Errorf("BasePath = %q, want %q", result.BasePath, wantBase) + } + for _, variant := range []string{"original", "w500", "w300"} { + want := wantBase + "/" + variant + ".webp" + if !hasKey(s3.keys(), want) { + t.Errorf("missing S3 key %q in %v", want, s3.keys()) + } + } +} + func TestCache_DownloadError(t *testing.T) { srv := startImageServer(t, nil, http.StatusNotFound) @@ -475,6 +538,34 @@ func TestCache_ResolvesPluginURL(t *testing.T) { } } +func TestCacheRetriesTransientPutObjectFailure(t *testing.T) { + jpeg := makeTestJPEG(t) + srv := startImageServer(t, jpeg, http.StatusOK) + + s3 := &mockS3{bucket: "media", failuresBeforeSuccess: 1} + c := newWithHTTPClient(s3, srv.Client()) + + _, err := c.Cache(context.Background(), CacheRequest{ + SourceURL: srv.URL + "/still.jpg", + ProviderID: "tmdb", + ContentType: "series", + ContentID: "1396", + ImageType: metadata.ImageStill, + SeasonNumber: intPointer(1), + EpisodeNumber: intPointer(1), + }) + if err != nil { + t.Fatalf("Cache() error = %v", err) + } + if len(s3.keys()) == 0 { + t.Fatal("expected uploads after retry") + } +} + +func intPointer(v int) *int { + return &v +} + type stubResolver struct { httpURL string } diff --git a/internal/manga/enrichment.go b/internal/manga/enrichment.go index f1dd60fe..273931d6 100644 --- a/internal/manga/enrichment.go +++ b/internal/manga/enrichment.go @@ -103,15 +103,16 @@ type enrichmentItemRow struct { // Enricher drives the manga metadata enrichment sweep. type Enricher struct { - pool *pgxpool.Pool - chainRepo *metadata.ChainRepository - resolver *metadata.PluginResolverAdapter - itemRepo *catalog.ItemRepository - personRepo *catalog.PersonRepository - providerIDs *catalog.ProviderIDRepository - imageCacher metadata.ImageCacher - batchSize int - workers int + pool *pgxpool.Pool + chainRepo *metadata.ChainRepository + resolver *metadata.PluginResolverAdapter + itemRepo *catalog.ItemRepository + personRepo *catalog.PersonRepository + providerIDs *catalog.ProviderIDRepository + imageCacher metadata.ImageCacher + imageCacheJobs metadata.ImageCacheJobEnqueuer + batchSize int + workers int } func NewEnricher( @@ -141,6 +142,13 @@ func (e *Enricher) SetImageCacher(cacher metadata.ImageCacher) { e.imageCacher = cacher } +func (e *Enricher) SetImageCacheJobEnqueuer(enqueuer metadata.ImageCacheJobEnqueuer) { + if e == nil { + return + } + e.imageCacheJobs = enqueuer +} + func (e *Enricher) Run(ctx context.Context) (int, error) { if e == nil || e.pool == nil || e.chainRepo == nil { return 0, nil @@ -399,11 +407,10 @@ func (e *Enricher) enrichWithProviders(ctx context.Context, item enrichmentItemR return errEnrichmentNoMatch } - e.cacheRemoteImages(ctx, item.ContentID, accumulator) - if err := e.persist(ctx, item.ContentID, accumulatedIDs, accumulator); err != nil { return fmt.Errorf("persisting enrichment for %s: %w", item.ContentID, err) } + e.enqueueRemoteArtwork(ctx, item.ContentID, accumulator) slog.Info("manga enrichment: enriched", "content_id", item.ContentID, @@ -426,10 +433,9 @@ func (e *Enricher) enrichWithProviders(ctx context.Context, item enrichmentItemR func (e *Enricher) enrichSecondaryOnly(ctx context.Context, item enrichmentItemRow, result *metadata.MetadataResult, providerErrs []error) error { upd := &catalog.MetadataUpdate{} if result != nil && result.BackdropPath != "" && !item.HasBackdrop { - path, thumbhash := e.cacheRemoteImage(ctx, item.ContentID, result.BackdropPath, metadata.ImageBackdrop) - upd.BackdropPath = &path - if thumbhash != "" { - upd.BackdropThumbhash = &thumbhash + upd.BackdropPath = &result.BackdropPath + if isRemoteHTTPImage(result.BackdropPath) { + upd.BackdropSourcePath = &result.BackdropPath } } if result != nil { @@ -459,6 +465,9 @@ func (e *Enricher) enrichSecondaryOnly(ctx context.Context, item enrichmentItemR if err := e.updateMetadataAndTimestamps(ctx, item.ContentID, upd); err != nil { return fmt.Errorf("persisting secondary metadata for %s: %w", item.ContentID, err) } + if result != nil && result.BackdropPath != "" && !item.HasBackdrop { + e.enqueueRemoteImage(ctx, item.ContentID, result.BackdropPath, metadata.ImageBackdrop) + } slog.Info("manga enrichment: secondary metadata added", "content_id", item.ContentID, @@ -654,18 +663,27 @@ func (e *Enricher) persist(ctx context.Context, contentID string, providerIDs ma if result.PosterPath != "" { upd.PosterPath = &result.PosterPath + if isRemoteHTTPImage(result.PosterPath) { + upd.PosterSourcePath = &result.PosterPath + } } if result.PosterThumbhash != "" { upd.PosterThumbhash = &result.PosterThumbhash } if result.BackdropPath != "" { upd.BackdropPath = &result.BackdropPath + if isRemoteHTTPImage(result.BackdropPath) { + upd.BackdropSourcePath = &result.BackdropPath + } } if result.BackdropThumbhash != "" { upd.BackdropThumbhash = &result.BackdropThumbhash } if result.LogoPath != "" { upd.LogoPath = &result.LogoPath + if isRemoteHTTPImage(result.LogoPath) { + upd.LogoSourcePath = &result.LogoPath + } } if result.Overview != "" { upd.Overview = &result.Overview @@ -724,6 +742,44 @@ func (e *Enricher) persist(ctx context.Context, contentID string, providerIDs ma return nil } +func (e *Enricher) enqueueRemoteArtwork(ctx context.Context, contentID string, result *metadata.MetadataResult) { + if e == nil || result == nil || contentID == "" { + return + } + e.enqueueRemoteImage(ctx, contentID, result.PosterPath, metadata.ImagePoster) + e.enqueueRemoteImage(ctx, contentID, result.BackdropPath, metadata.ImageBackdrop) + e.enqueueRemoteImage(ctx, contentID, result.LogoPath, metadata.ImageLogo) +} + +func (e *Enricher) enqueueRemoteImage(ctx context.Context, contentID, sourcePath string, imageType metadata.ImageType) { + if e == nil || e.imageCacheJobs == nil || contentID == "" || !isRemoteHTTPImage(sourcePath) { + return + } + enqueueCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _, err := e.imageCacheJobs.EnqueueBatch(enqueueCtx, []metadata.EnqueueImageCacheJobInput{{ + TargetType: metadata.ImageCacheTargetItem, + TargetContentID: contentID, + SeriesID: contentID, + SourcePath: sourcePath, + ProviderID: mangaMetadataImageProviderID, + ProviderContentID: contentID, + ContentType: mangaContentType(), + ImageType: metadata.ImageTypeToString(imageType), + }}) + if err != nil { + slog.Warn("manga enrichment: failed to enqueue image cache job", + "content_id", contentID, + "image_type", metadata.ImageTypeToString(imageType), + "error", err, + ) + } +} + +func isRemoteHTTPImage(path string) bool { + return strings.HasPrefix(path, "http://") || strings.HasPrefix(path, "https://") +} + func (e *Enricher) updateMetadataAndTimestamps(ctx context.Context, contentID string, upd *catalog.MetadataUpdate) error { if e.itemRepo == nil { return nil diff --git a/internal/metadata/fallback_unification_test.go b/internal/metadata/fallback_unification_test.go index 45651381..1ee95571 100644 --- a/internal/metadata/fallback_unification_test.go +++ b/internal/metadata/fallback_unification_test.go @@ -177,7 +177,7 @@ func TestPersistSeasonsAndEpisodes_ScheduledRefreshPreservesExistingAndBackfills MetadataSource: "provider", } - service.persistSeasonsAndEpisodes(ctx, seriesID, "en", "en", + service.persistSeasonsAndEpisodes(ctx, &models.MediaItem{ContentID: seriesID, Type: "series"}, nil, "en", "en", []SeasonResult{{ SeasonNumber: 1, Title: "Provider Season", @@ -252,7 +252,7 @@ func TestPersistSeasonsAndEpisodes_ManualRefreshReplacesNonEmptyButPreservesBlan MetadataSource: "provider", } - service.persistSeasonsAndEpisodes(ctx, seriesID, "en", "en", + service.persistSeasonsAndEpisodes(ctx, &models.MediaItem{ContentID: seriesID, Type: "series"}, nil, "en", "en", []SeasonResult{{ SeasonNumber: 1, Title: "New Season", diff --git a/internal/metadata/image_cache_enqueue_test.go b/internal/metadata/image_cache_enqueue_test.go new file mode 100644 index 00000000..b9d74dd0 --- /dev/null +++ b/internal/metadata/image_cache_enqueue_test.go @@ -0,0 +1,134 @@ +package metadata + +import ( + "context" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +type recordingImageCacheJobEnqueuer struct { + inputs []EnqueueImageCacheJobInput +} + +func (r *recordingImageCacheJobEnqueuer) Enqueue(_ context.Context, in EnqueueImageCacheJobInput) error { + _, err := r.EnqueueBatch(context.Background(), []EnqueueImageCacheJobInput{in}) + return err +} + +func (r *recordingImageCacheJobEnqueuer) EnqueueBatch(_ context.Context, inputs []EnqueueImageCacheJobInput) (int, error) { + r.inputs = append(r.inputs, inputs...) + return len(inputs), nil +} + +func TestPreserveCachedArtworkKeepsCachedPathWhenSourceMatches(t *testing.T) { + path, thumb, source := preserveCachedArtwork( + "tvdb://banners/episodes/1.jpg", + "", + "tvdb/series/1/seasons/1/episodes/1/still/original.webp", + "tvdb://banners/episodes/1.jpg", + "thumb", + ) + if path != "tvdb/series/1/seasons/1/episodes/1/still/original.webp" { + t.Fatalf("path = %q", path) + } + if thumb != "thumb" { + t.Fatalf("thumb = %q", thumb) + } + if source != "tvdb://banners/episodes/1.jpg" { + t.Fatalf("source = %q", source) + } +} + +func TestPersistSeasonsAndEpisodesPersistsSourceBeforeEnqueue(t *testing.T) { + const seriesID = "series-tvdb-123" + service, _, seasonRepo, episodeRepo := newSeasonEpisodeServiceForTest(seriesID) + enqueuer := &recordingImageCacheJobEnqueuer{} + service.SetAutoCacheImages(true) + service.SetImageCacheJobEnqueuer(enqueuer) + + series := &models.MediaItem{ + ContentID: seriesID, + Type: "series", + TvdbID: "123", + } + service.persistSeasonsAndEpisodes( + context.Background(), + series, + map[string]string{"tvdb": "123"}, + "en", + "en", + []SeasonResult{{ + SeasonNumber: 1, + Title: "Season 1", + PosterPath: "tvdb://banners/seasons/1.jpg", + }}, + []EpisodeResult{{ + ProviderIDs: map[string]string{"tvdb": "ep-1"}, + SeasonNumber: 1, + EpisodeNumber: 1, + Title: "Pilot", + StillPath: "tvdb://banners/episodes/1.jpg", + StillThumbhash: "provider-thumb", + }}, + MergeFillEmpty, + ) + + season := seasonRepo.seasons[seasonKey(seriesID, 1)] + if season == nil { + t.Fatal("season was not persisted") + } + if season.PosterSourcePath != "tvdb://banners/seasons/1.jpg" { + t.Fatalf("season source = %q", season.PosterSourcePath) + } + episode := episodeRepo.episodes[episodeKey(seriesID, 1, 1)] + if episode == nil { + t.Fatal("episode was not persisted") + } + if episode.StillSourcePath != "tvdb://banners/episodes/1.jpg" { + t.Fatalf("episode source = %q", episode.StillSourcePath) + } + if len(enqueuer.inputs) != 2 { + t.Fatalf("queued jobs = %d, want 2", len(enqueuer.inputs)) + } + if enqueuer.inputs[0].TargetContentID != season.ContentID { + t.Fatalf("season job target = %q, want %q", enqueuer.inputs[0].TargetContentID, season.ContentID) + } + if enqueuer.inputs[1].TargetContentID != episode.ContentID { + t.Fatalf("episode job target = %q, want %q", enqueuer.inputs[1].TargetContentID, episode.ContentID) + } +} + +func TestPreserveCachedArtworkRecordsNewProviderSource(t *testing.T) { + path, thumb, source := preserveCachedArtwork( + "tvdb://banners/episodes/new.jpg", + "", + "tvdb/series/1/seasons/1/episodes/1/still/original.webp", + "tvdb://banners/episodes/old.jpg", + "old-thumb", + ) + if path != "tvdb/series/1/seasons/1/episodes/1/still/original.webp" { + t.Fatalf("cached path should remain visible until worker succeeds, got %q", path) + } + if thumb != "old-thumb" { + t.Fatalf("thumb = %q", thumb) + } + if source != "tvdb://banners/episodes/new.jpg" { + t.Fatalf("source = %q", source) + } +} + +func TestProviderImageSourcePathOnlyRecordsProviderScheme(t *testing.T) { + if got := providerImageSourcePath("tvdb://banners/episodes/1.jpg"); got != "tvdb://banners/episodes/1.jpg" { + t.Fatalf("provider source = %q", got) + } + if got := providerImageSourcePath("tmdb/series/1/poster/original.webp"); got != "" { + t.Fatalf("cached path source = %q, want empty", got) + } + if got := providerImageSourcePath("file:///media/poster.jpg"); got != "" { + t.Fatalf("file source = %q, want empty", got) + } + if got := providerImageSourcePath("https://image.tmdb.org/t/p/original/a.jpg"); got != "https://image.tmdb.org/t/p/original/a.jpg" { + t.Fatalf("http source = %q", got) + } +} diff --git a/internal/metadata/image_cache_job_repo.go b/internal/metadata/image_cache_job_repo.go new file mode 100644 index 00000000..cd7d5455 --- /dev/null +++ b/internal/metadata/image_cache_job_repo.go @@ -0,0 +1,796 @@ +package metadata + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/models" +) + +const ( + ImageCacheTargetItem = "item" + ImageCacheTargetItemLocalization = "item_localization" + ImageCacheTargetSeason = "season" + ImageCacheTargetSeasonLocalization = "season_localization" + ImageCacheTargetEpisode = "episode" + ImageCacheTargetPerson = "person" + + ImageCacheImagePoster = "poster" + ImageCacheImageBackdrop = "backdrop" + ImageCacheImageLogo = "logo" + ImageCacheImageStill = "still" + ImageCacheImageProfile = "profile" + + ImageCacheStatusQueued = "queued" + ImageCacheStatusRunning = "running" + ImageCacheStatusSucceeded = "succeeded" + ImageCacheStatusFailed = "failed" + + imageCacheLeaseDuration = 15 * time.Minute + imageCacheMaxAttempts = 8 + imageCacheFailedRetryAfter = 6 * time.Hour +) + +type EnqueueImageCacheJobInput struct { + TargetType string + TargetContentID string + TargetLanguage string + SeriesID string + SourcePath string + ProviderID string + ProviderContentID string + ContentType string + ImageType string + SeasonNumber *int + EpisodeNumber *int +} + +type ImageCacheJobRepository struct { + pool *pgxpool.Pool +} + +func NewImageCacheJobRepository(pool *pgxpool.Pool) *ImageCacheJobRepository { + return &ImageCacheJobRepository{pool: pool} +} + +func imageCacheRetryDelay(attempt int) time.Duration { + if attempt <= 1 { + return time.Minute + } + delay := time.Minute << min(attempt-1, 7) + if delay > 2*time.Hour { + return 2 * time.Hour + } + return delay +} + +func (r *ImageCacheJobRepository) Enqueue(ctx context.Context, in EnqueueImageCacheJobInput) error { + _, err := r.EnqueueBatch(ctx, []EnqueueImageCacheJobInput{in}) + return err +} + +func (r *ImageCacheJobRepository) EnqueueBatch(ctx context.Context, inputs []EnqueueImageCacheJobInput) (int, error) { + if r == nil || r.pool == nil { + return 0, nil + } + valid := make([]EnqueueImageCacheJobInput, 0, len(inputs)) + for _, in := range inputs { + normalized, ok := normalizeImageCacheJobInput(in) + if !ok { + continue + } + valid = append(valid, normalized) + } + if len(valid) == 0 { + return 0, nil + } + + total := 0 + for start := 0; start < len(valid); start += 250 { + end := start + 250 + if end > len(valid) { + end = len(valid) + } + affected, err := r.enqueueBatchChunk(ctx, valid[start:end]) + if err != nil { + return total, err + } + total += affected + } + return total, nil +} + +func normalizeImageCacheJobInput(in EnqueueImageCacheJobInput) (EnqueueImageCacheJobInput, bool) { + in.SourcePath = strings.TrimSpace(in.SourcePath) + in.TargetLanguage = strings.TrimSpace(in.TargetLanguage) + if in.SourcePath == "" || !strings.Contains(in.SourcePath, "://") || isNonProviderImageScheme(strings.ToLower(in.SourcePath)) { + return EnqueueImageCacheJobInput{}, false + } + if in.ContentType == "" { + in.ContentType = "series" + } + if in.ProviderID == "" { + in.ProviderID = imageCacheProviderIDFromSource(in.SourcePath, "") + } + if in.ProviderContentID == "" { + in.ProviderContentID = firstNonEmpty(in.SeriesID, in.TargetContentID) + } + return in, true +} + +func (r *ImageCacheJobRepository) enqueueBatchChunk(ctx context.Context, inputs []EnqueueImageCacheJobInput) (int, error) { + var sql strings.Builder + args := make([]any, 0, len(inputs)*11+1) + sql.WriteString(` + INSERT INTO metadata_image_cache_jobs ( + target_type, target_content_id, target_language, series_id, source_path, + provider_id, provider_content_id, content_type, image_type, + season_number, episode_number, status, attempt_count, + next_attempt_at, locked_at, locked_by, last_error, + created_at, updated_at, completed_at + ) VALUES `) + for i, in := range inputs { + if i > 0 { + sql.WriteString(", ") + } + base := len(args) + fmt.Fprintf(&sql, `($%d, $%d, $%d, $%d, $%d, $%d, $%d, $%d, $%d, $%d, $%d, 'queued', 0, NOW(), NULL, '', '', NOW(), NOW(), NULL)`, + base+1, base+2, base+3, base+4, base+5, + base+6, base+7, base+8, base+9, base+10, base+11) + args = append(args, + in.TargetType, in.TargetContentID, strings.TrimSpace(in.TargetLanguage), in.SeriesID, in.SourcePath, + in.ProviderID, in.ProviderContentID, in.ContentType, in.ImageType, + in.SeasonNumber, in.EpisodeNumber, + ) + } + retryArg := len(args) + 1 + args = append(args, intervalLiteral(imageCacheFailedRetryAfter)) + fmt.Fprintf(&sql, ` + ON CONFLICT (target_type, target_content_id, image_type, target_language) DO UPDATE SET + series_id = EXCLUDED.series_id, + source_path = EXCLUDED.source_path, + provider_id = EXCLUDED.provider_id, + provider_content_id = EXCLUDED.provider_content_id, + content_type = EXCLUDED.content_type, + season_number = EXCLUDED.season_number, + episode_number = EXCLUDED.episode_number, + status = CASE + WHEN metadata_image_cache_jobs.source_path IS DISTINCT FROM EXCLUDED.source_path + THEN 'queued' + WHEN metadata_image_cache_jobs.status = 'failed' + AND metadata_image_cache_jobs.updated_at < NOW() - $%d::interval + THEN 'queued' + WHEN metadata_image_cache_jobs.status = 'succeeded' + THEN 'succeeded' + ELSE metadata_image_cache_jobs.status + END, + attempt_count = CASE + WHEN metadata_image_cache_jobs.source_path IS DISTINCT FROM EXCLUDED.source_path + THEN 0 + WHEN metadata_image_cache_jobs.status = 'failed' + AND metadata_image_cache_jobs.updated_at < NOW() - $%d::interval + THEN 0 + ELSE metadata_image_cache_jobs.attempt_count + END, + next_attempt_at = CASE + WHEN metadata_image_cache_jobs.source_path IS DISTINCT FROM EXCLUDED.source_path + THEN NOW() + WHEN metadata_image_cache_jobs.status = 'failed' + AND metadata_image_cache_jobs.updated_at < NOW() - $%d::interval + THEN NOW() + ELSE metadata_image_cache_jobs.next_attempt_at + END, + locked_at = CASE + WHEN metadata_image_cache_jobs.source_path IS DISTINCT FROM EXCLUDED.source_path + THEN NULL + WHEN metadata_image_cache_jobs.status = 'failed' + AND metadata_image_cache_jobs.updated_at < NOW() - $%d::interval + THEN NULL + ELSE metadata_image_cache_jobs.locked_at + END, + locked_by = CASE + WHEN metadata_image_cache_jobs.source_path IS DISTINCT FROM EXCLUDED.source_path + THEN '' + WHEN metadata_image_cache_jobs.status = 'failed' + AND metadata_image_cache_jobs.updated_at < NOW() - $%d::interval + THEN '' + ELSE metadata_image_cache_jobs.locked_by + END, + last_error = CASE + WHEN metadata_image_cache_jobs.source_path IS DISTINCT FROM EXCLUDED.source_path + THEN '' + WHEN metadata_image_cache_jobs.status = 'failed' + AND metadata_image_cache_jobs.updated_at < NOW() - $%d::interval + THEN '' + ELSE metadata_image_cache_jobs.last_error + END, + completed_at = CASE + WHEN metadata_image_cache_jobs.source_path IS DISTINCT FROM EXCLUDED.source_path + THEN NULL + WHEN metadata_image_cache_jobs.status = 'failed' + AND metadata_image_cache_jobs.updated_at < NOW() - $%d::interval + THEN NULL + ELSE metadata_image_cache_jobs.completed_at + END, + updated_at = NOW() + WHERE metadata_image_cache_jobs.source_path IS DISTINCT FROM EXCLUDED.source_path + OR metadata_image_cache_jobs.status IN ('queued', 'running') + OR ( + metadata_image_cache_jobs.status = 'failed' + AND metadata_image_cache_jobs.updated_at < NOW() - $%d::interval + )`, + retryArg, retryArg, retryArg, retryArg, retryArg, retryArg, retryArg, retryArg) + + tag, err := r.pool.Exec(ctx, sql.String(), args...) + if err != nil { + return 0, fmt.Errorf("enqueuing metadata image cache jobs: %w", err) + } + return int(tag.RowsAffected()), nil +} + +func (r *ImageCacheJobRepository) recoverExpiredRunning(ctx context.Context) error { + _, err := r.pool.Exec(ctx, ` + UPDATE metadata_image_cache_jobs + SET status = CASE + WHEN attempt_count + 1 >= $2 THEN 'failed' + ELSE 'queued' + END, + attempt_count = attempt_count + 1, + next_attempt_at = CASE + WHEN attempt_count + 1 >= $2 THEN next_attempt_at + ELSE NOW() + END, + locked_at = NULL, + locked_by = '', + last_error = CASE + WHEN attempt_count + 1 >= $2 THEN left('worker lease expired too many times', 2000) + ELSE last_error + END, + updated_at = NOW() + WHERE status = 'running' + AND locked_at < NOW() - $1::interval + `, intervalLiteral(imageCacheLeaseDuration), imageCacheMaxAttempts) + if err != nil { + return fmt.Errorf("recovering expired metadata image cache jobs: %w", err) + } + return nil +} + +func (r *ImageCacheJobRepository) ClaimDue(ctx context.Context, workerID string, limit int) ([]*models.MetadataImageCacheJob, error) { + if r == nil || r.pool == nil || limit <= 0 { + return nil, nil + } + if err := r.recoverExpiredRunning(ctx); err != nil { + return nil, err + } + rows, err := r.pool.Query(ctx, ` + WITH due AS ( + SELECT id + FROM metadata_image_cache_jobs + WHERE status = 'queued' + AND next_attempt_at <= NOW() + ORDER BY next_attempt_at ASC, id ASC + LIMIT $1 + FOR UPDATE SKIP LOCKED + ) + UPDATE metadata_image_cache_jobs j + SET status = 'running', + locked_at = NOW(), + locked_by = $2, + updated_at = NOW() + FROM due + WHERE j.id = due.id + RETURNING + j.id, j.target_type, j.target_content_id, j.target_language, j.series_id, + j.source_path, j.provider_id, j.provider_content_id, + j.content_type, j.image_type, j.season_number, j.episode_number, + j.status, j.attempt_count, j.next_attempt_at, j.locked_at, + j.locked_by, j.last_error, j.created_at, j.updated_at, j.completed_at + `, limit, workerID) + if err != nil { + return nil, fmt.Errorf("claiming metadata image cache jobs: %w", err) + } + defer rows.Close() + + jobs := make([]*models.MetadataImageCacheJob, 0, limit) + for rows.Next() { + job := new(models.MetadataImageCacheJob) + if err := rows.Scan( + &job.ID, &job.TargetType, &job.TargetContentID, &job.TargetLanguage, &job.SeriesID, + &job.SourcePath, &job.ProviderID, &job.ProviderContentID, + &job.ContentType, &job.ImageType, &job.SeasonNumber, &job.EpisodeNumber, + &job.Status, &job.AttemptCount, &job.NextAttemptAt, &job.LockedAt, + &job.LockedBy, &job.LastError, &job.CreatedAt, &job.UpdatedAt, &job.CompletedAt, + ); err != nil { + return nil, fmt.Errorf("scanning metadata image cache job: %w", err) + } + jobs = append(jobs, job) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating metadata image cache jobs: %w", err) + } + return jobs, nil +} + +// MarkSucceeded finalizes a job only if the caller still owns the lease +// (status running, locked_by matches). EnqueueBatch can repurpose a running +// row with a new source and a cleared lease; the ownership guard stops a +// stale worker from marking that replacement job complete and dropping the +// new artwork. +func (r *ImageCacheJobRepository) MarkSucceeded(ctx context.Context, id int64, lockedBy string) error { + _, err := r.pool.Exec(ctx, ` + UPDATE metadata_image_cache_jobs + SET status = 'succeeded', + completed_at = NOW(), + locked_at = NULL, + locked_by = '', + last_error = '', + updated_at = NOW() + WHERE id = $1 + AND status = 'running' + AND locked_by = $2 + `, id, lockedBy) + if err != nil { + return fmt.Errorf("marking metadata image cache job succeeded: %w", err) + } + return nil +} + +// MarkFailed records a failed attempt with backoff, guarded by lease ownership +// for the same reason as MarkSucceeded. +func (r *ImageCacheJobRepository) MarkFailed(ctx context.Context, id int64, attemptCount int, lockedBy string, errText string) error { + nextAttempt := attemptCount + 1 + status := ImageCacheStatusQueued + if nextAttempt >= imageCacheMaxAttempts { + status = ImageCacheStatusFailed + } + delay := imageCacheRetryDelay(nextAttempt) + + _, err := r.pool.Exec(ctx, ` + UPDATE metadata_image_cache_jobs + SET status = $2, + attempt_count = $3, + next_attempt_at = NOW() + $4::interval, + locked_at = NULL, + locked_by = '', + last_error = left($5, 2000), + updated_at = NOW() + WHERE id = $1 + AND status = 'running' + AND locked_by = $6 + `, id, status, nextAttempt, intervalLiteral(delay), errText, lockedBy) + if err != nil { + return fmt.Errorf("marking metadata image cache job failed: %w", err) + } + return nil +} + +// RequeueClaimed returns claimed-but-unprocessed jobs to the queue without +// burning a retry attempt. Used when a run is cancelled before its workers +// start, so the jobs do not sit locked until the lease expires. +func (r *ImageCacheJobRepository) RequeueClaimed(ctx context.Context, ids []int64, workerID string) error { + if r == nil || r.pool == nil || len(ids) == 0 { + return nil + } + _, err := r.pool.Exec(ctx, ` + UPDATE metadata_image_cache_jobs + SET status = 'queued', + next_attempt_at = NOW(), + locked_at = NULL, + locked_by = '', + updated_at = NOW() + WHERE id = ANY($1) + AND status = 'running' + AND locked_by = $2 + `, ids, workerID) + if err != nil { + return fmt.Errorf("requeueing claimed metadata image cache jobs: %w", err) + } + return nil +} + +// CurrentTargetSourcePath reports the source path currently stored on the +// job's target row so the processor can confirm it still owns the artwork +// before uploading to the deterministic storage key. Returns ("", nil) when +// the row no longer exists or the target type is unknown. +func (r *ImageCacheJobRepository) CurrentTargetSourcePath(ctx context.Context, job *models.MetadataImageCacheJob) (string, error) { + if r == nil || r.pool == nil || job == nil { + return "", nil + } + query, args, ok := currentTargetSourceQuery(job) + if !ok { + return "", nil + } + var current string + err := r.pool.QueryRow(ctx, query, args...).Scan(¤t) + if errors.Is(err, pgx.ErrNoRows) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("reading current target source path: %w", err) + } + return current, nil +} + +func currentTargetSourceQuery(job *models.MetadataImageCacheJob) (string, []any, bool) { + switch job.TargetType { + case ImageCacheTargetItem: + col, ok := itemArtworkSourceColumn(job.ImageType) + if !ok { + return "", nil, false + } + return fmt.Sprintf("SELECT %s FROM media_items WHERE content_id = $1", col), + []any{job.TargetContentID}, true + case ImageCacheTargetItemLocalization: + col, ok := itemArtworkSourceColumn(job.ImageType) + if !ok { + return "", nil, false + } + return fmt.Sprintf("SELECT %s FROM media_item_localizations WHERE content_id = $1 AND language = $2", col), + []any{job.TargetContentID, job.TargetLanguage}, true + case ImageCacheTargetSeason: + return "SELECT poster_source_path FROM seasons WHERE content_id = $1", + []any{job.TargetContentID}, true + case ImageCacheTargetSeasonLocalization: + return "SELECT poster_source_path FROM season_localizations WHERE season_content_id = $1 AND language = $2", + []any{job.TargetContentID, job.TargetLanguage}, true + case ImageCacheTargetEpisode: + return "SELECT still_source_path FROM episodes WHERE content_id = $1", + []any{job.TargetContentID}, true + case ImageCacheTargetPerson: + return "SELECT photo_source_path FROM people WHERE id = $1::bigint", + []any{job.TargetContentID}, true + default: + return "", nil, false + } +} + +func itemArtworkSourceColumn(imageType string) (string, bool) { + switch imageType { + case ImageCacheImagePoster: + return "poster_source_path", true + case ImageCacheImageBackdrop: + return "backdrop_source_path", true + case ImageCacheImageLogo: + return "logo_source_path", true + default: + return "", false + } +} + +func (r *ImageCacheJobRepository) DeleteSucceededBefore(ctx context.Context, before time.Time, limit int) (int, error) { + if r == nil || r.pool == nil || limit <= 0 { + return 0, nil + } + tag, err := r.pool.Exec(ctx, ` + WITH doomed AS ( + SELECT id + FROM metadata_image_cache_jobs + WHERE status = 'succeeded' + AND completed_at < $1 + ORDER BY completed_at ASC, id ASC + LIMIT $2 + ) + DELETE FROM metadata_image_cache_jobs j + USING doomed + WHERE j.id = doomed.id + `, before, limit) + if err != nil { + return 0, fmt.Errorf("deleting old succeeded metadata image cache jobs: %w", err) + } + return int(tag.RowsAffected()), nil +} + +func (r *ImageCacheJobRepository) EnqueueExistingProviderArtwork(ctx context.Context, limit int) (int, error) { + if r == nil || r.pool == nil || limit <= 0 { + return 0, nil + } + // Each branch is restricted to provider-origin sources (LIKE '%://%' minus + // cached/system schemes) AND to targets whose stored *_path is not already a + // cached relative path. The destination check makes the cached row itself the + // durable dedup marker, so pruning succeeded job rows does not cause the whole + // catalog to be re-downloaded once the rows age out. + rows, err := r.pool.Query(ctx, ` + WITH all_candidates AS ( + SELECT + 'poster'::text AS image_type, + 'item'::text AS target_type, + mi.content_id AS target_content_id, + ''::text AS target_language, + mi.content_id AS series_id, + mi.poster_source_path AS source_path, + mi.type AS content_type, + NULL::integer AS season_number, + NULL::integer AS episode_number, + mi.tmdb_id, + mi.tvdb_id, + mi.imdb_id + FROM media_items mi + WHERE mi.poster_source_path LIKE '%://%' + AND lower(mi.poster_source_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']) + AND (mi.poster_path LIKE '%://%' OR coalesce(mi.poster_path, '') = '') + UNION ALL + SELECT + 'backdrop'::text, + 'item'::text, + mi.content_id, + ''::text, + mi.content_id, + mi.backdrop_source_path, + mi.type, + NULL::integer, + NULL::integer, + mi.tmdb_id, + mi.tvdb_id, + mi.imdb_id + FROM media_items mi + WHERE mi.backdrop_source_path LIKE '%://%' + AND lower(mi.backdrop_source_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']) + AND (mi.backdrop_path LIKE '%://%' OR coalesce(mi.backdrop_path, '') = '') + UNION ALL + SELECT + 'logo'::text, + 'item'::text, + mi.content_id, + ''::text, + mi.content_id, + mi.logo_source_path, + mi.type, + NULL::integer, + NULL::integer, + mi.tmdb_id, + mi.tvdb_id, + mi.imdb_id + FROM media_items mi + WHERE mi.logo_source_path LIKE '%://%' + AND lower(mi.logo_source_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']) + AND (mi.logo_path LIKE '%://%' OR coalesce(mi.logo_path, '') = '') + UNION ALL + SELECT + 'poster'::text, + 'item_localization'::text, + loc.content_id, + loc.language, + loc.content_id, + loc.poster_source_path, + mi.type, + NULL::integer, + NULL::integer, + mi.tmdb_id, + mi.tvdb_id, + mi.imdb_id + FROM media_item_localizations loc + JOIN media_items mi ON mi.content_id = loc.content_id + WHERE loc.poster_source_path LIKE '%://%' + AND lower(loc.poster_source_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']) + AND (loc.poster_path LIKE '%://%' OR coalesce(loc.poster_path, '') = '') + UNION ALL + SELECT + 'backdrop'::text, + 'item_localization'::text, + loc.content_id, + loc.language, + loc.content_id, + loc.backdrop_source_path, + mi.type, + NULL::integer, + NULL::integer, + mi.tmdb_id, + mi.tvdb_id, + mi.imdb_id + FROM media_item_localizations loc + JOIN media_items mi ON mi.content_id = loc.content_id + WHERE loc.backdrop_source_path LIKE '%://%' + AND lower(loc.backdrop_source_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']) + AND (loc.backdrop_path LIKE '%://%' OR coalesce(loc.backdrop_path, '') = '') + UNION ALL + SELECT + 'logo'::text, + 'item_localization'::text, + loc.content_id, + loc.language, + loc.content_id, + loc.logo_source_path, + mi.type, + NULL::integer, + NULL::integer, + mi.tmdb_id, + mi.tvdb_id, + mi.imdb_id + FROM media_item_localizations loc + JOIN media_items mi ON mi.content_id = loc.content_id + WHERE loc.logo_source_path LIKE '%://%' + AND lower(loc.logo_source_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']) + AND (loc.logo_path LIKE '%://%' OR coalesce(loc.logo_path, '') = '') + UNION ALL + SELECT + 'poster'::text, + 'season'::text, + s.content_id AS target_content_id, + ''::text AS target_language, + s.series_id, + s.poster_source_path AS source_path, + 'series'::text AS content_type, + s.season_number, + NULL::integer AS episode_number, + mi.tmdb_id, + mi.tvdb_id, + mi.imdb_id + FROM seasons s + JOIN media_items mi ON mi.content_id = s.series_id + WHERE s.poster_source_path LIKE '%://%' + AND lower(s.poster_source_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']) + AND (s.poster_path LIKE '%://%' OR coalesce(s.poster_path, '') = '') + UNION ALL + SELECT + 'poster'::text, + 'season_localization'::text, + s.content_id, + loc.language, + s.series_id, + loc.poster_source_path, + 'series'::text, + s.season_number, + NULL::integer, + mi.tmdb_id, + mi.tvdb_id, + mi.imdb_id + FROM season_localizations loc + JOIN seasons s ON s.content_id = loc.season_content_id + JOIN media_items mi ON mi.content_id = s.series_id + WHERE loc.poster_source_path LIKE '%://%' + AND lower(loc.poster_source_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']) + AND (loc.poster_path LIKE '%://%' OR coalesce(loc.poster_path, '') = '') + UNION ALL + SELECT + 'still'::text, + 'episode'::text, + e.content_id, + ''::text, + e.series_id, + e.still_source_path, + 'series'::text, + e.season_number, + e.episode_number, + mi.tmdb_id, + mi.tvdb_id, + mi.imdb_id + FROM episodes e + JOIN media_items mi ON mi.content_id = e.series_id + WHERE e.still_source_path LIKE '%://%' + AND lower(e.still_source_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']) + AND (e.still_path LIKE '%://%' OR coalesce(e.still_path, '') = '') + UNION ALL + SELECT + 'profile'::text, + 'person'::text, + p.id::text, + ''::text, + ''::text, + p.photo_source_path, + 'people'::text, + NULL::integer, + NULL::integer, + p.tmdb_id, + p.tvdb_id, + p.imdb_id + FROM people p + WHERE p.photo_source_path LIKE '%://%' + AND lower(p.photo_source_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']) + AND (p.photo_path LIKE '%://%' OR coalesce(p.photo_path, '') = '') + ), + candidates AS ( + SELECT ac.* + FROM all_candidates ac + LEFT JOIN metadata_image_cache_jobs j + ON j.target_type = ac.target_type + AND j.target_content_id = ac.target_content_id + AND j.image_type = ac.image_type + AND j.target_language = ac.target_language + WHERE j.id IS NULL + OR j.source_path IS DISTINCT FROM ac.source_path + OR ( + j.status = 'failed' + AND j.updated_at < NOW() - $2::interval + ) + ORDER BY ac.target_type, ac.target_content_id, ac.target_language, ac.image_type + LIMIT $1 + ) + SELECT image_type, target_type, target_content_id, target_language, series_id, source_path, + content_type, season_number, episode_number, + COALESCE(tmdb_id, '') AS tmdb_id, + COALESCE(tvdb_id, '') AS tvdb_id, + COALESCE(imdb_id, '') AS imdb_id + FROM candidates + `, limit, intervalLiteral(imageCacheFailedRetryAfter)) + if err != nil { + return 0, fmt.Errorf("enqueueing existing provider artwork: %w", err) + } + defer rows.Close() + + inputs := make([]EnqueueImageCacheJobInput, 0, limit) + for rows.Next() { + var in EnqueueImageCacheJobInput + var tmdbID, tvdbID, imdbID string + if err := rows.Scan( + &in.ImageType, + &in.TargetType, + &in.TargetContentID, + &in.TargetLanguage, + &in.SeriesID, + &in.SourcePath, + &in.ContentType, + &in.SeasonNumber, + &in.EpisodeNumber, + &tmdbID, + &tvdbID, + &imdbID, + ); err != nil { + return 0, fmt.Errorf("scanning existing provider artwork: %w", err) + } + fallbackProvider := imageCachePrimaryProvider(tmdbID, tvdbID, imdbID) + in.ProviderID = imageCacheProviderIDFromSource(in.SourcePath, fallbackProvider) + in.ProviderContentID = imageCacheProviderContentID(in.ProviderID, tmdbID, tvdbID, imdbID, firstNonEmpty(in.SeriesID, in.TargetContentID)) + in.ContentType = imageCacheContentType(in.ContentType) + inputs = append(inputs, in) + } + if err := rows.Err(); err != nil { + return 0, fmt.Errorf("iterating existing provider artwork: %w", err) + } + return r.EnqueueBatch(ctx, inputs) +} + +func imageCacheProviderIDFromSource(sourcePath, fallback string) string { + if provider := providerIDFromPluginURL(sourcePath); provider != "" { + return provider + } + if fallback != "" { + return fallback + } + return "remote" +} + +func imageCachePrimaryProvider(tmdbID, tvdbID, imdbID string) string { + switch { + case strings.TrimSpace(tmdbID) != "": + return "tmdb" + case strings.TrimSpace(tvdbID) != "": + return "tvdb" + case strings.TrimSpace(imdbID) != "": + return "imdb" + default: + return "" + } +} + +func imageCacheProviderContentID(providerID, tmdbID, tvdbID, imdbID, fallback string) string { + switch providerID { + case "tmdb": + return firstNonEmpty(tmdbID, tvdbID, imdbID, fallback) + case "tvdb": + return firstNonEmpty(tvdbID, tmdbID, imdbID, fallback) + case "imdb": + return firstNonEmpty(imdbID, tmdbID, tvdbID, fallback) + default: + return firstNonEmpty(tmdbID, tvdbID, imdbID, fallback) + } +} + +func imageCacheContentType(contentType string) string { + switch strings.TrimSpace(contentType) { + case "movie": + return "movies" + case "audiobook": + return "audiobooks" + case "ebook": + return "ebooks" + default: + return strings.TrimSpace(contentType) + } +} diff --git a/internal/metadata/image_cache_job_repo_test.go b/internal/metadata/image_cache_job_repo_test.go new file mode 100644 index 00000000..17faeaea --- /dev/null +++ b/internal/metadata/image_cache_job_repo_test.go @@ -0,0 +1,93 @@ +package metadata + +import ( + "os" + "strings" + "testing" + "time" +) + +func TestImageCacheRetryDelayCaps(t *testing.T) { + if got := imageCacheRetryDelay(1); got != time.Minute { + t.Fatalf("attempt 1 delay = %s, want 1m", got) + } + if got := imageCacheRetryDelay(20); got != 2*time.Hour { + t.Fatalf("attempt 20 delay = %s, want 2h", got) + } +} + +func TestNormalizeImageCacheJobInputSkipsNonProviderArtwork(t *testing.T) { + for _, sourcePath := range []string{ + "", + "tmdb/series/1396/poster/original.webp", + "s3://media/tmdb/series/1396/poster/original.webp", + "file:///media/poster.jpg", + "local://poster.jpg", + "generated://collections/1/poster.jpg", + } { + if got, ok := normalizeImageCacheJobInput(EnqueueImageCacheJobInput{ + TargetType: ImageCacheTargetItem, + TargetContentID: "series-1", + SourcePath: sourcePath, + ImageType: ImageCacheImagePoster, + }); ok { + t.Fatalf("normalizeImageCacheJobInput(%q) = %#v, want skipped", sourcePath, got) + } + } +} + +func TestNormalizeImageCacheJobInputKeepsLanguageAndDefaultsAttribution(t *testing.T) { + got, ok := normalizeImageCacheJobInput(EnqueueImageCacheJobInput{ + TargetType: ImageCacheTargetItemLocalization, + TargetContentID: "series-1", + TargetLanguage: " fr-CA ", + SeriesID: "series-1", + SourcePath: "https://image.tmdb.org/t/p/original/poster.jpg", + ImageType: ImageCacheImagePoster, + }) + if !ok { + t.Fatal("normalizeImageCacheJobInput skipped remote HTTP source") + } + if got.TargetLanguage != "fr-CA" { + t.Fatalf("TargetLanguage = %q, want fr-CA", got.TargetLanguage) + } + if got.ProviderID != "remote" { + t.Fatalf("ProviderID = %q, want remote for unattributed HTTP source", got.ProviderID) + } + if got.ProviderContentID != "series-1" { + t.Fatalf("ProviderContentID = %q, want series-1", got.ProviderContentID) + } + if got.ContentType != "series" { + t.Fatalf("ContentType = %q, want series", got.ContentType) + } +} + +func TestImageCacheProviderIDFromSourceDoesNotUseURLSchemeAsProvider(t *testing.T) { + if got := imageCacheProviderIDFromSource("https://image.tmdb.org/t/p/original/a.jpg", "tmdb"); got != "tmdb" { + t.Fatalf("provider from HTTP source with fallback = %q, want tmdb", got) + } + if got := imageCacheProviderIDFromSource("https://image.tmdb.org/t/p/original/a.jpg", ""); got != "remote" { + t.Fatalf("provider from HTTP source without fallback = %q, want remote", got) + } + if got := imageCacheProviderIDFromSource("tvdb://banners/poster.jpg", "tmdb"); got != "tvdb" { + t.Fatalf("provider from plugin URL = %q, want tvdb", got) + } +} + +func TestExpandedImageCacheMigrationDefinesTargetMatrixAndLanguageUniqueKey(t *testing.T) { + body, err := os.ReadFile("../../migrations/sql/20260617203000_expand_metadata_image_cache_jobs.sql") + if err != nil { + t.Fatalf("read migration: %v", err) + } + sql := string(body) + for _, want := range []string{ + "ADD COLUMN IF NOT EXISTS target_language text NOT NULL DEFAULT ''", + "target_type IN ('item', 'item_localization', 'season', 'season_localization', 'episode', 'person')", + "image_type IN ('poster', 'backdrop', 'logo', 'still', 'profile')", + "UNIQUE (target_type, target_content_id, image_type, target_language)", + } { + if !strings.Contains(sql, want) { + t.Fatalf("migration missing %q", want) + } + } +} diff --git a/internal/metadata/image_cache_processor.go b/internal/metadata/image_cache_processor.go new file mode 100644 index 00000000..0240a0cc --- /dev/null +++ b/internal/metadata/image_cache_processor.go @@ -0,0 +1,473 @@ +package metadata + +import ( + "context" + "fmt" + "log/slog" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// imageCacheDiscoveryInterval throttles the full-catalog backfill sweep so an +// idle installation does not re-scan every entity table on every task tick. +// Draining of already-queued jobs is unaffected and stays responsive. +const imageCacheDiscoveryInterval = 15 * time.Minute + +type ImageCacheJobClaimer interface { + ClaimDue(ctx context.Context, workerID string, limit int) ([]*models.MetadataImageCacheJob, error) + MarkSucceeded(ctx context.Context, id int64, lockedBy string) error + MarkFailed(ctx context.Context, id int64, attemptCount int, lockedBy string, errText string) error + RequeueClaimed(ctx context.Context, ids []int64, workerID string) error + CurrentTargetSourcePath(ctx context.Context, job *models.MetadataImageCacheJob) (string, error) + EnqueueExistingProviderArtwork(ctx context.Context, limit int) (int, error) + DeleteSucceededBefore(ctx context.Context, before time.Time, limit int) (int, error) +} + +type SeasonArtworkUpdater interface { + UpdateArtworkIfSourceMatches(ctx context.Context, contentID, sourcePath, cachedPath, thumbhash string) (bool, error) +} + +type EpisodeStillUpdater interface { + UpdateStillIfSourceMatches(ctx context.Context, contentID, sourcePath, cachedPath, thumbhash string) (bool, error) +} + +type ItemArtworkUpdater interface { + UpdateArtworkIfSourceMatches(ctx context.Context, contentID, imageType, sourcePath, cachedPath, thumbhash string) (bool, error) +} + +type ItemLocalizationArtworkUpdater interface { + UpdateArtworkIfSourceMatches(ctx context.Context, contentID, language, imageType, sourcePath, cachedPath, thumbhash string) (bool, error) +} + +type SeasonLocalizationArtworkUpdater interface { + UpdateArtworkIfSourceMatches(ctx context.Context, contentID, language, sourcePath, cachedPath, thumbhash string) (bool, error) +} + +type PersonPhotoUpdater interface { + UpdatePhotoIfSourceMatches(ctx context.Context, personID int64, sourcePath, cachedPath, thumbhash string) (bool, error) +} + +type ImageCacheProcessorTargets struct { + Items ItemArtworkUpdater + Seasons SeasonArtworkUpdater + Episodes EpisodeStillUpdater + ItemLocalizations ItemLocalizationArtworkUpdater + SeasonLocalizations SeasonLocalizationArtworkUpdater + People PersonPhotoUpdater +} + +type ImageCacheProcessor struct { + jobs ImageCacheJobClaimer + cacher ImageCacher + resolver interface { + ResolveImageURL(ctx context.Context, path string, variant string) string + } + targets ImageCacheProcessorTargets + logger *slog.Logger + + enabled atomic.Bool + + discoveryInterval time.Duration + discoveryMu sync.Mutex + lastDiscovery time.Time +} + +// SetEnabled toggles background caching. When disabled the processor performs +// no discovery, claiming, or uploading, honoring metadata.cache_images so that +// merely configuring object storage does not download the whole catalog. +func (p *ImageCacheProcessor) SetEnabled(enabled bool) { + if p == nil { + return + } + p.enabled.Store(enabled) +} + +func NewImageCacheProcessor( + jobs ImageCacheJobClaimer, + cacher ImageCacher, + resolver interface { + ResolveImageURL(ctx context.Context, path string, variant string) string + }, + seasons SeasonArtworkUpdater, + episodes EpisodeStillUpdater, +) *ImageCacheProcessor { + return NewImageCacheProcessorWithTargets(jobs, cacher, resolver, ImageCacheProcessorTargets{ + Seasons: seasons, + Episodes: episodes, + }) +} + +func NewImageCacheProcessorWithTargets( + jobs ImageCacheJobClaimer, + cacher ImageCacher, + resolver interface { + ResolveImageURL(ctx context.Context, path string, variant string) string + }, + targets ImageCacheProcessorTargets, +) *ImageCacheProcessor { + p := &ImageCacheProcessor{ + jobs: jobs, + cacher: cacher, + resolver: resolver, + targets: targets, + logger: slog.Default(), + discoveryInterval: imageCacheDiscoveryInterval, + } + // Default to enabled; callers gate on metadata.cache_images via SetEnabled. + p.enabled.Store(true) + return p +} + +type ImageCacheRunStats struct { + Batches int + EnqueuedExisting int + Claimed int + Succeeded int + Failed int + Skipped int + DeletedSucceeded int + RuntimeLimited bool +} + +func (s *ImageCacheRunStats) add(other ImageCacheRunStats) { + s.EnqueuedExisting += other.EnqueuedExisting + s.Claimed += other.Claimed + s.Succeeded += other.Succeeded + s.Failed += other.Failed + s.Skipped += other.Skipped + s.DeletedSucceeded += other.DeletedSucceeded +} + +// RunOnce claims and processes one batch of already-queued jobs. It does not +// run catalog discovery; callers (RunUntilIdle) drive discovery on a throttled +// cadence so backlog draining stays decoupled from full-table sweeps. +func (p *ImageCacheProcessor) RunOnce(ctx context.Context, workerID string, claimLimit int, concurrency int) (ImageCacheRunStats, error) { + var stats ImageCacheRunStats + if p == nil || p.jobs == nil || p.cacher == nil || !p.enabled.Load() { + return stats, nil + } + if claimLimit <= 0 { + claimLimit = 100 + } + if concurrency <= 0 { + concurrency = 4 + } + + jobs, err := p.jobs.ClaimDue(ctx, workerID, claimLimit) + if err != nil { + return stats, err + } + stats.Claimed = len(jobs) + if len(jobs) == 0 { + p.cleanupSucceeded(ctx, &stats) + return stats, nil + } + + sem := make(chan struct{}, concurrency) + var wg sync.WaitGroup + var mu sync.Mutex + var unstarted []int64 +loop: + for i, job := range jobs { + // Acquire the semaphore before spawning so cancellation is observed here + // rather than inside a goroutine that already holds a claimed job. Jobs we + // never start are requeued below instead of being left locked until the + // lease expires. + select { + case sem <- struct{}{}: + case <-ctx.Done(): + for _, rem := range jobs[i:] { + unstarted = append(unstarted, rem.ID) + } + break loop + } + wg.Add(1) + go func(job *models.MetadataImageCacheJob) { + defer wg.Done() + defer func() { <-sem }() + outcome := p.processOne(ctx, job) + mu.Lock() + switch outcome { + case "succeeded": + stats.Succeeded++ + case "skipped": + stats.Skipped++ + default: + stats.Failed++ + } + mu.Unlock() + }(job) + } + wg.Wait() + + if len(unstarted) > 0 { + requeueCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + if err := p.jobs.RequeueClaimed(requeueCtx, unstarted, workerID); err != nil { + p.logger.Warn("metadata image cache: failed to requeue unstarted jobs", "count", len(unstarted), "error", err) + } + cancel() + } + + p.cleanupSucceeded(ctx, &stats) + if ctxErr := ctx.Err(); ctxErr != nil && stats.Claimed == 0 { + return stats, ctxErr + } + return stats, nil +} + +func (p *ImageCacheProcessor) RunUntilIdle(ctx context.Context, workerID string, claimLimit int, concurrency int, maxRuntime time.Duration) (ImageCacheRunStats, error) { + var total ImageCacheRunStats + if p == nil || p.jobs == nil || p.cacher == nil || !p.enabled.Load() { + return total, nil + } + + if maxRuntime <= 0 { + enqueued, derr := p.discoverExisting(ctx, claimLimit) + total.EnqueuedExisting += enqueued + if derr != nil { + return total, derr + } + stats, err := p.RunOnce(ctx, workerID, claimLimit, concurrency) + total.add(stats) + total.Batches = 1 + return total, err + } + + // Decide once per run whether a full-catalog backfill sweep is due. Within a + // due run we keep sweeping until the catalog is exhausted; otherwise we only + // drain the existing queue. + sweep := p.discoveryDue() + deadline := time.Now().Add(maxRuntime) + for { + if err := ctx.Err(); err != nil { + return total, err + } + if !time.Now().Before(deadline) { + total.RuntimeLimited = true + return total, nil + } + + stats, err := p.RunOnce(ctx, workerID, claimLimit, concurrency) + total.Batches++ + total.add(stats) + if err != nil { + return total, err + } + if stats.Claimed > 0 { + // Keep draining the queue before spending a full-table sweep. + continue + } + if !sweep { + return total, nil + } + enqueued, err := p.jobs.EnqueueExistingProviderArtwork(ctx, claimLimit) + if err != nil { + return total, err + } + total.EnqueuedExisting += enqueued + if enqueued == 0 { + // Catalog fully swept; throttle the next sweep. + p.markDiscovered() + return total, nil + } + } +} + +// discoveryDue reports whether enough time has elapsed since the last completed +// sweep to run another one. +func (p *ImageCacheProcessor) discoveryDue() bool { + if p.discoveryInterval <= 0 { + return true + } + p.discoveryMu.Lock() + defer p.discoveryMu.Unlock() + return p.lastDiscovery.IsZero() || time.Since(p.lastDiscovery) >= p.discoveryInterval +} + +func (p *ImageCacheProcessor) markDiscovered() { + p.discoveryMu.Lock() + p.lastDiscovery = time.Now() + p.discoveryMu.Unlock() +} + +// discoverExisting runs an unthrottled sweep (single-pass path) and records the +// time so the throttle applies to subsequent interval-driven runs. +func (p *ImageCacheProcessor) discoverExisting(ctx context.Context, limit int) (int, error) { + enqueued, err := p.jobs.EnqueueExistingProviderArtwork(ctx, limit) + if err == nil { + p.markDiscovered() + } + return enqueued, err +} + +func (p *ImageCacheProcessor) cleanupSucceeded(ctx context.Context, stats *ImageCacheRunStats) { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + deleted, err := p.jobs.DeleteSucceededBefore(cleanupCtx, time.Now().Add(-30*24*time.Hour), 1000) + if err != nil { + p.logger.Warn("metadata image cache: failed to delete old succeeded jobs", "error", err) + } else { + stats.DeletedSucceeded = deleted + } +} + +func terminalJobContext(parent context.Context) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(parent), 10*time.Second) +} + +func (p *ImageCacheProcessor) markFailed(parent context.Context, job *models.MetadataImageCacheJob, errText string) { + writeCtx, cancel := terminalJobContext(parent) + defer cancel() + if err := p.jobs.MarkFailed(writeCtx, job.ID, job.AttemptCount, job.LockedBy, errText); err != nil { + p.logger.Warn("metadata image cache: failed to mark job failed", "job_id", job.ID, "error", err) + } +} + +func (p *ImageCacheProcessor) markSucceeded(parent context.Context, job *models.MetadataImageCacheJob) { + writeCtx, cancel := terminalJobContext(parent) + defer cancel() + if err := p.jobs.MarkSucceeded(writeCtx, job.ID, job.LockedBy); err != nil { + p.logger.Warn("metadata image cache: failed to mark job succeeded", "job_id", job.ID, "error", err) + } +} + +func (p *ImageCacheProcessor) processOne(ctx context.Context, job *models.MetadataImageCacheJob) string { + if job == nil { + return "skipped" + } + imageType, err := imageCacheJobImageType(job.ImageType) + if err != nil { + p.markFailed(ctx, job, err.Error()) + return "failed" + } + + // Confirm the target still references this job's source before uploading. + // CacheImage writes to a deterministic, source-independent storage key, so a + // stale job whose source an admin or newer refresh has already replaced would + // otherwise overwrite the live artwork object even though the conditional DB + // update later no-ops. Dropping the obsolete job here avoids that. + current, err := p.jobs.CurrentTargetSourcePath(ctx, job) + if err != nil { + p.markFailed(ctx, job, err.Error()) + return "failed" + } + if current != job.SourcePath { + p.markSucceeded(ctx, job) + return "skipped" + } + + downloadURL := job.SourcePath + if isProviderImagePath(downloadURL) { + if p.resolver == nil { + p.markFailed(ctx, job, "missing image resolver") + return "failed" + } + downloadURL = p.resolver.ResolveImageURL(ctx, job.SourcePath, "original") + if downloadURL == "" { + p.markFailed(ctx, job, "image resolver returned empty URL") + return "failed" + } + } + + result, err := p.cacher.CacheImage(ctx, CacheImageRequest{ + SourceURL: downloadURL, + ProviderID: job.ProviderID, + ContentType: job.ContentType, + ContentID: job.ProviderContentID, + ImageType: imageType, + SeasonNumber: job.SeasonNumber, + EpisodeNumber: job.EpisodeNumber, + Language: job.TargetLanguage, + }) + if err != nil { + p.markFailed(ctx, job, err.Error()) + return "failed" + } + + if result == nil { + p.markFailed(ctx, job, "image cache returned no result") + return "failed" + } + cachedPath := cachedOriginalImagePath(result.BasePath, result.Ext) + if cachedPath == "" { + p.markFailed(ctx, job, "image cache returned empty stored path") + return "failed" + } + var updated bool + switch job.TargetType { + case ImageCacheTargetItem: + if p.targets.Items == nil { + p.markFailed(ctx, job, "missing item updater") + return "failed" + } + updated, err = p.targets.Items.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.ImageType, job.SourcePath, cachedPath, result.Thumbhash) + case ImageCacheTargetItemLocalization: + if p.targets.ItemLocalizations == nil { + p.markFailed(ctx, job, "missing item localization updater") + return "failed" + } + updated, err = p.targets.ItemLocalizations.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.TargetLanguage, job.ImageType, job.SourcePath, cachedPath, result.Thumbhash) + case ImageCacheTargetSeason: + if p.targets.Seasons == nil { + p.markFailed(ctx, job, "missing season updater") + return "failed" + } + updated, err = p.targets.Seasons.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.SourcePath, cachedPath, result.Thumbhash) + case ImageCacheTargetSeasonLocalization: + if p.targets.SeasonLocalizations == nil { + p.markFailed(ctx, job, "missing season localization updater") + return "failed" + } + updated, err = p.targets.SeasonLocalizations.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.TargetLanguage, job.SourcePath, cachedPath, result.Thumbhash) + case ImageCacheTargetEpisode: + if p.targets.Episodes == nil { + p.markFailed(ctx, job, "missing episode updater") + return "failed" + } + updated, err = p.targets.Episodes.UpdateStillIfSourceMatches(ctx, job.TargetContentID, job.SourcePath, cachedPath, result.Thumbhash) + case ImageCacheTargetPerson: + if p.targets.People == nil { + p.markFailed(ctx, job, "missing person updater") + return "failed" + } + personID, parseErr := strconv.ParseInt(job.TargetContentID, 10, 64) + if parseErr != nil { + err = fmt.Errorf("invalid person image cache target %q: %w", job.TargetContentID, parseErr) + break + } + updated, err = p.targets.People.UpdatePhotoIfSourceMatches(ctx, personID, job.SourcePath, cachedPath, result.Thumbhash) + default: + err = fmt.Errorf("unknown image cache target type %q", job.TargetType) + } + if err != nil { + p.markFailed(ctx, job, err.Error()) + return "failed" + } + if !updated { + p.markSucceeded(ctx, job) + return "skipped" + } + p.markSucceeded(ctx, job) + return "succeeded" +} + +func imageCacheJobImageType(value string) (ImageType, error) { + switch value { + case ImageCacheImagePoster: + return ImagePoster, nil + case ImageCacheImageBackdrop: + return ImageBackdrop, nil + case ImageCacheImageLogo: + return ImageLogo, nil + case ImageCacheImageStill: + return ImageStill, nil + case ImageCacheImageProfile: + return ImageProfile, nil + default: + return ImagePoster, fmt.Errorf("unknown metadata image cache image type %q", value) + } +} diff --git a/internal/metadata/image_cache_processor_test.go b/internal/metadata/image_cache_processor_test.go new file mode 100644 index 00000000..7863e776 --- /dev/null +++ b/internal/metadata/image_cache_processor_test.go @@ -0,0 +1,560 @@ +package metadata + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" +) + +type fakeImageCacheJobs struct { + claimed []*models.MetadataImageCacheJob + succeededID int64 + failedID int64 + failedText string + deletedCount int + requeuedIDs []int64 + currentSource *string // when set, overrides CurrentTargetSourcePath +} + +func (f *fakeImageCacheJobs) ClaimDue(context.Context, string, int) ([]*models.MetadataImageCacheJob, error) { + return f.claimed, nil +} + +func (f *fakeImageCacheJobs) MarkSucceeded(_ context.Context, id int64, _ string) error { + f.succeededID = id + return nil +} + +func (f *fakeImageCacheJobs) MarkFailed(_ context.Context, id int64, _ int, _ string, errText string) error { + f.failedID = id + f.failedText = errText + return nil +} + +func (f *fakeImageCacheJobs) RequeueClaimed(_ context.Context, ids []int64, _ string) error { + f.requeuedIDs = append(f.requeuedIDs, ids...) + return nil +} + +func (f *fakeImageCacheJobs) CurrentTargetSourcePath(_ context.Context, job *models.MetadataImageCacheJob) (string, error) { + if f.currentSource != nil { + return *f.currentSource, nil + } + return job.SourcePath, nil +} + +func (f *fakeImageCacheJobs) EnqueueExistingProviderArtwork(context.Context, int) (int, error) { + return 0, nil +} + +func (f *fakeImageCacheJobs) DeleteSucceededBefore(context.Context, time.Time, int) (int, error) { + return f.deletedCount, nil +} + +type loopingImageCacheJobs struct { + enqueueResults []int + claimedResults [][]*models.MetadataImageCacheJob + succeededIDs []int64 + enqueueCalls int + claimCalls int +} + +func (f *loopingImageCacheJobs) EnqueueExistingProviderArtwork(context.Context, int) (int, error) { + result := 0 + if f.enqueueCalls < len(f.enqueueResults) { + result = f.enqueueResults[f.enqueueCalls] + } + f.enqueueCalls++ + return result, nil +} + +func (f *loopingImageCacheJobs) ClaimDue(context.Context, string, int) ([]*models.MetadataImageCacheJob, error) { + var result []*models.MetadataImageCacheJob + if f.claimCalls < len(f.claimedResults) { + result = f.claimedResults[f.claimCalls] + } + f.claimCalls++ + return result, nil +} + +func (f *loopingImageCacheJobs) MarkSucceeded(_ context.Context, id int64, _ string) error { + f.succeededIDs = append(f.succeededIDs, id) + return nil +} + +func (f *loopingImageCacheJobs) MarkFailed(context.Context, int64, int, string, string) error { + return nil +} + +func (f *loopingImageCacheJobs) RequeueClaimed(context.Context, []int64, string) error { + return nil +} + +func (f *loopingImageCacheJobs) CurrentTargetSourcePath(_ context.Context, job *models.MetadataImageCacheJob) (string, error) { + return job.SourcePath, nil +} + +func (f *loopingImageCacheJobs) DeleteSucceededBefore(context.Context, time.Time, int) (int, error) { + return 0, nil +} + +type fakeImageCacher struct { + result *CacheImageResult + err error + reqs []CacheImageRequest +} + +func (f *fakeImageCacher) CacheImage(_ context.Context, req CacheImageRequest) (*CacheImageResult, error) { + f.reqs = append(f.reqs, req) + if f.err != nil { + return nil, f.err + } + return f.result, nil +} + +type fakeImageResolver struct { + url string +} + +func (f *fakeImageResolver) ResolveImageURL(context.Context, string, string) string { + return f.url +} + +type fakeEpisodeStillUpdater struct { + updated bool + contentID string + sourcePath string + cachedPath string + thumbhash string +} + +func (f *fakeEpisodeStillUpdater) UpdateStillIfSourceMatches(_ context.Context, contentID, sourcePath, cachedPath, thumbhash string) (bool, error) { + f.contentID = contentID + f.sourcePath = sourcePath + f.cachedPath = cachedPath + f.thumbhash = thumbhash + return f.updated, nil +} + +type fakeItemArtworkUpdater struct { + updated bool + contentID string + imageType string + sourcePath string + cachedPath string + thumbhash string +} + +func (f *fakeItemArtworkUpdater) UpdateArtworkIfSourceMatches(_ context.Context, contentID, imageType, sourcePath, cachedPath, thumbhash string) (bool, error) { + f.contentID = contentID + f.imageType = imageType + f.sourcePath = sourcePath + f.cachedPath = cachedPath + f.thumbhash = thumbhash + return f.updated, nil +} + +type fakeItemLocalizationArtworkUpdater struct { + updated bool + contentID string + language string + imageType string + sourcePath string + cachedPath string + thumbhash string +} + +func (f *fakeItemLocalizationArtworkUpdater) UpdateArtworkIfSourceMatches(_ context.Context, contentID, language, imageType, sourcePath, cachedPath, thumbhash string) (bool, error) { + f.contentID = contentID + f.language = language + f.imageType = imageType + f.sourcePath = sourcePath + f.cachedPath = cachedPath + f.thumbhash = thumbhash + return f.updated, nil +} + +type fakePersonPhotoUpdater struct { + updated bool + personID int64 + sourcePath string + cachedPath string + thumbhash string +} + +func (f *fakePersonPhotoUpdater) UpdatePhotoIfSourceMatches(_ context.Context, personID int64, sourcePath, cachedPath, thumbhash string) (bool, error) { + f.personID = personID + f.sourcePath = sourcePath + f.cachedPath = cachedPath + f.thumbhash = thumbhash + return f.updated, nil +} + +func TestImageCacheProcessorUpdatesEpisodeOnSuccess(t *testing.T) { + jobs := &fakeImageCacheJobs{claimed: []*models.MetadataImageCacheJob{{ + ID: 1, + TargetType: ImageCacheTargetEpisode, + TargetContentID: "episode-tvdb-1-1-1", + SourcePath: "tvdb://banners/episode.jpg", + ProviderID: "tvdb", + ProviderContentID: "1", + ContentType: "series", + ImageType: ImageCacheImageStill, + SeasonNumber: intPointer(1), + EpisodeNumber: intPointer(1), + }}} + cacher := &fakeImageCacher{result: &CacheImageResult{ + BasePath: "tvdb/series/1/seasons/1/episodes/1/still", + Ext: ".webp", + Thumbhash: "thumb", + }} + resolver := &fakeImageResolver{url: "https://artworks.thetvdb.com/banners/episode.jpg"} + episodes := &fakeEpisodeStillUpdater{updated: true} + + processor := NewImageCacheProcessor(jobs, cacher, resolver, nil, episodes) + stats, err := processor.RunOnce(context.Background(), "test-worker", 10, 1) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if stats.Succeeded != 1 { + t.Fatalf("Succeeded = %d, want 1", stats.Succeeded) + } + if episodes.cachedPath != "tvdb/series/1/seasons/1/episodes/1/still/original.webp" { + t.Fatalf("cachedPath = %q", episodes.cachedPath) + } + if episodes.sourcePath != "tvdb://banners/episode.jpg" { + t.Fatalf("sourcePath = %q", episodes.sourcePath) + } + if jobs.succeededID != 1 { + t.Fatalf("succeededID = %d", jobs.succeededID) + } +} + +func TestImageCacheProcessorUpdatesItemArtworkOnSuccess(t *testing.T) { + jobs := &fakeImageCacheJobs{claimed: []*models.MetadataImageCacheJob{{ + ID: 20, + TargetType: ImageCacheTargetItem, + TargetContentID: "series-1", + SourcePath: "tmdb://poster/series.jpg", + ProviderID: "tmdb", + ProviderContentID: "1396", + ContentType: "series", + ImageType: ImageCacheImageBackdrop, + }}} + cacher := &fakeImageCacher{result: &CacheImageResult{ + BasePath: "tmdb/series/1396/backdrop", + Ext: ".webp", + Thumbhash: "thumb", + }} + resolver := &fakeImageResolver{url: "https://image.tmdb.org/t/p/original/backdrop.jpg"} + items := &fakeItemArtworkUpdater{updated: true} + + processor := NewImageCacheProcessorWithTargets(jobs, cacher, resolver, ImageCacheProcessorTargets{ + Items: items, + }) + stats, err := processor.RunOnce(context.Background(), "test-worker", 10, 1) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if stats.Succeeded != 1 { + t.Fatalf("Succeeded = %d, want 1", stats.Succeeded) + } + if items.imageType != ImageCacheImageBackdrop { + t.Fatalf("imageType = %q, want backdrop", items.imageType) + } + if items.cachedPath != "tmdb/series/1396/backdrop/original.webp" { + t.Fatalf("cachedPath = %q", items.cachedPath) + } +} + +func TestImageCacheProcessorPassesLanguageToLocalizedItemArtwork(t *testing.T) { + jobs := &fakeImageCacheJobs{claimed: []*models.MetadataImageCacheJob{{ + ID: 21, + TargetType: ImageCacheTargetItemLocalization, + TargetContentID: "series-1", + TargetLanguage: "fr", + SourcePath: "tmdb://logo/fr.png", + ProviderID: "tmdb", + ProviderContentID: "1396", + ContentType: "series", + ImageType: ImageCacheImageLogo, + }}} + cacher := &fakeImageCacher{result: &CacheImageResult{ + BasePath: "tmdb/series/1396/localizations/fr/logo", + Ext: ".webp", + }} + resolver := &fakeImageResolver{url: "https://image.tmdb.org/t/p/original/logo.png"} + localizations := &fakeItemLocalizationArtworkUpdater{updated: true} + + processor := NewImageCacheProcessorWithTargets(jobs, cacher, resolver, ImageCacheProcessorTargets{ + ItemLocalizations: localizations, + }) + stats, err := processor.RunOnce(context.Background(), "test-worker", 10, 1) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if stats.Succeeded != 1 { + t.Fatalf("Succeeded = %d, want 1", stats.Succeeded) + } + if len(cacher.reqs) != 1 || cacher.reqs[0].Language != "fr" { + t.Fatalf("cached request language = %#v, want fr", cacher.reqs) + } + if localizations.language != "fr" || localizations.imageType != ImageCacheImageLogo { + t.Fatalf("localization update = language %q image %q", localizations.language, localizations.imageType) + } +} + +func TestImageCacheProcessorUpdatesPersonProfileOnSuccess(t *testing.T) { + jobs := &fakeImageCacheJobs{claimed: []*models.MetadataImageCacheJob{{ + ID: 22, + TargetType: ImageCacheTargetPerson, + TargetContentID: "287", + SourcePath: "tmdb://profile/287.jpg", + ProviderID: "tmdb", + ProviderContentID: "287", + ContentType: "people", + ImageType: ImageCacheImageProfile, + }}} + cacher := &fakeImageCacher{result: &CacheImageResult{ + BasePath: "tmdb/people/287/profile", + Ext: ".webp", + Thumbhash: "person-thumb", + }} + resolver := &fakeImageResolver{url: "https://image.tmdb.org/t/p/original/person.jpg"} + people := &fakePersonPhotoUpdater{updated: true} + + processor := NewImageCacheProcessorWithTargets(jobs, cacher, resolver, ImageCacheProcessorTargets{ + People: people, + }) + stats, err := processor.RunOnce(context.Background(), "test-worker", 10, 1) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if stats.Succeeded != 1 { + t.Fatalf("Succeeded = %d, want 1", stats.Succeeded) + } + if people.personID != 287 { + t.Fatalf("personID = %d, want 287", people.personID) + } + if people.cachedPath != "tmdb/people/287/profile/original.webp" { + t.Fatalf("cachedPath = %q", people.cachedPath) + } + if people.thumbhash != "person-thumb" { + t.Fatalf("thumbhash = %q", people.thumbhash) + } +} + +func TestImageCacheProcessorMarksSkippedWhenSourceNoLongerMatches(t *testing.T) { + jobs := &fakeImageCacheJobs{claimed: []*models.MetadataImageCacheJob{{ + ID: 2, + TargetType: ImageCacheTargetEpisode, + TargetContentID: "episode-tvdb-1-1-1", + SourcePath: "tvdb://banners/episode.jpg", + ProviderID: "tvdb", + ProviderContentID: "1", + ContentType: "series", + ImageType: ImageCacheImageStill, + SeasonNumber: intPointer(1), + EpisodeNumber: intPointer(1), + }}} + cacher := &fakeImageCacher{result: &CacheImageResult{BasePath: "tvdb/series/1/seasons/1/episodes/1/still", Ext: ".webp"}} + resolver := &fakeImageResolver{url: "https://artworks.thetvdb.com/banners/episode.jpg"} + episodes := &fakeEpisodeStillUpdater{updated: false} + + processor := NewImageCacheProcessor(jobs, cacher, resolver, nil, episodes) + stats, err := processor.RunOnce(context.Background(), "test-worker", 10, 1) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if stats.Skipped != 1 { + t.Fatalf("Skipped = %d, want 1", stats.Skipped) + } + if jobs.succeededID != 2 { + t.Fatalf("succeededID = %d, want 2", jobs.succeededID) + } +} + +func TestImageCacheProcessorMarksFailureOnCacheError(t *testing.T) { + jobs := &fakeImageCacheJobs{claimed: []*models.MetadataImageCacheJob{{ + ID: 3, + TargetType: ImageCacheTargetEpisode, + TargetContentID: "episode-tvdb-1-1-1", + SourcePath: "tvdb://banners/episode.jpg", + ProviderID: "tvdb", + ProviderContentID: "1", + ContentType: "series", + ImageType: ImageCacheImageStill, + SeasonNumber: intPointer(1), + EpisodeNumber: intPointer(1), + }}} + cacher := &fakeImageCacher{err: errors.New("cache failed")} + resolver := &fakeImageResolver{url: "https://artworks.thetvdb.com/banners/episode.jpg"} + episodes := &fakeEpisodeStillUpdater{updated: false} + + processor := NewImageCacheProcessor(jobs, cacher, resolver, nil, episodes) + stats, err := processor.RunOnce(context.Background(), "test-worker", 10, 1) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if stats.Failed != 1 { + t.Fatalf("Failed = %d, want 1", stats.Failed) + } + if jobs.failedID != 3 { + t.Fatalf("failedID = %d, want 3", jobs.failedID) + } +} + +func TestImageCacheProcessorMarksFailureOnEmptyCacheResult(t *testing.T) { + jobs := &fakeImageCacheJobs{claimed: []*models.MetadataImageCacheJob{{ + ID: 4, + TargetType: ImageCacheTargetEpisode, + TargetContentID: "episode-tvdb-1-1-1", + SourcePath: "tvdb://banners/episode.jpg", + ProviderID: "tvdb", + ProviderContentID: "1", + ContentType: "series", + ImageType: ImageCacheImageStill, + SeasonNumber: intPointer(1), + EpisodeNumber: intPointer(1), + }}} + cacher := &fakeImageCacher{result: &CacheImageResult{}} + resolver := &fakeImageResolver{url: "https://artworks.thetvdb.com/banners/episode.jpg"} + episodes := &fakeEpisodeStillUpdater{updated: true} + + processor := NewImageCacheProcessor(jobs, cacher, resolver, nil, episodes) + stats, err := processor.RunOnce(context.Background(), "test-worker", 10, 1) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if stats.Failed != 1 { + t.Fatalf("Failed = %d, want 1", stats.Failed) + } + if jobs.failedID != 4 { + t.Fatalf("failedID = %d, want 4", jobs.failedID) + } + if episodes.cachedPath != "" { + t.Fatalf("episode updater was called with cachedPath = %q", episodes.cachedPath) + } +} + +func TestImageCacheProcessorDeletesOldSucceededJobsWithoutClaimedJobs(t *testing.T) { + jobs := &fakeImageCacheJobs{deletedCount: 7} + processor := NewImageCacheProcessor(jobs, &fakeImageCacher{}, nil, nil, nil) + + stats, err := processor.RunOnce(context.Background(), "test-worker", 10, 1) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if stats.DeletedSucceeded != 7 { + t.Fatalf("DeletedSucceeded = %d, want 7", stats.DeletedSucceeded) + } +} + +func TestImageCacheProcessorRunUntilIdleDrainsNewWorkAddedDuringRun(t *testing.T) { + job1 := &models.MetadataImageCacheJob{ + ID: 10, + TargetType: ImageCacheTargetEpisode, + TargetContentID: "episode-tvdb-1-1-1", + SourcePath: "tvdb://banners/episode-1.jpg", + ProviderID: "tvdb", + ProviderContentID: "1", + ContentType: "series", + ImageType: ImageCacheImageStill, + SeasonNumber: intPointer(1), + EpisodeNumber: intPointer(1), + } + job2 := &models.MetadataImageCacheJob{ + ID: 11, + TargetType: ImageCacheTargetEpisode, + TargetContentID: "episode-tvdb-1-1-2", + SourcePath: "tvdb://banners/episode-2.jpg", + ProviderID: "tvdb", + ProviderContentID: "1", + ContentType: "series", + ImageType: ImageCacheImageStill, + SeasonNumber: intPointer(1), + EpisodeNumber: intPointer(2), + } + // Discovery now runs only when the queue drains, not on every batch: drain + // job1, find the queue empty and sweep (enqueues 1 more), drain job2, find + // the queue empty and sweep again (enqueues 0 -> idle). + jobs := &loopingImageCacheJobs{ + enqueueResults: []int{1, 0}, + claimedResults: [][]*models.MetadataImageCacheJob{ + {job1}, + {}, + {job2}, + {}, + }, + } + cacher := &fakeImageCacher{result: &CacheImageResult{ + BasePath: "tvdb/series/1/seasons/1/episodes/1/still", + Ext: ".webp", + }} + resolver := &fakeImageResolver{url: "https://artworks.thetvdb.com/banners/episode.jpg"} + episodes := &fakeEpisodeStillUpdater{updated: true} + + processor := NewImageCacheProcessor(jobs, cacher, resolver, nil, episodes) + stats, err := processor.RunUntilIdle(context.Background(), "test-worker", 1000, 2, time.Minute) + if err != nil { + t.Fatalf("RunUntilIdle() error = %v", err) + } + if stats.Batches != 4 { + t.Fatalf("Batches = %d, want 4", stats.Batches) + } + if stats.EnqueuedExisting != 1 || stats.Claimed != 2 || stats.Succeeded != 2 { + t.Fatalf("stats = %+v, want enqueued=1 claimed=2 succeeded=2", stats) + } + if jobs.enqueueCalls != 2 || jobs.claimCalls != 4 { + t.Fatalf("calls enqueue=%d claim=%d, want enqueue=2 claim=4", jobs.enqueueCalls, jobs.claimCalls) + } + if len(jobs.succeededIDs) != 2 || jobs.succeededIDs[0] != 10 || jobs.succeededIDs[1] != 11 { + t.Fatalf("succeededIDs = %#v, want [10 11]", jobs.succeededIDs) + } +} + +func TestImageCacheProcessorSkipsWhenTargetSourceChanged(t *testing.T) { + // A stale job whose target no longer references its source must not upload. + changed := "tmdb://poster/new.jpg" + jobs := &fakeImageCacheJobs{ + claimed: []*models.MetadataImageCacheJob{{ + ID: 40, + TargetType: ImageCacheTargetItem, + TargetContentID: "series-1", + SourcePath: "tmdb://poster/old.jpg", + ProviderID: "tmdb", + ProviderContentID: "1396", + ContentType: "series", + ImageType: ImageCacheImagePoster, + }}, + currentSource: &changed, + } + cacher := &fakeImageCacher{result: &CacheImageResult{BasePath: "tmdb/series/1396/poster", Ext: ".webp"}} + resolver := &fakeImageResolver{url: "https://image.tmdb.org/t/p/original/poster.jpg"} + items := &fakeItemArtworkUpdater{updated: true} + + processor := NewImageCacheProcessorWithTargets(jobs, cacher, resolver, ImageCacheProcessorTargets{Items: items}) + stats, err := processor.RunOnce(context.Background(), "test-worker", 10, 1) + if err != nil { + t.Fatalf("RunOnce() error = %v", err) + } + if stats.Skipped != 1 { + t.Fatalf("Skipped = %d, want 1", stats.Skipped) + } + if len(cacher.reqs) != 0 { + t.Fatalf("CacheImage called %d times, want 0 (stale job must not upload)", len(cacher.reqs)) + } + if items.cachedPath != "" { + t.Fatalf("item updater called with cachedPath = %q, want none", items.cachedPath) + } + if jobs.succeededID != 40 { + t.Fatalf("succeededID = %d, want 40", jobs.succeededID) + } +} + +func intPointer(v int) *int { + return &v +} diff --git a/internal/metadata/merge.go b/internal/metadata/merge.go index b06c2e11..589fae6f 100644 --- a/internal/metadata/merge.go +++ b/internal/metadata/merge.go @@ -196,6 +196,7 @@ func MergePersonDetail(source, target *PersonDetailResult, mode MergeMode) { mergeScalar(&target.Birthplace, source.Birthplace, mode) mergeScalar(&target.Homepage, source.Homepage, mode) mergeScalar(&target.PhotoPath, source.PhotoPath, mode) + mergeScalar(&target.PhotoSourcePath, source.PhotoSourcePath, mode) mergeScalar(&target.PhotoThumbhash, source.PhotoThumbhash, mode) mergeProviderIDMap(&target.ProviderIDs, source.ProviderIDs) } @@ -364,6 +365,9 @@ func mergePersonFields(dst *models.ItemPerson, src models.ItemPerson) { if dst.PhotoPath == "" { dst.PhotoPath = src.PhotoPath } + if dst.PhotoSourcePath == "" { + dst.PhotoSourcePath = src.PhotoSourcePath + } if dst.PhotoThumbhash == "" { dst.PhotoThumbhash = src.PhotoThumbhash } diff --git a/internal/metadata/person_refresh.go b/internal/metadata/person_refresh.go index f40c7975..813a8a24 100644 --- a/internal/metadata/person_refresh.go +++ b/internal/metadata/person_refresh.go @@ -32,6 +32,7 @@ type PersonRefreshService struct { pluginResolver pluginMetadataResolver repo personRefreshRepo imageCacher ImageCacher + imageCacheJobs ImageCacheJobEnqueuer imageResolver interface { ResolveImageURL(ctx context.Context, path string, variant string) string } @@ -53,6 +54,10 @@ func (s *PersonRefreshService) SetImageCacher(cacher ImageCacher) { s.imageCacher = cacher } +func (s *PersonRefreshService) SetImageCacheJobEnqueuer(enqueuer ImageCacheJobEnqueuer) { + s.imageCacheJobs = enqueuer +} + func (s *PersonRefreshService) SetImageResolver(resolver interface { ResolveImageURL(ctx context.Context, path string, variant string) string }) { @@ -141,17 +146,15 @@ func (s *PersonRefreshService) refreshPersonWithProviders( return nil, ErrPersonMetadataNotFound } - if cachedPath, thumbhash, err := s.cachePersonPhoto(ctx, *person, accumulator, photoProviderID); err != nil { - slog.Warn("person refresh: photo cache failed", - "person_id", id, - "provider", photoProviderID, - "error", err, + accumulator.PhotoSourcePath = providerImageSourcePath(accumulator.PhotoPath) + if accumulator.PhotoSourcePath != "" { + accumulator.PhotoPath, accumulator.PhotoThumbhash, accumulator.PhotoSourcePath = preserveCachedArtwork( + accumulator.PhotoPath, + accumulator.PhotoThumbhash, + person.PhotoPath, + person.PhotoSourcePath, + person.PhotoThumbhash, ) - } else { - accumulator.PhotoPath = cachedPath - if thumbhash != "" { - accumulator.PhotoThumbhash = thumbhash - } } existingDetail := personToPersonDetailResult(*person) @@ -166,10 +169,45 @@ func (s *PersonRefreshService) refreshPersonWithProviders( if err := s.repo.Update(ctx, refreshed); err != nil { return nil, fmt.Errorf("update person %d: %w", id, err) } + s.enqueuePersonPhoto(ctx, refreshed, accumulator.ProviderIDs, photoProviderID) return &refreshed, nil } +func (s *PersonRefreshService) enqueuePersonPhoto(ctx context.Context, person models.Person, providerIDs map[string]string, photoProviderID string) { + if s == nil || s.imageCacheJobs == nil || !isRemoteImageSourcePath(person.PhotoSourcePath) { + return + } + providerID := providerIDFromPluginURL(person.PhotoSourcePath) + if providerID == "" { + providerID = photoProviderID + } + if providerID == "" { + providerID = primaryPersonProviderID(providerIDs) + } + if providerID == "" { + providerID = "remote" + } + contentID := personCacheContentID(person, providerIDs, providerID) + enqueueCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if err := s.imageCacheJobs.Enqueue(enqueueCtx, EnqueueImageCacheJobInput{ + TargetType: ImageCacheTargetPerson, + TargetContentID: strconv.FormatInt(person.ID, 10), + SourcePath: person.PhotoSourcePath, + ProviderID: providerID, + ProviderContentID: contentID, + ContentType: "people", + ImageType: ImageCacheImageProfile, + }); err != nil { + slog.Warn("person refresh: failed to enqueue photo cache job", + "person_id", person.ID, + "provider", providerID, + "error", err, + ) + } +} + func (s *PersonRefreshService) cachePersonPhoto( ctx context.Context, person models.Person, @@ -239,14 +277,15 @@ func personProviderIDs(person models.Person) map[string]string { func personToPersonDetailResult(person models.Person) PersonDetailResult { result := PersonDetailResult{ - Name: person.Name, - SortName: person.SortName, - Bio: person.Bio, - Birthplace: person.Birthplace, - Homepage: person.Homepage, - PhotoPath: person.PhotoPath, - PhotoThumbhash: person.PhotoThumbhash, - ProviderIDs: copyMap(personProviderIDs(person)), + Name: person.Name, + SortName: person.SortName, + Bio: person.Bio, + Birthplace: person.Birthplace, + Homepage: person.Homepage, + PhotoPath: person.PhotoPath, + PhotoSourcePath: person.PhotoSourcePath, + PhotoThumbhash: person.PhotoThumbhash, + ProviderIDs: copyMap(personProviderIDs(person)), } if person.BirthDate != nil { result.BirthDate = person.BirthDate.Format("2006-01-02") @@ -275,6 +314,7 @@ func mergePersonIntoRecord(person models.Person, detail PersonDetailResult) (mod person.Birthplace = detail.Birthplace person.Homepage = detail.Homepage person.PhotoPath = detail.PhotoPath + person.PhotoSourcePath = detail.PhotoSourcePath person.PhotoThumbhash = detail.PhotoThumbhash person.TmdbID = detail.ProviderIDs["tmdb"] person.ImdbID = detail.ProviderIDs["imdb"] diff --git a/internal/metadata/plugin_provider.go b/internal/metadata/plugin_provider.go index 80be70f4..5c98c17e 100644 --- a/internal/metadata/plugin_provider.go +++ b/internal/metadata/plugin_provider.go @@ -640,6 +640,8 @@ func imageTypeFromKind(kind string) ImageType { return ImageLogo case "still": return ImageStill + case "profile": + return ImageProfile default: return ImagePoster } diff --git a/internal/metadata/service.go b/internal/metadata/service.go index 27b65862..62b22ea1 100644 --- a/internal/metadata/service.go +++ b/internal/metadata/service.go @@ -310,6 +310,7 @@ type MetadataService struct { seriesWork map[string]*seriesEpisodeWork hooks metadataServiceHooks imageCacher ImageCacher + imageCacheJobs ImageCacheJobEnqueuer autoCacheImages atomic.Bool // hot-reloaded from metadata.cache_images imageResolver interface { ResolveImageURL(ctx context.Context, path string, variant string) string @@ -435,6 +436,10 @@ func (s *MetadataService) SetImageCacher(c ImageCacher) { s.imageCacher = c } +func (s *MetadataService) SetImageCacheJobEnqueuer(enqueuer ImageCacheJobEnqueuer) { + s.imageCacheJobs = enqueuer +} + // SetAutoCacheImages controls whether refresh pipelines automatically cache // provider images into object storage. Explicit admin image applies still use // the configured image cacher when available. Safe for concurrent use. @@ -1507,15 +1512,8 @@ func (s *MetadataService) mergeAndPersist( ) } - // Cache images to S3 if enabled. - if s.autoCacheImages.Load() && s.imageCacher != nil && isCanonicalWrite { - s.cacheItemImages(ctx, item, images) - } - // Refreshes that keep the already-cached poster (locked field, no new - // candidates) never pass through cacheItemImages' source capture, so the - // provider-origin path must survive from the existing row. - if item.PosterSourcePath == "" && existingItem != nil && item.PosterPath == existingItem.PosterPath { - item.PosterSourcePath = existingItem.PosterSourcePath + if isCanonicalWrite { + prepareItemImagesForQueue(item, existingItem) } if isNew && contentID == "" { @@ -1549,6 +1547,9 @@ func (s *MetadataService) mergeAndPersist( item.ContentID = contentID unlockProviderDedup() providerDedupReleased = true + if isCanonicalWrite { + s.enqueueItemImages(ctx, item, accumulator.ProviderIDs, images) + } if !isCanonicalWrite && s.itemLocalizationRepo != nil { existingLoc, err := s.itemLocalizationRepo.Get(ctx, contentID, req.Language) @@ -1562,6 +1563,7 @@ func (s *MetadataService) mergeAndPersist( if err := s.itemLocalizationRepo.Upsert(ctx, loc); err != nil { return nil, fmt.Errorf("upserting item localization: %w", err) } + s.enqueueItemLocalizationImages(ctx, item, loc, accumulator.ProviderIDs, images) } // Persist people to the unified people table. @@ -1601,12 +1603,10 @@ func (s *MetadataService) mergeAndPersist( } } - s.cacheSeriesChildImages(ctx, item, accumulator.ProviderIDs, seasons, episodes) - // Persist seasons and episodes for series. if contentType == "series" { if len(seasons) > 0 { - s.persistSeasonsAndEpisodes(ctx, contentID, canonicalLanguage, req.Language, seasons, episodes, mergeMode) + s.persistSeasonsAndEpisodes(ctx, item, accumulator.ProviderIDs, canonicalLanguage, req.Language, seasons, episodes, mergeMode) } else if err := s.SynthesizeFallbackEpisodes(ctx, contentID); err != nil { slog.Warn("metadata: failed to synthesize fallback series structure", "content_id", contentID, "error", err) @@ -2336,8 +2336,7 @@ func (s *MetadataService) refreshSeriesChildTarget( if len(seasons) == 0 && len(episodes) == 0 { continue } - s.cacheSeriesChildImages(ctx, series, providerIDs, seasons, episodes) - s.persistSeasonsAndEpisodes(ctx, seriesID, canonicalLanguage, language, seasons, episodes, mergeMode) + s.persistSeasonsAndEpisodes(ctx, series, providerIDs, canonicalLanguage, language, seasons, episodes, mergeMode) updated = true } if !updated { @@ -2498,101 +2497,27 @@ func (s *MetadataService) fetchTargetEpisodeResults(ctx context.Context, provide return flattenEpisodeResults(episodeResults), nil } -// cacheSeriesChildImages downloads, processes, and uploads season poster -// and episode still images to S3 concurrently (max 6 in flight). Plugin- -// prefixed URLs (e.g. tmdb://, tvdb://) are resolved via s.imageResolver -// before downloading. The provider used to construct each S3 key is -// derived from that image's source URL scheme so the key reflects the -// actual contributing provider (e.g. a TVDB-sourced season poster lands -// under tvdb/series/{tvdbID}/seasons/{n}/poster/...), not whichever -// provider primaryProviderID happens to prefer. Failures are logged and -// the original path is kept. -func (s *MetadataService) cacheSeriesChildImages(ctx context.Context, series *models.MediaItem, providerIDs map[string]string, seasons []SeasonResult, episodes []EpisodeResult) { - if s == nil || !s.autoCacheImages.Load() || s.imageCacher == nil || series == nil { +func (s *MetadataService) enqueueSeriesChildImages(ctx context.Context, seriesID string, inputs []EnqueueImageCacheJobInput) { + s.enqueueImageCacheJobs(ctx, "series child", seriesID, inputs) +} + +func (s *MetadataService) enqueueImageCacheJobs(ctx context.Context, targetKind, targetID string, inputs []EnqueueImageCacheJobInput) { + if s == nil || !s.autoCacheImages.Load() || s.imageCacheJobs == nil || len(inputs) == 0 { return } - fallbackProvider := primaryProviderID(providerIDs) - sem := make(chan struct{}, 6) - var wg sync.WaitGroup - - // keyAttribution returns the (providerID, contentID) pair to use when - // building the S3 key for an image with the given source path. The - // provider is derived from the plugin URL scheme when present and falls - // back to the series-level primary provider for raw HTTP URLs. - keyAttribution := func(sourcePath string) (string, string) { - providerID := providerIDFromPluginURL(sourcePath) - if providerID == "" { - providerID = fallbackProvider - } - return providerID, findContentID(series, providerID) + enqueueCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + if _, err := s.imageCacheJobs.EnqueueBatch(enqueueCtx, inputs); err != nil { + slog.Warn("metadata: failed to enqueue image cache jobs", + "target_kind", targetKind, "target_id", targetID, "count", len(inputs), "error", err) } - - for i := range seasons { - downloadURL, ok := s.resolveImageURLForCache(ctx, seasons[i].PosterPath) - if !ok { - continue - } - providerID, contentID := keyAttribution(seasons[i].PosterPath) - seasonNum := seasons[i].SeasonNumber - wg.Add(1) - go func(idx int, sourceURL, providerID, contentID string, seasonNum int) { - defer wg.Done() - sem <- struct{}{} - defer func() { <-sem }() - result, err := s.imageCacher.CacheImage(ctx, CacheImageRequest{ - SourceURL: sourceURL, - ProviderID: providerID, - ContentType: "series", - ContentID: contentID, - ImageType: ImagePoster, - SeasonNumber: &seasonNum, - }) - if err != nil { - slog.Warn("metadata: season poster cache failed", "season", seasonNum, "error", err) - return - } - seasons[idx].PosterPath = cachedOriginalImagePath(result.BasePath, result.Ext) - seasons[idx].PosterThumbhash = result.Thumbhash - }(i, downloadURL, providerID, contentID, seasonNum) - } - for i := range episodes { - downloadURL, ok := s.resolveImageURLForCache(ctx, episodes[i].StillPath) - if !ok { - continue - } - providerID, contentID := keyAttribution(episodes[i].StillPath) - seasonNum := episodes[i].SeasonNumber - episodeNum := episodes[i].EpisodeNumber - wg.Add(1) - go func(idx int, sourceURL, providerID, contentID string, seasonNum, episodeNum int) { - defer wg.Done() - sem <- struct{}{} - defer func() { <-sem }() - result, err := s.imageCacher.CacheImage(ctx, CacheImageRequest{ - SourceURL: sourceURL, - ProviderID: providerID, - ContentType: "series", - ContentID: contentID, - ImageType: ImageStill, - SeasonNumber: &seasonNum, - EpisodeNumber: &episodeNum, - }) - if err != nil { - slog.Warn("metadata: episode still cache failed", - "season", seasonNum, "episode", episodeNum, "error", err) - return - } - episodes[idx].StillPath = cachedOriginalImagePath(result.BasePath, result.Ext) - episodes[idx].StillThumbhash = result.Thumbhash - }(i, downloadURL, providerID, contentID, seasonNum, episodeNum) - } - wg.Wait() } // providerIDFromPluginURL extracts the plugin slug from a plugin-scheme URL // (e.g. "tvdb://banners/..." -> "tvdb"). Returns "" for HTTP(S) URLs and // any other input lacking a scheme. func providerIDFromPluginURL(url string) string { + url = strings.TrimSpace(url) if url == "" { return "" } @@ -2603,7 +2528,67 @@ func providerIDFromPluginURL(url string) string { if i <= 0 { return "" } - return url[:i] + return strings.ToLower(strings.TrimSpace(url[:i])) +} + +func isProviderImagePath(path string) bool { + path = strings.TrimSpace(path) + lower := strings.ToLower(path) + return path != "" && + strings.Contains(path, "://") && + !strings.HasPrefix(lower, "http://") && + !strings.HasPrefix(lower, "https://") && + !isNonProviderImageScheme(lower) +} + +func isCachedImagePath(path string) bool { + path = strings.TrimSpace(path) + return path != "" && + path != "-" && + !strings.HasPrefix(path, "http://") && + !strings.HasPrefix(path, "https://") && + !isProviderImagePath(path) +} + +func isRemoteImageSourcePath(path string) bool { + path = strings.TrimSpace(path) + lower := strings.ToLower(path) + return path != "" && + path != "-" && + strings.Contains(path, "://") && + !isNonProviderImageScheme(lower) +} + +func isNonProviderImageScheme(lowerPath string) bool { + return strings.HasPrefix(lowerPath, "s3://") || + strings.HasPrefix(lowerPath, "file://") || + strings.HasPrefix(lowerPath, "local://") || + strings.HasPrefix(lowerPath, "upload://") || + strings.HasPrefix(lowerPath, "generated://") +} + +func providerImageSourcePath(path string) string { + path = strings.TrimSpace(path) + if isRemoteImageSourcePath(path) { + return path + } + return "" +} + +func preserveCachedArtwork(providerPath, providerThumbhash, existingCachedPath, existingSourcePath, existingThumbhash string) (string, string, string) { + if !isRemoteImageSourcePath(providerPath) { + if strings.TrimSpace(providerPath) == "" && isCachedImagePath(existingCachedPath) { + return existingCachedPath, existingThumbhash, existingSourcePath + } + return providerPath, providerThumbhash, "" + } + if isCachedImagePath(existingCachedPath) && existingSourcePath == providerPath { + return existingCachedPath, existingThumbhash, providerPath + } + if isCachedImagePath(existingCachedPath) { + return existingCachedPath, existingThumbhash, providerPath + } + return providerPath, providerThumbhash, providerPath } // resolveImageURLForCache normalizes a stored image path into a downloadable @@ -2730,26 +2715,39 @@ func buildItemLocalizationRecord( mergeScalar(&loc.Overview, accumulator.Overview, mergeMode) mergeScalar(&loc.Tagline, accumulator.Tagline, mergeMode) - previousPosterPath := loc.PosterPath - previousPosterThumbhash := loc.PosterThumbhash - previousBackdropPath := loc.BackdropPath - previousBackdropThumbhash := loc.BackdropThumbhash - + existingLocItem := &models.MediaItem{ + Type: contentType, + PosterPath: loc.PosterPath, + PosterSourcePath: loc.PosterSourcePath, + PosterThumbhash: loc.PosterThumbhash, + BackdropPath: loc.BackdropPath, + BackdropSourcePath: loc.BackdropSourcePath, + BackdropThumbhash: loc.BackdropThumbhash, + LogoPath: loc.LogoPath, + LogoSourcePath: loc.LogoSourcePath, + } locItem := &models.MediaItem{ - Type: contentType, - PosterPath: loc.PosterPath, - PosterThumbhash: loc.PosterThumbhash, - BackdropPath: loc.BackdropPath, - BackdropThumbhash: loc.BackdropThumbhash, - LogoPath: loc.LogoPath, + Type: contentType, + PosterPath: loc.PosterPath, + PosterSourcePath: loc.PosterSourcePath, + PosterThumbhash: loc.PosterThumbhash, + BackdropPath: loc.BackdropPath, + BackdropSourcePath: loc.BackdropSourcePath, + BackdropThumbhash: loc.BackdropThumbhash, + LogoPath: loc.LogoPath, + LogoSourcePath: loc.LogoSourcePath, } applyBestImages(locItem, images, mergeMode, preferredLanguage) + prepareItemImagesForQueue(locItem, existingLocItem) loc.PosterPath = locItem.PosterPath - loc.PosterThumbhash = mergedImageThumbhash(previousPosterPath, previousPosterThumbhash, locItem.PosterPath, "") + loc.PosterSourcePath = locItem.PosterSourcePath + loc.PosterThumbhash = locItem.PosterThumbhash loc.BackdropPath = locItem.BackdropPath - loc.BackdropThumbhash = mergedImageThumbhash(previousBackdropPath, previousBackdropThumbhash, locItem.BackdropPath, "") + loc.BackdropSourcePath = locItem.BackdropSourcePath + loc.BackdropThumbhash = locItem.BackdropThumbhash loc.LogoPath = locItem.LogoPath + loc.LogoSourcePath = locItem.LogoSourcePath ApplyDefaultSortTitleToLocalization(loc, titleLocked) @@ -2774,16 +2772,23 @@ func buildSeasonLocalizationRecord( } previousPosterPath := loc.PosterPath + previousPosterSourcePath := loc.PosterSourcePath previousPosterThumbhash := loc.PosterThumbhash mergeScalar(&loc.Title, season.Title, mergeMode) mergeScalar(&loc.Overview, season.Overview, mergeMode) mergeScalar(&loc.PosterPath, season.PosterPath, mergeMode) - loc.PosterThumbhash = mergedImageThumbhash( - previousPosterPath, - previousPosterThumbhash, + if isCachedImagePath(loc.PosterPath) && loc.PosterPath == previousPosterPath { + loc.PosterSourcePath = previousPosterSourcePath + loc.PosterThumbhash = previousPosterThumbhash + return loc + } + loc.PosterPath, loc.PosterThumbhash, loc.PosterSourcePath = preserveCachedArtwork( loc.PosterPath, season.PosterThumbhash, + previousPosterPath, + previousPosterSourcePath, + previousPosterThumbhash, ) return loc @@ -2818,11 +2823,12 @@ func seasonResultFromModel(season *models.Season) SeasonResult { } result := SeasonResult{ - SeasonNumber: season.SeasonNumber, - Title: season.Title, - Overview: season.Overview, - PosterPath: season.PosterPath, - PosterThumbhash: season.PosterThumbhash, + SeasonNumber: season.SeasonNumber, + Title: season.Title, + Overview: season.Overview, + PosterPath: season.PosterPath, + PosterSourcePath: season.PosterSourcePath, + PosterThumbhash: season.PosterThumbhash, } if season.AirDate != nil { result.AirDate = season.AirDate.Format("2006-01-02") @@ -2836,13 +2842,14 @@ func episodeResultFromModel(episode *models.Episode) EpisodeResult { } result := EpisodeResult{ - SeasonNumber: episode.SeasonNumber, - EpisodeNumber: episode.EpisodeNumber, - Title: episode.Title, - Overview: episode.Overview, - Runtime: episode.Runtime, - StillPath: episode.StillPath, - StillThumbhash: episode.StillThumbhash, + SeasonNumber: episode.SeasonNumber, + EpisodeNumber: episode.EpisodeNumber, + Title: episode.Title, + Overview: episode.Overview, + Runtime: episode.Runtime, + StillPath: episode.StillPath, + StillSourcePath: episode.StillSourcePath, + StillThumbhash: episode.StillThumbhash, ProviderIDs: map[string]string{ "imdb": episode.ImdbID, "tmdb": episode.TmdbID, @@ -2889,6 +2896,22 @@ func existingImageThumbhash(item *models.MediaItem, imageType ImageType) string } } +func existingImageSourcePath(item *models.MediaItem, imageType ImageType) string { + if item == nil { + return "" + } + switch imageType { + case ImagePoster: + return item.PosterSourcePath + case ImageBackdrop: + return item.BackdropSourcePath + case ImageLogo: + return item.LogoSourcePath + default: + return "" + } +} + func mergedImageThumbhash(previousPath, previousThumbhash, nextPath, nextThumbhash string) string { if nextThumbhash != "" { return nextThumbhash @@ -2933,15 +2956,86 @@ func (s *MetadataService) SearchProviders(ctx context.Context, query SearchQuery // persistSeasonsAndEpisodes creates/updates seasons and episodes in the DB. func (s *MetadataService) persistSeasonsAndEpisodes( ctx context.Context, - seriesID string, + series *models.MediaItem, + providerIDs map[string]string, canonicalLanguage string, language string, seasons []SeasonResult, episodes []EpisodeResult, mergeMode MergeMode, ) { + if series == nil || strings.TrimSpace(series.ContentID) == "" { + return + } + seriesID := series.ContentID seasonIDs := make(map[int]string, len(seasons)) isCanonicalWrite := strings.EqualFold(canonicalLanguage, language) + imageJobs := make([]EnqueueImageCacheJobInput, 0, len(seasons)+len(episodes)) + fallbackProvider := primaryProviderID(providerIDs) + keyAttribution := func(sourcePath string) (string, string) { + providerID := providerIDFromPluginURL(sourcePath) + if providerID == "" { + providerID = fallbackProvider + } + return providerID, findContentID(series, providerID) + } + addSeasonImageJob := func(season *models.Season) { + if season == nil || !isRemoteImageSourcePath(season.PosterSourcePath) { + return + } + providerID, providerContentID := keyAttribution(season.PosterSourcePath) + seasonNumber := season.SeasonNumber + imageJobs = append(imageJobs, EnqueueImageCacheJobInput{ + TargetType: ImageCacheTargetSeason, + TargetContentID: season.ContentID, + SeriesID: seriesID, + SourcePath: season.PosterSourcePath, + ProviderID: providerID, + ProviderContentID: providerContentID, + ContentType: "series", + ImageType: ImageCacheImagePoster, + SeasonNumber: &seasonNumber, + }) + } + addEpisodeImageJob := func(episode *models.Episode) { + if episode == nil || !isRemoteImageSourcePath(episode.StillSourcePath) { + return + } + providerID, providerContentID := keyAttribution(episode.StillSourcePath) + seasonNumber := episode.SeasonNumber + episodeNumber := episode.EpisodeNumber + imageJobs = append(imageJobs, EnqueueImageCacheJobInput{ + TargetType: ImageCacheTargetEpisode, + TargetContentID: episode.ContentID, + SeriesID: seriesID, + SourcePath: episode.StillSourcePath, + ProviderID: providerID, + ProviderContentID: providerContentID, + ContentType: "series", + ImageType: ImageCacheImageStill, + SeasonNumber: &seasonNumber, + EpisodeNumber: &episodeNumber, + }) + } + addSeasonLocalizationImageJob := func(season *models.Season, loc *models.SeasonLocalization) { + if season == nil || loc == nil || !isRemoteImageSourcePath(loc.PosterSourcePath) { + return + } + providerID, providerContentID := keyAttribution(loc.PosterSourcePath) + seasonNumber := season.SeasonNumber + imageJobs = append(imageJobs, EnqueueImageCacheJobInput{ + TargetType: ImageCacheTargetSeasonLocalization, + TargetContentID: season.ContentID, + TargetLanguage: loc.Language, + SeriesID: seriesID, + SourcePath: loc.PosterSourcePath, + ProviderID: providerID, + ProviderContentID: providerContentID, + ContentType: "series", + ImageType: ImageCacheImagePoster, + SeasonNumber: &seasonNumber, + }) + } // Phase 1: Upsert explicit seasons. if len(seasons) > 0 { @@ -2953,15 +3047,20 @@ func (s *MetadataService) persistSeasonsAndEpisodes( continue } providerSeason := season + providerSeason.PosterSourcePath = providerImageSourcePath(providerSeason.PosterPath) if existingSeason != nil && isCanonicalWrite { mergedSeason := seasonResultFromModel(existingSeason) MergeSeasonResult(&providerSeason, &mergedSeason, mergeMode) - mergedSeason.PosterThumbhash = mergedImageThumbhash( - existingSeason.PosterPath, - existingSeason.PosterThumbhash, - mergedSeason.PosterPath, + nextPath, nextThumbhash, nextSourcePath := preserveCachedArtwork( + providerSeason.PosterPath, providerSeason.PosterThumbhash, + existingSeason.PosterPath, + existingSeason.PosterSourcePath, + existingSeason.PosterThumbhash, ) + mergedSeason.PosterPath = nextPath + mergedSeason.PosterThumbhash = nextThumbhash + mergedSeason.PosterSourcePath = nextSourcePath providerSeason = mergedSeason } dbSeason := &models.Season{ @@ -2971,6 +3070,7 @@ func (s *MetadataService) persistSeasonsAndEpisodes( DefaultMetadataLanguage: canonicalLanguage, Overview: providerSeason.Overview, PosterPath: providerSeason.PosterPath, + PosterSourcePath: providerSeason.PosterSourcePath, PosterThumbhash: providerSeason.PosterThumbhash, MetadataSource: "provider", } @@ -2980,6 +3080,7 @@ func (s *MetadataService) persistSeasonsAndEpisodes( dbSeason.Title = existingSeason.Title dbSeason.Overview = existingSeason.Overview dbSeason.PosterPath = existingSeason.PosterPath + dbSeason.PosterSourcePath = existingSeason.PosterSourcePath dbSeason.PosterThumbhash = existingSeason.PosterThumbhash dbSeason.DefaultMetadataLanguage = existingSeason.DefaultMetadataLanguage } @@ -3003,21 +3104,25 @@ func (s *MetadataService) persistSeasonsAndEpisodes( continue } seasonIDs[dbSeason.SeasonNumber] = dbSeason.ContentID + addSeasonImageJob(dbSeason) if !isCanonicalWrite && s.seasonLocalizationRepo != nil { existingLoc, locErr := s.seasonLocalizationRepo.Get(ctx, dbSeason.ContentID, language) if locErr != nil { slog.Warn("metadata: failed to load season localization", "series_id", seriesID, "season", season.SeasonNumber, "error", locErr) } - if err := s.seasonLocalizationRepo.Upsert(ctx, buildSeasonLocalizationRecord( + loc := buildSeasonLocalizationRecord( existingLoc, dbSeason.ContentID, language, providerSeason, mergeMode, - )); err != nil { + ) + if err := s.seasonLocalizationRepo.Upsert(ctx, loc); err != nil { slog.Warn("metadata: failed to upsert season localization", "series_id", seriesID, "season", season.SeasonNumber, "error", err) + } else { + addSeasonLocalizationImageJob(dbSeason, loc) } } } @@ -3061,11 +3166,13 @@ func (s *MetadataService) persistSeasonsAndEpisodes( seasonModel.Title = mergedSeason.Title seasonModel.Overview = mergedSeason.Overview seasonModel.PosterPath = mergedSeason.PosterPath + seasonModel.PosterSourcePath = mergedSeason.PosterSourcePath seasonModel.PosterThumbhash = mergedSeason.PosterThumbhash } else { seasonModel.Title = existingSeason.Title seasonModel.Overview = existingSeason.Overview seasonModel.PosterPath = existingSeason.PosterPath + seasonModel.PosterSourcePath = existingSeason.PosterSourcePath seasonModel.PosterThumbhash = existingSeason.PosterThumbhash } } else { @@ -3083,6 +3190,7 @@ func (s *MetadataService) persistSeasonsAndEpisodes( continue } seasonIDs[seasonModel.SeasonNumber] = seasonModel.ContentID + addSeasonImageJob(seasonModel) } } @@ -3096,15 +3204,20 @@ func (s *MetadataService) persistSeasonsAndEpisodes( continue } providerEpisode := ep + providerEpisode.StillSourcePath = providerImageSourcePath(providerEpisode.StillPath) if existingEpisode != nil && isCanonicalWrite { mergedEpisode := episodeResultFromModel(existingEpisode) MergeEpisodeResult(&providerEpisode, &mergedEpisode, mergeMode) - mergedEpisode.StillThumbhash = mergedImageThumbhash( - existingEpisode.StillPath, - existingEpisode.StillThumbhash, - mergedEpisode.StillPath, + nextPath, nextThumbhash, nextSourcePath := preserveCachedArtwork( + providerEpisode.StillPath, providerEpisode.StillThumbhash, + existingEpisode.StillPath, + existingEpisode.StillSourcePath, + existingEpisode.StillThumbhash, ) + mergedEpisode.StillPath = nextPath + mergedEpisode.StillThumbhash = nextThumbhash + mergedEpisode.StillSourcePath = nextSourcePath providerEpisode = mergedEpisode } dbEp := &models.Episode{ @@ -3120,6 +3233,7 @@ func (s *MetadataService) persistSeasonsAndEpisodes( TmdbID: providerEpisode.ProviderIDs["tmdb"], TvdbID: providerEpisode.ProviderIDs["tvdb"], StillPath: providerEpisode.StillPath, + StillSourcePath: providerEpisode.StillSourcePath, StillThumbhash: providerEpisode.StillThumbhash, MetadataSource: "provider", } @@ -3130,6 +3244,7 @@ func (s *MetadataService) persistSeasonsAndEpisodes( dbEp.Overview = existingEpisode.Overview dbEp.DefaultMetadataLanguage = existingEpisode.DefaultMetadataLanguage dbEp.StillPath = existingEpisode.StillPath + dbEp.StillSourcePath = existingEpisode.StillSourcePath dbEp.StillThumbhash = existingEpisode.StillThumbhash } } else { @@ -3161,6 +3276,7 @@ func (s *MetadataService) persistSeasonsAndEpisodes( "episode", ep.EpisodeNumber, "error", err) continue } + addEpisodeImageJob(dbEp) if !isCanonicalWrite && s.episodeLocalizationRepo != nil { existingLoc, locErr := s.episodeLocalizationRepo.Get(ctx, dbEp.ContentID, language) if locErr != nil { @@ -3181,8 +3297,11 @@ func (s *MetadataService) persistSeasonsAndEpisodes( } } } + } + s.enqueueSeriesChildImages(ctx, seriesID, imageJobs) + if err := s.ensureSeriesEpisodeLinks(ctx, seriesID); err != nil { slog.Warn("metadata: failed to ensure series episode links", "series_id", seriesID, "error", err) @@ -5166,6 +5285,131 @@ func applyBestImages(item *models.MediaItem, images []RemoteImage, mode MergeMod applyIfBetter(&item.LogoPath, bestByType[ImageLogo]) } +type itemArtworkField struct { + imageType ImageType + path *string + source *string + thumbhash *string +} + +func itemArtworkFields(item *models.MediaItem) []itemArtworkField { + if item == nil { + return nil + } + return []itemArtworkField{ + {imageType: ImagePoster, path: &item.PosterPath, source: &item.PosterSourcePath, thumbhash: &item.PosterThumbhash}, + {imageType: ImageBackdrop, path: &item.BackdropPath, source: &item.BackdropSourcePath, thumbhash: &item.BackdropThumbhash}, + {imageType: ImageLogo, path: &item.LogoPath, source: &item.LogoSourcePath}, + } +} + +func prepareItemImagesForQueue(item, existing *models.MediaItem) { + for _, field := range itemArtworkFields(item) { + existingPath := existingImagePath(existing, field.imageType) + existingThumbhash := existingImageThumbhash(existing, field.imageType) + existingSource := existingImageSourcePath(existing, field.imageType) + currentThumbhash := "" + if field.thumbhash != nil { + currentThumbhash = *field.thumbhash + } + if isCachedImagePath(*field.path) && *field.path == existingPath && existingSource != "" { + *field.source = existingSource + if field.thumbhash != nil && currentThumbhash == "" { + *field.thumbhash = existingThumbhash + } + continue + } + nextPath, nextThumbhash, nextSource := preserveCachedArtwork( + *field.path, + currentThumbhash, + existingPath, + existingSource, + existingThumbhash, + ) + *field.path = nextPath + *field.source = nextSource + if field.thumbhash != nil { + *field.thumbhash = nextThumbhash + } + } +} + +func (s *MetadataService) enqueueItemImages(ctx context.Context, item *models.MediaItem, providerIDs map[string]string, images []RemoteImage) { + if s == nil || !s.autoCacheImages.Load() || s.imageCacheJobs == nil || item == nil || item.ContentID == "" { + return + } + inputs := make([]EnqueueImageCacheJobInput, 0, 3) + for _, field := range itemArtworkFields(item) { + sourcePath := strings.TrimSpace(*field.source) + if !isRemoteImageSourcePath(sourcePath) { + continue + } + providerID, providerContentID := itemImageCacheAttribution(item, providerIDs, images, sourcePath) + inputs = append(inputs, EnqueueImageCacheJobInput{ + TargetType: ImageCacheTargetItem, + TargetContentID: item.ContentID, + SeriesID: item.ContentID, + SourcePath: sourcePath, + ProviderID: providerID, + ProviderContentID: providerContentID, + ContentType: imageCacheContentType(item.Type), + ImageType: ImageTypeToString(field.imageType), + }) + } + s.enqueueImageCacheJobs(ctx, "item", item.ContentID, inputs) +} + +func (s *MetadataService) enqueueItemLocalizationImages(ctx context.Context, item *models.MediaItem, loc *models.MediaItemLocalization, providerIDs map[string]string, images []RemoteImage) { + if s == nil || !s.autoCacheImages.Load() || s.imageCacheJobs == nil || item == nil || loc == nil || loc.ContentID == "" || loc.Language == "" { + return + } + locItem := &models.MediaItem{ + ContentID: loc.ContentID, + Type: item.Type, + PosterSourcePath: loc.PosterSourcePath, + BackdropSourcePath: loc.BackdropSourcePath, + LogoSourcePath: loc.LogoSourcePath, + } + inputs := make([]EnqueueImageCacheJobInput, 0, 3) + for _, field := range itemArtworkFields(locItem) { + sourcePath := strings.TrimSpace(*field.source) + if !isRemoteImageSourcePath(sourcePath) { + continue + } + providerID, providerContentID := itemImageCacheAttribution(item, providerIDs, images, sourcePath) + inputs = append(inputs, EnqueueImageCacheJobInput{ + TargetType: ImageCacheTargetItemLocalization, + TargetContentID: loc.ContentID, + TargetLanguage: loc.Language, + SeriesID: item.ContentID, + SourcePath: sourcePath, + ProviderID: providerID, + ProviderContentID: providerContentID, + ContentType: imageCacheContentType(item.Type), + ImageType: ImageTypeToString(field.imageType), + }) + } + s.enqueueImageCacheJobs(ctx, "item localization", loc.ContentID, inputs) +} + +func itemImageCacheAttribution(item *models.MediaItem, providerIDs map[string]string, images []RemoteImage, sourcePath string) (string, string) { + providerID := providerIDFromPluginURL(sourcePath) + if providerID == "" { + providerID = findProviderID(images, sourcePath) + } + if providerID == "" { + providerID = primaryProviderID(providerIDs) + } + if providerID == "" { + providerID = "remote" + } + providerContentID := findContentID(item, providerID) + if providerContentID == "" && item != nil { + providerContentID = item.ContentID + } + return providerID, providerContentID +} + // cacheItemImages downloads, processes, and uploads item images to S3 // concurrently. Failures are logged and the original CDN URL is preserved. func (s *MetadataService) cacheItemImages(ctx context.Context, item *models.MediaItem, images []RemoteImage) { @@ -5428,6 +5672,8 @@ func ImageTypeToString(t ImageType) string { return "logo" case ImageStill: return "still" + case ImageProfile: + return "profile" default: return "poster" } @@ -5442,6 +5688,8 @@ func ImageTypeFromString(s string) ImageType { return ImageLogo case "still": return ImageStill + case "profile": + return ImageProfile default: return ImagePoster } diff --git a/internal/metadata/types.go b/internal/metadata/types.go index a5f05d4d..faada3ca 100644 --- a/internal/metadata/types.go +++ b/internal/metadata/types.go @@ -135,16 +135,17 @@ type PersonDetailRequest struct { // PersonDetailResult carries person-level metadata from a provider. type PersonDetailResult struct { - Name string - SortName string - Bio string - BirthDate string - DeathDate string - Birthplace string - Homepage string - PhotoPath string - PhotoThumbhash string - ProviderIDs map[string]string + Name string + SortName string + Bio string + BirthDate string + DeathDate string + Birthplace string + Homepage string + PhotoPath string + PhotoSourcePath string + PhotoThumbhash string + ProviderIDs map[string]string } // MetadataResult carries structured metadata from a single provider. @@ -219,6 +220,7 @@ const ( ImageBackdrop ImageLogo ImageStill // Episode stills + ImageProfile ) // CacheImageRequest describes an image to be cached. For season posters @@ -233,6 +235,7 @@ type CacheImageRequest struct { ImageType ImageType SeasonNumber *int EpisodeNumber *int + Language string } // CacheImageResult is returned by ImageCacher on success. @@ -247,6 +250,11 @@ type ImageCacher interface { CacheImage(ctx context.Context, req CacheImageRequest) (*CacheImageResult, error) } +type ImageCacheJobEnqueuer interface { + Enqueue(ctx context.Context, in EnqueueImageCacheJobInput) error + EnqueueBatch(ctx context.Context, inputs []EnqueueImageCacheJobInput) (int, error) +} + // SeasonsRequest is passed to EpisodeProvider.GetSeasons(). type SeasonsRequest struct { ProviderIDs map[string]string @@ -263,27 +271,29 @@ type EpisodesRequest struct { // SeasonResult carries season data from a provider. type SeasonResult struct { - ContentID string - SeasonNumber int - Title string - Overview string - AirDate string - PosterPath string - PosterThumbhash string - Episodes []EpisodeResult + ContentID string + SeasonNumber int + Title string + Overview string + AirDate string + PosterPath string + PosterSourcePath string + PosterThumbhash string + Episodes []EpisodeResult } // EpisodeResult carries episode data from a provider. type EpisodeResult struct { - ContentID string - ProviderIDs map[string]string - SeasonNumber int - EpisodeNumber int - Title string - Overview string - AirDate string - Runtime int - Ratings Ratings - StillPath string - StillThumbhash string + ContentID string + ProviderIDs map[string]string + SeasonNumber int + EpisodeNumber int + Title string + Overview string + AirDate string + Runtime int + Ratings Ratings + StillPath string + StillSourcePath string + StillThumbhash string } diff --git a/internal/models/media.go b/internal/models/media.go index b211697f..b441825a 100644 --- a/internal/models/media.go +++ b/internal/models/media.go @@ -262,22 +262,23 @@ func PersonKindFromJob(job string) PersonKind { // Person represents a deduplicated person entity. type Person struct { - ID int64 - Name string - SortName string - Bio string - BirthDate *time.Time - DeathDate *time.Time - Birthplace string - Homepage string - PhotoPath string - PhotoThumbhash string - TmdbID string - ImdbID string - TvdbID string - PlexGUID string - CreatedAt time.Time - UpdatedAt time.Time + ID int64 + Name string + SortName string + Bio string + BirthDate *time.Time + DeathDate *time.Time + Birthplace string + Homepage string + PhotoPath string + PhotoSourcePath string + PhotoThumbhash string + TmdbID string + ImdbID string + TvdbID string + PlexGUID string + CreatedAt time.Time + UpdatedAt time.Time } // ItemPerson represents a person's credit on a specific media item. @@ -320,8 +321,10 @@ type MediaItem struct { PosterSourcePath string // provider-origin path kept when caching rewrites PosterPath; feeds outbound embeds PosterThumbhash string BackdropPath string + BackdropSourcePath string BackdropThumbhash string LogoPath string + LogoSourcePath string MetadataS3Path string MetadataEtag string SeasonCount *int // series only @@ -362,6 +365,7 @@ type Season struct { Overview string AirDate *time.Time PosterPath string + PosterSourcePath string PosterThumbhash string MetadataS3Path string MetadataEtag string @@ -388,6 +392,7 @@ type Episode struct { TmdbID string TvdbID string StillPath string + StillSourcePath string StillThumbhash string MetadataS3Path string MetadataEtag string @@ -441,33 +446,37 @@ const ( ) type MediaItemLocalization struct { - ContentID string - Language string - Title string - SortTitle string - Overview string - Tagline string - PosterPath string - PosterThumbhash string - BackdropPath string - BackdropThumbhash string - LogoPath string - OverviewSource string // provider | ai | manual - TaglineSource string // provider | ai | manual - CreatedAt time.Time - UpdatedAt time.Time + ContentID string + Language string + Title string + SortTitle string + Overview string + Tagline string + PosterPath string + PosterSourcePath string + PosterThumbhash string + BackdropPath string + BackdropSourcePath string + BackdropThumbhash string + LogoPath string + LogoSourcePath string + OverviewSource string // provider | ai | manual + TaglineSource string // provider | ai | manual + CreatedAt time.Time + UpdatedAt time.Time } type SeasonLocalization struct { - SeasonContentID string - Language string - Title string - Overview string - PosterPath string - PosterThumbhash string - OverviewSource string // provider | ai | manual - CreatedAt time.Time - UpdatedAt time.Time + SeasonContentID string + Language string + Title string + Overview string + PosterPath string + PosterSourcePath string + PosterThumbhash string + OverviewSource string // provider | ai | manual + CreatedAt time.Time + UpdatedAt time.Time } type EpisodeLocalization struct { diff --git a/internal/models/metadata_image_cache_job.go b/internal/models/metadata_image_cache_job.go new file mode 100644 index 00000000..fe292e31 --- /dev/null +++ b/internal/models/metadata_image_cache_job.go @@ -0,0 +1,27 @@ +package models + +import "time" + +type MetadataImageCacheJob struct { + ID int64 + TargetType string + TargetContentID string + TargetLanguage string + SeriesID string + SourcePath string + ProviderID string + ProviderContentID string + ContentType string + ImageType string + SeasonNumber *int + EpisodeNumber *int + Status string + AttemptCount int + NextAttemptAt time.Time + LockedAt *time.Time + LockedBy string + LastError string + CreatedAt time.Time + UpdatedAt time.Time + CompletedAt *time.Time +} diff --git a/internal/taskmanager/tasks/cache_metadata_images.go b/internal/taskmanager/tasks/cache_metadata_images.go new file mode 100644 index 00000000..95704f26 --- /dev/null +++ b/internal/taskmanager/tasks/cache_metadata_images.go @@ -0,0 +1,77 @@ +package tasks + +import ( + "context" + "fmt" + "os" + "time" + + "github.com/Silo-Server/silo-server/internal/metadata" + "github.com/Silo-Server/silo-server/internal/taskmanager" +) + +const ( + cacheMetadataImagesIntervalMs = int64(60 * 1000) + cacheMetadataImagesBatchSize = 1000 + cacheMetadataImagesWorkers = 12 + cacheMetadataImagesMaxRuntime = 10 * time.Minute +) + +type MetadataImageCacheRunner interface { + RunUntilIdle(ctx context.Context, workerID string, claimLimit int, concurrency int, maxRuntime time.Duration) (metadata.ImageCacheRunStats, error) +} + +type CacheMetadataImagesTask struct { + runner MetadataImageCacheRunner +} + +func NewCacheMetadataImagesTask(runner MetadataImageCacheRunner) *CacheMetadataImagesTask { + return &CacheMetadataImagesTask{runner: runner} +} + +func (t *CacheMetadataImagesTask) Key() string { return "cache_metadata_images" } +func (t *CacheMetadataImagesTask) Name() string { return "Cache Metadata Images" } +func (t *CacheMetadataImagesTask) Description() string { + return "Caches provider metadata artwork into object storage" +} +func (t *CacheMetadataImagesTask) Category() taskmanager.TaskCategory { + return taskmanager.TaskCategoryMetadata +} +func (t *CacheMetadataImagesTask) IsHidden() bool { return false } + +func (t *CacheMetadataImagesTask) DefaultTriggers() []taskmanager.TriggerConfig { + return []taskmanager.TriggerConfig{ + {Type: taskmanager.TriggerTypeStartup}, + {Type: taskmanager.TriggerTypeInterval, IntervalMs: cacheMetadataImagesIntervalMs}, + } +} + +func (t *CacheMetadataImagesTask) Execute(ctx context.Context, progress taskmanager.ProgressReporter) error { + if t.runner == nil { + progress.Report(100, "Metadata image cache is not configured") + return nil + } + hostname, _ := os.Hostname() + if hostname == "" { + hostname = "silo" + } + stats, err := t.runner.RunUntilIdle(ctx, hostname, cacheMetadataImagesBatchSize, cacheMetadataImagesWorkers, cacheMetadataImagesMaxRuntime) + if err != nil { + return fmt.Errorf("caching metadata images: %w", err) + } + message := fmt.Sprintf( + "Batches %d, enqueued %d existing, claimed %d, cached %d, failed %d, skipped %d, deleted %d old successes", + stats.Batches, + stats.EnqueuedExisting, + stats.Claimed, + stats.Succeeded, + stats.Failed, + stats.Skipped, + stats.DeletedSucceeded, + ) + if stats.RuntimeLimited { + message += ", runtime budget reached" + } + progress.Report(100, message) + return nil +} diff --git a/internal/taskmanager/tasks/cache_metadata_images_test.go b/internal/taskmanager/tasks/cache_metadata_images_test.go new file mode 100644 index 00000000..a8f9c393 --- /dev/null +++ b/internal/taskmanager/tasks/cache_metadata_images_test.go @@ -0,0 +1,78 @@ +package tasks + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/metadata" + "github.com/Silo-Server/silo-server/internal/taskmanager" +) + +type fakeMetadataImageCacheRunner struct { + stats metadata.ImageCacheRunStats + err error + claimLimit int + concurrency int + maxRuntime time.Duration +} + +func (f *fakeMetadataImageCacheRunner) RunUntilIdle(_ context.Context, _ string, claimLimit int, concurrency int, maxRuntime time.Duration) (metadata.ImageCacheRunStats, error) { + f.claimLimit = claimLimit + f.concurrency = concurrency + f.maxRuntime = maxRuntime + return f.stats, f.err +} + +type recordingProgress struct { + message string +} + +func (r *recordingProgress) Report(_ float64, message string) { + r.message = message +} + +func (r *recordingProgress) SetResultData(json.RawMessage) {} + +func TestCacheMetadataImagesTaskProperties(t *testing.T) { + task := NewCacheMetadataImagesTask(&fakeMetadataImageCacheRunner{}) + if task.Key() != "cache_metadata_images" { + t.Fatalf("Key() = %q", task.Key()) + } + if task.Category() != taskmanager.TaskCategoryMetadata { + t.Fatalf("Category() = %q", task.Category()) + } + if len(task.DefaultTriggers()) != 2 { + t.Fatalf("DefaultTriggers count = %d, want 2", len(task.DefaultTriggers())) + } +} + +func TestCacheMetadataImagesTaskReportsStats(t *testing.T) { + runner := &fakeMetadataImageCacheRunner{ + stats: metadata.ImageCacheRunStats{ + Batches: 3, + EnqueuedExisting: 5, + Claimed: 4, + Succeeded: 3, + Failed: 1, + }, + } + task := NewCacheMetadataImagesTask(runner) + progress := &recordingProgress{} + if err := task.Execute(context.Background(), progress); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if runner.claimLimit != 1000 { + t.Fatalf("claimLimit = %d, want 1000", runner.claimLimit) + } + if runner.concurrency != 12 { + t.Fatalf("concurrency = %d, want 12", runner.concurrency) + } + if runner.maxRuntime != 10*time.Minute { + t.Fatalf("maxRuntime = %s, want 10m", runner.maxRuntime) + } + if progress.message != "Batches 3, enqueued 5 existing, claimed 4, cached 3, failed 1, skipped 0, deleted 0 old successes" { + t.Fatalf("progress message = %q", progress.message) + } +} diff --git a/internal/taskmanager/triggers/daily.go b/internal/taskmanager/triggers/daily.go index 5951f0b2..315b00d4 100644 --- a/internal/taskmanager/triggers/daily.go +++ b/internal/taskmanager/triggers/daily.go @@ -49,21 +49,23 @@ func (d *DailyTrigger) Start(_ *taskmanager.ExecutionResult) { default: } - d.stopCh = make(chan struct{}) + stopCh := make(chan struct{}) d.nextRun = d.calcNextRun(time.Now()) - d.timer = time.NewTimer(time.Until(d.nextRun)) + timer := time.NewTimer(time.Until(d.nextRun)) + d.stopCh = stopCh + d.timer = timer go func() { select { - case <-d.stopCh: - if !d.timer.Stop() { + case <-stopCh: + if !timer.Stop() { select { - case <-d.timer.C: + case <-timer.C: default: } } return - case <-d.timer.C: + case <-timer.C: select { case d.ch <- struct{}{}: default: diff --git a/internal/taskmanager/triggers/interval.go b/internal/taskmanager/triggers/interval.go index 7f48e688..f3f1168b 100644 --- a/internal/taskmanager/triggers/interval.go +++ b/internal/taskmanager/triggers/interval.go @@ -38,8 +38,6 @@ func (t *IntervalTrigger) Start(lastResult *taskmanager.ExecutionResult) { default: } - t.stopCh = make(chan struct{}) - var base time.Time if lastResult != nil && !lastResult.CompletedAt.IsZero() { base = lastResult.CompletedAt @@ -54,19 +52,22 @@ func (t *IntervalTrigger) Start(lastResult *taskmanager.ExecutionResult) { t.nextRun = time.Now() } - t.timer = time.NewTimer(delay) + stopCh := make(chan struct{}) + timer := time.NewTimer(delay) + t.stopCh = stopCh + t.timer = timer go func() { select { - case <-t.stopCh: - if !t.timer.Stop() { + case <-stopCh: + if !timer.Stop() { select { - case <-t.timer.C: + case <-timer.C: default: } } return - case <-t.timer.C: + case <-timer.C: select { case t.ch <- struct{}{}: default: diff --git a/internal/taskmanager/triggers/interval_test.go b/internal/taskmanager/triggers/interval_test.go new file mode 100644 index 00000000..3bbf13ae --- /dev/null +++ b/internal/taskmanager/triggers/interval_test.go @@ -0,0 +1,25 @@ +package triggers + +import ( + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/taskmanager" +) + +func TestIntervalTriggerFiresAfterStopStart(t *testing.T) { + trigger := NewIntervalTrigger(taskmanager.TriggerConfig{ + Type: taskmanager.TriggerTypeInterval, + IntervalMs: 10, + }) + + trigger.Start(nil) + trigger.Stop() + trigger.Start(&taskmanager.ExecutionResult{CompletedAt: time.Now()}) + + select { + case <-trigger.C(): + case <-time.After(time.Second): + t.Fatal("interval trigger did not fire after stop/start") + } +} diff --git a/internal/taskmanager/triggers/startup.go b/internal/taskmanager/triggers/startup.go index ee2ad3f6..2362a781 100644 --- a/internal/taskmanager/triggers/startup.go +++ b/internal/taskmanager/triggers/startup.go @@ -46,21 +46,23 @@ func (s *StartupTrigger) Start(_ *taskmanager.ExecutionResult) { } s.fired = true - s.stopCh = make(chan struct{}) + stopCh := make(chan struct{}) s.nextRun = time.Now().Add(s.delay) - s.timer = time.NewTimer(s.delay) + timer := time.NewTimer(s.delay) + s.stopCh = stopCh + s.timer = timer go func() { select { - case <-s.stopCh: - if !s.timer.Stop() { + case <-stopCh: + if !timer.Stop() { select { - case <-s.timer.C: + case <-timer.C: default: } } return - case <-s.timer.C: + case <-timer.C: s.mu.Lock() s.nextRun = time.Time{} s.mu.Unlock() diff --git a/internal/taskmanager/triggers/weekly.go b/internal/taskmanager/triggers/weekly.go index 9e2eafb6..41e318b3 100644 --- a/internal/taskmanager/triggers/weekly.go +++ b/internal/taskmanager/triggers/weekly.go @@ -52,21 +52,23 @@ func (w *WeeklyTrigger) Start(_ *taskmanager.ExecutionResult) { default: } - w.stopCh = make(chan struct{}) + stopCh := make(chan struct{}) w.nextRun = w.calcNextRun(time.Now()) - w.timer = time.NewTimer(time.Until(w.nextRun)) + timer := time.NewTimer(time.Until(w.nextRun)) + w.stopCh = stopCh + w.timer = timer go func() { select { - case <-w.stopCh: - if !w.timer.Stop() { + case <-stopCh: + if !timer.Stop() { select { - case <-w.timer.C: + case <-timer.C: default: } } return - case <-w.timer.C: + case <-timer.C: select { case w.ch <- struct{}{}: default: diff --git a/migrations/sql/20260617184537_metadata_image_cache_queue.sql b/migrations/sql/20260617184537_metadata_image_cache_queue.sql new file mode 100644 index 00000000..65ca2554 --- /dev/null +++ b/migrations/sql/20260617184537_metadata_image_cache_queue.sql @@ -0,0 +1,87 @@ +-- +goose Up +ALTER TABLE public.seasons + ADD COLUMN IF NOT EXISTS poster_source_path text NOT NULL DEFAULT ''; + +ALTER TABLE public.episodes + ADD COLUMN IF NOT EXISTS still_source_path text NOT NULL DEFAULT ''; + +UPDATE public.seasons +SET poster_source_path = poster_path +WHERE poster_source_path = '' + AND poster_path LIKE '%://%' + AND lower(poster_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']); + +UPDATE public.episodes +SET still_source_path = still_path +WHERE still_source_path = '' + AND still_path LIKE '%://%' + AND lower(still_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']); + +CREATE TABLE public.metadata_image_cache_jobs ( + id bigserial PRIMARY KEY, + target_type text NOT NULL, + target_content_id text NOT NULL, + series_id text NOT NULL, + source_path text NOT NULL, + provider_id text NOT NULL, + provider_content_id text NOT NULL, + content_type text NOT NULL DEFAULT 'series', + image_type text NOT NULL, + season_number integer, + episode_number integer, + status text NOT NULL DEFAULT 'queued', + attempt_count integer NOT NULL DEFAULT 0, + next_attempt_at timestamp with time zone NOT NULL DEFAULT now(), + locked_at timestamp with time zone, + locked_by text NOT NULL DEFAULT '', + last_error text NOT NULL DEFAULT '', + created_at timestamp with time zone NOT NULL DEFAULT now(), + updated_at timestamp with time zone NOT NULL DEFAULT now(), + completed_at timestamp with time zone, + CONSTRAINT metadata_image_cache_jobs_target_check + CHECK (target_type IN ('season', 'episode')), + CONSTRAINT metadata_image_cache_jobs_image_type_check + CHECK (image_type IN ('poster', 'still')), + CONSTRAINT metadata_image_cache_jobs_status_check + CHECK (status IN ('queued', 'running', 'succeeded', 'failed')), + CONSTRAINT metadata_image_cache_jobs_shape_check CHECK ( + ( + target_type = 'season' + AND image_type = 'poster' + AND season_number IS NOT NULL + AND episode_number IS NULL + ) + OR ( + target_type = 'episode' + AND image_type = 'still' + AND season_number IS NOT NULL + AND episode_number IS NOT NULL + ) + ), + CONSTRAINT metadata_image_cache_jobs_target_unique + UNIQUE (target_type, target_content_id, image_type) +); + +CREATE INDEX metadata_image_cache_jobs_due_idx + ON public.metadata_image_cache_jobs (next_attempt_at, id) + WHERE status = 'queued'; + +CREATE INDEX metadata_image_cache_jobs_running_lease_idx + ON public.metadata_image_cache_jobs (locked_at, id) + WHERE status = 'running'; + +CREATE INDEX metadata_image_cache_jobs_series_idx + ON public.metadata_image_cache_jobs (series_id, status); + +CREATE INDEX metadata_image_cache_jobs_succeeded_retention_idx + ON public.metadata_image_cache_jobs (completed_at, id) + WHERE status = 'succeeded'; + +-- +goose Down +DROP TABLE IF EXISTS public.metadata_image_cache_jobs; + +ALTER TABLE public.episodes + DROP COLUMN IF EXISTS still_source_path; + +ALTER TABLE public.seasons + DROP COLUMN IF EXISTS poster_source_path; diff --git a/migrations/sql/20260617203000_expand_metadata_image_cache_jobs.sql b/migrations/sql/20260617203000_expand_metadata_image_cache_jobs.sql new file mode 100644 index 00000000..470c924e --- /dev/null +++ b/migrations/sql/20260617203000_expand_metadata_image_cache_jobs.sql @@ -0,0 +1,184 @@ +-- +goose Up +ALTER TABLE public.media_items + ALTER COLUMN poster_source_path SET DEFAULT ''; + +UPDATE public.media_items +SET poster_source_path = '' +WHERE poster_source_path IS NULL; + +ALTER TABLE public.media_items + ALTER COLUMN poster_source_path SET NOT NULL, + ADD COLUMN IF NOT EXISTS backdrop_source_path text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS logo_source_path text NOT NULL DEFAULT ''; + +ALTER TABLE public.people + ADD COLUMN IF NOT EXISTS photo_source_path text NOT NULL DEFAULT ''; + +ALTER TABLE public.media_item_localizations + ADD COLUMN IF NOT EXISTS poster_source_path text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS backdrop_source_path text NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS logo_source_path text NOT NULL DEFAULT ''; + +ALTER TABLE public.season_localizations + ADD COLUMN IF NOT EXISTS poster_source_path text NOT NULL DEFAULT ''; + +UPDATE public.media_items +SET poster_source_path = poster_path +WHERE poster_source_path = '' + AND poster_path LIKE '%://%' + AND lower(poster_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']); + +UPDATE public.media_items +SET backdrop_source_path = backdrop_path +WHERE backdrop_source_path = '' + AND backdrop_path LIKE '%://%' + AND lower(backdrop_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']); + +UPDATE public.media_items +SET logo_source_path = logo_path +WHERE logo_source_path = '' + AND logo_path LIKE '%://%' + AND lower(logo_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']); + +UPDATE public.people +SET photo_source_path = photo_path +WHERE photo_source_path = '' + AND photo_path LIKE '%://%' + AND lower(photo_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']); + +UPDATE public.media_item_localizations +SET poster_source_path = poster_path +WHERE poster_source_path = '' + AND poster_path LIKE '%://%' + AND lower(poster_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']); + +UPDATE public.media_item_localizations +SET backdrop_source_path = backdrop_path +WHERE backdrop_source_path = '' + AND backdrop_path LIKE '%://%' + AND lower(backdrop_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']); + +UPDATE public.media_item_localizations +SET logo_source_path = logo_path +WHERE logo_source_path = '' + AND logo_path LIKE '%://%' + AND lower(logo_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']); + +UPDATE public.season_localizations +SET poster_source_path = poster_path +WHERE poster_source_path = '' + AND poster_path LIKE '%://%' + AND lower(poster_path) NOT LIKE ALL (ARRAY['s3://%', 'file://%', 'local://%', 'upload://%', 'generated://%']); + +ALTER TABLE public.metadata_image_cache_jobs + ADD COLUMN IF NOT EXISTS target_language text NOT NULL DEFAULT ''; + +ALTER TABLE public.metadata_image_cache_jobs + DROP CONSTRAINT IF EXISTS metadata_image_cache_jobs_target_unique, + DROP CONSTRAINT IF EXISTS metadata_image_cache_jobs_target_check, + DROP CONSTRAINT IF EXISTS metadata_image_cache_jobs_image_type_check, + DROP CONSTRAINT IF EXISTS metadata_image_cache_jobs_shape_check; + +ALTER TABLE public.metadata_image_cache_jobs + ADD CONSTRAINT metadata_image_cache_jobs_target_check + CHECK (target_type IN ('item', 'item_localization', 'season', 'season_localization', 'episode', 'person')), + ADD CONSTRAINT metadata_image_cache_jobs_image_type_check + CHECK (image_type IN ('poster', 'backdrop', 'logo', 'still', 'profile')), + ADD CONSTRAINT metadata_image_cache_jobs_shape_check CHECK ( + ( + target_type = 'item' + AND image_type IN ('poster', 'backdrop', 'logo') + AND target_language = '' + AND season_number IS NULL + AND episode_number IS NULL + ) + OR ( + target_type = 'item_localization' + AND image_type IN ('poster', 'backdrop', 'logo') + AND target_language <> '' + AND season_number IS NULL + AND episode_number IS NULL + ) + OR ( + target_type = 'season' + AND image_type = 'poster' + AND target_language = '' + AND season_number IS NOT NULL + AND episode_number IS NULL + ) + OR ( + target_type = 'season_localization' + AND image_type = 'poster' + AND target_language <> '' + AND season_number IS NOT NULL + AND episode_number IS NULL + ) + OR ( + target_type = 'episode' + AND image_type = 'still' + AND target_language = '' + AND season_number IS NOT NULL + AND episode_number IS NOT NULL + ) + OR ( + target_type = 'person' + AND image_type = 'profile' + AND target_language = '' + AND season_number IS NULL + AND episode_number IS NULL + ) + ), + ADD CONSTRAINT metadata_image_cache_jobs_target_unique + UNIQUE (target_type, target_content_id, image_type, target_language); + +-- +goose Down +DELETE FROM public.metadata_image_cache_jobs +WHERE target_type NOT IN ('season', 'episode') + OR image_type NOT IN ('poster', 'still') + OR target_language <> ''; + +ALTER TABLE public.metadata_image_cache_jobs + DROP CONSTRAINT IF EXISTS metadata_image_cache_jobs_target_unique, + DROP CONSTRAINT IF EXISTS metadata_image_cache_jobs_target_check, + DROP CONSTRAINT IF EXISTS metadata_image_cache_jobs_image_type_check, + DROP CONSTRAINT IF EXISTS metadata_image_cache_jobs_shape_check; + +ALTER TABLE public.metadata_image_cache_jobs + ADD CONSTRAINT metadata_image_cache_jobs_target_check + CHECK (target_type IN ('season', 'episode')), + ADD CONSTRAINT metadata_image_cache_jobs_image_type_check + CHECK (image_type IN ('poster', 'still')), + ADD CONSTRAINT metadata_image_cache_jobs_shape_check CHECK ( + ( + target_type = 'season' + AND image_type = 'poster' + AND season_number IS NOT NULL + AND episode_number IS NULL + ) + OR ( + target_type = 'episode' + AND image_type = 'still' + AND season_number IS NOT NULL + AND episode_number IS NOT NULL + ) + ), + ADD CONSTRAINT metadata_image_cache_jobs_target_unique + UNIQUE (target_type, target_content_id, image_type); + +ALTER TABLE public.metadata_image_cache_jobs + DROP COLUMN IF EXISTS target_language; + +ALTER TABLE public.season_localizations + DROP COLUMN IF EXISTS poster_source_path; + +ALTER TABLE public.media_item_localizations + DROP COLUMN IF EXISTS logo_source_path, + DROP COLUMN IF EXISTS backdrop_source_path, + DROP COLUMN IF EXISTS poster_source_path; + +ALTER TABLE public.people + DROP COLUMN IF EXISTS photo_source_path; + +ALTER TABLE public.media_items + DROP COLUMN IF EXISTS logo_source_path, + DROP COLUMN IF EXISTS backdrop_source_path;