Files
silo-server/internal/catalog/episode_catalog_source.go
T
14ffc91dfb [codex] Expand provider image cache queue (#176)
* feat(metadata): expand provider image cache queue

* fix(metadata): harden provider image cache queue

Addresses bug-review feedback from Codex/CodeRabbit on the metadata image
cache pipeline. All findings validated against the code before fixing;
false positives (rows/connection deadlock, PhotoSourcePath merge coupling)
were confirmed non-issues and left unchanged.

- Honor metadata.cache_images for the background processor. The
  cache_metadata_images task was registered whenever S3 was configured,
  so merely enabling object storage downloaded the entire provider-artwork
  catalog even with caching disabled. Add ImageCacheProcessor.SetEnabled,
  gate RunOnce/RunUntilIdle on it, and wire it (with hot reload) from
  cfg.Metadata.CacheImages in main.go.
- Guard terminal job updates with lease ownership. EnqueueBatch can
  repurpose a running row with a new source; MarkSucceeded/MarkFailed
  keyed on id alone let a stale worker finalize the replacement job and
  drop the new artwork. Thread locked_by through and add
  status='running' AND locked_by=$n guards.
- Avoid uploading stale jobs onto the live artwork key. Verify the
  target still references the job's source (CurrentTargetSourcePath)
  before CacheImage, so a job whose source an admin/refresh already
  replaced cannot overwrite the deterministic storage object.
- COALESCE nullable external IDs in EnqueueExistingProviderArtwork. A
  NULL tmdb_id/tvdb_id/imdb_id on any candidate failed the scan and
  aborted the whole cache run; matches the existing item_repo pattern.
- Stop re-downloading the catalog every 30 days. Discovery now skips
  targets whose *_path is already a cached relative path, making the
  cached row the durable dedup marker instead of the prunable job row.
- Decouple catalog sweeps from queue draining. RunOnce no longer runs
  discovery per batch; RunUntilIdle sweeps only when the queue drains and
  throttles full sweeps to every 15m, so idle installs stop full-scanning
  every entity table each minute.
- Requeue claimed-but-unstarted jobs on cancellation. Acquire the
  semaphore before spawning workers and RequeueClaimed any jobs not yet
  started, instead of leaving them locked until the 15m lease expires.
- Skip the backoff sleep after the final upload attempt in
  putObjectWithRetry (saves ~1.5s on permanent failures).
- Add the s3/file/local/upload/generated exclusion to the seasons and
  episodes backfill in migration 20260617184537 for consistency with the
  later migration (the bad backfill was inert downstream, but the
  asymmetry is removed).

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-18 10:07:58 -04:00

161 lines
5.0 KiB
Go

package catalog
import (
"fmt"
"strings"
)
const episodeCatalogSelectBody = `(
SELECT
e.content_id,
e.air_date AS episode_air_date,
'episode'::text AS type,
COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text) AS title,
COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text) AS sort_title,
LOWER(COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text)) AS sort_key,
COALESCE(NULLIF(BTRIM(e.default_metadata_language), ''), COALESCE(si.default_metadata_language, '')) AS default_metadata_language,
''::text AS original_title,
COALESCE(si.year, EXTRACT(YEAR FROM e.air_date)::integer, 0) AS year,
COALESCE(si.genres, '{}'::text[]) AS genres,
COALESCE(si.content_rating, '') AS content_rating,
COALESCE(NULLIF(e.runtime, 0), COALESCE(si.runtime, 0)) AS runtime,
COALESCE(e.overview, '') AS overview,
''::text AS tagline,
e.rating_imdb,
e.rating_tmdb,
NULL::integer AS rating_rt_critic,
NULL::integer AS rating_rt_audience,
COALESCE(e.imdb_id, '') AS imdb_id,
COALESCE(e.tmdb_id, '') AS tmdb_id,
COALESCE(e.tvdb_id, '') AS tvdb_id,
COALESCE(NULLIF(s.poster_path, ''), NULLIF(si.poster_path, ''), NULLIF(e.still_path, ''), '') AS poster_path,
''::text AS poster_source_path,
COALESCE(NULLIF(s.poster_thumbhash, ''), NULLIF(si.poster_thumbhash, ''), NULLIF(e.still_thumbhash, ''), '') AS poster_thumbhash,
COALESCE(si.backdrop_path, '') AS backdrop_path,
COALESCE(si.backdrop_source_path, '') AS backdrop_source_path,
COALESCE(si.backdrop_thumbhash, '') AS backdrop_thumbhash,
COALESCE(si.logo_path, '') AS logo_path,
COALESCE(si.logo_source_path, '') AS logo_source_path,
COALESCE(e.metadata_s3_path, '') AS metadata_s3_path,
COALESCE(e.metadata_etag, '') AS metadata_etag,
NULL::integer AS season_count,
COALESCE(si.studios, '{}'::text[]) AS studios,
COALESCE(si.networks, '{}'::text[]) AS networks,
COALESCE(si.countries, '{}'::text[]) AS countries,
COALESCE(si.keywords, '{}'::text[]) AS keywords,
COALESCE(si.original_language, '') AS original_language,
CASE WHEN e.air_date IS NULL THEN NULL ELSE e.air_date::text END AS release_date,
NULL::text AS first_air_date,
NULL::text AS last_air_date,
e.air_date AS last_air_date_at,
si.air_time,
si.air_timezone,
COALESCE(si.show_status, '') AS show_status,
si.matched_at,
si.last_refreshed,
si.refresh_failures,
si.episode_metadata_incomplete,
si.episode_metadata_last_checked_at,
COALESCE(si.locked_fields, '{}'::integer[]) AS locked_fields,
COALESCE(NULLIF(BTRIM(si.status), ''), 'matched') AS status,
e.created_at,
e.updated_at
FROM episodes e
JOIN media_items si ON si.content_id = e.series_id
LEFT JOIN seasons s ON s.content_id = e.season_id
WHERE %s
) mi`
const episodeCatalogActiveLibraryExists = `EXISTS (
SELECT 1
FROM episode_libraries el
WHERE el.episode_id = e.content_id
)`
var episodeCatalogBaseRelation = fmt.Sprintf(episodeCatalogSelectBody, episodeCatalogActiveLibraryExists)
func isEpisodeCatalogScope(scope string) bool {
return scope == "episode"
}
func catalogBaseRelationForScope(scope string) string {
if isEpisodeCatalogScope(scope) {
return episodeCatalogBaseRelation
}
return "media_items mi"
}
func episodeCatalogBaseRelationForLibraries(
allowedLibraryIDs []int,
disabledLibraryIDs []int,
argIdx int,
) (string, []any, bool) {
if len(allowedLibraryIDs) == 0 && len(disabledLibraryIDs) == 0 {
return episodeCatalogBaseRelation, nil, false
}
libraryConditions := []string{
"el.episode_id = e.content_id",
}
args := make([]any, 0, len(allowedLibraryIDs)+len(disabledLibraryIDs))
if len(allowedLibraryIDs) > 0 {
libraryConditions = append(
libraryConditions,
fmt.Sprintf("el.media_folder_id = ANY($%d)", argIdx),
)
args = append(args, allowedLibraryIDs)
argIdx++
}
if len(disabledLibraryIDs) > 0 {
libraryConditions = append(
libraryConditions,
fmt.Sprintf("NOT (el.media_folder_id = ANY($%d))", argIdx),
)
args = append(args, disabledLibraryIDs)
argIdx++
}
relation := fmt.Sprintf(
episodeCatalogSelectBody,
fmt.Sprintf(
`EXISTS (
SELECT 1
FROM episode_libraries el
WHERE %s
)`,
strings.Join(libraryConditions, "\n\t\t AND "),
),
)
return relation, args, true
}
func catalogLibraryContentExprForScope(scope, alias string) string {
if isEpisodeCatalogScope(scope) {
return alias + ".content_id"
}
return alias + ".content_id"
}
func catalogLibraryMembershipTableAndKeyForScope(scope string) (string, string) {
if isEpisodeCatalogScope(scope) {
return "episode_libraries", "episode_id"
}
return "media_item_libraries", "content_id"
}
func catalogMediaFileJoinConditionForScope(scope, mediaFileAlias, itemAlias string) string {
if isEpisodeCatalogScope(scope) {
return mediaFileAlias + ".episode_id = " + itemAlias + ".content_id"
}
return mediaFileAlias + ".content_id = " + itemAlias + ".content_id"
}
func catalogMediaFileGroupExprForScope(scope, mediaFileAlias string) string {
if isEpisodeCatalogScope(scope) {
return mediaFileAlias + ".episode_id"
}
return mediaFileAlias + ".content_id"
}