From 84b3318ca1e4fa0fdd9bcfc44d24c7186b1d3f60 Mon Sep 17 00:00:00 2001 From: Silo Server Migration Date: Sun, 24 May 2026 22:27:18 -0400 Subject: [PATCH] feat(catalog): lookup media presence by external ids --- internal/catalog/item_repo.go | 143 ++++++++++++++++++---- internal/catalog/item_repo_test.go | 20 +++ internal/catalog/provider_id_repo.go | 48 ++++++++ internal/catalog/provider_id_repo_test.go | 14 +++ 4 files changed, 202 insertions(+), 23 deletions(-) diff --git a/internal/catalog/item_repo.go b/internal/catalog/item_repo.go index f398c006..2b48e0a7 100644 --- a/internal/catalog/item_repo.go +++ b/internal/catalog/item_repo.go @@ -1330,6 +1330,20 @@ type MediaTMDBRow struct { Title string } +type ExternalIDLookupCandidate struct { + TMDBID string + TVDBID string + IMDbID string +} + +type ExternalIDMatchRow struct { + QueryTMDBID string + MediaID string + MatchedProvider string + LibraryID string + Title string +} + // LookupTMDBIDs returns one row per matching media item that has its tmdb_id // in the supplied list and is linked to at least one enabled library. // mediaType is "movie" or "series" (silo's internal naming). @@ -1341,38 +1355,121 @@ func (r *ItemRepository) LookupTMDBIDs(ctx context.Context, mediaType string, tm if len(tmdbIDs) == 0 { return nil, nil } - // Convert string IDs to int: TMDB IDs are stored as text in media_items - // but the plugin sends them as strings. We do an ANY match directly on - // the text column so no int conversion is required. - rows, err := r.pool.Query(ctx, ` - SELECT DISTINCT ON (mi.content_id) - mi.content_id, - COALESCE(mi.tmdb_id, ''), - mil.media_folder_id::text, - mi.title - FROM media_items mi - JOIN media_item_libraries mil ON mil.content_id = mi.content_id - JOIN media_folders mf ON mf.id = mil.media_folder_id - WHERE mi.tmdb_id = ANY($1) - AND mi.type = $2 - AND mf.enabled = true - ORDER BY mi.content_id, mil.media_folder_id ASC - `, tmdbIDs, mediaType) + candidates := make([]ExternalIDLookupCandidate, 0, len(tmdbIDs)) + for _, id := range tmdbIDs { + if strings.TrimSpace(id) != "" { + candidates = append(candidates, ExternalIDLookupCandidate{TMDBID: id}) + } + } + rows, err := r.LookupExternalIDs(ctx, mediaType, candidates) if err != nil { - return nil, fmt.Errorf("lookup tmdb ids: %w", err) + return nil, err + } + out := make([]MediaTMDBRow, 0, len(rows)) + for _, row := range rows { + out = append(out, MediaTMDBRow{ + MediaID: row.MediaID, + TMDBID: row.QueryTMDBID, + LibraryID: row.LibraryID, + Title: row.Title, + }) + } + return out, nil +} + +func lookupExternalIDsSQL() string { + return ` + WITH requested(query_tmdb_id, provider, provider_id, ord) AS ( + SELECT * FROM unnest($1::text[], $2::text[], $3::text[], $4::int[]) + ), + direct_matches AS ( + SELECT r.query_tmdb_id, mi.content_id, r.provider, mil.media_folder_id::text, mi.title, r.ord, + CASE r.provider WHEN 'tmdb' THEN 0 WHEN 'tvdb' THEN 1 WHEN 'imdb' THEN 2 ELSE 3 END AS provider_rank + FROM requested r + JOIN media_items mi + ON mi.type = $5 + AND ( + (r.provider = 'tmdb' AND COALESCE(mi.tmdb_id, '') = r.provider_id) + OR (r.provider = 'tvdb' AND COALESCE(mi.tvdb_id, '') = r.provider_id) + OR (r.provider = 'imdb' AND COALESCE(mi.imdb_id, '') = r.provider_id) + ) + JOIN media_item_libraries mil ON mil.content_id = mi.content_id + JOIN media_folders mf ON mf.id = mil.media_folder_id + WHERE mf.enabled = true + ), + provider_matches AS ( + SELECT r.query_tmdb_id, mi.content_id, r.provider, mil.media_folder_id::text, mi.title, r.ord, + CASE r.provider WHEN 'tmdb' THEN 0 WHEN 'tvdb' THEN 1 WHEN 'imdb' THEN 2 ELSE 3 END AS provider_rank + FROM requested r + JOIN media_item_provider_ids mip + ON mip.provider = r.provider + AND mip.provider_id = r.provider_id + AND mip.item_type = $5 + JOIN media_items mi ON mi.content_id = mip.content_id AND mi.type = $5 + JOIN media_item_libraries mil ON mil.content_id = mi.content_id + JOIN media_folders mf ON mf.id = mil.media_folder_id + WHERE mf.enabled = true + ) + SELECT DISTINCT ON (query_tmdb_id) + query_tmdb_id, content_id, provider, media_folder_id, title + FROM ( + SELECT * FROM direct_matches + UNION ALL + SELECT * FROM provider_matches + ) matches + ORDER BY query_tmdb_id, provider_rank ASC, ord ASC, content_id ASC, media_folder_id ASC` +} + +func (r *ItemRepository) LookupExternalIDs( + ctx context.Context, + mediaType string, + candidates []ExternalIDLookupCandidate, +) ([]ExternalIDMatchRow, error) { + if len(candidates) == 0 { + return nil, nil + } + + queryTMDBIDs := make([]string, 0, len(candidates)*3) + providers := make([]string, 0, len(candidates)*3) + providerIDs := make([]string, 0, len(candidates)*3) + ordinals := make([]int32, 0, len(candidates)*3) + + appendID := func(candidate ExternalIDLookupCandidate, provider, providerID string, ordinal int) { + providerID = strings.TrimSpace(providerID) + if providerID == "" { + return + } + queryTMDBIDs = append(queryTMDBIDs, strings.TrimSpace(candidate.TMDBID)) + providers = append(providers, provider) + providerIDs = append(providerIDs, providerID) + ordinals = append(ordinals, int32(ordinal)) + } + + for i, candidate := range candidates { + appendID(candidate, "tmdb", candidate.TMDBID, i) + appendID(candidate, "tvdb", candidate.TVDBID, i) + appendID(candidate, "imdb", candidate.IMDbID, i) + } + if len(providerIDs) == 0 { + return nil, nil + } + + rows, err := r.pool.Query(ctx, lookupExternalIDsSQL(), queryTMDBIDs, providers, providerIDs, ordinals, mediaType) + if err != nil { + return nil, fmt.Errorf("lookup external ids: %w", err) } defer rows.Close() - var out []MediaTMDBRow + out := make([]ExternalIDMatchRow, 0) for rows.Next() { - var row MediaTMDBRow - if err := rows.Scan(&row.MediaID, &row.TMDBID, &row.LibraryID, &row.Title); err != nil { - return nil, fmt.Errorf("scanning tmdb lookup row: %w", err) + var row ExternalIDMatchRow + if err := rows.Scan(&row.QueryTMDBID, &row.MediaID, &row.MatchedProvider, &row.LibraryID, &row.Title); err != nil { + return nil, fmt.Errorf("scanning external id lookup row: %w", err) } out = append(out, row) } if err := rows.Err(); err != nil { - return nil, fmt.Errorf("iterating tmdb lookup rows: %w", err) + return nil, fmt.Errorf("iterating external id lookup rows: %w", err) } return out, nil } diff --git a/internal/catalog/item_repo_test.go b/internal/catalog/item_repo_test.go index d39f2784..0be6a48e 100644 --- a/internal/catalog/item_repo_test.go +++ b/internal/catalog/item_repo_test.go @@ -191,6 +191,26 @@ func TestItemRepo_GetByExternalIDs_NilSliceStillBindsArg(t *testing.T) { _ = args } +func TestLookupExternalIDsSQLChecksProviderTableAndDirectColumns(t *testing.T) { + sql := lookupExternalIDsSQL() + + for _, want := range []string{ + "FROM requested r", + "JOIN media_item_provider_ids mip", + "mip.provider = r.provider", + "mip.provider_id = r.provider_id", + "COALESCE(mi.tmdb_id, '') = r.provider_id", + "COALESCE(mi.tvdb_id, '') = r.provider_id", + "COALESCE(mi.imdb_id, '') = r.provider_id", + "JOIN media_folders mf ON mf.id = mil.media_folder_id", + "mf.enabled = true", + } { + if !strings.Contains(sql, want) { + t.Fatalf("lookupExternalIDsSQL missing %q:\n%s", want, sql) + } + } +} + // TestItemRepo_Search_UsesWindowCount asserts that buildSearchSQL emits a // single-pass paged SELECT that includes COUNT(*) OVER () so Search no longer // needs a separate count query before the data fetch (audit 2026-05-01 ยง3.11). diff --git a/internal/catalog/provider_id_repo.go b/internal/catalog/provider_id_repo.go index 6d0386ca..bc5da5e5 100644 --- a/internal/catalog/provider_id_repo.go +++ b/internal/catalog/provider_id_repo.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "sort" + "strconv" "strings" "github.com/jackc/pgx/v5" @@ -30,6 +31,53 @@ func NewProviderIDRepository(pool *pgxpool.Pool) *ProviderIDRepository { return &ProviderIDRepository{pool: pool} } +func (r *ProviderIDRepository) AttachTMDBID(ctx context.Context, contentID, itemType string, tmdbID int) error { + contentID = strings.TrimSpace(contentID) + itemType = strings.TrimSpace(itemType) + if contentID == "" { + return fmt.Errorf("content_id is required") + } + if itemType == "" { + return fmt.Errorf("item_type is required") + } + if tmdbID <= 0 { + return fmt.Errorf("tmdb_id must be positive") + } + + tx, err := r.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin attach tmdb transaction: %w", err) + } + defer tx.Rollback(ctx) //nolint:errcheck + + tmdbText := strconv.Itoa(tmdbID) + if _, err := tx.Exec(ctx, ` + UPDATE media_items + SET tmdb_id = COALESCE(NULLIF(tmdb_id, ''), $1), + updated_at = NOW() + WHERE content_id = $2 + AND type = $3 + `, tmdbText, contentID, itemType); err != nil { + return fmt.Errorf("updating media item tmdb id: %w", err) + } + + if _, err := tx.Exec(ctx, ` + INSERT INTO media_item_provider_ids (content_id, item_type, provider, provider_id, created_at, updated_at) + VALUES ($1, $2, 'tmdb', $3, NOW(), NOW()) + ON CONFLICT (content_id, provider) DO UPDATE + SET item_type = EXCLUDED.item_type, + provider_id = EXCLUDED.provider_id, + updated_at = NOW() + `, contentID, itemType, tmdbText); err != nil { + return fmt.Errorf("upserting media item tmdb provider id: %w", err) + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit attach tmdb transaction: %w", err) + } + return nil +} + const providerIDColumns = `content_id, item_type, provider, provider_id, created_at, updated_at` var excludedProviderIDs = map[string]struct{}{ diff --git a/internal/catalog/provider_id_repo_test.go b/internal/catalog/provider_id_repo_test.go index 29472b9a..6f8c016d 100644 --- a/internal/catalog/provider_id_repo_test.go +++ b/internal/catalog/provider_id_repo_test.go @@ -40,3 +40,17 @@ func TestNormalizeDurableProviderIDsOrdersCanonicalKeysFirst(t *testing.T) { } } } + +func TestNormalizeDurableProviderIDsKeepsTMDBFirstForBackfill(t *testing.T) { + entries := normalizeDurableProviderIDs(map[string]string{ + "tvdb": "420105", + "imdb": "tt18076310", + "tmdb": "201992", + }) + if len(entries) != 3 { + t.Fatalf("len(entries) = %d, want 3", len(entries)) + } + if entries[0].Provider != "tmdb" || entries[0].ProviderID != "201992" { + t.Fatalf("first entry = (%q, %q), want tmdb/201992", entries[0].Provider, entries[0].ProviderID) + } +}