diff --git a/cmd/silo/main.go b/cmd/silo/main.go index cc4232aa..6f5a61d8 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -1939,6 +1939,7 @@ func main() { taskMgr.Register(tasks.NewScanLibrariesTask(deps.FolderRepo, deps.LibraryScanQueue, deps.EventBus)) } taskMgr.Register(tasks.NewCleanupOrphanedMediaItemsTask(catalog.NewOrphanedProvisionalCleaner(deps.DB))) + taskMgr.Register(tasks.NewBackfillMediaItemAliasesTask(catalog.NewItemAliasRepository(deps.DB))) if deps.S3Public != nil { taskMgr.Register(tasks.NewCleanupArtworkRevisionsTask( metadata.NewArtworkRevisionGarbageCollector(deps.DB, deps.S3Public), diff --git a/go.mod b/go.mod index 669174e9..82add34b 100644 --- a/go.mod +++ b/go.mod @@ -109,7 +109,7 @@ require ( ) require ( - github.com/Silo-Server/silo-plugin-sdk v0.10.0 + github.com/Silo-Server/silo-plugin-sdk v0.12.0 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect diff --git a/go.sum b/go.sum index 5c5c4ac2..6b4d337d 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= github.com/SherClockHolmes/webpush-go v1.4.0 h1:ocnzNKWN23T9nvHi6IfyrQjkIc0oJWv1B1pULsf9i3s= github.com/SherClockHolmes/webpush-go v1.4.0/go.mod h1:XSq8pKX11vNV8MJEMwjrlTkxhAj1zKfxmyhdV7Pd6UA= -github.com/Silo-Server/silo-plugin-sdk v0.10.0 h1:OU2PfQwnUQAnf8FEZSTfWI+3x0OBiVfjKHHaVWhD/E0= -github.com/Silo-Server/silo-plugin-sdk v0.10.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA= +github.com/Silo-Server/silo-plugin-sdk v0.12.0 h1:7MnaSIJBVdRE8FhfLXWx7Pe+Q2apJiaPMotfhb1VHoE= +github.com/Silo-Server/silo-plugin-sdk v0.12.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA= github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4= github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc= github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM= diff --git a/internal/catalog/item_alias_repo.go b/internal/catalog/item_alias_repo.go new file mode 100644 index 00000000..d97bb665 --- /dev/null +++ b/internal/catalog/item_alias_repo.go @@ -0,0 +1,324 @@ +package catalog + +import ( + "context" + "fmt" + "strings" + + "github.com/Silo-Server/silo-server/internal/lang" + "github.com/Silo-Server/silo-server/internal/models" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + itemAliasKindOriginal = "original" + itemAliasKindLocalized = "localized" + itemAliasKindAlternate = "alternate" +) + +// ItemAliasRepository persists and backfills provider-confirmed item titles. +type ItemAliasRepository struct { + pool *pgxpool.Pool + events *SearchIndexEventRepository +} + +func NewItemAliasRepository(pool *pgxpool.Pool) *ItemAliasRepository { + return &ItemAliasRepository{pool: pool, events: NewSearchIndexEventRepository(pool)} +} + +// ReplaceProvider replaces one provider's aliases atomically. Other providers' +// aliases are deliberately untouched so a partial provider outage cannot erase +// titles learned from successful sources. +func (r *ItemAliasRepository) ReplaceProvider(ctx context.Context, contentID, provider string, aliases []models.MediaItemAlias) error { + return r.writeProviderAliases(ctx, contentID, provider, "", true, aliases, true) +} + +// ReplaceProviderLanguage refreshes one provider response for a particular +// requested language. The request scope is independent of each alias's own +// language, so a multilingual response can be replaced without erasing aliases +// learned from another library-language request. +func (r *ItemAliasRepository) ReplaceProviderLanguage(ctx context.Context, contentID, provider, language string, aliases []models.MediaItemAlias) error { + language = normalizeAliasLanguage(language) + if language == "" { + return r.ReplaceProvider(ctx, contentID, provider, aliases) + } + return r.writeProviderAliases(ctx, contentID, provider, language, false, aliases, true) +} + +// RefreshProviderLanguage persists one provider response. Complete snapshots +// replace the request-language/provider scope; legacy or partial responses +// merge only, preserving aliases that the response cannot authoritatively +// declare stale. +func (r *ItemAliasRepository) RefreshProviderLanguage(ctx context.Context, contentID, provider, language string, aliases []models.MediaItemAlias, complete bool) error { + language = normalizeAliasLanguage(language) + if language == "" { + return r.writeProviderAliases(ctx, contentID, provider, "", true, aliases, complete) + } + return r.writeProviderAliases(ctx, contentID, provider, language, false, aliases, complete) +} + +func (r *ItemAliasRepository) writeProviderAliases( + ctx context.Context, + contentID string, + provider string, + snapshotLanguage string, + replaceProvider bool, + aliases []models.MediaItemAlias, + replace bool, +) error { + contentID = strings.TrimSpace(contentID) + provider = strings.ToLower(strings.TrimSpace(provider)) + snapshotLanguage = normalizeAliasLanguage(snapshotLanguage) + if r == nil || r.pool == nil || contentID == "" || provider == "" { + return nil + } + // An empty legacy/partial response carries no deletion authority. An empty + // complete snapshot is authoritative and must clear the refreshed provider + // scope so stale aliases do not remain searchable indefinitely. + if len(aliases) == 0 && !replace { + return nil + } + tx, err := r.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("begin alias replacement: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + if replace { + deleteSQL := `DELETE FROM media_item_aliases WHERE content_id = $1 AND provider = $2` + deleteArgs := []any{contentID, provider} + if !replaceProvider { + // NULL is legacy provenance from before request-scope tracking. + // Adopt only the rows the old scoped replacement contract could + // authoritatively replace; other localized legacy aliases may + // belong to an independent library-language refresh. + deleteSQL += ` AND ( + snapshot_language = $3 OR + ( + snapshot_language IS NULL AND + (language = '' OR kind = 'original' OR language = $3) + ) + )` + deleteArgs = append(deleteArgs, snapshotLanguage) + } + if _, err := tx.Exec(ctx, deleteSQL, deleteArgs...); err != nil { + return fmt.Errorf("delete provider aliases: %w", err) + } + } + for _, alias := range aliases { + title := strings.TrimSpace(alias.Title) + if title == "" { + continue + } + kind := strings.ToLower(strings.TrimSpace(alias.Kind)) + switch kind { + case itemAliasKindOriginal, itemAliasKindLocalized, itemAliasKindAlternate: + default: + kind = itemAliasKindAlternate + } + language := normalizeAliasLanguage(alias.Language) + if _, err := tx.Exec(ctx, ` + INSERT INTO media_item_aliases (content_id, title, language, kind, provider, snapshot_language) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (content_id, normalized_title, language, kind, provider, snapshot_language) + DO UPDATE SET title = EXCLUDED.title, updated_at = now() + `, contentID, title, language, kind, provider, snapshotLanguage); err != nil { + return fmt.Errorf("insert provider alias: %w", err) + } + } + if r.events != nil { + if err := r.events.EnqueueUpsert(ctx, tx, contentID); err != nil { + return fmt.Errorf("enqueue alias search index update: %w", err) + } + } + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("commit alias replacement: %w", err) + } + return nil +} + +// BackfillBatch seeds aliases from existing original and localized titles. +// Its cursor is persisted in the same transaction as every batch, making an +// interrupted run resumable without relying on task-manager process state. +func (r *ItemAliasRepository) BackfillBatch(ctx context.Context, afterContentID string, limit int) (string, int, error) { + if r == nil || r.pool == nil || limit <= 0 { + return afterContentID, 0, nil + } + tx, err := r.pool.Begin(ctx) + if err != nil { + return afterContentID, 0, fmt.Errorf("begin item alias backfill: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + const taskKey = "media_item_aliases_v1" + if _, err := tx.Exec(ctx, ` + INSERT INTO media_item_alias_backfill_state (task_key) + VALUES ($1) + ON CONFLICT (task_key) DO NOTHING + `, taskKey); err != nil { + return afterContentID, 0, fmt.Errorf("initialize item alias backfill state: %w", err) + } + var persistedCursor string + var completed bool + if err := tx.QueryRow(ctx, ` + SELECT last_content_id, completed + FROM media_item_alias_backfill_state + WHERE task_key = $1 + FOR UPDATE + `, taskKey).Scan(&persistedCursor, &completed); err != nil { + return afterContentID, 0, fmt.Errorf("lock item alias backfill state: %w", err) + } + if completed { + if err := tx.Commit(ctx); err != nil { + return persistedCursor, 0, fmt.Errorf("commit completed item alias backfill check: %w", err) + } + return persistedCursor, 0, nil + } + if persistedCursor > afterContentID { + afterContentID = persistedCursor + } + + rows, err := tx.Query(ctx, ` + WITH batch AS ( + SELECT content_id, original_title, original_language + FROM media_items + WHERE content_id > $1 + ORDER BY content_id + LIMIT $2 + ), inserted_originals AS ( + INSERT INTO media_item_aliases (content_id, title, language, kind, provider, snapshot_language) + SELECT content_id, original_title, + public.canonical_language_code(split_part(replace(original_language, '_', '-'), '-', 1)), + 'original', 'silo.backfill', '' + FROM batch + WHERE btrim(COALESCE(original_title, '')) <> '' + ON CONFLICT (content_id, normalized_title, language, kind, provider, snapshot_language) DO NOTHING + ), inserted_localizations AS ( + INSERT INTO media_item_aliases (content_id, title, language, kind, provider, snapshot_language) + SELECT loc.content_id, loc.title, + public.canonical_language_code(split_part(replace(loc.language, '_', '-'), '-', 1)), + 'localized', 'silo.backfill', '' + FROM media_item_localizations loc + JOIN batch USING (content_id) + WHERE btrim(COALESCE(loc.title, '')) <> '' + ON CONFLICT (content_id, normalized_title, language, kind, provider, snapshot_language) DO NOTHING + ) + SELECT content_id FROM batch ORDER BY content_id + `, afterContentID, limit) + if err != nil { + return afterContentID, 0, fmt.Errorf("backfill item aliases: %w", err) + } + defer rows.Close() + last := afterContentID + count := 0 + for rows.Next() { + if err := rows.Scan(&last); err != nil { + return afterContentID, count, err + } + count++ + } + if err := rows.Err(); err != nil { + return afterContentID, count, err + } + rows.Close() + + if count > 0 && r.events != nil { + rows, err := tx.Query(ctx, ` + SELECT content_id + FROM media_items + WHERE content_id > $1 AND content_id <= $2 + ORDER BY content_id + `, afterContentID, last) + if err != nil { + return afterContentID, count, fmt.Errorf("load alias backfill search index ids: %w", err) + } + contentIDs := make([]string, 0, count) + for rows.Next() { + var contentID string + if err := rows.Scan(&contentID); err != nil { + rows.Close() + return afterContentID, count, fmt.Errorf("scan alias backfill search index id: %w", err) + } + contentIDs = append(contentIDs, contentID) + } + if err := rows.Err(); err != nil { + rows.Close() + return afterContentID, count, fmt.Errorf("iterate alias backfill search index ids: %w", err) + } + rows.Close() + if err := r.events.EnqueueUpserts(ctx, tx, contentIDs); err != nil { + return afterContentID, count, fmt.Errorf("enqueue alias backfill search index updates: %w", err) + } + } + + batchCompleted := count < limit + if _, err := tx.Exec(ctx, ` + UPDATE media_item_alias_backfill_state + SET last_content_id = $2, completed = $3, updated_at = now() + WHERE task_key = $1 + `, taskKey, last, batchCompleted); err != nil { + return afterContentID, count, fmt.Errorf("save item alias backfill state: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return afterContentID, count, fmt.Errorf("commit item alias backfill: %w", err) + } + return last, count, nil +} + +// ResetCompletedBackfill re-arms the backfill for a fresh full pass when a +// prior run finished. An interrupted run (completed = false) keeps its cursor +// so it resumes instead of rescanning; without this reset a manually +// re-triggered task would report success while processing zero items forever. +func (r *ItemAliasRepository) ResetCompletedBackfill(ctx context.Context) error { + if r == nil || r.pool == nil { + return nil + } + if _, err := r.pool.Exec(ctx, ` + UPDATE media_item_alias_backfill_state + SET last_content_id = '', completed = false, updated_at = now() + WHERE task_key = 'media_item_aliases_v1' AND completed = true + `); err != nil { + return fmt.Errorf("reset item alias backfill state: %w", err) + } + return nil +} + +func normalizeAliasLanguage(language string) string { + return lang.Canonical(strings.ReplaceAll(language, "_", "-")) +} + +func (r *ItemAliasRepository) ListByContentIDs(ctx context.Context, contentIDs []string) (map[string][]models.MediaItemAlias, error) { + out := make(map[string][]models.MediaItemAlias) + if r == nil || r.pool == nil || len(contentIDs) == 0 { + return out, nil + } + rows, err := r.pool.Query(ctx, ` + SELECT content_id, title, language, kind, provider + FROM ( + SELECT DISTINCT ON (content_id, normalized_title, language, kind, provider) + content_id, title, language, kind, provider + FROM media_item_aliases + WHERE content_id = ANY($1) + ORDER BY + content_id, + normalized_title, + language, + kind, + provider, + updated_at DESC, + id DESC + ) deduplicated + ORDER BY content_id, language, kind, title + `, contentIDs) + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + var alias models.MediaItemAlias + if err := rows.Scan(&alias.ContentID, &alias.Title, &alias.Language, &alias.Kind, &alias.Provider); err != nil { + return nil, err + } + out[alias.ContentID] = append(out[alias.ContentID], alias) + } + return out, rows.Err() +} diff --git a/internal/catalog/item_alias_repo_test.go b/internal/catalog/item_alias_repo_test.go new file mode 100644 index 00000000..c89bb2ba --- /dev/null +++ b/internal/catalog/item_alias_repo_test.go @@ -0,0 +1,459 @@ +package catalog + +import ( + "context" + "fmt" + "slices" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" +) + +const ( + catalogTestContentTypeSeries = "series" + catalogTestProviderTMDB = "tmdb" + catalogTestProviderTVDB = "tvdb" +) + +func TestItemAliasRepositoryReplacesOnlySuccessfulProviderAndCascades(t *testing.T) { + pool := newSemanticCoverageTestPool(t) + ctx := context.Background() + var aliasTable *string + if err := pool.QueryRow(ctx, `SELECT to_regclass('public.media_item_aliases')::text`).Scan(&aliasTable); err != nil { + t.Fatalf("check media_item_aliases table: %v", err) + } + if aliasTable == nil || *aliasTable == "" { + t.Skip("test database has not applied media item aliases migration") + } + contentID := fmt.Sprintf("alias-repo-%d", time.Now().UnixNano()) + seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched") + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) }) + repo := NewItemAliasRepository(pool) + + if err := repo.ReplaceProvider(ctx, contentID, catalogTestProviderTVDB, []models.MediaItemAlias{ + {Title: "10 Tokyo Warriors", Language: "en", Kind: itemAliasKindAlternate}, + }); err != nil { + t.Fatalf("seed TVDB aliases: %v", err) + } + if err := repo.ReplaceProvider(ctx, contentID, catalogTestProviderTMDB, []models.MediaItemAlias{ + {Title: "Old TMDB Title", Language: "en", Kind: itemAliasKindLocalized}, + }); err != nil { + t.Fatalf("seed TMDB aliases: %v", err) + } + if err := repo.ReplaceProvider(ctx, contentID, catalogTestProviderTMDB, []models.MediaItemAlias{ + {Title: "New TMDB Title", Language: "en", Kind: itemAliasKindLocalized}, + {Title: "New TMDB Title", Language: "en", Kind: itemAliasKindLocalized}, + }); err != nil { + t.Fatalf("replace TMDB aliases: %v", err) + } + + aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID}) + if err != nil { + t.Fatalf("ListByContentIDs(): %v", err) + } + titles := make([]string, 0, len(aliasesByID[contentID])) + for _, alias := range aliasesByID[contentID] { + titles = append(titles, alias.Title) + } + if !slices.Contains(titles, "10 Tokyo Warriors") || !slices.Contains(titles, "New TMDB Title") || slices.Contains(titles, "Old TMDB Title") { + t.Fatalf("titles = %v", titles) + } + if len(titles) != 2 { + t.Fatalf("deduplicated titles = %v, want 2", titles) + } + + if _, err := pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID); err != nil { + t.Fatalf("delete media item: %v", err) + } + var count int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM media_item_aliases WHERE content_id = $1`, contentID).Scan(&count); err != nil { + t.Fatalf("count cascaded aliases: %v", err) + } + if count != 0 { + t.Fatalf("alias count after item delete = %d, want 0", count) + } +} + +func TestItemAliasRepositoryLanguageRefreshPreservesOtherLibraryLanguages(t *testing.T) { + pool := newSemanticCoverageTestPool(t) + ctx := context.Background() + var aliasTable *string + if err := pool.QueryRow(ctx, `SELECT to_regclass('public.media_item_aliases')::text`).Scan(&aliasTable); err != nil { + t.Fatalf("check media_item_aliases table: %v", err) + } + if aliasTable == nil || *aliasTable == "" { + t.Skip("test database has not applied media item aliases migration") + } + contentID := fmt.Sprintf("alias-language-%d", time.Now().UnixNano()) + seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched") + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) }) + repo := NewItemAliasRepository(pool) + + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "ja", []models.MediaItemAlias{ + {Title: "日本語", Language: "ja", Kind: itemAliasKindLocalized}, + }, true); err != nil { + t.Fatalf("seed Japanese aliases: %v", err) + } + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en", []models.MediaItemAlias{ + {Title: "Old English", Language: "en", Kind: itemAliasKindLocalized}, + {Title: "Old Untagged", Kind: itemAliasKindAlternate}, + }, true); err != nil { + t.Fatalf("seed English aliases: %v", err) + } + if err := repo.ReplaceProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en-US", []models.MediaItemAlias{ + {Title: "New English", Language: "en", Kind: itemAliasKindLocalized}, + {Title: "日本語原題", Language: "ja", Kind: itemAliasKindOriginal}, + {Title: "New Untagged", Kind: itemAliasKindAlternate}, + }); err != nil { + t.Fatalf("refresh English aliases: %v", err) + } + + aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID}) + if err != nil { + t.Fatalf("ListByContentIDs(): %v", err) + } + titles := make([]string, 0, len(aliasesByID[contentID])) + for _, alias := range aliasesByID[contentID] { + titles = append(titles, alias.Title) + } + for _, want := range []string{"New English", "日本語", "日本語原題", "New Untagged"} { + if !slices.Contains(titles, want) { + t.Fatalf("titles = %v, want preserved/refreshed %q", titles, want) + } + } + for _, stale := range []string{"Old English", "Old Untagged"} { + if slices.Contains(titles, stale) { + t.Fatalf("titles = %v, stale alias %q was not replaced", titles, stale) + } + } +} + +func TestItemAliasRepositoryCompleteSnapshotReplacesCrossLanguageAliasesInItsScope(t *testing.T) { + pool := newSemanticCoverageTestPool(t) + ctx := context.Background() + contentID := fmt.Sprintf("alias-cross-language-%d", time.Now().UnixNano()) + seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched") + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) }) + repo := NewItemAliasRepository(pool) + + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "ja", []models.MediaItemAlias{ + {Title: "日本語ライブラリ", Language: "ja", Kind: itemAliasKindLocalized}, + }, true); err != nil { + t.Fatalf("seed Japanese snapshot: %v", err) + } + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en", []models.MediaItemAlias{ + {Title: "English Title", Language: "en", Kind: itemAliasKindLocalized}, + {Title: "Ancienne française", Language: "fr", Kind: itemAliasKindAlternate}, + }, true); err != nil { + t.Fatalf("seed multilingual English snapshot: %v", err) + } + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en", []models.MediaItemAlias{ + {Title: "Updated English Title", Language: "en", Kind: itemAliasKindLocalized}, + }, true); err != nil { + t.Fatalf("replace English snapshot: %v", err) + } + + aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID}) + if err != nil { + t.Fatalf("ListByContentIDs(): %v", err) + } + titles := make([]string, 0, len(aliasesByID[contentID])) + for _, alias := range aliasesByID[contentID] { + titles = append(titles, alias.Title) + } + for _, want := range []string{"日本語ライブラリ", "Updated English Title"} { + if !slices.Contains(titles, want) { + t.Fatalf("titles = %v, want %q", titles, want) + } + } + for _, stale := range []string{"English Title", "Ancienne française"} { + if slices.Contains(titles, stale) { + t.Fatalf("titles = %v, stale cross-language alias %q survived", titles, stale) + } + } +} + +func TestItemAliasRepositoryListDeduplicatesAliasesAcrossSnapshotScopes(t *testing.T) { + pool := newSemanticCoverageTestPool(t) + ctx := context.Background() + contentID := fmt.Sprintf("alias-snapshot-dedup-%d", time.Now().UnixNano()) + seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched") + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) }) + repo := NewItemAliasRepository(pool) + + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en", []models.MediaItemAlias{ + {Title: "The Office", Language: "en", Kind: itemAliasKindAlternate}, + }, true); err != nil { + t.Fatalf("seed English snapshot: %v", err) + } + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "ja", []models.MediaItemAlias{ + {Title: "the office", Language: "en", Kind: itemAliasKindAlternate}, + }, true); err != nil { + t.Fatalf("seed Japanese snapshot: %v", err) + } + + aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID}) + if err != nil { + t.Fatalf("ListByContentIDs(): %v", err) + } + aliases := aliasesByID[contentID] + if len(aliases) != 1 { + t.Fatalf("aliases = %#v, want one normalized alias", aliases) + } + if aliases[0].Title != "the office" { + t.Fatalf("alias title = %q, want newest spelling", aliases[0].Title) + } +} + +func TestItemAliasRepositoryScopedRefreshAdoptsLegacyWithoutErasingProviderWideSnapshot(t *testing.T) { + pool := newSemanticCoverageTestPool(t) + ctx := context.Background() + contentID := fmt.Sprintf("alias-scope-provenance-%d", time.Now().UnixNano()) + seedSemanticCoverageMediaItem(t, pool, contentID, "movie", "matched") + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) }) + repo := NewItemAliasRepository(pool) + + if err := repo.ReplaceProvider(ctx, contentID, catalogTestProviderTMDB, []models.MediaItemAlias{ + {Title: "Provider-wide title", Kind: itemAliasKindAlternate}, + }); err != nil { + t.Fatalf("seed provider-wide snapshot: %v", err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO media_item_aliases ( + content_id, title, language, kind, provider, snapshot_language + ) VALUES + ($1, 'Legacy unscoped title', '', 'alternate', $2, NULL), + ($1, '従来の日本語タイトル', 'ja', 'localized', $2, NULL) + `, contentID, catalogTestProviderTMDB); err != nil { + t.Fatalf("seed legacy alias: %v", err) + } + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", []models.MediaItemAlias{ + {Title: "Current English title", Language: "en", Kind: itemAliasKindLocalized}, + }, true); err != nil { + t.Fatalf("refresh English snapshot: %v", err) + } + + aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID}) + if err != nil { + t.Fatalf("ListByContentIDs(): %v", err) + } + titles := make([]string, 0, len(aliasesByID[contentID])) + for _, alias := range aliasesByID[contentID] { + titles = append(titles, alias.Title) + } + for _, want := range []string{"Provider-wide title", "Current English title", "従来の日本語タイトル"} { + if !slices.Contains(titles, want) { + t.Fatalf("titles = %v, want preserved/refreshed %q", titles, want) + } + } + if slices.Contains(titles, "Legacy unscoped title") { + t.Fatalf("titles = %v, legacy alias was not adopted by complete refresh", titles) + } +} + +func TestItemAliasRepositoryPartialRefreshMergesWithoutErasing(t *testing.T) { + pool := newSemanticCoverageTestPool(t) + ctx := context.Background() + contentID := fmt.Sprintf("alias-partial-%d", time.Now().UnixNano()) + seedSemanticCoverageMediaItem(t, pool, contentID, "movie", "matched") + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) }) + repo := NewItemAliasRepository(pool) + + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", []models.MediaItemAlias{ + {Title: "Authoritative English", Language: "en", Kind: itemAliasKindLocalized}, + {Title: "Provider Alternate", Language: "en", Kind: itemAliasKindAlternate}, + }, true); err != nil { + t.Fatalf("seed complete aliases: %v", err) + } + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", []models.MediaItemAlias{ + {Title: "Legacy Primary Title", Language: "en", Kind: itemAliasKindLocalized}, + }, false); err != nil { + t.Fatalf("merge partial aliases: %v", err) + } + + aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID}) + if err != nil { + t.Fatalf("ListByContentIDs(): %v", err) + } + titles := make([]string, 0, len(aliasesByID[contentID])) + for _, alias := range aliasesByID[contentID] { + titles = append(titles, alias.Title) + } + for _, want := range []string{"Authoritative English", "Provider Alternate", "Legacy Primary Title"} { + if !slices.Contains(titles, want) { + t.Fatalf("titles = %v, partial refresh erased %q", titles, want) + } + } +} + +func TestItemAliasRepositoryEmptySnapshotHonorsCompleteness(t *testing.T) { + pool := newSemanticCoverageTestPool(t) + ctx := context.Background() + contentID := fmt.Sprintf("alias-empty-%d", time.Now().UnixNano()) + seedSemanticCoverageMediaItem(t, pool, contentID, "movie", "matched") + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) }) + repo := NewItemAliasRepository(pool) + + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", []models.MediaItemAlias{ + {Title: "Stale English", Language: "en", Kind: itemAliasKindLocalized}, + }, true); err != nil { + t.Fatalf("seed TMDB aliases: %v", err) + } + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en", []models.MediaItemAlias{ + {Title: "Keep TVDB", Language: "en", Kind: itemAliasKindAlternate}, + }, true); err != nil { + t.Fatalf("seed TVDB aliases: %v", err) + } + + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", nil, false); err != nil { + t.Fatalf("merge empty partial snapshot: %v", err) + } + aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID}) + if err != nil { + t.Fatalf("list aliases after partial snapshot: %v", err) + } + if !slices.ContainsFunc(aliasesByID[contentID], func(alias models.MediaItemAlias) bool { + return alias.Title == "Stale English" + }) { + t.Fatalf("partial empty snapshot removed aliases: %+v", aliasesByID[contentID]) + } + + if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", nil, true); err != nil { + t.Fatalf("replace with empty complete snapshot: %v", err) + } + aliasesByID, err = repo.ListByContentIDs(ctx, []string{contentID}) + if err != nil { + t.Fatalf("list aliases after complete snapshot: %v", err) + } + for _, alias := range aliasesByID[contentID] { + if alias.Provider == catalogTestProviderTMDB { + t.Fatalf("complete empty snapshot retained TMDB alias: %+v", alias) + } + } + if !slices.ContainsFunc(aliasesByID[contentID], func(alias models.MediaItemAlias) bool { + return alias.Title == "Keep TVDB" && alias.Provider == catalogTestProviderTVDB + }) { + t.Fatalf("complete TMDB snapshot erased another provider: %+v", aliasesByID[contentID]) + } +} + +func TestItemAliasRepositoryBackfillResumesAndEnqueuesIndexEvents(t *testing.T) { + pool := newSemanticCoverageTestPool(t) + ctx := context.Background() + var stateTable *string + if err := pool.QueryRow(ctx, `SELECT to_regclass('public.media_item_alias_backfill_state')::text`).Scan(&stateTable); err != nil { + t.Fatalf("check alias backfill state table: %v", err) + } + if stateTable == nil || *stateTable == "" { + t.Skip("test database has not applied alias backfill state migration") + } + + prefix := fmt.Sprintf("zz-alias-backfill-%d-", time.Now().UnixNano()) + firstID, secondID := prefix+"a", prefix+"b" + for _, contentID := range []string{firstID, secondID} { + seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched") + if _, err := pool.Exec(ctx, ` + UPDATE media_items SET original_title = '倒凶十将伝', original_language = 'jpn' + WHERE content_id = $1 + `, contentID); err != nil { + t.Fatalf("seed original title: %v", err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO media_item_localizations (content_id, language, title) + VALUES ($1, 'en_US', '10 Tokyo Warriors') + `, contentID); err != nil { + t.Fatalf("seed localization: %v", err) + } + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM catalog_search_index_events WHERE content_id = ANY($1)`, []string{firstID, secondID}) + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, []string{firstID, secondID}) + _, _ = pool.Exec(ctx, `DELETE FROM media_item_alias_backfill_state WHERE task_key = 'media_item_aliases_v1'`) + }) + if _, err := pool.Exec(ctx, ` + INSERT INTO media_item_alias_backfill_state (task_key, last_content_id, completed) + VALUES ('media_item_aliases_v1', $1, false) + ON CONFLICT (task_key) DO UPDATE + SET last_content_id = EXCLUDED.last_content_id, completed = false + `, prefix); err != nil { + t.Fatalf("seed backfill cursor: %v", err) + } + + repo := NewItemAliasRepository(pool) + repo.events.WithActiveProvider(SearchProviderMeilisearch) + firstCursor, processed, err := repo.BackfillBatch(ctx, "", 1) + if err != nil || processed != 1 || firstCursor != firstID { + t.Fatalf("first BackfillBatch() = (%q, %d, %v), want (%q, 1, nil)", firstCursor, processed, err, firstID) + } + secondCursor, processed, err := repo.BackfillBatch(ctx, "", 1) + if err != nil || processed != 1 || secondCursor != secondID { + t.Fatalf("resumed BackfillBatch() = (%q, %d, %v), want (%q, 1, nil)", secondCursor, processed, err, secondID) + } + + for _, contentID := range []string{firstID, secondID} { + aliases, err := repo.ListByContentIDs(ctx, []string{contentID}) + if err != nil { + t.Fatalf("list aliases for %s: %v", contentID, err) + } + languages := make([]string, 0, len(aliases[contentID])) + for _, alias := range aliases[contentID] { + languages = append(languages, alias.Language) + } + if !slices.Contains(languages, "ja") || !slices.Contains(languages, "en") { + t.Fatalf("alias languages for %s = %v, want canonical ja and en", contentID, languages) + } + var eventCount int + if err := pool.QueryRow(ctx, ` + SELECT count(*) FROM catalog_search_index_events + WHERE content_id = $1 AND action = 'upsert' + `, contentID).Scan(&eventCount); err != nil { + t.Fatalf("count index events for %s: %v", contentID, err) + } + if eventCount == 0 { + t.Fatalf("no search-index upsert event for %s", contentID) + } + } +} + +func TestItemRepositorySearchesPersistedAliasesAcrossExactFTSAndFuzzyPaths(t *testing.T) { + pool := newSemanticCoverageTestPool(t) + ctx := context.Background() + var aliasTable *string + if err := pool.QueryRow(ctx, `SELECT to_regclass('public.media_item_aliases')::text`).Scan(&aliasTable); err != nil { + t.Fatalf("check media_item_aliases table: %v", err) + } + if aliasTable == nil || *aliasTable == "" { + t.Skip("test database has not applied media item aliases migration") + } + contentID := fmt.Sprintf("alias-search-%d", time.Now().UnixNano()) + seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched") + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) }) + if _, err := pool.Exec(ctx, ` + UPDATE media_items SET title = '倒凶十将伝', original_title = '倒凶十将伝' WHERE content_id = $1 + `, contentID); err != nil { + t.Fatalf("seed native title: %v", err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO media_item_aliases (content_id, title, language, kind, provider) + VALUES ($1, '10 Tokyo Warriors', 'en', 'alternate', 'tvdb') + `, contentID); err != nil { + t.Fatalf("seed searchable alias: %v", err) + } + + repo := NewItemRepository(pool) + // The final query deliberately verifies fuzzy matching for a misspelling. + for _, query := range []string{"10 Tokyo Warriors", "Tokyo Warriors", "10 Tokyo Warriros"} { //nolint:misspell + items, _, err := repo.Search(ctx, query, []string{catalogTestContentTypeSeries}, 20, 0, AccessFilter{}) + if err != nil { + t.Fatalf("Search(%q): %v", query, err) + } + found := false + for _, item := range items { + found = found || item.ContentID == contentID + } + if !found { + t.Fatalf("Search(%q) did not return alias-backed item %s", query, contentID) + } + } +} diff --git a/internal/catalog/item_repo.go b/internal/catalog/item_repo.go index 1dee0fed..3be5392a 100644 --- a/internal/catalog/item_repo.go +++ b/internal/catalog/item_repo.go @@ -1325,13 +1325,26 @@ func (r *ItemRepository) buildFuzzySearchFromParsed(parsed parsedSearchQuery, it // word-boundary extent of the title instead, and at equal thresholds it is // a strict superset of % (the whole string is itself a valid extent). The // same gin_trgm_ops index serves both operators. - conditions := []string{"public.normalize_search_text($1) <<% mi.title_normalized"} + // The alias arm is a scalar array subquery (InitPlan) for the same reason + // as the FTS builder's alias arms: `= ANY(param)` keeps the outer OR a + // BitmapOr over the two gin_trgm_ops indexes, where an `IN (subquery)` arm + // would force a seq scan evaluating <<% against every media_items row. + conditions := []string{`( + public.normalize_search_text($1) <<% mi.title_normalized OR + mi.content_id = ANY(COALESCE(( + SELECT array_agg(DISTINCT mia.content_id) FROM media_item_aliases mia + WHERE public.normalize_search_text($1) <<% mia.normalized_title + ), '{}'::text[])) + )`} if minSimilarity > 0 { // The floor is whole-title similarity, NOT word similarity: augmenting // a query that already has hits must only admit near-identical titles, // and word similarity rates embedded prefix words far too high (see // fuzzyAugmentSimilarityFloor). - conditions = append(conditions, fmt.Sprintf("similarity(public.normalize_search_text($1), mi.title_normalized) >= $%d", argIdx)) + conditions = append(conditions, fmt.Sprintf(`GREATEST( + similarity(public.normalize_search_text($1), mi.title_normalized), + COALESCE((SELECT MAX(similarity(public.normalize_search_text($1), mia.normalized_title)) FROM media_item_aliases mia WHERE mia.content_id = mi.content_id), 0) + ) >= $%d`, argIdx)) args = append(args, minSimilarity) argIdx++ } @@ -1360,8 +1373,14 @@ func (r *ItemRepository) buildFuzzySearchFromParsed(parsed parsedSearchQuery, it WITH scored AS ( SELECT %s, - MAX(strict_word_similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_rank, - MAX(similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_full_rank + MAX(GREATEST( + strict_word_similarity(public.normalize_search_text($1), mi.title_normalized), + COALESCE((SELECT MAX(strict_word_similarity(public.normalize_search_text($1), mia.normalized_title)) FROM media_item_aliases mia WHERE mia.content_id = mi.content_id), 0) + )) AS fuzzy_rank, + MAX(GREATEST( + similarity(public.normalize_search_text($1), mi.title_normalized), + COALESCE((SELECT MAX(similarity(public.normalize_search_text($1), mia.normalized_title)) FROM media_item_aliases mia WHERE mia.content_id = mi.content_id), 0) + )) AS fuzzy_full_rank FROM %s %s GROUP BY %s diff --git a/internal/catalog/item_repo_test.go b/internal/catalog/item_repo_test.go index 9e6e6dbe..6ca7cb99 100644 --- a/internal/catalog/item_repo_test.go +++ b/internal/catalog/item_repo_test.go @@ -463,7 +463,7 @@ func TestItemRepo_Search_FTSQueryHasNoFuzzyArm(t *testing.T) { } // TestItemRepo_BuildFuzzySearchSQL asserts that the fuzzy fallback query scores -// only on the indexed title_normalized column (no title tsvector rebuild), +// only on indexed normalized title/alias columns (no title tsvector rebuild), // matches via strict word similarity so long titles stay reachable, ranks by // descending word similarity with whole-title closeness as tie-break, excludes // already-seen content_ids, and applies the same scope filters (type, manga @@ -477,12 +477,15 @@ func TestItemRepo_BuildFuzzySearchSQL(t *testing.T) { if !strings.Contains(dataSQL, "public.normalize_search_text($1) <<% mi.title_normalized") { t.Fatalf("expected strict-word-similarity arm against title_normalized; got:\n%s", dataSQL) } - if !strings.Contains(dataSQL, "MAX(strict_word_similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_rank") { + if !strings.Contains(dataSQL, "strict_word_similarity(public.normalize_search_text($1), mi.title_normalized)") || !strings.Contains(dataSQL, "AS fuzzy_rank") { t.Fatalf("expected strict word similarity ranking on title_normalized; got:\n%s", dataSQL) } - if !strings.Contains(dataSQL, "MAX(similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_full_rank") { + if !strings.Contains(dataSQL, "similarity(public.normalize_search_text($1), mi.title_normalized)") || !strings.Contains(dataSQL, "AS fuzzy_full_rank") { t.Fatalf("expected whole-title similarity tie-break rank; got:\n%s", dataSQL) } + if !strings.Contains(dataSQL, "<<% mia.normalized_title") { + t.Fatalf("expected alias trigram candidates; got:\n%s", dataSQL) + } // The whole point of the separate query: it must never rebuild the title // tsvectors that made the fused query slow. if strings.Contains(dataSQL, "to_tsvector") || strings.Contains(dataSQL, "ts_rank_cd") { @@ -539,7 +542,7 @@ func TestItemRepo_BuildFuzzySearchSQL_SimilarityFloor(t *testing.T) { // Whole-title similarity, not word similarity: the augment floor must not // admit embedded prefix words ("coral" scores 0.5 word-similarity to // "coraline"). - if !strings.Contains(floorSQL, "AND similarity(public.normalize_search_text($1), mi.title_normalized) >= $2") { + if !strings.Contains(floorSQL, ") >= $2") || !strings.Contains(floorSQL, "mia.normalized_title") { t.Fatalf("expected explicit whole-title similarity floor predicate as $2; got:\n%s", floorSQL) } if len(floorArgs) < 2 || floorArgs[1] != fuzzyAugmentSimilarityFloor { @@ -551,7 +554,7 @@ func TestItemRepo_BuildFuzzySearchSQL_SimilarityFloor(t *testing.T) { } baseSQL, _, baseArgs := repo.buildFuzzySearchSQL("avegners", []string{"movie"}, 20, 0, AccessFilter{}, true, nil, 0) - if strings.Contains(baseSQL, "AND similarity(public.normalize_search_text($1), mi.title_normalized) >= $2") { + if strings.Contains(baseSQL, ") >= $2") { t.Fatalf("zero floor must not add a similarity predicate; got:\n%s", baseSQL) } if len(baseArgs) != len(floorArgs)-1 { diff --git a/internal/catalog/search_indexer.go b/internal/catalog/search_indexer.go index 2d73a56d..22742af7 100644 --- a/internal/catalog/search_indexer.go +++ b/internal/catalog/search_indexer.go @@ -613,6 +613,7 @@ type catalogSearchDocument struct { Title string `json:"title"` SortTitle string `json:"sort_title,omitempty"` OriginalTitle string `json:"original_title,omitempty"` + Aliases []string `json:"-"` TitleVariants []string `json:"title_variants,omitempty"` Year int `json:"year,omitempty"` Overview string `json:"overview,omitempty"` @@ -718,6 +719,7 @@ func mixedCatalogSearchDocumentSQL(mediaPredicate, episodePredicate string, type COALESCE(mi.title, ''), COALESCE(mi.sort_title, ''), COALESCE(mi.original_title, ''), + COALESCE(aliases.titles, ARRAY[]::text[]), COALESCE(mi.year, 0), COALESCE(mi.overview, ''), COALESCE(mi.tagline, ''), @@ -730,6 +732,11 @@ func mixedCatalogSearchDocumentSQL(mediaPredicate, episodePredicate string, type COALESCE(libraries.ids, ARRAY[]::integer[]) FROM candidates c JOIN media_items mi ON c.type <> 'episode' AND mi.content_id = c.content_id + LEFT JOIN LATERAL ( + SELECT array_agg(DISTINCT mia.title ORDER BY mia.title) FILTER (WHERE btrim(mia.title) <> '') AS titles + FROM media_item_aliases mia + WHERE mia.content_id = mi.content_id + ) aliases ON true LEFT JOIN LATERAL ( SELECT array_agg(DISTINCT p.name) FILTER (WHERE p.name IS NOT NULL AND p.name <> '') AS names FROM item_people ip @@ -748,6 +755,7 @@ func mixedCatalogSearchDocumentSQL(mediaPredicate, episodePredicate string, type COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text), COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text), ''::text, + ARRAY[]::text[], COALESCE(EXTRACT(YEAR FROM e.air_date)::integer, 0), COALESCE(e.overview, ''), ''::text, @@ -776,6 +784,7 @@ func scanCatalogSearchDocuments(rows pgx.Rows) ([]catalogSearchDocument, error) &doc.Title, &doc.SortTitle, &doc.OriginalTitle, + &doc.Aliases, &doc.Year, &doc.Overview, &doc.Tagline, @@ -965,12 +974,16 @@ func countCatalogSearchEligibleDocuments(ctx context.Context, q coverageQuerier, } func catalogSearchTitleVariants(doc catalogSearchDocument) []string { - return compactNonEmptyStrings([]string{ + variants := []string{ doc.Title, doc.SortTitle, doc.OriginalTitle, normalizeTitleForComparison(doc.Title), normalizeTitleForComparison(doc.SortTitle), normalizeTitleForComparison(doc.OriginalTitle), - }) + } + for _, alias := range doc.Aliases { + variants = append(variants, alias, normalizeTitleForComparison(alias)) + } + return compactNonEmptyStrings(variants) } diff --git a/internal/catalog/search_indexer_test.go b/internal/catalog/search_indexer_test.go index 4519e6f1..063047af 100644 --- a/internal/catalog/search_indexer_test.go +++ b/internal/catalog/search_indexer_test.go @@ -137,6 +137,37 @@ func TestAttachDocumentVectorsSkipsWhenSemanticDisabled(t *testing.T) { } } +func TestCatalogSearchDocumentIncludesPersistedAliases(t *testing.T) { + pool := newSemanticCoverageTestPool(t) + ctx := context.Background() + var aliasTable *string + if err := pool.QueryRow(ctx, `SELECT to_regclass('public.media_item_aliases')::text`).Scan(&aliasTable); err != nil { + t.Fatalf("check media_item_aliases table: %v", err) + } + if aliasTable == nil || *aliasTable == "" { + t.Skip("test database has not applied media item aliases migration") + } + contentID := fmt.Sprintf("alias-doc-%d", time.Now().UnixNano()) + seedSemanticCoverageMediaItem(t, pool, contentID, "series", "matched") + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) }) + if _, err := pool.Exec(ctx, `UPDATE media_items SET title = '倒凶十将伝', original_title = '倒凶十将伝' WHERE content_id = $1`, contentID); err != nil { + t.Fatalf("seed native title: %v", err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO media_item_aliases (content_id, title, language, kind, provider) + VALUES ($1, '10 Tokyo Warriors', 'en', 'alternate', 'tvdb')`, contentID); err != nil { + t.Fatalf("seed alias: %v", err) + } + + docs, err := NewCatalogSearchIndexer(pool, nil).LoadDocumentsByIDs(ctx, []string{contentID}, []string{catalogTestContentTypeSeries}, "", false, false) + if err != nil { + t.Fatalf("LoadDocumentsByIDs(): %v", err) + } + if len(docs) != 1 || !slices.Contains(docs[0].TitleVariants, "10 Tokyo Warriors") { + t.Fatalf("documents = %#v, want English alias in title variants", docs) + } +} + // TestCountCatalogSearchVectorDocumentsModelCoverage verifies case (a): an // embedding stored under a different model is excluded when a model is // requested, but counted when the model filter is empty. diff --git a/internal/catalog/search_postgres_mixed.go b/internal/catalog/search_postgres_mixed.go index 871b924d..7d73831e 100644 --- a/internal/catalog/search_postgres_mixed.go +++ b/internal/catalog/search_postgres_mixed.go @@ -103,7 +103,7 @@ func (r *ItemRepository) buildMixedSearchSQLFromParsed( setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B') )` mediaOverviewVector := `to_tsvector('english', COALESCE(mi.overview, ''))` - mediaConditions = append(mediaConditions, searchMatchCondition(mediaTitleVector, mediaOverviewVector)) + mediaConditions = append(mediaConditions, searchMatchCondition(mediaTitleVector, mediaOverviewVector, mediaSearchAliasMatchArms()...)) if len(itemTypes) > 0 { mediaConditions = append(mediaConditions, fmt.Sprintf("mi.type = ANY($%d)", argIdx)) args = append(args, mediaTypes) @@ -157,6 +157,7 @@ func (r *ItemRepository) buildMixedSearchSQLFromParsed( argIdx++ if includeMediaItems { + mediaAliasArms := mediaSearchAliasArms(exactIdx) branches = append(branches, buildMixedSearchCandidateBranch( "mi.content_id", "mi.type", "mi.title", "mi.year", `( @@ -167,6 +168,7 @@ func (r *ItemRepository) buildMixedSearchSQLFromParsed( `to_tsvector('english', COALESCE(mi.overview, ''))`, []string{`mi.title_normalized`, `public.normalize_search_text(mi.original_title)`, `public.normalize_search_text(mi.sort_title)`}, "media_items mi", mediaConditions, exactIdx, yearIdx, phraseIdx, + &mediaAliasArms, )) } if includeEpisodes { @@ -178,6 +180,7 @@ func (r *ItemRepository) buildMixedSearchSQLFromParsed( []string{episodeNormalizedTitle}, "episodes e JOIN media_items si ON si.content_id = e.series_id", episodeConditions, exactIdx, yearIdx, phraseIdx, + nil, )) } @@ -252,11 +255,73 @@ func splitSearchItemTypes(itemTypes []string) (mediaTypes []string, includeEpiso return mediaTypes, includeEpisodes } -func searchMatchCondition(titleVector, overviewVector string) string { +func searchMatchCondition(titleVector, overviewVector string, extraArms ...string) string { titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))` prefixQuery := `to_tsquery('simple', $2)` - return fmt.Sprintf(`((%s) @@ %s OR ($2 <> '' AND (%s) @@ %s) OR (%s) @@ websearch_to_tsquery('english', $1))`, - titleVector, titleQuery, titleVector, prefixQuery, overviewVector) + arms := []string{ + fmt.Sprintf(`(%s) @@ %s`, titleVector, titleQuery), + fmt.Sprintf(`($2 <> '' AND (%s) @@ %s)`, titleVector, prefixQuery), + fmt.Sprintf(`(%s) @@ websearch_to_tsquery('english', $1)`, overviewVector), + } + arms = append(arms, extraArms...) + return "(" + strings.Join(arms, " OR ") + ")" +} + +// mediaSearchAliasMatchArms are the alias arms OR'd into the media branch's +// match condition. They are scalar array subqueries (InitPlans), NOT `IN +// (subquery)` arms: an uncorrelated subquery result is a plain parameter, so +// `content_id = ANY(...)` stays index-served and participates in the BitmapOr +// with the title/overview GIN arms. An `IN (SELECT ...)` arm inside an OR has +// no index path and forces a seq scan of media_items that rebuilds every +// tsvector per row. +func mediaSearchAliasMatchArms() []string { + return []string{ + `mi.content_id = ANY(COALESCE(( + SELECT array_agg(DISTINCT mia.content_id) FROM media_item_aliases mia + WHERE to_tsvector('simple', mia.normalized_title) @@ websearch_to_tsquery('simple', public.normalize_search_text($1)) + ), '{}'::text[]))`, + `($2 <> '' AND mi.content_id = ANY(COALESCE(( + SELECT array_agg(DISTINCT mia.content_id) FROM media_item_aliases mia + WHERE to_tsvector('simple', mia.normalized_title) @@ to_tsquery('simple', $2) + ), '{}'::text[])))`, + } +} + +// mixedSearchAliasArms carries the media branch's provider-alias extensions to +// the per-branch ranking SELECT. Episodes have no aliases and pass nil. +type mixedSearchAliasArms struct { + exactArm string // OR'd into exact_title_match + contiguousArm string // OR'd into contiguous_title_match + titleRank string // GREATEST'd with title_rank + prefixRank string // GREATEST'd with title_prefix_rank +} + +// mediaSearchAliasArms builds the alias ranking arms for the media branch. +// The correlated rank subqueries run per matched row only (content_id +// lookups), and setweight(..., 'A') makes an alias hit rank like a real title +// hit — an unweighted tsvector defaults to weight D (0.1), which buried exact +// alias matches ~10x below partial real-title matches. The WHERE arms above +// stay unweighted to keep matching idx_media_item_aliases_search_vector's +// indexed expression. +func mediaSearchAliasArms(exactIdx int) mixedSearchAliasArms { + return mixedSearchAliasArms{ + exactArm: fmt.Sprintf(`mi.content_id IN ( + SELECT mia.content_id FROM media_item_aliases mia + WHERE mia.normalized_title = $%d + )`, exactIdx), + contiguousArm: fmt.Sprintf(`mi.content_id IN ( + SELECT mia.content_id FROM media_item_aliases mia + WHERE mia.normalized_title LIKE '%%' || $%d || '%%' + )`, exactIdx), + titleRank: `COALESCE(( + SELECT MAX(ts_rank_cd(setweight(to_tsvector('simple', mia.normalized_title), 'A'), websearch_to_tsquery('simple', public.normalize_search_text($1)))) + FROM media_item_aliases mia WHERE mia.content_id = mi.content_id + ), 0)`, + prefixRank: `COALESCE(( + SELECT MAX(ts_rank_cd(setweight(to_tsvector('simple', mia.normalized_title), 'A'), to_tsquery('simple', $2))) + FROM media_item_aliases mia WHERE mia.content_id = mi.content_id + ), 0)`, + } } func buildMixedSearchCandidateBranch( @@ -265,15 +330,27 @@ func buildMixedSearchCandidateBranch( fromClause string, conditions []string, exactIdx, yearIdx, phraseIdx int, + aliasArms *mixedSearchAliasArms, ) string { - exactArms := make([]string, 0, len(exactTitleExprs)) - contiguousArms := make([]string, 0, len(exactTitleExprs)) + exactArms := make([]string, 0, len(exactTitleExprs)+1) + contiguousArms := make([]string, 0, len(exactTitleExprs)+1) for _, expr := range exactTitleExprs { exactArms = append(exactArms, fmt.Sprintf("%s = $%d", expr, exactIdx)) contiguousArms = append(contiguousArms, fmt.Sprintf("%s LIKE '%%' || $%d || '%%'", expr, exactIdx)) } titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))` prefixQuery := `to_tsquery('simple', $2)` + titleRankExpr := fmt.Sprintf("ts_rank_cd(%s, %s)", titleVector, titleQuery) + prefixRankExpr := fmt.Sprintf("ts_rank_cd(%s, %s)", titleVector, prefixQuery) + if aliasArms != nil { + // Alias exact/contiguous arms are hashed subplans in the CASE select + // list (evaluated per grouped row, hash built once); the rank arms are + // per-row correlated content_id lookups over the matched set only. + exactArms = append(exactArms, aliasArms.exactArm) + contiguousArms = append(contiguousArms, aliasArms.contiguousArm) + titleRankExpr = fmt.Sprintf("GREATEST(%s, %s)", titleRankExpr, aliasArms.titleRank) + prefixRankExpr = fmt.Sprintf("GREATEST(%s, %s)", prefixRankExpr, aliasArms.prefixRank) + } return fmt.Sprintf(` SELECT %s AS content_id, @@ -282,8 +359,8 @@ func buildMixedSearchCandidateBranch( CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS exact_title_match, CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS contiguous_title_match, CASE WHEN $%d::int IS NOT NULL AND (%s) = $%d::int THEN 1 ELSE 0 END AS year_match, - ts_rank_cd(%s, %s) AS title_rank, - CASE WHEN $2 <> '' THEN ts_rank_cd(%s, %s) ELSE 0 END AS title_prefix_rank, + %s AS title_rank, + CASE WHEN $2 <> '' THEN %s ELSE 0 END AS title_prefix_rank, ts_rank_cd(%s, websearch_to_tsquery('english', $1)) AS overview_rank, CASE WHEN $%d <> '' THEN ts_rank_cd(%s, phraseto_tsquery('simple', public.normalize_search_text($%d))) ELSE 0 END AS phrase_rank FROM %s @@ -292,8 +369,8 @@ func buildMixedSearchCandidateBranch( exactIdx, strings.Join(exactArms, " OR "), exactIdx, strings.Join(contiguousArms, " OR "), yearIdx, yearExpr, yearIdx, - titleVector, titleQuery, - titleVector, prefixQuery, + titleRankExpr, + prefixRankExpr, overviewVector, phraseIdx, titleVector, phraseIdx, fromClause, strings.Join(conditions, " AND ")) diff --git a/internal/metadata/episode_match_validation.go b/internal/metadata/episode_match_validation.go new file mode 100644 index 00000000..ef500414 --- /dev/null +++ b/internal/metadata/episode_match_validation.go @@ -0,0 +1,577 @@ +package metadata + +import ( + "context" + "fmt" + "path/filepath" + "regexp" + "slices" + "sort" + "strings" + "sync" + "time" + "unicode" + "unicode/utf8" + + "github.com/Silo-Server/silo-server/internal/naming" + "golang.org/x/text/unicode/norm" +) + +const ( + // Search APIs often return many same-title remakes. Three candidates was too + // shallow for real libraries (the correct "Memories" result was below it), + // while eight keeps the worst-case provider work bounded. + maxEpisodeValidationCandidates = 8 + maxEpisodeValidationProviders = 2 + maxEpisodeValidationSamples = 3 + maxEpisodeValidationSeasons = 3 + episodeValidationTimeout = 10 * time.Second +) + +var ( + episodeCodePattern = regexp.MustCompile(`(?i)s\d{1,4}e\d{1,3}(?:e\d{1,3})?`) + episodeReleaseSuffixPattern = regexp.MustCompile( + `(?i)(?:^|[ ._-])(?:2160p|1080p|720p|576p|480p|webrip|web[ ._-]?dl|bluray|blu[ ._-]?ray|bdrip|hdtv|dvdrip|remux|x26[45]|h26[45]|hevc|av1|aac|ac3|eac3|dts(?:[ ._-]?hd)?|flac|hdr10?|dolby[ ._-]?vision)(?:$|[ ._-])`, + ) + episodeGenericTitlePattern = regexp.MustCompile(`(?i)^(?:tba|tbd|pilot|episode|episode\s+\d+|ep\.?\s*\d+|part\s+\d+|chapter\s+\d+|\d+)$`) + episodeTechnicalTitlePattern = regexp.MustCompile(`(?i)^(?:\d{3,4}p|web(?:rip|dl)?|bluray|hdtv|remux|x26[45]|h26[45]|hevc)(?:\b|[ ._-])`) +) + +type localEpisodeMatchHint struct { + SeasonNumber int + EpisodeNumber int + Title string +} + +type episodeCandidateValidation struct { + candidate MatchCandidate + provider string + coordinates int + compared int + matches int + exactMatches int + similarityTotal float64 + evaluated bool + err error +} + +type episodeValidationProvider struct { + provider EpisodeProvider + slug string +} + +// validateSeriesMatchByEpisodes uses a small amount of episode-title evidence +// to resolve otherwise ambiguous exact-title series searches. It is deliberately +// positive-only: unavailable, generic, or disagreeing episode data leaves the +// item unmatched instead of turning absence of evidence into a rejection. +func validateSeriesMatchByEpisodes( + ctx context.Context, + hints *MatchHints, + candidates []MatchCandidate, + providers []Provider, + language string, +) (*MatchCandidate, []error) { + if hints == nil || hints.Type != matchContentTypeSeries || trustedHintIDsPresent(hints) { + return nil, nil + } + + // Coordinate selection keeps useful episode titles when present, but its + // primary purpose is to build a bounded series-shape fingerprint: the last + // locally present episode from up to three spread-out seasons. + localEpisodes := selectLocalEpisodeCoordinateHints(hints.AllGroupFilePaths) + if len(localEpisodes) == 0 { + return nil, nil + } + + contenders, contendersExhaustive := episodeValidationContenders(hints, candidates) + if len(contenders) == 0 { + return nil, nil + } + + validationCtx, cancel := context.WithTimeout(ctx, episodeValidationTimeout) + defer cancel() + + validations := make([]episodeCandidateValidation, len(contenders)) + var wg sync.WaitGroup + for i := range contenders { + candidateProviders := selectEpisodeValidationProviders(contenders[i], providers) + validations[i].candidate = contenders[i] + if len(candidateProviders) == 0 { + continue + } + + wg.Add(1) + go func(index int, episodeProviders []episodeValidationProvider) { + defer wg.Done() + validations[index] = validateEpisodeCandidateAcrossProviders( + validationCtx, + validations[index].candidate, + episodeProviders, + localEpisodes, + language, + ) + }(i, candidateProviders) + } + wg.Wait() + + errs := make([]error, 0, len(validations)) + for _, validation := range validations { + if validation.err != nil { + errs = append(errs, validation.err) + } + // Every contender must have been checked successfully. Otherwise a + // temporarily unavailable rival could be incorrectly treated as a zero. + if !validation.evaluated || validation.err != nil { + return nil, errs + } + } + + titleValidations := append([]episodeCandidateValidation(nil), validations...) + sort.SliceStable(titleValidations, func(i, j int) bool { + if titleValidations[i].matches != titleValidations[j].matches { + return titleValidations[i].matches > titleValidations[j].matches + } + if titleValidations[i].exactMatches != titleValidations[j].exactMatches { + return titleValidations[i].exactMatches > titleValidations[j].exactMatches + } + return titleValidations[i].similarityTotal > titleValidations[j].similarityTotal + }) + + best := titleValidations[0] + if episodeEvidenceIsSufficient(best, localEpisodes) && + (len(titleValidations) == 1 || episodeEvidenceClearlyWins(best, titleValidations[1])) { + winner := best.candidate + winner.MatchReasons = append(winner.MatchReasons, + fmt.Sprintf("episode_title_corroboration:%d_of_%d", best.matches, len(localEpisodes))) + return &winner, errs + } + + coordinateValidations := append([]episodeCandidateValidation(nil), validations...) + sort.SliceStable(coordinateValidations, func(i, j int) bool { + if coordinateValidations[i].coordinates != coordinateValidations[j].coordinates { + return coordinateValidations[i].coordinates > coordinateValidations[j].coordinates + } + return episodeValidationEvidenceBetter(coordinateValidations[i], coordinateValidations[j]) + }) + best = coordinateValidations[0] + var next episodeCandidateValidation + if len(coordinateValidations) > 1 { + next = coordinateValidations[1] + } + if !episodeCoordinateEvidenceIsSufficient( + best, next, localEpisodes, len(coordinateValidations), contendersExhaustive, + ) { + return nil, errs + } + + winner := best.candidate + winner.MatchReasons = append(winner.MatchReasons, + fmt.Sprintf("episode_coordinate_corroboration:%d_of_%d", best.coordinates, len(localEpisodes))) + return &winner, errs +} + +func episodeValidationContenders(hints *MatchHints, candidates []MatchCandidate) ([]MatchCandidate, bool) { + scored := make([]scoredMatchCandidate, 0, len(candidates)) + for _, candidate := range candidates { + annotateCandidateMatch(&candidate, hints) + titleSimilarity, _ := bestCandidateTitleSimilarity(hints.Title, candidate, 0) + exactTitleCandidate := candidate.MatchScore >= automaticMatchAcceptanceFloor && titleSimilarity == 1 + crossProviderYearCandidate := hints.Year != 0 && candidate.Year == hints.Year && + candidateCorroboratingSourceCount(candidate) >= 2 + if (!exactTitleCandidate && !crossProviderYearCandidate) || + !candidateTypeMatchesHint(hints.Type, candidate.ContentType) || providerIDRichness(candidate.ProviderIDs) == 0 { + continue + } + scored = append(scored, scoredMatchCandidate{candidate: candidate, score: candidate.MatchScore}) + } + sort.SliceStable(scored, func(i, j int) bool { return scored[i].score > scored[j].score }) + exhaustive := len(scored) <= maxEpisodeValidationCandidates + if len(scored) > maxEpisodeValidationCandidates { + scored = scored[:maxEpisodeValidationCandidates] + } + out := make([]MatchCandidate, 0, len(scored)) + for _, candidate := range scored { + out = append(out, candidate.candidate) + } + return out, exhaustive +} + +func selectEpisodeValidationProviders(candidate MatchCandidate, providers []Provider) []episodeValidationProvider { + // TMDB fetches exactly one requested season, making it the cheapest + // validator when both first-party IDs are present. The configured chain + // order remains the fallback for TVDB-only and third-party candidates. + ordered := make([]Provider, 0, len(providers)) + for _, provider := range providers { + if strings.EqualFold(provider.Slug(), "tmdb") { + ordered = append(ordered, provider) + } + } + for _, provider := range providers { + if !strings.EqualFold(provider.Slug(), "tmdb") { + ordered = append(ordered, provider) + } + } + + selected := make([]episodeValidationProvider, 0, maxEpisodeValidationProviders) + for _, provider := range ordered { + if _, local := provider.(IdentityHintProvider); local { + continue + } + episodeProvider, ok := provider.(EpisodeProvider) + if !ok { + continue + } + providerSlug := strings.ToLower(strings.TrimSpace(provider.Slug())) + if strings.TrimSpace(candidate.ProviderIDs[providerSlug]) == "" { + continue + } + selected = append(selected, episodeValidationProvider{provider: episodeProvider, slug: providerSlug}) + if len(selected) == maxEpisodeValidationProviders { + break + } + } + return selected +} + +func validateEpisodeCandidateAcrossProviders( + ctx context.Context, + candidate MatchCandidate, + providers []episodeValidationProvider, + localEpisodes []localEpisodeMatchHint, + language string, +) episodeCandidateValidation { + best := episodeCandidateValidation{candidate: candidate} + var lastErr error + for _, provider := range providers { + validation := validateEpisodeCandidate( + ctx, candidate, provider.slug, provider.provider, localEpisodes, language, + ) + if validation.err != nil { + lastErr = validation.err + continue + } + if !best.evaluated || episodeValidationEvidenceBetter(validation, best) { + best = validation + } + if episodeEvidenceIsSufficient(validation, localEpisodes) { + return validation + } + } + if best.evaluated { + return best + } + best.err = lastErr + return best +} + +func episodeValidationEvidenceBetter(left, right episodeCandidateValidation) bool { + if left.matches != right.matches { + return left.matches > right.matches + } + if left.exactMatches != right.exactMatches { + return left.exactMatches > right.exactMatches + } + if left.similarityTotal != right.similarityTotal { + return left.similarityTotal > right.similarityTotal + } + if left.compared != right.compared { + return left.compared > right.compared + } + return left.coordinates > right.coordinates +} + +func validateEpisodeCandidate( + ctx context.Context, + candidate MatchCandidate, + providerSlug string, + provider EpisodeProvider, + localEpisodes []localEpisodeMatchHint, + language string, +) episodeCandidateValidation { + validation := episodeCandidateValidation{candidate: candidate, provider: providerSlug} + validation.evaluated = true + + localBySeason := make(map[int][]localEpisodeMatchHint) + seasonNumbers := make([]int, 0, maxEpisodeValidationSeasons) + for _, episode := range localEpisodes { + if _, exists := localBySeason[episode.SeasonNumber]; !exists { + seasonNumbers = append(seasonNumbers, episode.SeasonNumber) + } + localBySeason[episode.SeasonNumber] = append(localBySeason[episode.SeasonNumber], episode) + } + sort.Ints(seasonNumbers) + + for _, seasonNumber := range seasonNumbers { + episodes, err := provider.GetEpisodes(ctx, EpisodesRequest{ + ProviderIDs: copyMap(candidate.ProviderIDs), + SeasonNumber: seasonNumber, + Language: language, + }) + if err != nil { + if isProvider404(err) { + // A missing season is valid negative evidence for this candidate. + continue + } + validation.err = fmt.Errorf("episode validation via %s for %s season %d: %w", providerSlug, candidate.Title, seasonNumber, err) + return validation + } + + providerEpisodes := make(map[int]EpisodeResult, len(episodes)) + for _, episode := range episodes { + if episode.SeasonNumber == seasonNumber { + providerEpisodes[episode.EpisodeNumber] = episode + } + } + for _, localEpisode := range localBySeason[seasonNumber] { + providerEpisode, ok := providerEpisodes[localEpisode.EpisodeNumber] + if !ok { + continue + } + validation.coordinates++ + if isGenericEpisodeMatchTitle(localEpisode.Title) || isGenericEpisodeMatchTitle(providerEpisode.Title) { + continue + } + validation.compared++ + similarity := episodeTitleSimilarity(localEpisode.Title, providerEpisode.Title) + if similarity < 0.8 { + continue + } + validation.matches++ + validation.similarityTotal += similarity + if similarity == 1 { + validation.exactMatches++ + } + } + } + return validation +} + +func episodeEvidenceIsSufficient(validation episodeCandidateValidation, localEpisodes []localEpisodeMatchHint) bool { + if len(localEpisodes) >= 2 { + return validation.matches >= 2 + } + return len(localEpisodes) == 1 && validation.matches == 1 && validation.exactMatches == 1 && + isDistinctiveSingleEpisodeTitle(localEpisodes[0].Title) +} + +func episodeEvidenceClearlyWins(best, next episodeCandidateValidation) bool { + if best.matches != next.matches { + return best.matches > next.matches + } + if best.exactMatches != next.exactMatches { + return best.exactMatches > next.exactMatches + } + return best.similarityTotal-next.similarityTotal >= 0.2 +} + +func episodeCoordinateEvidenceIsSufficient( + validation episodeCandidateValidation, + next episodeCandidateValidation, + localEpisodes []localEpisodeMatchHint, + contenderCount int, + contendersExhaustive bool, +) bool { + if len(localEpisodes) < 2 || validation.coordinates != len(localEpisodes) { + return false + } + // Two coordinates spread across two seasons carry meaningful series-shape + // evidence. Within one season, require all three spread-out samples; merely + // finding two early episodes is common across unrelated same-title series. + if distinctEpisodeSeasons(localEpisodes) < 2 && len(localEpisodes) < maxEpisodeValidationSamples { + return false + } + // Coordinates cannot prove that a differently named candidate is an alias. + // Cross-provider exact-year title variants are eligible for validation, but + // they must win through matching episode titles. Coordinate-only promotion + // remains limited to an exact normalized series title. + if !slices.Contains(validation.candidate.MatchReasons, "exact_title") { + return false + } + if contenderCount > 1 { + // Shape evidence can only choose between multiple exact-title candidates + // when the bounded candidate set was exhaustive and exactly one candidate + // contains every sampled coordinate. A truncated search result remains + // unmatched because an unchecked rival may have the same shape. + if !contendersExhaustive || next.coordinates == len(localEpisodes) { + return false + } + if distinctEpisodeSeasons(localEpisodes) < 2 { + return false + } + } + discriminating := episodeCoordinatesAreDiscriminating(localEpisodes) + return discriminating || candidateCorroboratingSourceCount(validation.candidate) >= 2 +} + +func distinctEpisodeSeasons(episodes []localEpisodeMatchHint) int { + seasons := make(map[int]struct{}) + for _, episode := range episodes { + seasons[episode.SeasonNumber] = struct{}{} + } + return len(seasons) +} + +func episodeCoordinatesAreDiscriminating(episodes []localEpisodeMatchHint) bool { + for _, episode := range episodes { + if episode.SeasonNumber > 1 || episode.EpisodeNumber >= 6 { + return true + } + } + return false +} + +func candidateCorroboratingSourceCount(candidate MatchCandidate) int { + seen := make(map[string]struct{}, len(candidate.Sources)) + for _, source := range candidate.Sources { + source = strings.ToLower(strings.TrimSpace(source)) + if source == "" || nonCorroboratingSources[source] { + continue + } + seen[source] = struct{}{} + } + return len(seen) +} + +func selectLocalEpisodeCoordinateHints(paths []string) []localEpisodeMatchHint { + bySeason := make(map[int]map[int]localEpisodeMatchHint) + for _, path := range paths { + parsed := naming.ParseFilename(path, "series") + if parsed == nil || parsed.EpisodeNum <= 0 { + continue + } + if bySeason[parsed.SeasonNum] == nil { + bySeason[parsed.SeasonNum] = make(map[int]localEpisodeMatchHint) + } + title := extractEpisodeMatchTitle(path) + if isGenericEpisodeMatchTitle(title) { + title = "" + } + existing, exists := bySeason[parsed.SeasonNum][parsed.EpisodeNum] + if exists && utf8.RuneCountInString(existing.Title) >= utf8.RuneCountInString(title) { + continue + } + bySeason[parsed.SeasonNum][parsed.EpisodeNum] = localEpisodeMatchHint{ + SeasonNumber: parsed.SeasonNum, EpisodeNumber: parsed.EpisodeNum, Title: title, + } + } + + seasonNumbers := make([]int, 0, len(bySeason)) + for season, episodes := range bySeason { + if len(episodes) > 0 { + seasonNumbers = append(seasonNumbers, season) + } + } + if len(seasonNumbers) == 0 { + return nil + } + sort.Ints(seasonNumbers) + + // Specials are weak shape evidence. Prefer numbered seasons whenever they + // are available, retaining season zero only for specials-only libraries. + numberedSeasons := make([]int, 0, len(seasonNumbers)) + for _, season := range seasonNumbers { + if season > 0 { + numberedSeasons = append(numberedSeasons, season) + } + } + if len(numberedSeasons) > 0 { + seasonNumbers = numberedSeasons + } + + if len(seasonNumbers) == 1 { + selectedSeason := seasonNumbers[0] + episodeNumbers := sortedEpisodeNumbers(bySeason[selectedSeason]) + if len(episodeNumbers) > maxEpisodeValidationSamples { + episodeNumbers = []int{ + episodeNumbers[0], + episodeNumbers[len(episodeNumbers)/2], + episodeNumbers[len(episodeNumbers)-1], + } + } + out := make([]localEpisodeMatchHint, 0, len(episodeNumbers)) + for _, episodeNumber := range episodeNumbers { + out = append(out, bySeason[selectedSeason][episodeNumber]) + } + return out + } + + if len(seasonNumbers) > maxEpisodeValidationSeasons { + seasonNumbers = []int{ + seasonNumbers[0], + seasonNumbers[len(seasonNumbers)/2], + seasonNumbers[len(seasonNumbers)-1], + } + } + + // The last locally present episode of several seasons is far more + // discriminating than S01E01-style existence checks: same-name remakes + // rarely share both season count and episode count per season. + out := make([]localEpisodeMatchHint, 0, len(seasonNumbers)) + for _, seasonNumber := range seasonNumbers { + episodeNumbers := sortedEpisodeNumbers(bySeason[seasonNumber]) + out = append(out, bySeason[seasonNumber][episodeNumbers[len(episodeNumbers)-1]]) + } + return out +} + +func sortedEpisodeNumbers(episodes map[int]localEpisodeMatchHint) []int { + numbers := make([]int, 0, len(episodes)) + for episodeNumber := range episodes { + numbers = append(numbers, episodeNumber) + } + sort.Ints(numbers) + return numbers +} + +func extractEpisodeMatchTitle(path string) string { + base := filepath.Base(path) + stem := strings.TrimSuffix(base, filepath.Ext(base)) + location := episodeCodePattern.FindStringIndex(stem) + if location == nil { + return "" + } + title := strings.TrimLeft(stem[location[1]:], " ._-") + if cut := strings.IndexAny(title, "[{"); cut >= 0 { + title = title[:cut] + } + if location := episodeReleaseSuffixPattern.FindStringIndex(title); location != nil { + title = title[:location[0]] + } + title = strings.NewReplacer(".", " ", "_", " ").Replace(title) + return strings.Join(strings.Fields(strings.Trim(title, " -")), " ") +} + +func isGenericEpisodeMatchTitle(title string) bool { + normalized := strings.Join(strings.Fields(strings.TrimSpace(title)), " ") + return normalized == "" || episodeGenericTitlePattern.MatchString(normalized) || episodeTechnicalTitlePattern.MatchString(normalized) +} + +func isDistinctiveSingleEpisodeTitle(title string) bool { + normalized := normalizeTitleForScoring(title) + return !isGenericEpisodeMatchTitle(normalized) && utf8.RuneCountInString(normalized) >= 10 && len(strings.Fields(normalized)) >= 2 +} + +func episodeTitleSimilarity(left, right string) float64 { + leftComparable := foldEpisodeTitleDiacritics(normalizeTitleForScoring(left)) + rightComparable := foldEpisodeTitleDiacritics(normalizeTitleForScoring(right)) + if leftComparable == "" || rightComparable == "" { + return 0 + } + if leftComparable == rightComparable { + return 1 + } + if naming.InferTitlesCoherent(leftComparable, rightComparable) { + return 0.8 + } + return 0 +} + +func foldEpisodeTitleDiacritics(value string) string { + var builder strings.Builder + for _, r := range norm.NFD.String(value) { + if !unicode.Is(unicode.Mn, r) { + builder.WriteRune(r) + } + } + return norm.NFC.String(builder.String()) +} diff --git a/internal/metadata/episode_match_validation_test.go b/internal/metadata/episode_match_validation_test.go new file mode 100644 index 00000000..ccd465f2 --- /dev/null +++ b/internal/metadata/episode_match_validation_test.go @@ -0,0 +1,745 @@ +//nolint:goconst // Repeated provider IDs and episode titles make these fixtures readable. +package metadata + +import ( + "context" + "errors" + "sync" + "testing" +) + +type episodeValidationStubProvider struct { + mu sync.Mutex + slug string + episodes map[string][]EpisodeResult + errors map[string]error + calls []string +} + +type episodeValidationPipelineProvider struct { + *episodeValidationStubProvider + searchResults []SearchResult +} + +func (p *episodeValidationPipelineProvider) Search(_ context.Context, _ SearchQuery) ([]SearchResult, error) { + return append([]SearchResult(nil), p.searchResults...), nil +} + +func (p *episodeValidationPipelineProvider) GetMetadata(_ context.Context, req MetadataRequest) (*MetadataResult, error) { + id := req.ProviderIDs["tmdb"] + year := 0 + switch id { + case "2015": + year = 2015 + case "2020": + year = 2020 + } + return &MetadataResult{ + HasMetadata: true, + Title: "Crims", + Year: year, + ProviderIDs: map[string]string{"tmdb": id}, + }, nil +} + +func (p *episodeValidationStubProvider) Slug() string { + if p.slug == "" { + return "tmdb" + } + return p.slug +} +func (p *episodeValidationStubProvider) Name() string { return p.Slug() } +func (p *episodeValidationStubProvider) ForTypes() []string { return []string{"series"} } +func (p *episodeValidationStubProvider) GetSeasons(context.Context, SeasonsRequest) ([]SeasonResult, error) { + return nil, nil +} +func (p *episodeValidationStubProvider) GetEpisodes(_ context.Context, req EpisodesRequest) ([]EpisodeResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + id := req.ProviderIDs[p.Slug()] + p.calls = append(p.calls, id) + if err := p.errors[id]; err != nil { + return nil, err + } + return append([]EpisodeResult(nil), p.episodes[id]...), nil +} + +func (p *episodeValidationStubProvider) callCount() int { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.calls) +} + +func TestExtractEpisodeMatchTitle(t *testing.T) { + t.Parallel() + tests := map[string]string{ + "standard radarr style": "/tv/Crims/Season 1/Crims - S01E01 - Day One[WEBDL-1080p.AAC.h264.NTb].mp4", + "dotted release": "/tv/Crims/Season 1/Crims.S01E01.Day.One.1080p.WEB-DL.x264-GROUP.mkv", + "no episode title": "/tv/Crims/Season 1/Crims.S01E01.1080p.WEB.x264-GROUP.mkv", + } + wants := map[string]string{ + "standard radarr style": "Day One", + "dotted release": "Day One", + "no episode title": "", + } + for name, path := range tests { + name, path := name, path + t.Run(name, func(t *testing.T) { + t.Parallel() + if got := extractEpisodeMatchTitle(path); got != wants[name] { + t.Fatalf("extractEpisodeMatchTitle() = %q, want %q", got, wants[name]) + } + }) + } +} + +func TestValidateSeriesMatchByEpisodes_SelectsSecondExactTitleCandidate(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "2020": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Arrival"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Departure"}, + }, + "2015": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Crime and Punishment"}, + }, + }} + hints := &MatchHints{ + Title: "Crims", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Crims/Season 1/Crims - S01E01 - Day One[WEBDL-1080p].mkv", + "/tv/Crims/Season 1/Crims - S01E02 - Crime and Punishment[WEBDL-1080p].mkv", + }, + } + candidates := []MatchCandidate{ + {Title: "Crims", Year: 2020, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "2020"}}, + {Title: "Crims", Year: 2015, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "2015"}}, + } + + winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if len(errs) != 0 { + t.Fatalf("validation errors = %v", errs) + } + if winner == nil || winner.ProviderIDs["tmdb"] != "2015" { + t.Fatalf("winner = %+v, want 2015 candidate", winner) + } + if !containsString(winner.MatchReasons, "episode_title_corroboration:2_of_2") { + t.Fatalf("winner reasons = %v", winner.MatchReasons) + } +} + +func TestValidateSeriesMatchByEpisodes_LeavesMemoriesUnmatched(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "2018": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "A New Beginning"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Home"}, + }, + "1998": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Old Friends"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Reunion"}, + }, + }} + hints := &MatchHints{ + Title: "Memories", Type: "series", + AllGroupFilePaths: []string{ + "/anime/Memories/Season 1/Memories - S01E01 - Magnetic Rose[Bluray-1080p].mkv", + "/anime/Memories/Season 1/Memories - S01E02 - Stink Bomb[Bluray-1080p].mkv", + }, + } + candidates := []MatchCandidate{ + {Title: "Memories", Year: 2018, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "2018"}}, + {Title: "Memories", Year: 1998, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "1998"}}, + } + + winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if winner != nil { + t.Fatalf("winner = %+v, want unmatched type conflict", winner) + } +} + +func TestValidateSeriesMatchByEpisodes_SearchesPastFormerTopThree(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "1": {{SeasonNumber: 1, EpisodeNumber: 1, Title: "Wrong One"}, {SeasonNumber: 1, EpisodeNumber: 2, Title: "Wrong Two"}}, + "2": {{SeasonNumber: 1, EpisodeNumber: 1, Title: "Wrong One"}, {SeasonNumber: 1, EpisodeNumber: 2, Title: "Wrong Two"}}, + "3": {{SeasonNumber: 1, EpisodeNumber: 1, Title: "Wrong One"}, {SeasonNumber: 1, EpisodeNumber: 2, Title: "Wrong Two"}}, + "4": {{SeasonNumber: 1, EpisodeNumber: 1, Title: "Right One"}, {SeasonNumber: 1, EpisodeNumber: 2, Title: "Right Two"}}, + }} + hints := &MatchHints{ + Title: "Shared", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Shared/Season 1/Shared - S01E01 - Right One[WEBDL-1080p].mkv", + "/tv/Shared/Season 1/Shared - S01E02 - Right Two[WEBDL-1080p].mkv", + }, + } + candidates := make([]MatchCandidate, 0, 4) + for _, id := range []string{"1", "2", "3", "4"} { + candidates = append(candidates, MatchCandidate{ + Title: "Shared", Year: 2000, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": id}, + }) + } + + winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if winner == nil || winner.ProviderIDs["tmdb"] != "4" { + t.Fatalf("winner = %+v, want corroborated fourth candidate", winner) + } + if got := provider.callCount(); got != len(candidates) { + t.Fatalf("provider calls = %d, want %d", got, len(candidates)) + } +} + +func TestValidateSeriesMatchByEpisodes_BoundsCandidateFetches(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{}} + hints := &MatchHints{ + Title: "Shared", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Shared/Shared - S01E01 - Right One.mkv", + "/tv/Shared/Shared - S01E02 - Right Two.mkv", + }, + } + candidates := make([]MatchCandidate, 0, maxEpisodeValidationCandidates+4) + for i := 0; i < maxEpisodeValidationCandidates+4; i++ { + id := string(rune('a' + i)) + provider.episodes[id] = []EpisodeResult{ + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Wrong One"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Wrong Two"}, + } + candidates = append(candidates, MatchCandidate{ + Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": id}, + }) + } + + validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if got := provider.callCount(); got != maxEpisodeValidationCandidates { + t.Fatalf("provider calls = %d, want bounded %d", got, maxEpisodeValidationCandidates) + } +} + +func TestValidateSeriesMatchByEpisodes_ValidatesCrossProviderExactYearAlias(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "show": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "The Beginning"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "The Journey"}, + }, + }} + hints := &MatchHints{ + Title: "Bunny Drop", Year: 2011, Type: "series", + AllGroupFilePaths: []string{ + "/tv/Bunny Drop/Bunny Drop - S01E01 - The Beginning.mkv", + "/tv/Bunny Drop/Bunny Drop - S01E02 - The Journey.mkv", + }, + } + candidates := []MatchCandidate{{ + Title: "Usagi Drop", Year: 2011, ContentType: "series", Sources: []string{"tmdb", "tvdb"}, + ProviderIDs: map[string]string{"tmdb": "show", "tvdb": "123"}, + }} + + winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if len(errs) != 0 || winner == nil || winner.ProviderIDs["tmdb"] != "show" { + t.Fatalf("winner = %+v errors = %v, want exact-year alias corroborated", winner, errs) + } +} + +func TestValidateSeriesMatchByEpisodes_AcceptsDistinctiveSoleEpisode(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "232926": {{SeasonNumber: 1, EpisodeNumber: 1, Title: "The Fiancé Who Killed Me"}}, + }} + hints := &MatchHints{ + Title: "7th Time Loop", Type: "series", + AllGroupFilePaths: []string{ + "/tv/7th Time Loop/Season 1/7th Time Loop - S01E01 - The Fiance Who Killed Me[WEBDL-1080p].mkv", + }, + } + candidates := []MatchCandidate{{ + Title: "7th Time Loop", Year: 2024, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "232926"}, + }} + + winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if winner == nil { + t.Fatal("distinctive exact episode title should corroborate the sole candidate") + } +} + +func TestValidateSeriesMatchByEpisodes_FallsBackWhenPreferredProviderHasGenericTitles(t *testing.T) { + t.Parallel() + tmdb := &episodeValidationStubProvider{slug: "tmdb", episodes: map[string][]EpisodeResult{ + "61834": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Episode 1"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Episode 2"}, + }, + }} + tvdb := &episodeValidationStubProvider{slug: "tvdb", episodes: map[string][]EpisodeResult{ + "289456": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Day Thirty-Six"}, + }, + }} + hints := &MatchHints{ + Title: "Crims", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Crims/Season 1/Crims - S01E01 - Day One.WEBDL-1080p.mkv", + "/tv/Crims/Season 1/Crims - S01E02 - Day Thirty-Six.WEBDL-1080p.mkv", + }, + } + candidates := []MatchCandidate{{ + Title: "Crims", Year: 2015, ContentType: "series", Sources: []string{"tmdb", "tvdb"}, + ProviderIDs: map[string]string{"tmdb": "61834", "tvdb": "289456"}, + }} + + winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{tvdb, tmdb}, "en") + if len(errs) != 0 { + t.Fatalf("validation errors = %v", errs) + } + if winner == nil || winner.ProviderIDs["tvdb"] != "289456" { + t.Fatalf("winner = %+v, want TVDB-corroborated Crims", winner) + } + if tmdb.callCount() != 1 || tvdb.callCount() != 1 { + t.Fatalf("provider calls tmdb=%d tvdb=%d, want one each", tmdb.callCount(), tvdb.callCount()) + } +} + +func TestValidateSeriesMatchByEpisodes_TreatsMissingRivalSeasonAsZeroEvidence(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{ + episodes: map[string][]EpisodeResult{ + "correct": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Day Thirty-Six"}, + }, + }, + errors: map[string]error{"rival": errors.New("tmdb: HTTP 404: season not found")}, + } + hints := &MatchHints{ + Title: "Crims", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Crims/Crims - S01E01 - Day One.mkv", + "/tv/Crims/Crims - S01E02 - Day Thirty-Six.mkv", + }, + } + candidates := []MatchCandidate{ + {Title: "Crims", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "rival"}}, + {Title: "Crims", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "correct"}}, + } + + winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if len(errs) != 0 { + t.Fatalf("validation errors = %v, want missing season treated as evaluated evidence", errs) + } + if winner == nil || winner.ProviderIDs["tmdb"] != "correct" { + t.Fatalf("winner = %+v, want corroborated candidate", winner) + } +} + +func TestValidateSeriesMatchByEpisodes_OperationalRivalErrorBlocksPromotion(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{ + episodes: map[string][]EpisodeResult{ + "correct": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Day Thirty-Six"}, + }, + }, + errors: map[string]error{"rival": errors.New("tmdb: HTTP 503: unavailable")}, + } + hints := &MatchHints{ + Title: "Crims", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Crims/Crims - S01E01 - Day One.mkv", + "/tv/Crims/Crims - S01E02 - Day Thirty-Six.mkv", + }, + } + candidates := []MatchCandidate{ + {Title: "Crims", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "rival"}}, + {Title: "Crims", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "correct"}}, + } + + winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if winner != nil { + t.Fatalf("winner = %+v, want operationally unavailable rival to block promotion", winner) + } + if len(errs) != 1 { + t.Fatalf("validation errors = %v, want one operational error", errs) + } +} + +func TestValidateSeriesMatchByEpisodes_AcceptsMultiSourceCoordinateCorroboration(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "show": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Episode 1"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Episode 2"}, + {SeasonNumber: 1, EpisodeNumber: 3, Title: "Episode 3"}, + }, + }} + hints := &MatchHints{ + Title: "Shared", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Shared/Shared - S01E01.mkv", + "/tv/Shared/Shared - S01E02.mkv", + "/tv/Shared/Shared - S01E03.mkv", + }, + } + candidates := []MatchCandidate{{ + Title: "Shared", ContentType: "series", Sources: []string{"tmdb", "tvdb"}, + ProviderIDs: map[string]string{"tmdb": "show", "tvdb": "other"}, + }} + + winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if len(errs) != 0 { + t.Fatalf("validation errors = %v", errs) + } + if winner == nil || !containsString(winner.MatchReasons, "episode_coordinate_corroboration:3_of_3") { + t.Fatalf("winner = %+v, want coordinate corroboration", winner) + } +} + +func TestValidateSeriesMatchByEpisodes_RejectsWeakSingleSourceCoordinates(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "show": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Episode 1"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Episode 2"}, + {SeasonNumber: 1, EpisodeNumber: 3, Title: "Episode 3"}, + }, + }} + hints := &MatchHints{ + Title: "Shared", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Shared/Shared - S01E01.mkv", + "/tv/Shared/Shared - S01E02.mkv", + "/tv/Shared/Shared - S01E03.mkv", + }, + } + candidates := []MatchCandidate{{ + Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "show"}, + }} + + winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if winner != nil { + t.Fatalf("winner = %+v, weak S01E01-E03 coordinates from one source must stay unmatched", winner) + } +} + +func TestValidateSeriesMatchByEpisodes_AcceptsDistinctiveSingleSourceCoordinates(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "show": { + {SeasonNumber: 2, EpisodeNumber: 1, Title: "Episode 1"}, + {SeasonNumber: 2, EpisodeNumber: 4, Title: "Episode 4"}, + {SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"}, + }, + }} + hints := &MatchHints{ + Title: "Shared", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Shared/Shared - S02E01.mkv", + "/tv/Shared/Shared - S02E04.mkv", + "/tv/Shared/Shared - S02E08.mkv", + }, + } + candidates := []MatchCandidate{{ + Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "show"}, + }} + + winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if winner == nil || !containsString(winner.MatchReasons, "episode_coordinate_corroboration:3_of_3") { + t.Fatalf("winner = %+v, want distinctive coordinate corroboration", winner) + } +} + +func TestValidateSeriesMatchByEpisodes_RejectsCoordinateOnlyChoiceBetweenExactTitles(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "first": { + {SeasonNumber: 2, EpisodeNumber: 1, Title: "Episode 1"}, + {SeasonNumber: 2, EpisodeNumber: 4, Title: "Episode 4"}, + {SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"}, + }, + "second": { + {SeasonNumber: 2, EpisodeNumber: 1, Title: "Episode 1"}, + }, + }} + hints := &MatchHints{ + Title: "Shared", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Shared/Shared - S02E01.mkv", + "/tv/Shared/Shared - S02E04.mkv", + "/tv/Shared/Shared - S02E08.mkv", + }, + } + candidates := []MatchCandidate{ + {Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "first"}}, + {Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "second"}}, + } + + winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if winner != nil { + t.Fatalf("winner = %+v, coordinates alone must not choose between exact-title candidates", winner) + } +} + +func TestSelectLocalEpisodeCoordinateHints_SamplesLastEpisodeAcrossSeasons(t *testing.T) { + t.Parallel() + + hints := selectLocalEpisodeCoordinateHints([]string{ + "/tv/Shared/Season 1/Shared - S01E01.mkv", + "/tv/Shared/Season 1/Shared - S01E10.mkv", + "/tv/Shared/Season 2/Shared - S02E08.mkv", + "/tv/Shared/Season 3/Shared - S03E12.mkv", + "/tv/Shared/Season 4/Shared - S04E04.mkv", + }) + + want := []localEpisodeMatchHint{ + {SeasonNumber: 1, EpisodeNumber: 10}, + {SeasonNumber: 3, EpisodeNumber: 12}, + {SeasonNumber: 4, EpisodeNumber: 4}, + } + if len(hints) != len(want) { + t.Fatalf("hints = %+v, want %+v", hints, want) + } + for i := range want { + if hints[i].SeasonNumber != want[i].SeasonNumber || hints[i].EpisodeNumber != want[i].EpisodeNumber { + t.Fatalf("hints = %+v, want %+v", hints, want) + } + } +} + +func TestValidateSeriesMatchByEpisodes_AcceptsUniqueMultiSeasonShape(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "original": { + {SeasonNumber: 1, EpisodeNumber: 10, Title: "Episode 10"}, + {SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"}, + {SeasonNumber: 4, EpisodeNumber: 6, Title: "Episode 6"}, + }, + "remake": { + {SeasonNumber: 1, EpisodeNumber: 10, Title: "Episode 10"}, + {SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"}, + }, + }} + hints := &MatchHints{ + Title: "Shared", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Shared/Season 1/Shared - S01E10.mkv", + "/tv/Shared/Season 2/Shared - S02E08.mkv", + "/tv/Shared/Season 4/Shared - S04E06.mkv", + }, + } + candidates := []MatchCandidate{ + {Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "remake"}}, + {Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "original"}}, + } + + winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if len(errs) != 0 { + t.Fatalf("validation errors = %v", errs) + } + if winner == nil || winner.ProviderIDs["tmdb"] != "original" { + t.Fatalf("winner = %+v, want original multi-season shape", winner) + } + if !containsString(winner.MatchReasons, "episode_coordinate_corroboration:3_of_3") { + t.Fatalf("winner reasons = %v", winner.MatchReasons) + } +} + +func TestValidateSeriesMatchByEpisodes_RejectsTwoCoordinateSingleSeasonShape(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "candidate": { + {SeasonNumber: 2, EpisodeNumber: 1, Title: "Episode 1"}, + {SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"}, + }, + }} + hints := &MatchHints{ + Title: "Shared", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Shared/Season 2/Shared - S02E01.mkv", + "/tv/Shared/Season 2/Shared - S02E08.mkv", + }, + } + candidates := []MatchCandidate{{ + Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, + ProviderIDs: map[string]string{"tmdb": "candidate"}, + }} + + winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if len(errs) != 0 { + t.Fatalf("validation errors = %v", errs) + } + if winner != nil { + t.Fatalf("winner = %+v, two coordinates from one season are insufficient shape evidence", winner) + } +} + +func TestValidateSeriesMatchByEpisodes_RejectsShapeWhenCandidatesAreTruncated(t *testing.T) { + t.Parallel() + provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{}} + hints := &MatchHints{ + Title: "Shared", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Shared/Season 1/Shared - S01E10.mkv", + "/tv/Shared/Season 2/Shared - S02E08.mkv", + }, + } + candidates := make([]MatchCandidate, 0, maxEpisodeValidationCandidates+1) + for i := 0; i < maxEpisodeValidationCandidates+1; i++ { + id := string(rune('a' + i)) + provider.episodes[id] = []EpisodeResult{{SeasonNumber: 1, EpisodeNumber: 10, Title: "Episode 10"}} + if i == 0 { + provider.episodes[id] = append(provider.episodes[id], EpisodeResult{SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"}) + } + candidates = append(candidates, MatchCandidate{ + Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": id}, + }) + } + + winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en") + if winner != nil { + t.Fatalf("winner = %+v, truncated candidate set must stay unmatched", winner) + } +} + +func TestInitialMatchPipelineUsesEpisodeCorroboration(t *testing.T) { + t.Parallel() + harness := newTestHarness() + harness.service.seasonRepo = newFakeSeasonRepo() + harness.service.episodeRepo = newFakeEpisodeRepo() + provider := &episodeValidationPipelineProvider{ + episodeValidationStubProvider: &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "2020": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Arrival"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Departure"}, + }, + "2015": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Crime and Punishment"}, + }, + }}, + searchResults: []SearchResult{ + {Name: "Crims", Year: 2020, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "2020"}}, + {Name: "Crims", Year: 2015, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "2015"}}, + }, + } + + result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{ + Hints: &MatchHints{ + Title: "Crims", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Crims/Season 1/Crims - S01E01 - Day One[WEBDL-1080p].mkv", + "/tv/Crims/Season 1/Crims - S01E02 - Crime and Punishment[WEBDL-1080p].mkv", + }, + }, + Language: "en", + Mode: ModeInitialMatch, + }, []Provider{provider}) + if err != nil { + t.Fatalf("ProcessWithProviders() error = %v", err) + } + if result == nil || !result.Updated || result.Decision == nil || result.Decision.Outcome != "matched" { + t.Fatalf("result = %#v, want matched update", result) + } + item, err := harness.itemRepo.GetByID(context.Background(), result.ContentID) + if err != nil { + t.Fatalf("load matched item: %v", err) + } + if item.TmdbID != "2015" || item.Year != 2015 { + t.Fatalf("matched item = tmdb:%q year:%d, want tmdb:2015 year:2015", item.TmdbID, item.Year) + } + foundReason := false + for _, candidate := range result.Decision.TopCandidates { + if candidate.ProviderIDs["tmdb"] == "2015" && containsString(candidate.Reasons, "episode_title_corroboration:2_of_2") { + foundReason = true + } + } + if !foundReason { + t.Fatalf("decision = %+v, want episode corroboration reason on selected candidate", result.Decision) + } +} + +func TestInitialMatchPipelineDoesNotAddEpisodeValidationFetchForDecisiveMatch(t *testing.T) { + t.Parallel() + harness := newTestHarness() + harness.service.seasonRepo = newFakeSeasonRepo() + harness.service.episodeRepo = newFakeEpisodeRepo() + provider := &episodeValidationPipelineProvider{ + episodeValidationStubProvider: &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{ + "2015": { + {SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"}, + {SeasonNumber: 1, EpisodeNumber: 2, Title: "Crime and Punishment"}, + }, + }}, + searchResults: []SearchResult{ + {Name: "Crims", Year: 2015, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "2015"}}, + }, + } + + result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{ + Hints: &MatchHints{ + Title: "Crims", Year: 2015, Type: "series", + AllGroupFilePaths: []string{ + "/tv/Crims/Season 1/Crims - S01E01 - Day One.mkv", + "/tv/Crims/Season 1/Crims - S01E02 - Crime and Punishment.mkv", + }, + }, + Language: "en", + Mode: ModeInitialMatch, + }, []Provider{provider}) + if err != nil { + t.Fatalf("ProcessWithProviders() error = %v", err) + } + if result == nil || result.Decision == nil || result.Decision.Outcome != "matched" { + t.Fatalf("result = %#v, want decisive match", result) + } + // The single call is the normal post-match episode enrichment. An episode + // validation fallback would add a second provider fetch before enrichment. + if calls := provider.callCount(); calls != 1 { + t.Fatalf("episode provider calls = %d, want only the normal enrichment fetch", calls) + } +} + +func TestInitialMatchPipelinePreservesCandidateRejectionForPermanentEpisodeValidationError(t *testing.T) { + t.Parallel() + harness := newTestHarness() + provider := &episodeValidationPipelineProvider{ + episodeValidationStubProvider: &episodeValidationStubProvider{errors: map[string]error{ + "first": errors.New("tmdb: HTTP 400: bad request"), + "second": errors.New("tmdb: HTTP 400: bad request"), + }}, + searchResults: []SearchResult{ + {Name: "Shared", Year: 2020, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "first"}}, + {Name: "Shared", Year: 2015, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "second"}}, + }, + } + + result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{ + Hints: &MatchHints{ + Title: "Shared", Type: "series", + AllGroupFilePaths: []string{ + "/tv/Shared/Shared - S01E01 - First Episode.mkv", + "/tv/Shared/Shared - S01E02 - Second Episode.mkv", + }, + }, + Language: "en", + Mode: ModeInitialMatch, + }, []Provider{provider}) + if err != nil { + t.Fatalf("ProcessWithProviders() error = %v", err) + } + if result == nil || result.Decision == nil || result.Decision.Outcome != "candidate_rejected" { + t.Fatalf("result = %#v decision = %+v, want deterministic candidate rejection", result, result.Decision) + } +} + +func containsString(values []string, want string) bool { + for _, value := range values { + if value == want { + return true + } + } + return false +} diff --git a/internal/metadata/id_consensus_test.go b/internal/metadata/id_consensus_test.go new file mode 100644 index 00000000..db9c96a2 --- /dev/null +++ b/internal/metadata/id_consensus_test.go @@ -0,0 +1,132 @@ +//nolint:goconst // Repeated canonical IDs are the evidence under test. +package metadata + +import ( + "context" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +type idConsensusPipelineProvider struct{} + +func (p *idConsensusPipelineProvider) Slug() string { return "tmdb" } +func (p *idConsensusPipelineProvider) Name() string { return "TMDB" } +func (p *idConsensusPipelineProvider) ForTypes() []string { return []string{"series"} } + +func (p *idConsensusPipelineProvider) Search(context.Context, SearchQuery) ([]SearchResult, error) { + return []SearchResult{ + { + Name: "A Teacher", Year: 2020, Provider: "tvdb", + ProviderIDs: map[string]string{ + "imdb": "tt10680614", "tmdb": "103992", "tvdb": "352440", + }, + }, + { + Name: "A Teacher", Year: 2020, Provider: "tmdb", + ProviderIDs: map[string]string{ + "imdb": "tt10680614", "tmdb": "103992", "tvdb": "473725", + }, + }, + }, nil +} + +func (p *idConsensusPipelineProvider) GetMetadata(_ context.Context, req MetadataRequest) (*MetadataResult, error) { + if req.ProviderIDs["tvdb"] != "" { + return nil, errUnexpectedQuarantinedProviderID + } + return &MetadataResult{ + HasMetadata: true, + Title: "A Teacher", + Year: 2020, + // A provider detail response may repeat its stale cross-reference. The + // pipeline must not let it undo the search-phase quarantine. + ProviderIDs: map[string]string{ + "imdb": "tt10680614", "tmdb": "103992", "tvdb": "473725", + }, + }, nil +} + +var errUnexpectedQuarantinedProviderID = &unexpectedQuarantinedProviderIDError{} + +type unexpectedQuarantinedProviderIDError struct{} + +func (*unexpectedQuarantinedProviderIDError) Error() string { + return "quarantined provider ID reached metadata phase" +} + +func TestInitialMatchPipelineQuarantinesConflictingConsensusID(t *testing.T) { + t.Parallel() + harness := newTestHarness() + provider := &idConsensusPipelineProvider{} + + result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{ + Hints: &MatchHints{Title: "A Teacher", Type: "series"}, + Mode: ModeInitialMatch, + }, []Provider{provider}) + if err != nil { + t.Fatalf("ProcessWithProviders() error = %v", err) + } + if result == nil || !result.Updated || result.Decision == nil || result.Decision.Outcome != "matched" { + t.Fatalf("result = %#v, want matched consensus", result) + } + item, err := harness.itemRepo.GetByID(context.Background(), result.ContentID) + if err != nil { + t.Fatalf("load matched item: %v", err) + } + if item.ImdbID != "tt10680614" || item.TmdbID != "103992" { + t.Fatalf("persisted consensus IDs = imdb:%q tmdb:%q", item.ImdbID, item.TmdbID) + } + if item.TvdbID != "" { + t.Fatalf("conflicting TVDB ID persisted as %q", item.TvdbID) + } +} + +func TestRefreshPipelinesQuarantineStoredConflictingConsensusID(t *testing.T) { + for name, mode := range map[string]RefreshMode{"scheduled": ModeScheduledRefresh, "manual": ModeManualRefresh} { + mode := mode + t.Run(name, func(t *testing.T) { + t.Parallel() + harness := newTestHarness() + const contentID = "series-refresh-consensus" + if err := harness.itemRepo.Upsert(context.Background(), &models.MediaItem{ + ContentID: contentID, Type: "series", Title: "A Teacher", Year: 2020, Status: "matched", + ImdbID: "tt10680614", TmdbID: "103992", TvdbID: "473725", + Studios: []string{}, Networks: []string{}, Countries: []string{}, Genres: []string{}, + }); err != nil { + t.Fatalf("seed item: %v", err) + } + + result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{ + ContentID: contentID, + Language: "en", + Mode: mode, + }, []Provider{&idConsensusPipelineProvider{}}) + if err != nil { + t.Fatalf("ProcessWithProviders() error = %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want updated refresh", result) + } + item, err := harness.itemRepo.GetByID(context.Background(), contentID) + if err != nil { + t.Fatalf("load refreshed item: %v", err) + } + if item.ImdbID != "tt10680614" || item.TmdbID != "103992" || item.TvdbID != "" { + t.Fatalf("refreshed IDs = imdb:%q tmdb:%q tvdb:%q, want disputed TVDB ID quarantined", item.ImdbID, item.TmdbID, item.TvdbID) + } + }) + } +} + +func TestApplyCandidateProviderIDConsensusRemovesDisputedStoredID(t *testing.T) { + ids := map[string]string{"imdb": "tt10680614", "tmdb": "103992", "tvdb": "473725"} + winner := &MatchCandidate{ + ProviderIDs: map[string]string{"imdb": "tt10680614", "tmdb": "103992", "tvdb": "473725"}, + ConflictingProviderIDKeys: []string{"tvdb"}, + } + applyCandidateProviderIDConsensus(ids, winner, nil) + if ids["tvdb"] != "" || ids["imdb"] != "tt10680614" || ids["tmdb"] != "103992" { + t.Fatalf("consensus IDs = %#v", ids) + } +} diff --git a/internal/metadata/match_candidates.go b/internal/metadata/match_candidates.go index 82e93e51..a3d369ee 100644 --- a/internal/metadata/match_candidates.go +++ b/internal/metadata/match_candidates.go @@ -5,34 +5,49 @@ import ( "fmt" "log/slog" "math" + "slices" "sort" "strconv" "strings" "unicode" + "github.com/Silo-Server/silo-server/internal/lang" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/naming" + "github.com/Silo-Server/silo-server/internal/providerid" ) // MatchCandidate represents a deduplicated search result grouped by normalized // provider IDs. Multiple raw SearchResult rows from different providers that // share the same TMDB/TVDB/IMDB IDs are collapsed into a single candidate. type MatchCandidate struct { - Title string `json:"title"` - Year int `json:"year"` - ContentType string `json:"content_type"` - ProviderIDs map[string]string `json:"provider_ids"` - ImageURL string `json:"image_url,omitempty"` - Overview string `json:"overview,omitempty"` - Sources []string `json:"sources"` - AgreementHints []string `json:"agreement_hints"` - DetailScore int `json:"-"` + Title string `json:"title"` + OriginalTitle string `json:"original_title,omitempty"` + TitleAliases []TitleAlias `json:"aliases,omitempty"` + TitleLanguage string `json:"title_language,omitempty"` + TitleIsFallback bool `json:"title_is_fallback,omitempty"` + MatchedTitle string `json:"matched_title,omitempty"` + MatchScore float64 `json:"match_score,omitempty"` + MatchReasons []string `json:"match_reasons,omitempty"` + Year int `json:"year"` + ContentType string `json:"content_type"` + ProviderIDs map[string]string `json:"provider_ids"` + ImageURL string `json:"image_url,omitempty"` + Overview string `json:"overview,omitempty"` + Sources []string `json:"sources"` + AgreementHints []string `json:"agreement_hints"` + DetailScore int `json:"-"` + // ConflictingProviderIDKeys contains canonical provider IDs deliberately + // excluded after two other canonical IDs proved that provider results refer + // to the same work. These keys remain quarantined for the rest of the match + // pipeline so a later detail response cannot silently reintroduce them. + ConflictingProviderIDKeys []string `json:"-"` + titleRank int } var canonicalCandidateIDKeys = []string{"tmdb", "tvdb", "imdb"} -func compatibleProviderIDs(left, right map[string]string) bool { - overlap := false +func providerIDMergeEvidence(left, right map[string]string) (matches int, conflicts []string) { for _, key := range canonicalCandidateIDKeys { lv := strings.TrimSpace(left[key]) rv := strings.TrimSpace(right[key]) @@ -40,11 +55,23 @@ func compatibleProviderIDs(left, right map[string]string) bool { continue } if lv != rv { - return false + conflicts = append(conflicts, key) + continue } - overlap = true + matches++ } - return overlap + return matches, conflicts +} + +func compatibleProviderIDs(left, right map[string]string) bool { + matches, conflicts := providerIDMergeEvidence(left, right) + if matches == 0 { + return false + } + // One agreeing ID plus one conflict is not consensus. Two independently + // agreeing canonical IDs are enough to prove identity while quarantining a + // stale third-party cross-reference. + return len(conflicts) == 0 || matches >= 2 } func providerIDRichness(ids map[string]string) int { @@ -57,9 +84,82 @@ func providerIDRichness(ids map[string]string) int { return score } +func sanitizeCandidateProviderIDs(ids map[string]string) map[string]string { + if len(ids) == 0 { + return nil + } + sanitized := make(map[string]string, len(ids)) + for key, value := range ids { + key = strings.ToLower(strings.TrimSpace(key)) + value, valid := sanitizeProviderIDValue(key, value) + if !valid { + continue + } + sanitized[key] = value + } + return sanitized +} + +func sanitizeCanonicalProviderIDsInPlace(ids map[string]string) { + for _, key := range canonicalCandidateIDKeys { + value := strings.TrimSpace(ids[key]) + if value == "" { + delete(ids, key) + continue + } + sanitized, valid := sanitizeProviderIDValue(key, value) + if !valid { + delete(ids, key) + continue + } + ids[key] = sanitized + } +} + +func sanitizeProviderIDValue(key, value string) (string, bool) { + key = strings.ToLower(strings.TrimSpace(key)) + value = strings.TrimSpace(value) + if key == "" || value == "" { + return "", false + } + switch key { + case "tmdb", "tvdb": + if !providerid.IsPositiveDecimal(value) { + return "", false + } + case "imdb": + value = strings.ToLower(value) + if !isValidIMDbProviderID(value) { + return "", false + } + } + return value, true +} + +func sanitizedMatchHintProviderIDs(hints *MatchHints) *MatchHints { + if hints == nil { + return nil + } + sanitized := *hints + sanitized.TmdbID, _ = sanitizeProviderIDValue("tmdb", hints.TmdbID) + sanitized.TvdbID, _ = sanitizeProviderIDValue("tvdb", hints.TvdbID) + sanitized.ImdbID, _ = sanitizeProviderIDValue("imdb", hints.ImdbID) + return &sanitized +} + +func isValidIMDbProviderID(value string) bool { + value = strings.ToLower(strings.TrimSpace(value)) + if !strings.HasPrefix(value, "tt") { + return false + } + digits := strings.TrimPrefix(value, "tt") + return len(digits) >= 7 && len(digits) <= 10 && providerid.IsPositiveDecimal(digits) +} + const ( - minimumDetailTieBreakScore = 20 - minimumDetailTieBreakGap = 12 + automaticMatchAcceptanceFloor = 55 + minimumDetailTieBreakScore = 20 + minimumDetailTieBreakGap = 12 ) func duplicateTieBreakWinner(hints *MatchHints, scoredCandidates []scoredMatchCandidate) (*MatchCandidate, bool) { @@ -113,13 +213,13 @@ func duplicateTieBreakComparable(hints *MatchHints, left, right MatchCandidate) !strings.EqualFold(left.ContentType, right.ContentType) { return false } - if inferTitleSimilarity(left.Title, right.Title, hints.Year) != 1 { + if candidateToCandidateTitleSimilarity(left, right, hints.Year) != 1 { return false } - if inferTitleSimilarity(hints.Title, left.Title, hints.Year) != 1 { + if similarity, _ := bestCandidateTitleSimilarity(hints.Title, left, hints.Year); similarity != 1 { return false } - if inferTitleSimilarity(hints.Title, right.Title, hints.Year) != 1 { + if similarity, _ := bestCandidateTitleSimilarity(hints.Title, right, hints.Year); similarity != 1 { return false } return samePrimaryProvider(left.ProviderIDs, right.ProviderIDs) @@ -182,15 +282,23 @@ func normalizedKey(ids map[string]string) string { // provider IDs are unioned, sources list every provider slug that returned // the result, and agreement_hints notes when multiple providers agree. func NormalizeCandidates(results []SearchResult, contentType string) []MatchCandidate { + return NormalizeCandidatesForLanguage(results, contentType, "") +} + +// NormalizeCandidatesForLanguage deduplicates results while preferring an +// explicitly localized provider title over a provider-marked fallback. +func NormalizeCandidatesForLanguage(results []SearchResult, contentType, language string) []MatchCandidate { type bucket struct { - candidate MatchCandidate - sources map[string]bool + candidate MatchCandidate + sources map[string]bool + conflictingIDKeys map[string]bool } ordered := make([]string, 0) buckets := make(map[string]*bucket) for _, sr := range results { + sr.ProviderIDs = sanitizeCandidateProviderIDs(sr.ProviderIDs) key := "" for _, existingKey := range ordered { if compatibleProviderIDs(buckets[existingKey].candidate.ProviderIDs, sr.ProviderIDs) { @@ -208,26 +316,43 @@ func NormalizeCandidates(results []SearchResult, contentType string) []MatchCand b, exists := buckets[key] if !exists { + title, titleLanguage, fallback, titleRank := preferredSearchResultTitle(sr, language) b = &bucket{ candidate: MatchCandidate{ - Title: sr.Name, - Year: sr.Year, - ContentType: contentType, - ProviderIDs: make(map[string]string), - ImageURL: sr.ImageURL, - Overview: sr.Overview, + Title: title, + OriginalTitle: sr.OriginalTitle, + TitleAliases: copyTitleAliases(sr.TitleAliases, sr.Provider), + TitleLanguage: titleLanguage, + TitleIsFallback: fallback, + titleRank: titleRank, + Year: sr.Year, + ContentType: contentType, + ProviderIDs: make(map[string]string), + ImageURL: sr.ImageURL, + Overview: sr.Overview, }, - sources: make(map[string]bool), + sources: make(map[string]bool), + conflictingIDKeys: make(map[string]bool), } buckets[key] = b ordered = append(ordered, key) } + mergeCandidateTitles(&b.candidate, sr, language) - // Merge provider IDs. + // Merge provider IDs. When two canonical IDs agree but a third + // conflicts, the candidates are the same work and the disputed key is + // quarantined instead of allowing provider iteration order to choose it. for k, v := range sr.ProviderIDs { - if v != "" { - b.candidate.ProviderIDs[k] = v + v = strings.TrimSpace(v) + if v == "" || b.conflictingIDKeys[k] { + continue } + if existing := strings.TrimSpace(b.candidate.ProviderIDs[k]); existing != "" && existing != v && slices.Contains(canonicalCandidateIDKeys, k) { + delete(b.candidate.ProviderIDs, k) + b.conflictingIDKeys[k] = true + continue + } + b.candidate.ProviderIDs[k] = v } // Track source providers. @@ -242,6 +367,7 @@ func NormalizeCandidates(results []SearchResult, contentType string) []MatchCand if b.candidate.ImageURL == "" && sr.ImageURL != "" { b.candidate.ImageURL = sr.ImageURL } + b.candidate.DetailScore = candidateDetailScore(b.candidate) } // Build final list preserving insertion order. @@ -255,6 +381,12 @@ func NormalizeCandidates(results []SearchResult, contentType string) []MatchCand } sort.Strings(sources) b.candidate.Sources = sources + for _, key := range canonicalCandidateIDKeys { + if b.conflictingIDKeys[key] { + b.candidate.ConflictingProviderIDKeys = append(b.candidate.ConflictingProviderIDKeys, key) + b.candidate.AgreementHints = append(b.candidate.AgreementHints, "quarantined_"+key+"_id") + } + } // Compute agreement hints from corroborating sources only: a local // sidecar (nfo) echoing a remote result is not provider agreement. @@ -275,15 +407,122 @@ func NormalizeCandidates(results []SearchResult, contentType string) []MatchCand return candidates } +func preferredSearchResultTitle(result SearchResult, language string) (string, string, bool, int) { + requested := baseMetadataLanguage(language) + resultLanguage := baseMetadataLanguage(result.TitleLanguage) + if strings.TrimSpace(result.Name) != "" && requested != "" && resultLanguage == requested && !result.TitleIsFallback { + return result.Name, resultLanguage, false, 3 + } + for _, alias := range result.TitleAliases { + if strings.TrimSpace(alias.Title) != "" && requested != "" && baseMetadataLanguage(alias.Language) == requested { + return alias.Title, requested, false, 2 + } + } + // Older plugins predate the language contract. Their primary title keeps + // normal first-provider priority; its original_title must not silently + // replace a title that may already be localized. + if strings.TrimSpace(result.Name) != "" && resultLanguage == "" && !result.TitleIsFallback { + return result.Name, "", false, 3 + } + if requested != "" && strings.TrimSpace(result.OriginalTitle) != "" { + return result.OriginalTitle, baseMetadataLanguage(result.OriginalLanguage), true, 1 + } + if strings.TrimSpace(result.Name) != "" { + return result.Name, resultLanguage, result.TitleIsFallback, 1 + } + return result.OriginalTitle, baseMetadataLanguage(result.OriginalLanguage), true, 1 +} + +func baseMetadataLanguage(language string) string { + return lang.Canonical(strings.ReplaceAll(language, "_", "-")) +} + +func copyTitleAliases(aliases []TitleAlias, provider string) []TitleAlias { + out := make([]TitleAlias, 0, len(aliases)) + for _, alias := range aliases { + if alias.Provider == "" { + alias.Provider = provider + } + out = appendUniqueTitleAlias(out, alias) + } + return out +} + +func mergeCandidateTitles(candidate *MatchCandidate, result SearchResult, language string) { + if candidate == nil { + return + } + if candidate.OriginalTitle == "" && strings.TrimSpace(result.OriginalTitle) != "" { + candidate.OriginalTitle = result.OriginalTitle + } + for _, alias := range copyTitleAliases(result.TitleAliases, result.Provider) { + candidate.TitleAliases = appendUniqueTitleAlias(candidate.TitleAliases, alias) + } + if strings.TrimSpace(result.OriginalTitle) != "" && !strings.EqualFold(result.OriginalTitle, candidate.Title) { + candidate.TitleAliases = appendUniqueTitleAlias(candidate.TitleAliases, TitleAlias{ + Title: result.OriginalTitle, Language: baseMetadataLanguage(result.OriginalLanguage), Kind: titleAliasKindOriginal, Provider: result.Provider, + }) + } + if strings.TrimSpace(result.Name) != "" && !strings.EqualFold(result.Name, candidate.Title) { + candidate.TitleAliases = appendUniqueTitleAlias(candidate.TitleAliases, TitleAlias{ + Title: result.Name, Language: baseMetadataLanguage(result.TitleLanguage), Kind: titleAliasKindLocalized, Provider: result.Provider, + }) + } + + title, titleLanguage, fallback, titleRank := preferredSearchResultTitle(result, language) + if strings.TrimSpace(title) != "" && titleRank > candidate.titleRank { + candidate.Title = title + candidate.TitleLanguage = titleLanguage + candidate.TitleIsFallback = fallback + candidate.titleRank = titleRank + } +} + +func appendUniqueTitleAlias(aliases []TitleAlias, alias TitleAlias) []TitleAlias { + alias.Title = strings.TrimSpace(alias.Title) + if alias.Title == "" { + return aliases + } + for _, existing := range aliases { + if strings.EqualFold(existing.Title, alias.Title) && + baseMetadataLanguage(existing.Language) == baseMetadataLanguage(alias.Language) && + strings.EqualFold(existing.Kind, alias.Kind) && + strings.EqualFold(existing.Provider, alias.Provider) { + return aliases + } + } + return append(aliases, alias) +} + +func candidateDetailScore(candidate MatchCandidate) int { + score := providerIDRichness(candidate.ProviderIDs) * 10 + if candidate.Year != 0 { + score += 15 + } + if strings.TrimSpace(candidate.Overview) != "" { + score += 20 + } + if strings.TrimSpace(candidate.ImageURL) != "" { + score += 15 + } + return score +} + // SearchAndNormalize is a convenience method that calls SearchProviders and // normalizes the results into MatchCandidates. Plugin-prefixed image URLs // (e.g. "metadb://...") are resolved to presigned HTTP URLs before returning. func (s *MetadataService) SearchAndNormalize(ctx context.Context, query SearchQuery, folderID int) ([]MatchCandidate, error) { + if strings.TrimSpace(query.Language) == "" { + query.Language = s.resolveFolderLanguage(ctx, folderID) + } results, err := s.SearchProviders(ctx, query, folderID) if err != nil { return nil, err } - candidates := NormalizeCandidates(results, query.ContentType) + candidates := NormalizeCandidatesForLanguage(results, query.ContentType, query.Language) + for i := range candidates { + annotateCandidateMatch(&candidates[i], &MatchHints{Title: query.Title, Year: query.Year, Type: query.ContentType}) + } if s.imageResolver != nil { for i, c := range candidates { @@ -300,11 +539,17 @@ func (s *MetadataService) SearchAndNormalize(ctx context.Context, query SearchQu } func scoreMatchCandidate(hints *MatchHints, candidate MatchCandidate) float64 { + score, _, _ := scoreMatchCandidateDetailed(hints, candidate) + return score +} + +func scoreMatchCandidateDetailed(hints *MatchHints, candidate MatchCandidate) (float64, string, []string) { if hints == nil { - return 0 + return 0, "", nil } score := 0.0 + reasons := make([]string, 0, 5) trustedIDMatches := 0 for _, key := range trustedSearchIDKeys { hintValue := trustedIDValue(hints, key) @@ -314,26 +559,37 @@ func scoreMatchCandidate(hints *MatchHints, candidate MatchCandidate) float64 { if candidate.ProviderIDs[key] == hintValue { score += 100 trustedIDMatches++ + reasons = append(reasons, "trusted_"+key+"_id") } } if trustedIDMatches > 0 { score += float64(trustedIDMatches * 10) } - score += float64(len(candidate.Sources) * 12) + if sourceCount := candidateCorroboratingSourceCount(candidate); sourceCount > 0 { + score += float64(sourceCount * 12) + reasons = append(reasons, "provider_sources") + } - if strings.TrimSpace(hints.Title) != "" && strings.TrimSpace(candidate.Title) != "" { - titleSimilarity := inferTitleSimilarity(hints.Title, candidate.Title, hints.Year) + matchedTitle := "" + if strings.TrimSpace(hints.Title) != "" { + titleSimilarity, title := bestCandidateTitleSimilarity(hints.Title, candidate, hints.Year) + matchedTitle = title if titleSimilarity == 1 { score += 45 + reasons = append(reasons, "exact_title") } else { score += titleSimilarity * 35 + if titleSimilarity > 0 { + reasons = append(reasons, "coherent_title") + } } } switch { case hints.Year != 0 && candidate.Year == hints.Year: score += 20 + reasons = append(reasons, "exact_year") case hints.Year != 0 && candidate.Year != 0 && math.Abs(float64(candidate.Year-hints.Year)) == 1: score += 5 } @@ -343,7 +599,41 @@ func scoreMatchCandidate(hints *MatchHints, candidate MatchCandidate) float64 { score += float64(providerIDRichness(candidate.ProviderIDs)) } - return score + return score, matchedTitle, reasons +} + +func candidateTitles(candidate MatchCandidate) []string { + titles := []string{candidate.Title, candidate.OriginalTitle} + for _, alias := range candidate.TitleAliases { + titles = append(titles, alias.Title) + } + return titles +} + +func bestCandidateTitleSimilarity(hint string, candidate MatchCandidate, year int) (float64, string) { + bestScore := 0.0 + bestTitle := "" + for _, title := range candidateTitles(candidate) { + score := inferTitleSimilarity(hint, title, year) + if score > bestScore { + bestScore = score + bestTitle = title + } + } + return bestScore, bestTitle +} + +func annotateCandidateMatch(candidate *MatchCandidate, hints *MatchHints) { + if candidate == nil { + return + } + candidate.MatchScore, candidate.MatchedTitle, candidate.MatchReasons = scoreMatchCandidateDetailed(hints, *candidate) + if len(candidate.ConflictingProviderIDKeys) > 0 { + candidate.MatchReasons = append(candidate.MatchReasons, "provider_id_consensus") + for _, key := range candidate.ConflictingProviderIDKeys { + candidate.MatchReasons = append(candidate.MatchReasons, "quarantined_"+key+"_id") + } + } } type scoredMatchCandidate struct { @@ -384,7 +674,7 @@ func candidatesAreSingleDistinctShow(best MatchCandidate, scored []scoredMatchCa if c.candidate.Year == 0 || c.candidate.Year != best.Year { return false } - if inferTitleSimilarity(best.Title, c.candidate.Title, best.Year) != 1 { + if candidateToCandidateTitleSimilarity(best, c.candidate, best.Year) != 1 { return false } for _, key := range canonicalCandidateIDKeys { @@ -521,9 +811,10 @@ func selectInitialMatchCandidate(hints *MatchHints, candidates []MatchCandidate, scoredCandidates := make([]scoredMatchCandidate, 0, len(candidates)) for _, candidate := range candidates { + annotateCandidateMatch(&candidate, hints) scoredCandidates = append(scoredCandidates, scoredMatchCandidate{ candidate: candidate, - score: scoreMatchCandidate(hints, candidate), + score: candidate.MatchScore, }) } sort.SliceStable(scoredCandidates, func(i, j int) bool { @@ -556,15 +847,49 @@ func selectInitialMatchCandidate(hints *MatchHints, candidates []MatchCandidate, } } - best := scoredCandidates[0] if trustedHintIDsPresent(hints) { - if candidateMatchesTrustedIDs(hints, best.candidate) { - return &best.candidate, true + // Provider searches can return noisy title-ranked results even for an ID + // query. A trusted local ID is decisive, so choose the highest-scored + // candidate that actually carries the matching ID instead of requiring + // the provider's first result to be correct. + for i := range scoredCandidates { + if candidateMatchesTrustedIDs(hints, scoredCandidates[i].candidate) { + return &scoredCandidates[i].candidate, true + } } return nil, false } - if best.score < 55 { + // A title-only local sidecar is useful metadata, but it is not an + // independent search result. Once a remote candidate with canonical IDs is + // available, remove ID-less candidates from the automatic selection set so + // their presence cannot turn the remote result into a misleading + // multi-candidate score-gap win. They remain in the original candidate list + // for diagnostics. + if anyCandidateHasProviderIDs(scoredCandidates) { + eligible := make([]scoredMatchCandidate, 0, len(scoredCandidates)) + for _, scored := range scoredCandidates { + if providerIDRichness(scored.candidate.ProviderIDs) > 0 { + eligible = append(eligible, scored) + } + } + scoredCandidates = eligible + } + if len(scoredCandidates) == 0 { + return nil, false + } + + best := scoredCandidates[0] + if best.score < automaticMatchAcceptanceFloor { + return nil, false + } + // A known local year that conflicts with the provider by more than the + // tolerated release-date window is negative evidence, not something a high + // source-count score may erase. Trusted external IDs were handled above and + // remain decisive; title-only matches must respect this guard even when two + // providers return the same canonical item. + if hints != nil && hints.Year != 0 && best.candidate.Year != 0 && + absYearDelta(best.candidate.Year, hints.Year) > 2 { return nil, false } // A search that resolves to a single distinct show (one candidate, or the @@ -588,7 +913,7 @@ func selectInitialMatchCandidate(hints *MatchHints, candidates []MatchCandidate, // strong independent corroboration — it stands in for a missing hint year // (folders without a "(YYYY)"). A lone single-source no-year result is NOT // accepted here and stays subject to the single-candidate >=70 gate. - multiSourceCorroborated := distinctSourceCount(topGroup) >= 2 + multiSourceCorroborated := hints.Year == 0 && distinctSourceCount(topGroup) >= 2 // An exact normalized-title match on a sole distinct show is strong // corroboration on its own, even when the folder year is off by a year // or two (festival vs wide-release date, regional release) — e.g. @@ -596,13 +921,14 @@ func selectInitialMatchCandidate(hints *MatchHints, candidates []MatchCandidate, // Bounded to ±2 years so same-title remakes decades apart still require // a year or multi-source match. Uses the same normalizer as title scoring // so "exact" here means a perfect title-similarity component. + titleSimilarity, _ := bestCandidateTitleSimilarity(hints.Title, best.candidate, hints.Year) titleCorroborated := hints.Year != 0 && best.candidate.Year != 0 && absYearDelta(best.candidate.Year, hints.Year) <= 2 && - normalizeTitleForScoring(best.candidate.Title) == normalizeTitleForScoring(hints.Title) + titleSimilarity == 1 // Year or source-count corroboration is only meaningful when the winning // candidate is at least title-coherent with the scanner hint. Otherwise a // high source/provider score can auto-accept an unrelated same-year result. - hintTitleCoherent := inferTitleSimilarity(hints.Title, best.candidate.Title, hints.Year) > 0 + hintTitleCoherent := titleSimilarity > 0 if (hintTitleCoherent && (yearCorroborated || multiSourceCorroborated)) || titleCorroborated { return pickByProviderPriority(topGroup, providerPriority), true } @@ -670,7 +996,20 @@ func exactTitleYearTypeMatch(hints *MatchHints, candidate MatchCandidate) bool { if !candidateTypeMatchesHint(hints.Type, candidate.ContentType) { return false } - return inferTitleSimilarity(hints.Title, candidate.Title, hints.Year) == 1 + similarity, _ := bestCandidateTitleSimilarity(hints.Title, candidate, hints.Year) + return similarity == 1 +} + +func candidateToCandidateTitleSimilarity(left, right MatchCandidate, year int) float64 { + best := 0.0 + for _, leftTitle := range candidateTitles(left) { + for _, rightTitle := range candidateTitles(right) { + if similarity := inferTitleSimilarity(leftTitle, rightTitle, year); similarity > best { + best = similarity + } + } + } + return best } func candidatePrimaryProvider(candidate MatchCandidate) string { @@ -715,6 +1054,22 @@ func trustedHintIDsPresent(hints *MatchHints) bool { return false } +func candidatesConflictWithTrustedIDs(hints *MatchHints, candidates []MatchCandidate) bool { + if hints == nil { + return false + } + for _, candidate := range candidates { + for _, key := range trustedSearchIDKeys { + hintValue := strings.TrimSpace(trustedIDValue(hints, key)) + candidateValue := strings.TrimSpace(candidate.ProviderIDs[key]) + if hintValue != "" && candidateValue != "" && hintValue != candidateValue { + return true + } + } + } + return false +} + func candidateMatchesTrustedIDs(hints *MatchHints, candidate MatchCandidate) bool { matched := false for _, key := range trustedSearchIDKeys { @@ -825,7 +1180,60 @@ func normalizeTitleForScoring(title string) string { } } - return strings.Join(strings.Fields(builder.String()), " ") + fields := strings.Fields(builder.String()) + for i, field := range fields { + fields[i] = normalizeNumberWord(field) + } + return strings.Join(fields, " ") +} + +var normalizedNumberWords = map[string]string{ + "zero": "0", "zeroth": "0", + //nolint:goconst // This is a declarative number/ordinal lookup, not a domain label. + "one": "1", "first": "1", + "two": "2", "second": "2", //nolint:goconst // Ordinal word mapping, not a reusable domain value. + "three": "3", "third": "3", + "four": "4", "fourth": "4", + "five": "5", "fifth": "5", + "six": "6", "sixth": "6", + "seven": "7", "seventh": "7", + "eight": "8", "eighth": "8", + "nine": "9", "ninth": "9", + "ten": "10", "tenth": "10", + "eleven": "11", "eleventh": "11", + "twelve": "12", "twelfth": "12", + "thirteen": "13", "thirteenth": "13", + "fourteen": "14", "fourteenth": "14", + "fifteen": "15", "fifteenth": "15", + "sixteen": "16", "sixteenth": "16", + "seventeen": "17", "seventeenth": "17", + "eighteen": "18", "eighteenth": "18", + "nineteen": "19", "nineteenth": "19", + "twenty": "20", "twentieth": "20", +} + +func normalizeNumberWord(value string) string { + if normalized, ok := normalizedNumberWords[value]; ok { + return normalized + } + for _, suffix := range []string{"st", "nd", "rd", "th"} { + if stem, ok := strings.CutSuffix(value, suffix); ok && isASCIIDigits(stem) { + return stem + } + } + return value +} + +func isASCIIDigits(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if r < '0' || r > '9' { + return false + } + } + return true } func normalizeNumericRune(r rune) (rune, bool) { diff --git a/internal/metadata/match_candidates_test.go b/internal/metadata/match_candidates_test.go index f714b44b..c9a833ef 100644 --- a/internal/metadata/match_candidates_test.go +++ b/internal/metadata/match_candidates_test.go @@ -1,6 +1,8 @@ +//nolint:goconst // Repeated titles and provider keys keep scoring fixtures explicit. package metadata import ( + "errors" "testing" "github.com/Silo-Server/silo-server/internal/models" @@ -32,6 +34,186 @@ func TestSelectInitialMatchCandidate_IgnoresLocalContentIDForTrustedSelection(t } } +func TestSelectInitialMatchCandidate_TrustedIDCanWinBelowNoisyTopResult(t *testing.T) { + t.Parallel() + + winner, ok := selectInitialMatchCandidate( + &MatchHints{Title: "10 Tricks", Year: 2022, Type: "movie", ImdbID: "tt0473100"}, + []MatchCandidate{ + { + Title: "10 Tricks", + Year: 2022, + ContentType: "movie", + ProviderIDs: map[string]string{"imdb": "tt9999999", "tmdb": "1"}, + Sources: []string{"tmdb", "tvdb"}, + }, + { + Title: "Ten Tricks", + Year: 2021, + ContentType: "movie", + ProviderIDs: map[string]string{"imdb": "tt0473100", "tmdb": "2"}, + Sources: []string{"tmdb"}, + }, + }, + nil, + ) + if !ok || winner == nil { + t.Fatal("expected candidate carrying the trusted IMDb ID to win") + } + if got := winner.ProviderIDs["imdb"]; got != "tt0473100" { + t.Fatalf("winner IMDb id = %q, want tt0473100", got) + } +} + +func TestBuildMatchDecisionClassifiesProviderFailures(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + outcome string + }{ + {name: "rate limit is transient", err: errors.New("provider returned HTTP 429"), outcome: "provider_transient"}, + {name: "bad request is permanent", err: errors.New("provider returned HTTP 400"), outcome: "provider_permanent"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + decision := buildMatchDecision(&MatchHints{Title: "Example"}, nil, nil, false, []error{tt.err}) + if string(decision.Outcome) != tt.outcome { + t.Fatalf("outcome = %q, want %q", decision.Outcome, tt.outcome) + } + }) + } +} + +func TestBuildMatchDecisionOnlyReportsTrustedIDConflictForExplicitConflict(t *testing.T) { + t.Parallel() + hints := &MatchHints{Title: "10 Tricks", ImdbID: "tt0473100"} + + missingID := buildMatchDecision(hints, []MatchCandidate{{ + Title: "10 Tricks", ProviderIDs: map[string]string{"tmdb": "123"}, + }}, nil, false, nil) + if missingID.Outcome != "candidate_rejected" { + t.Fatalf("candidate with no IMDb id outcome = %q, want candidate_rejected", missingID.Outcome) + } + + conflictingID := buildMatchDecision(hints, []MatchCandidate{{ + Title: "10 Tricks", ProviderIDs: map[string]string{"imdb": "tt9999999"}, + }}, nil, false, nil) + if conflictingID.Outcome != "trusted_id_conflict" { + t.Fatalf("conflicting IMDb id outcome = %q, want trusted_id_conflict", conflictingID.Outcome) + } +} + +func TestBuildMatchDecisionAttachesWinnerReasonsOnlyToIDLessWinner(t *testing.T) { + t.Parallel() + hints := &MatchHints{Title: "Shared", Year: 2020, Type: "series"} + candidates := []MatchCandidate{ + {Title: "Shared", Year: 2020, ContentType: "series", Sources: []string{"first"}}, + {Title: "Shared", Year: 2020, ContentType: "series", Sources: []string{"second"}}, + } + winner := candidates[1] + winner.MatchReasons = append(winner.MatchReasons, "episode_title_corroboration:2_of_2") + + decision := buildMatchDecision(hints, candidates, &winner, true, nil) + if len(decision.TopCandidates) != 2 { + t.Fatalf("top candidates = %d, want 2", len(decision.TopCandidates)) + } + for _, candidate := range decision.TopCandidates { + hasWinnerReason := containsString(candidate.Reasons, "episode_title_corroboration:2_of_2") + if candidate.Sources[0] == "second" && !hasWinnerReason { + t.Fatalf("winner reasons = %v, want episode corroboration", candidate.Reasons) + } + if candidate.Sources[0] == "first" && hasWinnerReason { + t.Fatalf("non-winner reasons = %v, must not contain winner-only evidence", candidate.Reasons) + } + } +} + +func TestScoreMatchCandidate_DoesNotCountLocalNFOAsProviderCorroboration(t *testing.T) { + t.Parallel() + + hints := &MatchHints{Title: "Shared", Type: "series"} + remote := MatchCandidate{Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}} + remoteAndNFO := remote + remoteAndNFO.Sources = []string{"tmdb", "nfo"} + + remoteScore := scoreMatchCandidate(hints, remote) + if got := scoreMatchCandidate(hints, remoteAndNFO); got != remoteScore { + t.Fatalf("remote+nfo score = %.1f, want remote-only score %.1f", got, remoteScore) + } +} + +func TestMergePreferredTitleMetadataReplacesExplicitFallback(t *testing.T) { + t.Parallel() + accumulator := &MetadataResult{ + Title: "倒凶十将伝", TitleLanguage: "ja", TitleIsFallback: true, + } + mergePreferredTitleMetadata(accumulator, &MetadataResult{ + Title: "10 Tokyo Warriors", TitleLanguage: "en", + OriginalTitle: "倒凶十将伝", OriginalLanguage: "ja", + }, "en", "tmdb", true) + if accumulator.Title != "10 Tokyo Warriors" || accumulator.TitleLanguage != "en" || accumulator.TitleIsFallback { + t.Fatalf("localized title = (%q, %q, %v)", accumulator.Title, accumulator.TitleLanguage, accumulator.TitleIsFallback) + } +} + +func TestMergePreferredTitleMetadataPreservesUnclassifiedFirstProviderTitle(t *testing.T) { + t.Parallel() + accumulator := &MetadataResult{Title: "Sidecar Title"} + mergePreferredTitleMetadata(accumulator, &MetadataResult{ + Title: "Localized Provider Title", TitleLanguage: "en", + }, "en", "tmdb", true) + if accumulator.Title != "Sidecar Title" { + t.Fatalf("title = %q, want first-provider sidecar title", accumulator.Title) + } +} + +func TestMergePreferredTitleMetadataRequiresEveryProviderResponseToBeComplete(t *testing.T) { + t.Parallel() + accumulator := &MetadataResult{} + mergePreferredTitleMetadata(accumulator, &MetadataResult{ + Title: "Complete Title", TitleLanguage: "en", TitleAliasesComplete: true, + }, "en", "tmdb", true) + mergePreferredTitleMetadata(accumulator, &MetadataResult{ + Title: "Legacy Title", TitleLanguage: "en", TitleAliasesComplete: false, + }, "en", "tmdb", true) + + if complete := accumulator.titleAliasProviders["tmdb"]; complete { + t.Fatal("mixed complete and partial responses granted alias deletion authority") + } +} + +func TestPreferredTitlesPreserveOldPluginPrimaryWithoutLanguageMetadata(t *testing.T) { + t.Parallel() + search := SearchResult{Name: "Localized Legacy Title", OriginalTitle: "Native Title"} + title, language, fallback, rank := preferredSearchResultTitle(search, "en") + if title != "Localized Legacy Title" || language != "" || fallback || rank != 3 { + t.Fatalf("legacy search title = (%q, %q, %t, %d)", title, language, fallback, rank) + } + + metadata := &MetadataResult{Title: "Localized Legacy Title", OriginalTitle: "Native Title"} + title, language, fallback, rank = preferredMetadataResultTitle(metadata, "en") + if title != "Localized Legacy Title" || language != "" || fallback || rank != 3 { + t.Fatalf("legacy metadata title = (%q, %q, %t, %d)", title, language, fallback, rank) + } +} + +func TestSanitizeCandidateProviderIDsRequiresStrictPositiveIMDbID(t *testing.T) { + t.Parallel() + for _, invalid := range []string{"tt1", "tt0000000", "tt12345678901", "nm1234567", "1234567"} { + if got := sanitizeCandidateProviderIDs(map[string]string{"imdb": invalid}); len(got) != 0 { + t.Fatalf("sanitize imdb %q = %#v, want empty", invalid, got) + } + } + for _, valid := range []string{"tt0000001", "tt12345678", "tt123456789", "tt1234567890"} { + if got := sanitizeCandidateProviderIDs(map[string]string{"imdb": valid}); got["imdb"] != valid { + t.Fatalf("sanitize imdb %q = %#v", valid, got) + } + } +} + func TestSelectInitialMatchCandidate_SoleExactTitleYearOffByTwoMatches(t *testing.T) { t.Parallel() @@ -80,6 +262,31 @@ func TestSelectInitialMatchCandidate_SoleExactTitleYearOffByThreeRejected(t *tes } } +func TestSelectInitialMatchCandidate_CrossProviderAgreementDoesNotOverrideYearConflict(t *testing.T) { + t.Parallel() + + // Both providers know the same show by the local alias, but the five-year + // conflict is evidence that this is a different work. Source agreement may + // replace a missing local year; it must not override a known conflicting one. + winner, ok := selectInitialMatchCandidate( + &MatchHints{Title: "The Piano Forest", Year: 2007, Type: "series"}, + []MatchCandidate{ + { + Title: "Five Fingers", + TitleAliases: []TitleAlias{{Title: "Piano Forest", Language: "en", Kind: "alternate"}}, + Year: 2012, + ContentType: "series", + ProviderIDs: map[string]string{"imdb": "tt2242048", "tvdb": "261129"}, + Sources: []string{"tmdb", "tvdb"}, + }, + }, + nil, + ) + if ok || winner != nil { + t.Fatalf("expected cross-provider year conflict to be rejected, got ok=%v winner=%+v", ok, winner) + } +} + func TestSelectInitialMatchCandidate_SoleDifferentTitleExactYearStillFloored(t *testing.T) { t.Parallel() @@ -220,6 +427,67 @@ func TestNormalizeCandidates(t *testing.T) { } }, }, + { + name: "merge two-ID consensus and quarantine conflicting third ID", + results: []SearchResult{ + { + Name: "A Teacher", + Year: 2020, + Provider: "tvdb", + ProviderIDs: map[string]string{ + "imdb": "tt10680614", "tmdb": "103992", "tvdb": "352440", + }, + }, + { + Name: "A Teacher", + Year: 2020, + Provider: "tmdb", + ProviderIDs: map[string]string{ + "imdb": "tt10680614", "tmdb": "103992", "tvdb": "473725", + }, + }, + }, + content: "series", + wantLen: 1, + check: func(t *testing.T, candidates []MatchCandidate) { + candidate := candidates[0] + if candidate.ProviderIDs["imdb"] != "tt10680614" || candidate.ProviderIDs["tmdb"] != "103992" { + t.Fatalf("agreed provider IDs = %+v", candidate.ProviderIDs) + } + if candidate.ProviderIDs["tvdb"] != "" { + t.Fatalf("conflicting TVDB ID was retained: %+v", candidate.ProviderIDs) + } + if len(candidate.ConflictingProviderIDKeys) != 1 || candidate.ConflictingProviderIDKeys[0] != "tvdb" { + t.Fatalf("quarantined keys = %v, want [tvdb]", candidate.ConflictingProviderIDKeys) + } + annotateCandidateMatch(&candidate, &MatchHints{Title: "A Teacher", Type: "series"}) + if !containsString(candidate.MatchReasons, "provider_id_consensus") || + !containsString(candidate.MatchReasons, "quarantined_tvdb_id") { + t.Fatalf("match reasons = %v", candidate.MatchReasons) + } + }, + }, + { + name: "discard malformed canonical provider cross references", + results: []SearchResult{ + { + Name: "Fast & Furious: Spy Racers", + Year: 2019, + Provider: "tvdb", + ProviderIDs: map[string]string{ + "imdb": "TT8322592", "tmdb": "95594-fast-furious-spy-racers", "tvdb": "362429", + }, + }, + }, + content: "series", + wantLen: 1, + check: func(t *testing.T, candidates []MatchCandidate) { + ids := candidates[0].ProviderIDs + if ids["tmdb"] != "" || ids["tvdb"] != "362429" || ids["imdb"] != "tt8322592" { + t.Fatalf("sanitized provider IDs = %+v", ids) + } + }, + }, { name: "no recognized provider IDs gets synthetic key and stays separate", results: []SearchResult{ @@ -388,6 +656,144 @@ func TestNormalizeCandidates(t *testing.T) { } } +func TestNormalizeCandidatesForLanguage_PrefersKnownLibraryAlias(t *testing.T) { + results := []SearchResult{ + { + Name: "倒凶十将伝", OriginalTitle: "倒凶十将伝", OriginalLanguage: "ja", + TitleLanguage: "ja", TitleIsFallback: true, + TitleAliases: []TitleAlias{{Title: "10 Tokyo Warriors", Language: "en", Kind: "alternate"}}, + Year: 1999, Provider: "tvdb", ProviderIDs: map[string]string{"tvdb": "123"}, + }, + { + Name: "10 Tokyo Warriors", OriginalTitle: "倒凶十将伝", OriginalLanguage: "ja", + TitleLanguage: "en", + Year: 1999, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "456", "tvdb": "123"}, + }, + } + + candidates := NormalizeCandidatesForLanguage(results, "series", "en") + if len(candidates) != 1 { + t.Fatalf("candidates = %d, want 1: %+v", len(candidates), candidates) + } + if candidates[0].Title != "10 Tokyo Warriors" || candidates[0].OriginalTitle != "倒凶十将伝" { + t.Fatalf("localized candidate = %+v", candidates[0]) + } + + winner, ok := selectInitialMatchCandidate(&MatchHints{Title: "10 Tokyo Warriors", Type: "series"}, candidates, []string{"tvdb", "tmdb"}) + if !ok || winner == nil { + t.Fatal("expected alias-coherent multi-source candidate to be selected") + } + if winner.MatchedTitle != "10 Tokyo Warriors" || winner.MatchScore < 70 { + t.Fatalf("winner diagnostics = %+v", winner) + } +} + +func TestNormalizeCandidates_UnknownLanguageAliasMatchesButDoesNotBecomePrimary(t *testing.T) { + candidates := NormalizeCandidatesForLanguage([]SearchResult{{ + Name: "倒凶十将伝", OriginalTitle: "倒凶十将伝", OriginalLanguage: "ja", + TitleLanguage: "ja", TitleIsFallback: true, + TitleAliases: []TitleAlias{{Title: "10 Tokyo Warriors", Kind: "alternate"}}, + Year: 1999, Provider: "tvdb", ProviderIDs: map[string]string{"tvdb": "123"}, + }}, "series", "en") + if got := candidates[0].Title; got != "倒凶十将伝" { + t.Fatalf("primary title = %q, want native fallback", got) + } + annotateCandidateMatch(&candidates[0], &MatchHints{Title: "10 Tokyo Warriors", Type: "series"}) + if got := candidates[0].MatchedTitle; got != "10 Tokyo Warriors" { + t.Fatalf("matched title = %q", got) + } +} + +func TestNormalizeCandidatesForLanguage_PrefersNativeTitleOverNonNativeFallback(t *testing.T) { + candidates := NormalizeCandidatesForLanguage([]SearchResult{{ + Name: "Titre de secours", OriginalTitle: "Native Title", OriginalLanguage: "ja", + TitleLanguage: "fr", TitleIsFallback: true, + Provider: "tvdb", ProviderIDs: map[string]string{"tvdb": "123"}, + }}, "series", "en") + if len(candidates) != 1 { + t.Fatalf("candidates = %d, want 1", len(candidates)) + } + if got := candidates[0].Title; got != "Native Title" { + t.Fatalf("primary title = %q, want provider-confirmed native title", got) + } + if got := candidates[0].TitleLanguage; got != "ja" { + t.Fatalf("title language = %q, want ja", got) + } +} + +func TestPreferredMetadataResultTitle_PrefersNativeTitleOverNonNativeFallback(t *testing.T) { + title, language, fallback, rank := preferredMetadataResultTitle(&MetadataResult{ + Title: "Titre de secours", OriginalTitle: "Native Title", OriginalLanguage: "jpn", + TitleLanguage: "fr", TitleIsFallback: true, + }, "en") + if title != "Native Title" || language != "ja" || !fallback || rank != 1 { + t.Fatalf("preferred title = %q, %q, %t, %d", title, language, fallback, rank) + } +} + +func TestMergePreferredTitleMetadataClassifiesNativeFallbackAsOriginalAlias(t *testing.T) { + accumulator := &MetadataResult{} + mergePreferredTitleMetadata(accumulator, &MetadataResult{ + Title: "倒凶十将伝", OriginalTitle: "倒凶十将伝", OriginalLanguage: "ja", + TitleLanguage: "ja", TitleIsFallback: true, + }, "en", "tvdb", true) + if len(accumulator.TitleAliases) != 1 || accumulator.TitleAliases[0].Kind != "original" || accumulator.TitleAliases[0].Language != "ja" { + t.Fatalf("native aliases = %#v", accumulator.TitleAliases) + } +} + +func TestMergePreferredTitleMetadataDoesNotAttributeIdentityHintAliases(t *testing.T) { + t.Parallel() + accumulator := &MetadataResult{} + mergePreferredTitleMetadata(accumulator, &MetadataResult{ + Title: "Curated Sidecar Title", + TitleLanguage: "en", + OriginalTitle: "Native Sidecar Title", + OriginalLanguage: "ja", + TitleAliases: []TitleAlias{{Title: "Sidecar Alternate", Language: "en", Kind: "alternate"}}, + TitleAliasesComplete: true, + }, "en", "nfo", false) + + if accumulator.Title != "Curated Sidecar Title" || accumulator.OriginalTitle != "Native Sidecar Title" { + t.Fatalf("metadata fields were not retained: %#v", accumulator) + } + if len(accumulator.TitleAliases) != 0 { + t.Fatalf("identity-hint aliases = %#v, want none", accumulator.TitleAliases) + } + if _, attributed := accumulator.titleAliasProviders["nfo"]; attributed { + t.Fatal("identity-hint provider received alias persistence authority") + } +} + +func TestSelectInitialMatchCandidateDirectIMDbIDFor10Tricks(t *testing.T) { + t.Parallel() + candidates := NormalizeCandidatesForLanguage([]SearchResult{{ + Name: "Ten Tricks", Year: 2022, Provider: "tmdb", + ProviderIDs: map[string]string{"imdb": "tt0473100", "tmdb": "12345"}, + }}, "movie", "en") + winner, ok := selectInitialMatchCandidate(&MatchHints{ + Title: "10 Tricks", Year: 2022, Type: "movie", ImdbID: "tt0473100", + }, candidates, []string{"tmdb"}) + if !ok || winner == nil { + t.Fatal("trusted IMDb ID did not produce a decisive match") + } + if winner.ProviderIDs["imdb"] != "tt0473100" || winner.MatchScore < 100 { + t.Fatalf("winner = %#v", winner) + } +} + +func TestNormalizeTitleForScoring_NumberWordsAndOrdinals(t *testing.T) { + for _, pair := range [][2]string{ + {"10 Tricks", "Ten Tricks"}, + {"Dune Part Two", "Dune Part 2"}, + {"The 10th Kingdom", "The Tenth Kingdom"}, + } { + if got, want := normalizeTitleForScoring(pair[0]), normalizeTitleForScoring(pair[1]); got != want { + t.Errorf("normalize %q = %q, %q = %q", pair[0], got, pair[1], want) + } + } +} + func TestSelectInitialMatchCandidate_AcceptsSinglePunctuationEquivalentCandidate(t *testing.T) { tests := []struct { name string diff --git a/internal/metadata/match_identity_fallback.go b/internal/metadata/match_identity_fallback.go new file mode 100644 index 00000000..006d37b7 --- /dev/null +++ b/internal/metadata/match_identity_fallback.go @@ -0,0 +1,54 @@ +package metadata + +import ( + "strconv" + "strings" +) + +const maxProviderIdentityFallbacks = 3 + +func compactAlternateMatchIdentities(hints *MatchHints) []MatchIdentityHint { + if hints == nil || len(hints.AlternateIdentities) == 0 { + return nil + } + + seen := map[string]struct{}{ + matchIdentityComparisonKey(hints.Title, hints.Year): {}, + } + out := make([]MatchIdentityHint, 0, min(len(hints.AlternateIdentities), maxProviderIdentityFallbacks)) + for _, alternate := range hints.AlternateIdentities { + alternate.Title = strings.Join(strings.Fields(strings.TrimSpace(alternate.Title)), " ") + key := matchIdentityComparisonKey(alternate.Title, alternate.Year) + if alternate.Title == "" || key == "" { + continue + } + if _, exists := seen[key]; exists { + continue + } + seen[key] = struct{}{} + out = append(out, alternate) + if len(out) == maxProviderIdentityFallbacks { + break + } + } + return out +} + +func matchIdentityComparisonKey(title string, year int) string { + normalized := strings.Join(strings.Fields(normalizeTitleForScoring(title)), " ") + if normalized == "" { + return "" + } + return normalized + "\x00" + strconv.Itoa(year) +} + +func bestCandidateScore(hints *MatchHints, candidates []MatchCandidate) float64 { + best := 0.0 + for _, candidate := range candidates { + score, _, _ := scoreMatchCandidateDetailed(hints, candidate) + if score > best { + best = score + } + } + return best +} diff --git a/internal/metadata/match_identity_fallback_test.go b/internal/metadata/match_identity_fallback_test.go new file mode 100644 index 00000000..8c26a3f8 --- /dev/null +++ b/internal/metadata/match_identity_fallback_test.go @@ -0,0 +1,126 @@ +//nolint:goconst // Repeated titles and provider IDs document each fallback scenario. +package metadata + +import ( + "context" + "sync" + "testing" +) + +type identityFallbackProvider struct { + mu sync.Mutex + queries []SearchQuery +} + +type typeMismatchProvider struct{} + +func (p *typeMismatchProvider) Slug() string { return "tmdb" } +func (p *typeMismatchProvider) Name() string { return "TMDB" } +func (p *typeMismatchProvider) ForTypes() []string { return []string{"movie", "series"} } +func (p *typeMismatchProvider) Search(_ context.Context, query SearchQuery) ([]SearchResult, error) { + if query.ContentType != "series" || query.ProviderIDs["imdb"] != "tt10473306" { + return nil, nil + } + return []SearchResult{{ + Name: "Are You Afraid of the Dark?", Year: 2019, Provider: "tmdb", + ProviderIDs: map[string]string{"imdb": "tt10473306", "tmdb": "83755"}, + }}, nil +} +func (p *typeMismatchProvider) GetMetadata(context.Context, MetadataRequest) (*MetadataResult, error) { + return &MetadataResult{}, nil +} + +func (p *identityFallbackProvider) Slug() string { return "tmdb" } +func (p *identityFallbackProvider) Name() string { return "TMDB" } +func (p *identityFallbackProvider) ForTypes() []string { return []string{"movie"} } +func (p *identityFallbackProvider) Search(_ context.Context, query SearchQuery) ([]SearchResult, error) { + p.mu.Lock() + p.queries = append(p.queries, query) + p.mu.Unlock() + if query.Title != "Alien Invasion" || query.Year != 2018 { + return nil, nil + } + return []SearchResult{{ + Name: "Alien Invasion", Year: 2018, Provider: "tmdb", + ProviderIDs: map[string]string{"tmdb": "4242"}, + }}, nil +} +func (p *identityFallbackProvider) GetMetadata(_ context.Context, req MetadataRequest) (*MetadataResult, error) { + return &MetadataResult{ + HasMetadata: true, Title: "Alien Invasion", Year: 2018, + ProviderIDs: map[string]string{"tmdb": req.ProviderIDs["tmdb"]}, + }, nil +} + +func TestInitialMatchUsesAlternatePathIdentityAfterPrimaryMiss(t *testing.T) { + t.Parallel() + harness := newTestHarness() + provider := &identityFallbackProvider{} + + result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{ + Hints: &MatchHints{ + Title: "After the Lethargy", Year: 2018, Type: "movie", + AlternateIdentities: []MatchIdentityHint{{Title: "Alien Invasion", Year: 2018, Source: "filename"}}, + }, + Mode: ModeInitialMatch, + }, []Provider{provider}) + if err != nil { + t.Fatalf("ProcessWithProviders() error = %v", err) + } + if result == nil || !result.Updated || result.Decision == nil || result.Decision.Outcome != "matched" { + t.Fatalf("result = %#v, want alternate-identity match", result) + } + item, err := harness.itemRepo.GetByID(context.Background(), result.ContentID) + if err != nil { + t.Fatalf("load item: %v", err) + } + if item.Title != "Alien Invasion" || item.TmdbID != "4242" { + t.Fatalf("item = title %q tmdb %q", item.Title, item.TmdbID) + } + provider.mu.Lock() + defer provider.mu.Unlock() + if len(provider.queries) != 2 || provider.queries[1].Title != "Alien Invasion" { + t.Fatalf("queries = %#v, want primary then filename fallback", provider.queries) + } +} + +func TestCompactAlternateMatchIdentitiesDeduplicatesAndBounds(t *testing.T) { + t.Parallel() + hints := &MatchHints{ + Title: "Primary", Year: 2020, + AlternateIdentities: []MatchIdentityHint{ + {Title: "Primary", Year: 2020}, + {Title: "A", Year: 2020}, + {Title: "A", Year: 2020}, + {Title: "B", Year: 2021}, + {Title: "C", Year: 2022}, + {Title: "D", Year: 2023}, + }, + } + + got := compactAlternateMatchIdentities(hints) + if len(got) != maxProviderIdentityFallbacks || got[0].Title != "A" || got[2].Title != "C" { + t.Fatalf("compact identities = %#v", got) + } +} + +func TestInitialMatchDiagnosesTrustedIDInOppositeContentType(t *testing.T) { + t.Parallel() + harness := newTestHarness() + + result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{ + Hints: &MatchHints{ + Title: "Are You Afraid of the Dark!", Year: 2019, Type: "movie", ImdbID: "tt10473306", + }, + Mode: ModeInitialMatch, + }, []Provider{&typeMismatchProvider{}}) + if err != nil { + t.Fatalf("ProcessWithProviders() error = %v", err) + } + if result == nil || result.Decision == nil || result.Decision.Outcome != "trusted_id_type_mismatch" { + t.Fatalf("result = %#v, want trusted-ID type mismatch", result) + } + if result.Decision.CandidateCount != 1 { + t.Fatalf("decision = %+v, want one opposite-type candidate", result.Decision) + } +} diff --git a/internal/metadata/match_type_diagnostics.go b/internal/metadata/match_type_diagnostics.go new file mode 100644 index 00000000..69cbadc3 --- /dev/null +++ b/internal/metadata/match_type_diagnostics.go @@ -0,0 +1,64 @@ +package metadata + +import ( + "context" + "strings" +) + +func (s *MetadataService) trustedIDTypeMismatchCandidates( + ctx context.Context, + hints *MatchHints, + providerIDs map[string]string, + language string, + resolveChain chainResolver, +) []MatchCandidate { + if hints == nil || !trustedHintIDsPresent(hints) || resolveChain == nil { + return nil + } + + var oppositeType string + switch strings.ToLower(strings.TrimSpace(hints.Type)) { + case matchContentTypeMovie: + oppositeType = matchContentTypeSeries + case matchContentTypeSeries: + oppositeType = matchContentTypeMovie + default: + return nil + } + + chain, err := resolveChain(providerChainContentLevel(oppositeType)) + if err != nil { + return nil + } + query := SearchQuery{ + ContentType: oppositeType, + ProviderIDs: copyMap(providerIDs), + Language: language, + FilePath: hints.FilePath, + RepresentativeFilePath: hints.RepresentativeFilePath, + ObservedRootPath: hints.ObservedRootPath, + AllGroupFilePaths: append([]string(nil), hints.AllGroupFilePaths...), + PrimarySidecarSearchPaths: append([]string(nil), hints.PrimarySidecarSearchPaths...), + } + + results := make([]SearchResult, 0) + for _, provider := range chain { + searchProvider, ok := provider.(SearchProvider) + if !ok { + continue + } + providerResults, searchErr := searchProvider.Search(ctx, query) + if searchErr != nil { + // This is optional diagnosis. A provider outage must not turn a + // deterministic same-type miss into a transient queue failure. + continue + } + for _, result := range providerResults { + candidate := MatchCandidate{ProviderIDs: result.ProviderIDs} + if candidateMatchesTrustedIDs(hints, candidate) { + results = append(results, result) + } + } + } + return NormalizeCandidatesForLanguage(results, oppositeType, language) +} diff --git a/internal/metadata/nfo_identity_test.go b/internal/metadata/nfo_identity_test.go index 62ae0ffa..8cbf9c15 100644 --- a/internal/metadata/nfo_identity_test.go +++ b/internal/metadata/nfo_identity_test.go @@ -1,3 +1,4 @@ +//nolint:goconst // Repeated titles and provider IDs keep NFO scenarios recognizable. package metadata import ( @@ -99,6 +100,73 @@ func (p *remoteStubProvider) lastMetadataIDs() map[string]string { return copyMap(p.lastMetaIDs) } +func (p *remoteStubProvider) lastSearchProviderIDs() map[string]string { + p.mu.Lock() + defer p.mu.Unlock() + return copyMap(p.lastSearchIDs) +} + +func TestInitialMatch_MalformedNFOIdentityDoesNotSuppressTitleSearch(t *testing.T) { + t.Parallel() + h := newTestHarness() + nfo := &localHintStubProvider{ + hints: map[string]string{"tmdb": "not-a-number", "imdb": "nm1234567"}, + searchResults: []SearchResult{{Name: "The Matrix", Year: 1999, Provider: "nfo", ProviderIDs: map[string]string{"tmdb": "not-a-number"}}}, + metadata: &MetadataResult{HasMetadata: true, Title: "The Matrix"}, + } + remote := &remoteStubProvider{ + slug: "tmdb", + searchResults: []SearchResult{{ + Name: "The Matrix", Year: 1999, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "603", "imdb": "tt0133093"}, + }}, + metadata: &MetadataResult{HasMetadata: true, Title: "The Matrix", Year: 1999, ProviderIDs: map[string]string{"tmdb": "603", "imdb": "tt0133093"}}, + } + + result, err := h.service.ProcessWithProviders(context.Background(), ProcessRequest{ + Hints: &MatchHints{Title: "The Matrix", Year: 1999, Type: "movie"}, + Mode: ModeInitialMatch, + }, []Provider{nfo, remote}) + if err != nil { + t.Fatalf("ProcessWithProviders() error = %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want title-based remote match", result) + } + if ids := remote.lastSearchProviderIDs(); ids["tmdb"] != "" || ids["imdb"] != "" { + t.Fatalf("remote search received malformed trusted IDs: %#v", ids) + } +} + +func TestInitialMatch_MalformedScannerIdentityDoesNotSuppressTitleSearch(t *testing.T) { + t.Parallel() + h := newTestHarness() + remote := &remoteStubProvider{ + slug: "tmdb", + searchResults: []SearchResult{{ + Name: "The Matrix", Year: 1999, Provider: "tmdb", + ProviderIDs: map[string]string{"tmdb": "603", "imdb": "tt0133093"}, + }}, + metadata: &MetadataResult{HasMetadata: true, Title: "The Matrix", Year: 1999, ProviderIDs: map[string]string{"tmdb": "603", "imdb": "tt0133093"}}, + } + + result, err := h.service.ProcessWithProviders(context.Background(), ProcessRequest{ + Hints: &MatchHints{ + Title: "The Matrix", Year: 1999, Type: "movie", + TmdbID: "not-a-number", ImdbID: "nm1234567", + }, + Mode: ModeInitialMatch, + }, []Provider{remote}) + if err != nil { + t.Fatalf("ProcessWithProviders() error = %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want title-based remote match", result) + } + if ids := remote.lastSearchProviderIDs(); ids["tmdb"] != "" || ids["imdb"] != "" { + t.Fatalf("remote search received malformed scanner IDs: %#v", ids) + } +} + func seedMovieItem(t *testing.T, h *testHarness, contentID, title string, year int) { t.Helper() if err := h.itemRepo.Upsert(context.Background(), &models.MediaItem{ diff --git a/internal/metadata/plugin_provider.go b/internal/metadata/plugin_provider.go index 3bb1bd09..3a3b80e2 100644 --- a/internal/metadata/plugin_provider.go +++ b/internal/metadata/plugin_provider.go @@ -202,12 +202,17 @@ func (p *PluginProvider) Search(ctx context.Context, query SearchQuery) ([]Searc results := make([]SearchResult, 0, len(response.GetResults())) for _, result := range response.GetResults() { results = append(results, SearchResult{ - Name: result.GetTitle(), - Year: int(result.GetYear()), - ProviderIDs: mergePluginProviderIDs(p.capabilityID, result.GetProviderId(), result.GetProviderIds()), - ImageURL: result.GetImageUrl(), - Overview: result.GetOverview(), - Provider: p.Slug(), + Name: result.GetTitle(), + OriginalTitle: result.GetOriginalTitle(), + TitleAliases: titleAliasesFromPlugin(result.GetTitleAliases(), p.Slug()), + TitleLanguage: result.GetTitleLanguage(), + TitleIsFallback: result.GetTitleIsFallback(), + OriginalLanguage: result.GetOriginalLanguage(), + Year: int(result.GetYear()), + ProviderIDs: mergePluginProviderIDs(p.capabilityID, result.GetProviderId(), result.GetProviderIds()), + ImageURL: result.GetImageUrl(), + Overview: result.GetOverview(), + Provider: p.Slug(), }) } return results, nil @@ -244,39 +249,56 @@ func (p *PluginProvider) GetMetadata(ctx context.Context, req MetadataRequest) ( } return &MetadataResult{ - HasMetadata: true, - ProviderIDs: mergePluginProviderIDs(p.capabilityID, response.GetItem().GetProviderId(), response.GetItem().GetProviderIds()), - Title: response.GetItem().GetTitle(), - OriginalTitle: response.GetItem().GetOriginalTitle(), - SortTitle: response.GetItem().GetSortTitle(), - Overview: response.GetItem().GetOverview(), - Tagline: response.GetItem().GetTagline(), - Year: int(response.GetItem().GetYear()), - Runtime: int(response.GetItem().GetRuntime()), - Genres: append([]string(nil), response.GetItem().GetGenres()...), - Keywords: keywordsFromPluginMetadata(response.GetItem().GetMetadata()), - Studios: append([]string(nil), response.GetItem().GetStudios()...), - Networks: append([]string(nil), response.GetItem().GetNetworks()...), - Countries: append([]string(nil), response.GetItem().GetCountries()...), - OriginalLanguage: response.GetItem().GetOriginalLanguage(), - ContentRating: response.GetItem().GetContentRating(), - Ratings: ratingsFromStruct(response.GetItem().GetRatings()), - People: peopleFromRecords(response.GetItem().GetPeople()), - Videos: videosFromRecords(p.Slug(), response.GetItem().GetVideos()), - PosterPath: response.GetItem().GetPosterPath(), - PosterThumbhash: response.GetItem().GetPosterThumbhash(), - BackdropPath: response.GetItem().GetBackdropPath(), - ShowStatus: response.GetItem().GetStatus(), - BackdropThumbhash: response.GetItem().GetBackdropThumbhash(), - LogoPath: response.GetItem().GetLogoPath(), - SeasonCount: int(response.GetItem().GetSeasonCount()), - FirstAirDate: response.GetItem().GetFirstAirDate(), - LastAirDate: response.GetItem().GetLastAirDate(), - AirTime: response.GetItem().GetAirTime(), - ReleaseDate: response.GetItem().GetReleaseDate(), + HasMetadata: true, + ProviderIDs: mergePluginProviderIDs(p.capabilityID, response.GetItem().GetProviderId(), response.GetItem().GetProviderIds()), + Title: response.GetItem().GetTitle(), + OriginalTitle: response.GetItem().GetOriginalTitle(), + TitleAliases: titleAliasesFromPlugin(response.GetItem().GetTitleAliases(), p.Slug()), + TitleAliasesComplete: response.GetItem().GetTitleAliasesComplete(), + TitleLanguage: response.GetItem().GetTitleLanguage(), + TitleIsFallback: response.GetItem().GetTitleIsFallback(), + SortTitle: response.GetItem().GetSortTitle(), + Overview: response.GetItem().GetOverview(), + Tagline: response.GetItem().GetTagline(), + Year: int(response.GetItem().GetYear()), + Runtime: int(response.GetItem().GetRuntime()), + Genres: append([]string(nil), response.GetItem().GetGenres()...), + Keywords: keywordsFromPluginMetadata(response.GetItem().GetMetadata()), + Studios: append([]string(nil), response.GetItem().GetStudios()...), + Networks: append([]string(nil), response.GetItem().GetNetworks()...), + Countries: append([]string(nil), response.GetItem().GetCountries()...), + OriginalLanguage: response.GetItem().GetOriginalLanguage(), + ContentRating: response.GetItem().GetContentRating(), + Ratings: ratingsFromStruct(response.GetItem().GetRatings()), + People: peopleFromRecords(response.GetItem().GetPeople()), + Videos: videosFromRecords(p.Slug(), response.GetItem().GetVideos()), + PosterPath: response.GetItem().GetPosterPath(), + PosterThumbhash: response.GetItem().GetPosterThumbhash(), + BackdropPath: response.GetItem().GetBackdropPath(), + ShowStatus: response.GetItem().GetStatus(), + BackdropThumbhash: response.GetItem().GetBackdropThumbhash(), + LogoPath: response.GetItem().GetLogoPath(), + SeasonCount: int(response.GetItem().GetSeasonCount()), + FirstAirDate: response.GetItem().GetFirstAirDate(), + LastAirDate: response.GetItem().GetLastAirDate(), + AirTime: response.GetItem().GetAirTime(), + ReleaseDate: response.GetItem().GetReleaseDate(), }, nil } +func titleAliasesFromPlugin(aliases []*pluginv1.TitleAlias, provider string) []TitleAlias { + out := make([]TitleAlias, 0, len(aliases)) + for _, alias := range aliases { + if alias == nil || strings.TrimSpace(alias.GetTitle()) == "" { + continue + } + out = append(out, TitleAlias{ + Title: alias.GetTitle(), Language: alias.GetLanguage(), Kind: alias.GetKind(), Provider: provider, + }) + } + return out +} + func (p *PluginProvider) GetPersonDetail(ctx context.Context, req PersonDetailRequest) (*PersonDetailResult, error) { client, err := p.clientFactory(ctx, p.installationID, p.capabilityID) if err != nil { diff --git a/internal/metadata/service.go b/internal/metadata/service.go index 5cf39c96..32568b71 100644 --- a/internal/metadata/service.go +++ b/internal/metadata/service.go @@ -25,6 +25,8 @@ import ( "github.com/Silo-Server/silo-server/internal/scanner" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // FileContentUpdater updates content_id on media_files. @@ -297,6 +299,7 @@ type MetadataService struct { libraryRepo metadataLibraryRepo folderRepo metadataFolderRepo itemLocalizationRepo *catalog.MediaItemLocalizationRepository + itemAliasRepo *catalog.ItemAliasRepository seasonLocalizationRepo *catalog.SeasonLocalizationRepository episodeLocalizationRepo *catalog.EpisodeLocalizationRepository autoTranslator AutoTranslator // optional; set via SetAutoTranslator @@ -387,6 +390,7 @@ func NewMetadataService( rootClaimRepo *catalog.RootClaimRepository, ) *MetadataService { var itemLocalizationRepo *catalog.MediaItemLocalizationRepository + var itemAliasRepo *catalog.ItemAliasRepository var seasonLocalizationRepo *catalog.SeasonLocalizationRepository var episodeLocalizationRepo *catalog.EpisodeLocalizationRepository var scannedRootRepo metadataScannedRootRepo @@ -401,6 +405,7 @@ func NewMetadataService( dbPool = pool videoRepo = catalog.NewVideoRepository(pool) itemLocalizationRepo = catalog.NewMediaItemLocalizationRepository(pool) + itemAliasRepo = catalog.NewItemAliasRepository(pool) seasonLocalizationRepo = catalog.NewSeasonLocalizationRepository(pool) episodeLocalizationRepo = catalog.NewEpisodeLocalizationRepository(pool) scannedRootRepo = scanner.NewScannedRootRepository(pool) @@ -420,6 +425,7 @@ func NewMetadataService( libraryRepo: libraryRepo, folderRepo: folderRepo, itemLocalizationRepo: itemLocalizationRepo, + itemAliasRepo: itemAliasRepo, seasonLocalizationRepo: seasonLocalizationRepo, episodeLocalizationRepo: episodeLocalizationRepo, personRepo: personRepo, @@ -569,6 +575,9 @@ func (s *MetadataService) Process(ctx context.Context, req ProcessRequest) (*Pro } final.IsNew = final.IsNew || result.IsNew final.Updated = final.Updated || result.Updated + if result.Decision != nil { + final.Decision = result.Decision + } } s.maybeAutoTranslate(ctx, folderID, final.ContentID) @@ -1140,6 +1149,7 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques // Phase 1: Search — find provider IDs. accumulatedIDs := make(map[string]string) maps.Copy(accumulatedIDs, req.ProviderIDs) + sanitizeCanonicalProviderIDsInPlace(accumulatedIDs) // Track provider 404s as stale external IDs. This applies to initial match // as well so bad embedded folder/file IDs can be recorded and dropped @@ -1148,25 +1158,29 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques if req.ContentID != "" { provider404s = make(map[string]string) } + var matchDecision *MatchDecision + var providerMatchErrors []error + quarantinedProviderIDKeys := make(map[string]struct{}) switch req.Mode { case ModeInitialMatch: if req.Hints == nil { return nil, fmt.Errorf("initial match requires hints") } + selectionHints := sanitizedMatchHintProviderIDs(req.Hints) // Seed external IDs from hints. Hints.ContentID is Silo's local // skeleton item ID, not a searchable provider ID. - if req.Hints.FileHash != "" { - accumulatedIDs["oshash"] = req.Hints.FileHash + if selectionHints.FileHash != "" { + accumulatedIDs["oshash"] = selectionHints.FileHash } - if req.Hints.TmdbID != "" { - accumulatedIDs["tmdb"] = req.Hints.TmdbID + if selectionHints.TmdbID != "" { + accumulatedIDs["tmdb"] = selectionHints.TmdbID } - if req.Hints.TvdbID != "" { - accumulatedIDs["tvdb"] = req.Hints.TvdbID + if selectionHints.TvdbID != "" { + accumulatedIDs["tvdb"] = selectionHints.TvdbID } - if req.Hints.ImdbID != "" { - accumulatedIDs["imdb"] = req.Hints.ImdbID + if selectionHints.ImdbID != "" { + accumulatedIDs["imdb"] = selectionHints.ImdbID } // Add filepath for local providers. if req.Hints.RepresentativeFilePath != "" { @@ -1200,53 +1214,58 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques protectedKeys[key] = true } wonHints := applyBuiltinIdentityHints(ctx, itemChain, searchQuery, accumulatedIDs, protectedKeys) - selectionHints := req.Hints if len(wonHints) > 0 { - hintsCopy := *req.Hints + hintsCopy := *selectionHints overrideHintIDs(&hintsCopy, wonHints) selectionHints = &hintsCopy } searchQuery = suppressTitleYearFallbackForTrustedIDs(searchQuery) - allResults := make([]SearchResult, 0) - for _, p := range itemChain { - sp, ok := p.(SearchProvider) - if !ok { - continue - } - results, err := sp.Search(ctx, searchQuery) - if err != nil { - if handleProvider404(provider404s, accumulatedIDs, p.Slug(), err, - "title", req.Hints.Title, - "year", req.Hints.Year, - ) { + searchProviders := func(query SearchQuery) []SearchResult { + resultsForQuery := make([]SearchResult, 0) + for _, p := range itemChain { + sp, ok := p.(SearchProvider) + if !ok { continue } - slog.WarnContext(ctx, "metadata: search provider error", "component", "metadata", - "provider", p.Slug(), "error", err) - continue - } - slog.DebugContext(ctx, "metadata: provider search result", "component", "metadata", - "provider", p.Slug(), - "query_title", searchQuery.Title, - "query_year", searchQuery.Year, - "result_count", len(results), - ) - for _, result := range results { - if searchResultConflictsWithTrustedIDs(accumulatedIDs, result.ProviderIDs) { - slog.WarnContext(ctx, "metadata: skipping conflicting search result", "component", "metadata", - "provider", p.Slug(), - "title", req.Hints.Title, - "year", req.Hints.Year, - "hinted_ids", accumulatedIDs, - "candidate_ids", result.ProviderIDs, - ) + results, searchErr := sp.Search(ctx, query) + if searchErr != nil { + if handleProvider404(provider404s, accumulatedIDs, p.Slug(), searchErr, + "title", query.Title, + "year", query.Year, + ) { + providerMatchErrors = append(providerMatchErrors, searchErr) + continue + } + slog.WarnContext(ctx, "metadata: search provider error", "component", "metadata", + "provider", p.Slug(), "error", searchErr) + providerMatchErrors = append(providerMatchErrors, searchErr) continue } - allResults = append(allResults, result) + slog.DebugContext(ctx, "metadata: provider search result", "component", "metadata", + "provider", p.Slug(), + "query_title", query.Title, + "query_year", query.Year, + "result_count", len(results), + ) + for _, result := range results { + if searchResultConflictsWithTrustedIDs(accumulatedIDs, result.ProviderIDs) { + slog.WarnContext(ctx, "metadata: retaining conflicting search result for rejection diagnostics", "component", "metadata", + "provider", p.Slug(), + "title", query.Title, + "year", query.Year, + "hinted_ids", accumulatedIDs, + "candidate_ids", result.ProviderIDs, + ) + } + resultsForQuery = append(resultsForQuery, result) + } } + return resultsForQuery } - candidates := NormalizeCandidates(allResults, contentType) + allResults := searchProviders(searchQuery) + + candidates := NormalizeCandidatesForLanguage(allResults, contentType, searchQuery.Language) slog.DebugContext(ctx, "metadata: search candidates assembled", "component", "metadata", "query_title", searchQuery.Title, "query_year", searchQuery.Year, @@ -1257,17 +1276,74 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques for _, p := range itemChain { providerPriority = append(providerPriority, p.Slug()) } - if winner, ok := selectInitialMatchCandidate(selectionHints, candidates, providerPriority); ok && winner != nil { - for k, v := range winner.ProviderIDs { - if v != "" { - accumulatedIDs[k] = v + trySelection := func(hints *MatchHints, selectionCandidates []MatchCandidate) (*MatchCandidate, bool) { + selected, ok := selectInitialMatchCandidate(hints, selectionCandidates, providerPriority) + if ok { + return selected, true + } + episodeWinner, validationErrors := validateSeriesMatchByEpisodes( + ctx, hints, selectionCandidates, itemChain, searchQuery.Language, + ) + providerMatchErrors = append(providerMatchErrors, validationErrors...) + return episodeWinner, episodeWinner != nil + } + + effectiveHints := selectionHints + winner, matched := trySelection(selectionHints, candidates) + // Structured IDs remain decisive and never fall back to a title search. + // For title/year identities, try independently parsed path hypotheses only + // after the primary identity fails. Each attempt is bounded and scored + // against the identity that produced its provider query. + if !matched && !trustedHintIDsPresent(selectionHints) { + for _, alternate := range compactAlternateMatchIdentities(selectionHints) { + alternateHints := *selectionHints + alternateHints.Title = alternate.Title + alternateHints.Year = alternate.Year + alternateHints.HintSource = alternate.Source + alternateHints.AlternateIdentities = nil + + alternateQuery := searchQuery + alternateQuery.Title = alternate.Title + alternateQuery.Year = alternate.Year + alternateResults := searchProviders(alternateQuery) + alternateCandidates := NormalizeCandidatesForLanguage(alternateResults, contentType, alternateQuery.Language) + if len(alternateCandidates) == 0 { + continue + } + alternateWinner, alternateMatched := trySelection(&alternateHints, alternateCandidates) + if len(candidates) == 0 || alternateMatched || bestCandidateScore(&alternateHints, alternateCandidates) > bestCandidateScore(effectiveHints, candidates) { + candidates = alternateCandidates + effectiveHints = &alternateHints + } + if alternateMatched { + winner = alternateWinner + matched = true + break } } } + trustedIDTypeMismatch := false + if !matched && len(candidates) == 0 && trustedHintIDsPresent(selectionHints) { + if mismatchCandidates := s.trustedIDTypeMismatchCandidates( + ctx, selectionHints, accumulatedIDs, searchQuery.Language, resolveChain, + ); len(mismatchCandidates) > 0 { + candidates = mismatchCandidates + effectiveHints = selectionHints + trustedIDTypeMismatch = true + } + } + matchDecision = buildMatchDecision(effectiveHints, candidates, winner, matched, providerMatchErrors) + if trustedIDTypeMismatch { + matchDecision.Outcome = MatchOutcomeTrustedIDTypeMismatch + } + if matched && winner != nil { + applyCandidateProviderIDConsensus(accumulatedIDs, winner, quarantinedProviderIDKeys) + } case ModeIdentify: // Use user-provided IDs directly. maps.Copy(accumulatedIDs, req.ProviderIDs) + sanitizeCanonicalProviderIDsInPlace(accumulatedIDs) if contentType == "" && req.ContentID != "" { if existing, err := s.itemRepo.GetByID(ctx, req.ContentID); err == nil { contentType = existing.Type @@ -1297,6 +1373,7 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques if existing.ImdbID != "" { accumulatedIDs["imdb"] = existing.ImdbID } + sanitizeCanonicalProviderIDsInPlace(accumulatedIDs) contentType = existing.Type if err := s.suppressRecordedStaleProviderIDs(ctx, req.ContentID, accumulatedIDs); err != nil { return nil, err @@ -1355,18 +1432,16 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques } for _, result := range results { if searchResultConflictsWithTrustedIDs(accumulatedIDs, result.ProviderIDs) { - continue + slog.WarnContext(ctx, "metadata: retaining conflicting refresh search result for trusted-id rejection", "component", "metadata", + "provider", p.Slug(), "content_id", req.ContentID, + "hinted_ids", accumulatedIDs, "candidate_ids", result.ProviderIDs) } allResults = append(allResults, result) } } - candidates := NormalizeCandidates(allResults, contentType) + candidates := NormalizeCandidatesForLanguage(allResults, contentType, searchQuery.Language) if winner, ok := selectRefreshMatchCandidate(existing, wonHints, candidates); ok && winner != nil { - for k, v := range winner.ProviderIDs { - if v != "" { - accumulatedIDs[k] = v - } - } + applyCandidateProviderIDConsensus(accumulatedIDs, winner, quarantinedProviderIDKeys) } } if req.Mode != ModeIdentify { @@ -1429,15 +1504,22 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques if handleProvider404(provider404s, accumulatedIDs, p.Slug(), err, "content_id", req.ContentID, ) { + if req.Mode == ModeInitialMatch { + providerMatchErrors = append(providerMatchErrors, err) + } continue } slog.WarnContext(ctx, "metadata: provider error", "component", "metadata", "provider", p.Slug(), "error", err) + if req.Mode == ModeInitialMatch { + providerMatchErrors = append(providerMatchErrors, err) + } continue } if result == nil || !result.HasMetadata { continue } + result.ProviderIDs = sanitizeCandidateProviderIDs(result.ProviderIDs) // Identity-hint providers contribute IDs exclusively through the // trusted-hint phase: their Phase-2 results merge metadata fields but // never inject provider-id keys that conflict with or extend the @@ -1445,12 +1527,19 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques if isIdentityHinter { result.ProviderIDs = nil } + for key := range quarantinedProviderIDKeys { + delete(result.ProviderIDs, key) + } + mergePreferredTitleMetadata(accumulator, result, req.Language, p.Slug(), !isIdentityHinter) // Bootstrap: feed new IDs to subsequent providers. mergeProviderIDs(accumulator, result) accumulatedIDs = accumulator.ProviderIDs // Merge fields into accumulator (FillEmpty — first provider wins). MergeMetadata(result, accumulator, nil, MergeFillEmpty) } + if len(quarantinedProviderIDKeys) > 0 { + accumulator.quarantinedProviderIDKeys = maps.Clone(quarantinedProviderIDKeys) + } // Phase 3: Images — all ImageProviders run, collect all available images. var allImages []RemoteImage for _, p := range itemChain { @@ -1603,13 +1692,32 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques } } } - return &ProcessResult{Updated: false}, nil + if matchDecision == nil { + matchDecision = &MatchDecision{Outcome: MatchOutcomeMetadataEmpty, Threshold: automaticMatchAcceptanceFloor} + } else if matchDecision.Outcome == MatchOutcomeMatched { + matchDecision.Outcome = MatchOutcomeMetadataEmpty + } + if hasTransientMatchError(providerMatchErrors) { + matchDecision.Outcome = MatchOutcomeProviderTransient + } else if len(providerMatchErrors) > 0 && + (matchDecision.Outcome == MatchOutcomeNoCandidates || matchDecision.Outcome == MatchOutcomeMetadataEmpty) { + // Optional corroboration failures must not turn a successfully + // completed search with rejected candidates into a permanent provider + // failure. The candidate decision remains actionable and deterministic. + matchDecision.Outcome = MatchOutcomeProviderPermanent + } + return &ProcessResult{Updated: false, Decision: matchDecision}, nil } result, err := s.mergeAndPersist(ctx, req, accumulator, allImages, allSeasons, allEpisodes, contentType) if err != nil { return nil, err } + if result != nil && strings.TrimSpace(result.ContentID) != "" { + if err := s.persistItemAliases(ctx, result.ContentID, req.Language, accumulator); err != nil { + return nil, err + } + } // Refresh stale ID records on successful refresh: clear anything resolved, // then keep only the providers that still 404ed during this run. @@ -1649,9 +1757,211 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques } } + if result != nil { + result.Decision = matchDecision + } return result, nil } +func buildMatchDecision(hints *MatchHints, candidates []MatchCandidate, winner *MatchCandidate, matched bool, providerErrors []error) *MatchDecision { + decision := &MatchDecision{CandidateCount: len(candidates), Threshold: automaticMatchAcceptanceFloor} + switch { + case matched && winner != nil: + decision.Outcome = MatchOutcomeMatched + case hasTransientMatchError(providerErrors): + decision.Outcome = MatchOutcomeProviderTransient + case len(providerErrors) > 0 && len(candidates) == 0: + decision.Outcome = MatchOutcomeProviderPermanent + case len(candidates) == 0: + decision.Outcome = MatchOutcomeNoCandidates + case candidatesConflictWithTrustedIDs(hints, candidates): + decision.Outcome = MatchOutcomeTrustedIDConflict + default: + decision.Outcome = MatchOutcomeCandidateRejected + } + + scored := make([]MatchCandidate, len(candidates)) + copy(scored, candidates) + for i := range scored { + annotateCandidateMatch(&scored[i], hints) + } + sort.SliceStable(scored, func(i, j int) bool { return scored[i].MatchScore > scored[j].MatchScore }) + if len(scored) > 3 { + scored = scored[:3] + } + for _, candidate := range scored { + reasons := append([]string(nil), candidate.MatchReasons...) + if matched && winner != nil && sameMatchCandidate(candidate, *winner) { + for _, reason := range winner.MatchReasons { + if !slices.Contains(reasons, reason) { + reasons = append(reasons, reason) + } + } + } + decision.TopCandidates = append(decision.TopCandidates, MatchDecisionCandidate{ + Title: candidate.Title, MatchedTitle: candidate.MatchedTitle, Year: candidate.Year, + ProviderIDs: copyMap(candidate.ProviderIDs), Sources: append([]string(nil), candidate.Sources...), + Score: candidate.MatchScore, Reasons: reasons, + }) + } + return decision +} + +func sameMatchCandidate(left, right MatchCandidate) bool { + leftKey, rightKey := normalizedKey(left.ProviderIDs), normalizedKey(right.ProviderIDs) + if leftKey != "" || rightKey != "" { + return leftKey != "" && leftKey == rightKey + } + if left.Title != right.Title || left.OriginalTitle != right.OriginalTitle || + left.Year != right.Year || left.ContentType != right.ContentType || + !slices.Equal(left.Sources, right.Sources) || len(left.ProviderIDs) != len(right.ProviderIDs) { + return false + } + for key, value := range left.ProviderIDs { + if right.ProviderIDs[key] != value { + return false + } + } + return true +} + +func hasTransientMatchError(errs []error) bool { + for _, err := range errs { + if err == nil { + continue + } + switch status.Code(err) { + case codes.DeadlineExceeded, codes.ResourceExhausted, codes.Aborted, codes.Unavailable: + return true + } + message := strings.ToLower(err.Error()) + for _, marker := range []string{"timeout", "deadline exceeded", "connection reset", "connection refused", "temporarily unavailable", "unavailable", "rate limit", "too many requests", "http 429", "http 500", "http 502", "http 503", "http 504"} { + if strings.Contains(message, marker) { + return true + } + } + } + return false +} + +func mergePreferredTitleMetadata(accumulator, result *MetadataResult, language, provider string, attributeAliases bool) { + if accumulator == nil || result == nil { + return + } + if attributeAliases { + if accumulator.titleAliasProviders == nil { + accumulator.titleAliasProviders = make(map[string]bool) + } + providerComplete, providerSeen := accumulator.titleAliasProviders[provider] + if !providerSeen { + accumulator.titleAliasProviders[provider] = result.TitleAliasesComplete + } else { + // Multiple installations can share a provider slug. The aggregate is + // authoritative only when every contributing response says its alias + // snapshot is complete; one legacy/partial response removes deletion + // authority for the shared provider scope. + accumulator.titleAliasProviders[provider] = providerComplete && result.TitleAliasesComplete + } + if strings.TrimSpace(result.Title) != "" { + kind := titleAliasKindLocalized + if strings.TrimSpace(result.OriginalTitle) != "" && strings.EqualFold(strings.TrimSpace(result.Title), strings.TrimSpace(result.OriginalTitle)) && + (baseMetadataLanguage(result.OriginalLanguage) == "" || baseMetadataLanguage(result.TitleLanguage) == baseMetadataLanguage(result.OriginalLanguage)) { + kind = titleAliasKindOriginal + } + accumulator.TitleAliases = appendUniqueTitleAlias(accumulator.TitleAliases, TitleAlias{ + Title: result.Title, Language: baseMetadataLanguage(result.TitleLanguage), Kind: kind, Provider: provider, + }) + } + for _, alias := range result.TitleAliases { + if alias.Provider == "" { + alias.Provider = provider + } + accumulator.TitleAliases = appendUniqueTitleAlias(accumulator.TitleAliases, alias) + } + if result.OriginalTitle != "" && !strings.EqualFold(result.OriginalTitle, result.Title) { + accumulator.TitleAliases = appendUniqueTitleAlias(accumulator.TitleAliases, TitleAlias{ + Title: result.OriginalTitle, Language: baseMetadataLanguage(result.OriginalLanguage), Kind: titleAliasKindOriginal, Provider: provider, + }) + } + } + if accumulator.OriginalTitle == "" && strings.TrimSpace(result.OriginalTitle) != "" { + accumulator.OriginalTitle = result.OriginalTitle + } + + title, titleLanguage, fallback, rank := preferredMetadataResultTitle(result, language) + currentRank := metadataTitlePreferenceRank(accumulator.TitleLanguage, accumulator.TitleIsFallback, language) + if strings.TrimSpace(accumulator.Title) != "" && currentRank == 0 { + currentRank = 1 + } + if strings.TrimSpace(accumulator.Title) == "" || rank > currentRank { + accumulator.Title = title + accumulator.TitleLanguage = titleLanguage + accumulator.TitleIsFallback = fallback + } +} + +func (s *MetadataService) persistItemAliases(ctx context.Context, contentID, language string, result *MetadataResult) error { + if s == nil || s.itemAliasRepo == nil || result == nil { + return nil + } + byProvider := make(map[string][]models.MediaItemAlias) + for _, alias := range result.TitleAliases { + provider := strings.TrimSpace(alias.Provider) + if provider == "" { + continue + } + byProvider[provider] = append(byProvider[provider], models.MediaItemAlias{ + ContentID: contentID, Title: alias.Title, Language: alias.Language, Kind: alias.Kind, Provider: provider, + }) + } + for provider, complete := range result.titleAliasProviders { + if err := s.itemAliasRepo.RefreshProviderLanguage(ctx, contentID, provider, language, byProvider[provider], complete); err != nil { + return fmt.Errorf("persisting %s title aliases: %w", provider, err) + } + } + return nil +} + +func preferredMetadataResultTitle(result *MetadataResult, language string) (string, string, bool, int) { + requested := baseMetadataLanguage(language) + titleLanguage := baseMetadataLanguage(result.TitleLanguage) + if strings.TrimSpace(result.Title) != "" && requested != "" && titleLanguage == requested && !result.TitleIsFallback { + return result.Title, titleLanguage, false, 3 + } + for _, alias := range result.TitleAliases { + if strings.TrimSpace(alias.Title) != "" && requested != "" && baseMetadataLanguage(alias.Language) == requested { + return alias.Title, requested, false, 2 + } + } + if strings.TrimSpace(result.Title) != "" && titleLanguage == "" && !result.TitleIsFallback { + return result.Title, "", false, 3 + } + if requested != "" && strings.TrimSpace(result.OriginalTitle) != "" { + return result.OriginalTitle, baseMetadataLanguage(result.OriginalLanguage), true, 1 + } + if strings.TrimSpace(result.Title) != "" { + return result.Title, titleLanguage, result.TitleIsFallback, 1 + } + return result.OriginalTitle, baseMetadataLanguage(result.OriginalLanguage), true, 1 +} + +func metadataTitlePreferenceRank(titleLanguage string, fallback bool, language string) int { + if strings.TrimSpace(titleLanguage) == "" { + // A provider that does not participate in the language contract (notably + // the local NFO provider and older plugins) retains normal first-provider + // priority. Only an explicitly marked fallback may be displaced by a + // later requested-language result. + if fallback { + return 1 + } + return 3 + } + if baseMetadataLanguage(titleLanguage) == baseMetadataLanguage(language) && !fallback { + return 3 + } + return 1 +} + // refreshFollowUpContentID returns the content ID that a completed refresh's // follow-up writes (stale-ID clear/record, debt sync) should target: the // canonical ID the item was persisted or merged into when available, falling @@ -1675,6 +1985,12 @@ func (s *MetadataService) mergeAndPersist( episodes []EpisodeResult, contentType string, ) (*ProcessResult, error) { + // Quarantined IDs are unsafe for deduplication, canonical ID derivation, and + // persistence regardless of whether this is a new item or a refresh. + for key := range accumulator.quarantinedProviderIDKeys { + delete(accumulator.ProviderIDs, key) + } + // Determine merge mode. var mergeMode MergeMode switch req.Mode { @@ -1805,6 +2121,9 @@ func (s *MetadataService) mergeAndPersist( accumulator.ProviderIDs = make(map[string]string, len(durableIDs)) } for key, value := range durableIDs { + if _, quarantined := accumulator.quarantinedProviderIDKeys[key]; quarantined { + continue + } if _, exists := accumulator.ProviderIDs[key]; !exists { accumulator.ProviderIDs[key] = value } @@ -1845,6 +2164,10 @@ func (s *MetadataService) mergeAndPersist( } else { MergeGlobalMetadata(accumulator, existingResult, locked, mergeMode) } + for key := range accumulator.quarantinedProviderIDKeys { + delete(existingResult.ProviderIDs, key) + } + existingResult.quarantinedProviderIDKeys = accumulator.quarantinedProviderIDKeys accumulator = existingResult } @@ -2749,6 +3072,7 @@ func (s *MetadataService) resolveSeriesRefreshProviderIDs(ctx context.Context, s return nil, err } maps.Copy(accumulatedIDs, durableIDs) + sanitizeCanonicalProviderIDsInPlace(accumulatedIDs) if err := s.suppressRecordedStaleProviderIDs(ctx, series.ContentID, accumulatedIDs); err != nil { return nil, err } @@ -2792,18 +3116,16 @@ func (s *MetadataService) resolveSeriesRefreshProviderIDs(ctx context.Context, s } for _, result := range results { if searchResultConflictsWithTrustedIDs(accumulatedIDs, result.ProviderIDs) { - continue + slog.WarnContext(ctx, "metadata: retaining conflicting target-refresh result for consensus", "component", "metadata", + "provider", p.Slug(), "content_id", series.ContentID, + "hinted_ids", accumulatedIDs, "candidate_ids", result.ProviderIDs) } allResults = append(allResults, result) } } - candidates := NormalizeCandidates(allResults, series.Type) + candidates := NormalizeCandidatesForLanguage(allResults, series.Type, searchQuery.Language) if winner, ok := selectRefreshMatchCandidate(series, nil, candidates); ok && winner != nil { - for k, v := range winner.ProviderIDs { - if v != "" { - accumulatedIDs[k] = v - } - } + applyCandidateProviderIDConsensus(accumulatedIDs, winner, nil) } if err := s.suppressRecordedStaleProviderIDs(ctx, series.ContentID, accumulatedIDs); err != nil { return nil, err @@ -6357,6 +6679,42 @@ func searchResultConflictsWithTrustedIDs(hintedIDs, candidateIDs map[string]stri return false } +// applyCandidateProviderIDConsensus replaces accumulated canonical IDs with a +// normalized winner while removing any cross-provider IDs quarantined during +// candidate grouping. The optional quarantine map carries the decision into +// Phase 2 so detail responses cannot reintroduce the disputed ID. +func applyCandidateProviderIDConsensus(accumulatedIDs map[string]string, winner *MatchCandidate, quarantine map[string]struct{}) { + if winner == nil { + return + } + locallyQuarantined := make(map[string]struct{}, len(winner.ConflictingProviderIDKeys)) + for _, key := range winner.ConflictingProviderIDKeys { + key = strings.ToLower(strings.TrimSpace(key)) + if key == "" { + continue + } + delete(accumulatedIDs, key) + locallyQuarantined[key] = struct{}{} + if quarantine != nil { + quarantine[key] = struct{}{} + } + } + for key, value := range winner.ProviderIDs { + key = strings.ToLower(strings.TrimSpace(key)) + value = strings.TrimSpace(value) + if key == "" || value == "" { + continue + } + if _, quarantined := locallyQuarantined[key]; quarantined { + continue + } + if _, quarantined := quarantine[key]; quarantined { + continue + } + accumulatedIDs[key] = value + } +} + // applyBuiltinIdentityHints consults the chain's IdentityHintProviders (the // built-in NFO provider) and folds their curated tmdb/imdb/tvdb ids into // accumulatedIDs ahead of Phase-1 search. Keys in protected (stored durable @@ -6379,8 +6737,12 @@ func applyBuiltinIdentityHints( } hints := hinter.IdentityHints(ctx, query) for _, key := range trustedSearchIDKeys { - value := strings.TrimSpace(hints[key]) - if value == "" { + value, valid := sanitizeProviderIDValue(key, hints[key]) + if !valid { + if strings.TrimSpace(hints[key]) != "" { + slog.WarnContext(ctx, "metadata: ignoring malformed local identity hint", "component", "metadata", + "provider", p.Slug(), "key", key, "value", strings.TrimSpace(hints[key])) + } continue } current := strings.TrimSpace(accumulatedIDs[key]) diff --git a/internal/metadata/types.go b/internal/metadata/types.go index 60fa078e..92b949a6 100644 --- a/internal/metadata/types.go +++ b/internal/metadata/types.go @@ -78,8 +78,50 @@ type ProcessResult struct { ContentID string IsNew bool // True if this was an initial match (new item created) Updated bool // True if any fields were changed + Decision *MatchDecision } +// MatchDecision is a bounded, persistence-safe explanation of an automatic +// matcher outcome. It intentionally excludes artwork and provider payloads. +type MatchDecision struct { + Outcome MatchOutcome `json:"outcome"` + CandidateCount int `json:"candidate_count"` + Threshold float64 `json:"threshold"` + TopCandidates []MatchDecisionCandidate `json:"top_candidates,omitempty"` +} + +type MatchDecisionCandidate struct { + Title string `json:"title"` + MatchedTitle string `json:"matched_title,omitempty"` + Year int `json:"year,omitempty"` + ProviderIDs map[string]string `json:"provider_ids,omitempty"` + Sources []string `json:"sources,omitempty"` + Score float64 `json:"score"` + Reasons []string `json:"reasons,omitempty"` +} + +type MatchFailure struct { + Kind MatchOutcome + Message string + Decision *MatchDecision +} + +// MatchOutcome is the stable matcher/queue failure taxonomy exposed in +// bounded diagnostics. Keep additions backward-compatible with persisted +// failure_kind values and the admin API. +type MatchOutcome string + +const ( + MatchOutcomeMatched MatchOutcome = "matched" + MatchOutcomeNoCandidates MatchOutcome = "no_candidates" + MatchOutcomeCandidateRejected MatchOutcome = "candidate_rejected" + MatchOutcomeTrustedIDConflict MatchOutcome = "trusted_id_conflict" + MatchOutcomeTrustedIDTypeMismatch MatchOutcome = "trusted_id_type_mismatch" + MatchOutcomeMetadataEmpty MatchOutcome = "metadata_empty" + MatchOutcomeProviderTransient MatchOutcome = "provider_transient" + MatchOutcomeProviderPermanent MatchOutcome = "provider_permanent" +) + // MatchHints carries scanner-extracted data into the pipeline. // Adapted from matcher.MatchHints with FilePath added for local providers. type MatchHints struct { @@ -99,6 +141,17 @@ type MatchHints struct { ObservedRootPath string AllGroupFilePaths []string PrimarySidecarSearchPaths []string + // AlternateIdentities are independently parsed title/year hypotheses from + // the filename and surrounding directories. They are tried only after the + // primary scanner identity fails, keeping provider traffic bounded while + // allowing a good filename to recover from a stale or release-like folder. + AlternateIdentities []MatchIdentityHint +} + +type MatchIdentityHint struct { + Title string + Year int + Source string } // SearchQuery is passed to SearchProvider.Search(). @@ -118,14 +171,34 @@ type SearchQuery struct { // SearchResult is returned from SearchProvider.Search(). type SearchResult struct { - Name string - Year int - ProviderIDs map[string]string - ImageURL string - Overview string - Provider string // Slug of the provider that returned this + Name string + OriginalTitle string + TitleAliases []TitleAlias + TitleLanguage string + TitleIsFallback bool + OriginalLanguage string + Year int + ProviderIDs map[string]string + ImageURL string + Overview string + Provider string // Slug of the provider that returned this } +// TitleAlias is a provider-confirmed title for the same work. +type TitleAlias struct { + Title string `json:"title"` + Language string `json:"language,omitempty"` + Kind string `json:"kind"` + Provider string `json:"provider,omitempty"` +} + +const ( + titleAliasKindOriginal = "original" + titleAliasKindLocalized = "localized" + matchContentTypeMovie = "movie" + matchContentTypeSeries = "series" +) + // MetadataRequest is passed to MetadataProvider.GetMetadata(). type MetadataRequest struct { ProviderIDs map[string]string @@ -163,24 +236,33 @@ type PersonDetailResult struct { // MetadataResult carries structured metadata from a single provider. type MetadataResult struct { - HasMetadata bool - ProviderIDs map[string]string - Title string - OriginalTitle string - SortTitle string - Overview string - Tagline string - Year int - Runtime int - Genres []string - Studios []string - Networks []string - Countries []string - Keywords []string - OriginalLanguage string - ContentRating string - Ratings Ratings - People []models.ItemPerson + HasMetadata bool + ProviderIDs map[string]string + Title string + OriginalTitle string + SortTitle string + Overview string + Tagline string + Year int + Runtime int + Genres []string + Studios []string + Networks []string + Countries []string + Keywords []string + OriginalLanguage string + TitleAliases []TitleAlias + TitleAliasesComplete bool + TitleLanguage string + TitleIsFallback bool + // titleAliasProviders records whether each contributing provider declared + // its aliases to be an authoritative snapshot. False means merge-only, + // which is the safe default for legacy plugins and partial responses. + titleAliasProviders map[string]bool + quarantinedProviderIDKeys map[string]struct{} + ContentRating string + Ratings Ratings + People []models.ItemPerson // Images (S3 paths or URLs). PosterPath string PosterThumbhash string diff --git a/internal/metadata/worker.go b/internal/metadata/worker.go index 3322cf94..3b0dcf0a 100644 --- a/internal/metadata/worker.go +++ b/internal/metadata/worker.go @@ -16,6 +16,7 @@ import ( "github.com/Silo-Server/silo-server/internal/catalog" "github.com/Silo-Server/silo-server/internal/librarykind" "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/naming" "github.com/Silo-Server/silo-server/internal/notifications" ) @@ -332,6 +333,7 @@ func (w *MatchWorker) buildProcessRequestForGroup(ctx context.Context, represent ImdbID: skeleton.ImdbID, TvdbID: skeleton.TvdbID, HintSource: "scanner", + AlternateIdentities: queuedMatchIdentityAlternates(representative, skeleton), } return ProcessRequest{ @@ -342,6 +344,62 @@ func (w *MatchWorker) buildProcessRequestForGroup(ctx context.Context, represent } } +const maxAlternateMatchIdentities = 3 + +func queuedMatchIdentityAlternates(file *models.MediaFile, skeleton *skeletonResult) []MatchIdentityHint { + if file == nil || skeleton == nil { + return nil + } + + seen := map[string]struct{}{ + matchIdentityKey(skeleton.Title, skeleton.Year): {}, + } + out := make([]MatchIdentityHint, 0, maxAlternateMatchIdentities) + add := func(title string, year int, source string) { + title = strings.TrimSpace(naming.StripComparisonSafeEditionSuffix(title)) + key := matchIdentityKey(title, year) + if title == "" || key == "" { + return + } + if _, exists := seen[key]; exists { + return + } + seen[key] = struct{}{} + out = append(out, MatchIdentityHint{Title: title, Year: year, Source: source}) + } + + itemType := strings.ToLower(strings.TrimSpace(skeleton.Type)) + if parsed := naming.ParseFilename(file.FilePath, itemType); parsed != nil { + add(parsed.Title, parsed.Year, "current_path") + } + + if itemType == matchContentTypeMovie { + base := strings.TrimSuffix(filepath.Base(file.FilePath), filepath.Ext(file.FilePath)) + stem := naming.ParseInferMovieStem(base, skeleton.Title, skeleton.Year) + add(stem.Title, stem.Year, "filename") + + observedRoot := strings.TrimSpace(skeleton.ObservedRootPath) + if observedRoot == "" { + observedRoot = filepath.Dir(file.FilePath) + } + rootStem := naming.ParseInferMovieStem(filepath.Base(observedRoot), "", 0) + add(rootStem.Title, rootStem.Year, "release_folder") + } + + if len(out) > maxAlternateMatchIdentities { + out = out[:maxAlternateMatchIdentities] + } + return out +} + +func matchIdentityKey(title string, year int) string { + title = strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(title)), " ")) + if title == "" { + return "" + } + return title + "\x00" + strconv.Itoa(year) +} + func compactUniquePaths(paths []string) []string { if len(paths) == 0 { return nil @@ -728,7 +786,8 @@ func (w *MatchWorker) queuedMovieSkeleton(ctx context.Context, file *models.Medi return skeleton, true, nil } - skeleton, err := w.service.createOrFindSkeleton(ctx, file, file.MediaFolderID) + currentFile := reparseQueuedFileIdentity(file, "movie") + skeleton, err := w.service.createOrFindSkeleton(ctx, currentFile, currentFile.MediaFolderID) if err != nil { return nil, false, err } @@ -776,6 +835,34 @@ func (w *MatchWorker) reusableQueuedMovieSkeleton(ctx context.Context, file *mod if year == 0 { year = file.BaseYear } + currentFile := reparseQueuedFileIdentity(file, itemType) + if currentFile.BaseTitle != "" { + title = currentFile.BaseTitle + } + if currentFile.BaseYear != 0 { + year = currentFile.BaseYear + } + if currentFile.BaseType != "" { + itemType = currentFile.BaseType + } + + refreshedIDs := trustedStructuredIDsForSkeleton(file.FilePath, observedRootPath, rootPath) + // This is an unmatched-style skeleton being re-evaluated. Structured path + // IDs are current input; IDs that disappeared from the path must not remain + // trusted forever merely because an older parser copied them onto the + // provisional item. + tmdbID, imdbID, tvdbID := "", "", "" + if refreshedIDs != nil { + if refreshedIDs.TmdbID != "" { + tmdbID = refreshedIDs.TmdbID + } + if refreshedIDs.ImdbID != "" { + imdbID = refreshedIDs.ImdbID + } + if refreshedIDs.TvdbID != "" { + tvdbID = refreshedIDs.TvdbID + } + } return &skeletonResult{ ContentID: item.ContentID, @@ -787,12 +874,37 @@ func (w *MatchWorker) reusableQueuedMovieSkeleton(ctx context.Context, file *mod Title: title, Year: year, Type: itemType, - TmdbID: item.TmdbID, - ImdbID: item.ImdbID, - TvdbID: item.TvdbID, + TmdbID: tmdbID, + ImdbID: imdbID, + TvdbID: tvdbID, }, true } +// reparseQueuedFileIdentity refreshes the in-memory scanner identity from the +// current path without mutating the persisted scan row. Queue entries can live +// across parser releases, including entries that have not created a skeleton +// yet, so both fresh and reusable skeleton paths must use this view. +func reparseQueuedFileIdentity(file *models.MediaFile, fallbackType string) *models.MediaFile { + if file == nil { + return nil + } + current := *file + parseType := strings.TrimSpace(fallbackType) + if parseType == "" { + parseType = strings.TrimSpace(current.BaseType) + } + if parsed := naming.ParseFilename(current.FilePath, parseType); parsed != nil { + if parsed.Title != "" { + current.BaseTitle = parsed.Title + } + current.BaseYear = parsed.Year + if parsed.Type != "" { + current.BaseType = parsed.Type + } + } + return ¤t +} + func scopedMatcherPath(useSeriesQueue bool, useMovieQueue bool) string { switch { case useSeriesQueue && useMovieQueue: @@ -946,7 +1058,8 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo return len(groupFiles), nil } - skeleton, err := w.service.createOrFindSkeleton(ctx, representative, job.MediaFolderID) + currentRepresentative := reparseQueuedFileIdentity(representative, "series") + skeleton, err := w.service.createOrFindSkeleton(ctx, currentRepresentative, job.MediaFolderID) if err != nil { queueErr := truncateSeriesQueueError(err.Error()) if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, queueErr); updateErr != nil { @@ -971,7 +1084,22 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo return 0, fmt.Errorf("relinking series root %d/%s: %w", job.MediaFolderID, job.ObservedRootPath, err) } - if skeleton.IsNew && skeleton.ItemStatus != "ambiguous" { + needsInitialMatch := skeleton.IsNew + if !needsInitialMatch && strings.TrimSpace(skeleton.ContentID) != "" && w.service.itemRepo != nil { + item, loadErr := w.service.itemRepo.GetByID(ctx, skeleton.ContentID) + if loadErr != nil { + queueErr := truncateSeriesQueueError(loadErr.Error()) + if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, queueErr); updateErr != nil { + return 0, updateErr + } + return 0, fmt.Errorf("loading linked series skeleton %s: %w", skeleton.ContentID, loadErr) + } + if item != nil { + skeleton.ItemStatus = item.Status + needsInitialMatch = isSkeletonLikeStatus(item.Status) + } + } + if needsInitialMatch && skeleton.ItemStatus != "ambiguous" { req := w.buildProcessRequestForGroup(ctx, representative, skeleton, groupFiles) result, processErr := w.service.Process(ctx, req) if processErr != nil { diff --git a/internal/metadata/worker_test.go b/internal/metadata/worker_test.go index 0ecb4d15..5b476007 100644 --- a/internal/metadata/worker_test.go +++ b/internal/metadata/worker_test.go @@ -12,6 +12,8 @@ import ( "github.com/Silo-Server/silo-server/internal/models" ) +const queuedShowName = "Show Name" + type fakeWorkerFolderRepo struct { folders map[int]*models.MediaFolder } @@ -1474,3 +1476,132 @@ func TestProcessSeriesRoot_Site2_NonNotFoundErrorStillFails(t *testing.T) { t.Fatalf("unexpected error message: %v", err) } } + +func TestReparseQueuedFileIdentityUsesCurrentPathWithoutMutatingScanRow(t *testing.T) { + t.Parallel() + const expectedTitle = "10 Tricks" + file := &models.MediaFile{ + FilePath: "/movies/10 Tricks (2022)/10 Tricks (2022) (tt0473100) [WEBDL-1080p].mp4", + BaseTitle: "stale scanner title", + BaseYear: 0, + BaseType: "movie", + } + current := reparseQueuedFileIdentity(file, "movie") + if current == file { + t.Fatal("reparse returned the persisted model pointer") + } + if current.BaseTitle != expectedTitle || current.BaseYear != 2022 || current.BaseType != matchContentTypeMovie { + t.Fatalf("reparsed identity = %q/%d/%q", current.BaseTitle, current.BaseYear, current.BaseType) + } + if file.BaseTitle != "stale scanner title" || file.BaseYear != 0 { + t.Fatalf("persisted scan model was mutated: %#v", file) + } +} + +func TestReparseQueuedFileIdentityClearsStaleYearAndUsesQueueType(t *testing.T) { + t.Parallel() + file := &models.MediaFile{ + FilePath: "/movies/Show Name/Show Name.mkv", + BaseTitle: "stale", + BaseYear: 1999, + BaseType: "series", + } + current := reparseQueuedFileIdentity(file, "movie") + if current.BaseTitle != queuedShowName || current.BaseYear != 0 || current.BaseType != "movie" { + t.Fatalf("reparsed identity = %q/%d/%q, want Show Name/0/movie", current.BaseTitle, current.BaseYear, current.BaseType) + } + if file.BaseYear != 1999 || file.BaseType != "series" { + t.Fatalf("persisted scan model was mutated: %#v", file) + } +} + +func TestReusableQueuedMovieSkeletonPreservesKnownYearWhenCurrentPathHasNone(t *testing.T) { + t.Parallel() + h := newTestHarness() + ctx := context.Background() + const contentID = "local-year-fallback" + if err := h.itemRepo.Upsert(ctx, &models.MediaItem{ + ContentID: contentID, + Status: "pending", + Title: queuedShowName, + Year: 1999, + Type: "movie", + Studios: []string{}, + Networks: []string{}, + Countries: []string{}, + Genres: []string{}, + }); err != nil { + t.Fatalf("seed item: %v", err) + } + file := &models.MediaFile{ + ContentID: contentID, + FilePath: "/movies/Show Name/Show Name.mkv", + BaseTitle: queuedShowName, + BaseYear: 1987, + BaseType: "movie", + } + + worker := NewMatchWorker(h.service, h.fileRepo, 1, 1, 0) + skeleton, ok := worker.reusableQueuedMovieSkeleton(ctx, file) + if !ok || skeleton == nil { + t.Fatal("expected reusable skeleton") + } + if skeleton.Year != 1999 { + t.Fatalf("skeleton year = %d, want preserved item year 1999", skeleton.Year) + } +} + +func TestQueuedMatchIdentityAlternatesRecoversSeriesParentShow(t *testing.T) { + file := &models.MediaFile{ + FilePath: "/shows/Becoming You/Becoming.You.S01.HDR.2160p.WEB-DL-GROUP/Becoming.You.S01E01.2160p.WEB-DL.mkv", + } + skeleton := &skeletonResult{ + Title: "Becoming.You.S01.HDR.2160p.WEB-DL-GROUP", + Type: "series", + ObservedRootPath: "/shows/Becoming You/Becoming.You.S01.HDR.2160p.WEB-DL-GROUP", + } + + got := queuedMatchIdentityAlternates(file, skeleton) + if len(got) == 0 || got[0].Title != "Becoming You" || got[0].Source != "current_path" { + t.Fatalf("alternate identities = %#v, want current-path Becoming You", got) + } +} + +func TestQueuedMatchIdentityAlternatesRecoversMovieFilenameAndReleaseFolder(t *testing.T) { + const alienInvasionTitle = "Alien Invasion" + tests := []struct { + name string + file *models.MediaFile + skeleton *skeletonResult + wantTitle string + wantYear int + wantSource string + }{ + { + name: "filename beats divergent parent", + file: &models.MediaFile{FilePath: "/movies/After the Lethargy (2018)/Alien Invasion (2018).mkv"}, + skeleton: &skeletonResult{Title: "After the Lethargy", Year: 2018, Type: "movie", + ObservedRootPath: "/movies/After the Lethargy (2018)"}, + wantTitle: alienInvasionTitle, wantYear: 2018, wantSource: "filename", + }, + { + name: "hash filename uses release folder", + file: &models.MediaFile{FilePath: "/movies/Jurassic.World.2015.1080p.WEB-DL-GROUP/291684abcdef012345.mkv"}, + skeleton: &skeletonResult{Title: "291684abcdef012345", Type: "movie", + ObservedRootPath: "/movies/Jurassic.World.2015.1080p.WEB-DL-GROUP"}, + wantTitle: "Jurassic World", wantYear: 2015, wantSource: "release_folder", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := queuedMatchIdentityAlternates(tt.file, tt.skeleton) + for _, identity := range got { + if identity.Title == tt.wantTitle && identity.Year == tt.wantYear && identity.Source == tt.wantSource { + return + } + } + t.Fatalf("alternate identities = %#v, want %q/%d from %s", got, tt.wantTitle, tt.wantYear, tt.wantSource) + }) + } +} diff --git a/internal/models/media.go b/internal/models/media.go index 8ee7ae6c..2b994c15 100644 --- a/internal/models/media.go +++ b/internal/models/media.go @@ -418,6 +418,15 @@ type MediaItem struct { AddedAt *time.Time // populated by browse queries (MIN(mil.first_seen_at)) } +// MediaItemAlias is a provider-confirmed searchable title for a media item. +type MediaItemAlias struct { + ContentID string + Title string + Language string + Kind string + Provider string +} + // Season represents a row in the seasons table. type Season struct { ContentID string diff --git a/internal/naming/filename.go b/internal/naming/filename.go index 27b8def5..ddbc904f 100644 --- a/internal/naming/filename.go +++ b/internal/naming/filename.go @@ -10,9 +10,9 @@ import ( ) var ( - // folderTagRe matches bracketed or braced provider-ID tags like - // [tmdbid-27205], {tvdb-81189}, [imdbid-tt1375666], etc. - folderTagRe = regexp.MustCompile(`\s*[{\[](tmdb|tmdbid|imdb|imdbid|tvdb|tvdbid)-[\w]+[}\]]`) + // folderTagRe matches strict provider-ID tags in square, curly, or round + // brackets, including Plex-style bare IMDb tags such as (tt0473100). + folderTagRe = regexp.MustCompile(`(?i)\s*(?:\((?:(?:tmdb|tmdbid|tvdb|tvdbid)-\d+|(?:imdb|imdbid)-tt\d{7,10}|tt\d{7,10})\)|\[(?:(?:tmdb|tmdbid|tvdb|tvdbid)-\d+|(?:imdb|imdbid)-tt\d{7,10}|tt\d{7,10})\]|\{(?:(?:tmdb|tmdbid|tvdb|tvdbid)-\d+|(?:imdb|imdbid)-tt\d{7,10}|tt\d{7,10})\})`) // titleYearRe matches "Title (Year)" with optional trailing content. titleYearRe = regexp.MustCompile(`^(.+?)\s*\((\d{4})\)`) @@ -34,6 +34,13 @@ var ( // specialsDirRe matches common specials/extras folders. specialsDirRe = regexp.MustCompile(`(?i)^(?:specials?|extras?)$`) + + // seasonReleaseDirRe recognizes release-pack directories such as + // "Show.Name.S01.2160p.WEB-DL-GROUP". These are season containers, not + // show roots; treating them as roots fragments one show into one metadata + // item per release directory and searches providers with the release name. + seasonReleaseDirRe = regexp.MustCompile(`(?i)(?:^|[ ._-])s\d{1,4}(?:[ ._-]|$)`) + seasonReleaseTechRe = regexp.MustCompile(`(?i)(?:^|[ ._-])(?:2160p|1080p|720p|576p|480p|web(?:[ ._-]?dl|rip)?|blu[ ._-]?ray|bluray|hdtv|remux|x26[45]|h26[45]|hevc)(?:[ ._-]|$)`) ) // ResolvePathContext classifies a media path using both naming heuristics and @@ -272,7 +279,7 @@ func detectMovieFolderEvidence(parentBase string, nameNoExt string, hasSeasonStr } func hasExplicitFolderIDs(name string) bool { - return folderIDPattern.MatchString(name) + return ParseStructuredFolderIDs(name) != nil } func parseTitleYearCandidate(name string) (string, int) { @@ -307,6 +314,12 @@ func collapseWhitespace(value string) string { func deriveSeriesRoot(filePath string, hasEpisodePattern bool, forceParent bool) (*SeriesRoot, bool) { parentDir := path.Dir(filePath) + if isSeasonReleaseDirectoryForParent(path.Base(parentDir), path.Base(path.Dir(parentDir))) { + rootPath := path.Dir(parentDir) + if rootPath != "." && rootPath != "/" && rootPath != "" { + return &SeriesRoot{RootPath: rootPath, FolderName: path.Base(rootPath)}, true + } + } for current := parentDir; current != "." && current != "/" && current != ""; current = path.Dir(current) { segment := path.Base(current) if isSeasonDirSegment(segment, hasEpisodePattern || forceParent) || specialsDirRe.MatchString(segment) { @@ -334,6 +347,26 @@ func deriveSeriesRoot(filePath string, hasEpisodePattern bool, forceParent bool) return nil, false } +func isSeasonReleaseDirectory(segment string) bool { + return seasonReleaseDirRe.MatchString(segment) && seasonReleaseTechRe.MatchString(segment) +} + +func isSeasonReleaseDirectoryForParent(segment, parentSegment string) bool { + if !isSeasonReleaseDirectory(segment) { + return false + } + location := seasonReleaseDirRe.FindStringIndex(segment) + if location == nil || location[0] == 0 { + return false + } + releaseTitle := strings.Trim(segment[:location[0]], " ._-") + parentTitle, _ := parseTitleYearCandidate(parentSegment) + if parentTitle == "" { + parentTitle = parentSegment + } + return inferTitlesCoherent(releaseTitle, parentTitle) +} + func deriveMovieRoot(filePath string) (*CanonicalRoot, bool) { baseName := path.Base(filePath) nameNoExt := strings.TrimSuffix(baseName, filepath.Ext(baseName)) diff --git a/internal/naming/filename_test.go b/internal/naming/filename_test.go index 66ce242f..4191a6fe 100644 --- a/internal/naming/filename_test.go +++ b/internal/naming/filename_test.go @@ -282,6 +282,60 @@ func TestParseFilename(t *testing.T) { } } +func TestParseFilename_StripsStructuredProviderIDsFromTitles(t *testing.T) { + t.Parallel() + for _, test := range []struct { + path string + wantTitle string + wantYear int + }{ + { + path: "/movies/10 Tricks (2022)/10 Tricks (2022) (tt0473100) [WEBDL-1080p.AAC.h264.GNOME].mp4", + wantTitle: "10 Tricks", + wantYear: 2022, + }, + {path: "/movies/Some Movie [tt1234567]/Some Movie.mkv", wantTitle: "Some Movie"}, + {path: "/movies/Some Movie (tmdb-12345)/Some Movie.mkv", wantTitle: "Some Movie"}, + } { + hints := ParseFilename(test.path, "movie") + if hints.Title != test.wantTitle || hints.Year != test.wantYear { + t.Errorf("ParseFilename(%q) = title %q year %d, want %q/%d", test.path, hints.Title, hints.Year, test.wantTitle, test.wantYear) + } + } +} + +func TestParseFilename_SeriesReleasePackDirectoryUsesParentShowRoot(t *testing.T) { + path := "/library/Becoming You/Becoming.You.S01.HDR.2160p.WEB-DL.DDP5.1.H.265-ROCCaT/Becoming.You.S01E01.HDR.2160p.WEB-DL.mkv" + + hints := ParseFilename(path, "series") + if hints == nil { + t.Fatal("ParseFilename returned nil") + return + } + if hints.Title != "Becoming You" || hints.Year != 0 || hints.Type != "series" { + t.Fatalf("ParseFilename() = title %q year %d type %q, want Becoming You/0/series", hints.Title, hints.Year, hints.Type) + } + root, ok := DetectSeriesRoot(path, "series") + if !ok || root.RootPath != "/library/Becoming You" { + t.Fatalf("DetectSeriesRoot() = %+v, %v", root, ok) + } +} + +func TestParseFilename_SeriesReleasePackDirectoryRequiresCoherentParent(t *testing.T) { + t.Parallel() + flatPath := "/library/Becoming.You.S01.HDR.2160p.WEB-DL-GROUP/Becoming.You.S01E01.mkv" + root, ok := DetectSeriesRoot(flatPath, "series") + if !ok || root.RootPath != "/library/Becoming.You.S01.HDR.2160p.WEB-DL-GROUP" { + t.Fatalf("flat release root = %+v, %v; must not collapse to the library root", root, ok) + } + + nestedPath := "/library/Becoming You (2020)/Becoming.You.S01.1080p.WEB-DL/Becoming.You.S01E01.mkv" + root, ok = DetectSeriesRoot(nestedPath, "series") + if !ok || root.RootPath != "/library/Becoming You (2020)" { + t.Fatalf("nested release root = %+v, %v; want coherent parent show folder", root, ok) + } +} + func TestResolvePathContext(t *testing.T) { tests := []struct { name string diff --git a/internal/naming/folderid.go b/internal/naming/folderid.go index 5a4a96a3..e82173b9 100644 --- a/internal/naming/folderid.go +++ b/internal/naming/folderid.go @@ -3,18 +3,20 @@ package naming import ( "regexp" "strings" + + "github.com/Silo-Server/silo-server/internal/providerid" ) // folderIDPattern matches patterns like [tmdbid-27205], {tmdb-27205}, // [imdbid-tt1375666], {imdb-tt1375666}, [tvdbid-81189], {tvdb-81189}, etc. // The regex captures the provider prefix and the ID value. -var folderIDPattern = regexp.MustCompile(`[{\[](tmdb|tmdbid|imdb|imdbid|tvdb|tvdbid)-([\w]+)[}\]]`) -var trailingImdbIDPattern = regexp.MustCompile(`(?i)(?:^|\s)(tt\d{7,8})$`) +var folderIDPattern = regexp.MustCompile(`(?i)(?:\((tmdb|tmdbid|imdb|imdbid|tvdb|tvdbid)-([a-z0-9]+)\)|\[(tmdb|tmdbid|imdb|imdbid|tvdb|tvdbid)-([a-z0-9]+)\]|\{(tmdb|tmdbid|imdb|imdbid|tvdb|tvdbid)-([a-z0-9]+)\})`) +var trailingImdbIDPattern = regexp.MustCompile(`(?i)(?:^|\s)(tt\d{7,10})$`) // bracketedBareImdbPattern matches a bare IMDb id wrapped in brackets without a // provider prefix, e.g. [tt10011226] or {tt0095016} (Plex/Kodi-style tags). A // tt-prefixed number is unambiguously IMDb. -var bracketedBareImdbPattern = regexp.MustCompile(`(?i)[{\[](tt\d{7,8})[}\]]`) +var bracketedBareImdbPattern = regexp.MustCompile(`(?i)(?:\((tt\d{7,10})\)|\[(tt\d{7,10})\]|\{(tt\d{7,10})\})`) // ParseStructuredFolderIDs extracts only explicit provider IDs from a folder or // file name, such as {tmdb-27205}, [imdbid-tt1375666], or [tt1375666]. It does @@ -23,21 +25,37 @@ func ParseStructuredFolderIDs(name string) *FolderIDHints { matches := folderIDPattern.FindAllStringSubmatch(name, -1) hints := &FolderIDHints{} for _, m := range matches { - provider := strings.ToLower(m[1]) - id := m[2] + provider, id := "", "" + for i := 1; i+1 < len(m); i += 2 { + if m[i] != "" { + provider, id = strings.ToLower(m[i]), strings.ToLower(m[i+1]) + break + } + } switch provider { case "tmdb", "tmdbid": - hints.TmdbID = id + if providerid.IsPositiveDecimal(id) { + hints.TmdbID = id + } case "imdb", "imdbid": - hints.ImdbID = id + if isIMDbProviderID(id) { + hints.ImdbID = id + } case "tvdb", "tvdbid": - hints.TvdbID = id + if providerid.IsPositiveDecimal(id) { + hints.TvdbID = id + } } } if m := bracketedBareImdbPattern.FindStringSubmatch(name); m != nil && hints.ImdbID == "" { - hints.ImdbID = strings.ToLower(m[1]) + for _, value := range m[1:] { + if isIMDbProviderID(value) { + hints.ImdbID = strings.ToLower(value) + break + } + } } if hints.TmdbID == "" && hints.ImdbID == "" && hints.TvdbID == "" { @@ -46,6 +64,14 @@ func ParseStructuredFolderIDs(name string) *FolderIDHints { return hints } +func isIMDbProviderID(value string) bool { + value = strings.ToLower(strings.TrimSpace(value)) + if !trailingImdbIDPattern.MatchString(value) { + return false + } + return providerid.IsPositiveDecimal(strings.TrimPrefix(value, "tt")) +} + // ParseFolderIDs extracts external provider IDs from a folder name. Mirroring // Jellyfin's path-attribute model, only explicit evidence is honored: bracket // tags with provider prefixes ({tmdb-27205}, [tvdbid-81189], [imdbid-tt1375666]), diff --git a/internal/naming/folderid_test.go b/internal/naming/folderid_test.go index c0d30985..5283c452 100644 --- a/internal/naming/folderid_test.go +++ b/internal/naming/folderid_test.go @@ -9,6 +9,9 @@ func TestParseFolderIDs_BracketedBareImdb(t *testing.T) { {"square brackets", "17 Blocks (2021) [tt10011226]", "tt10011226"}, {"curly braces", "Some Show {tt1234567}", "tt1234567"}, {"8-digit tt", "Movie (2024) [tt12345678]", "tt12345678"}, + {"9-digit tt", "Movie (2024) [tt123456789]", "tt123456789"}, + {"10-digit tt", "Movie (2024) [tt1234567890]", "tt1234567890"}, + {"parentheses production pattern", "10 Tricks (2022) (tt0473100) [WEBDL-1080p.AAC.h264.GNOME].mp4", "tt0473100"}, } for _, c := range cases { got := ParseFolderIDs(c.folder) @@ -28,6 +31,34 @@ func TestParseFolderIDs_BracketedBareImdb(t *testing.T) { } } +func TestParseStructuredFolderIDs_ParenthesizedProviderIDs(t *testing.T) { + tests := []struct { + name string + want FolderIDHints + }{ + {"Movie (imdb-tt1375666)", FolderIDHints{ImdbID: "tt1375666"}}, + {"Movie (tmdb-27205)", FolderIDHints{TmdbID: "27205"}}, + {"Show (tvdbid-81189)", FolderIDHints{TvdbID: "81189"}}, + } + for _, tt := range tests { + got := ParseStructuredFolderIDs(tt.name) + if got == nil || *got != tt.want { + t.Errorf("ParseStructuredFolderIDs(%q) = %+v, want %+v", tt.name, got, tt.want) + } + } + + for _, invalid := range []string{ + "Movie (tt123)", "Movie (tt12345678901)", "Movie (tmdb-12x)", + "Movie (imdb-1234567)", "Movie [tmdb-123)", "Movie {tt1234567]", + "Movie (tmdb-0)", "Show (tvdb-000)", "Movie (tt0000000)", + "Movie (imdb-tt0000000)", "District 9", "Beverly Hills 90210", + } { + if got := ParseStructuredFolderIDs(invalid); got != nil { + t.Errorf("ParseStructuredFolderIDs(%q) = %+v, want nil", invalid, got) + } + } +} + func TestParseStructuredFolderIDs_BracketedBareImdb(t *testing.T) { got := ParseStructuredFolderIDs("17 Blocks (2021) [tt10011226]") if got == nil || got.ImdbID != "tt10011226" { diff --git a/internal/providerid/validate.go b/internal/providerid/validate.go new file mode 100644 index 00000000..8e780aa3 --- /dev/null +++ b/internal/providerid/validate.go @@ -0,0 +1,23 @@ +// Package providerid contains validation shared by provider-ID ingestion +// boundaries. +package providerid + +// IsPositiveDecimal reports whether value is a non-zero base-10 integer +// written using ASCII digits only. Leading zeroes are accepted because +// providers may treat the original text as their canonical identifier. +func IsPositiveDecimal(value string) bool { + if value == "" { + return false + } + + nonZero := false + for _, r := range value { + if r < '0' || r > '9' { + return false + } + if r != '0' { + nonZero = true + } + } + return nonZero +} diff --git a/internal/providerid/validate_test.go b/internal/providerid/validate_test.go new file mode 100644 index 00000000..69703c6e --- /dev/null +++ b/internal/providerid/validate_test.go @@ -0,0 +1,25 @@ +package providerid + +import "testing" + +func TestIsPositiveDecimal(t *testing.T) { + t.Parallel() + + tests := map[string]bool{ + "": false, + "0": false, + "000": false, + "1": true, + "001": true, + "123456": true, + "-1": false, + "1.0": false, + "１２３": false, + "12a": false, + } + for value, want := range tests { + if got := IsPositiveDecimal(value); got != want { + t.Errorf("IsPositiveDecimal(%q) = %v, want %v", value, got, want) + } + } +} diff --git a/internal/taskmanager/tasks/backfill_media_item_aliases.go b/internal/taskmanager/tasks/backfill_media_item_aliases.go new file mode 100644 index 00000000..66f95c0c --- /dev/null +++ b/internal/taskmanager/tasks/backfill_media_item_aliases.go @@ -0,0 +1,80 @@ +package tasks + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/Silo-Server/silo-server/internal/taskmanager" +) + +const mediaItemAliasBackfillBatchSize = 500 + +type MediaItemAliasBackfiller interface { + BackfillBatch(ctx context.Context, afterContentID string, limit int) (nextContentID string, processed int, err error) + // ResetCompletedBackfill re-arms a finished backfill so a manual re-run + // performs a fresh pass instead of a permanent no-op; an interrupted run + // keeps its persisted cursor and resumes. + ResetCompletedBackfill(ctx context.Context) error +} + +// BackfillMediaItemAliasesTask seeds the alias table from metadata Silo +// already owns. It is manual and idempotent; provider refreshes subsequently +// replace only their own authoritative aliases. +type BackfillMediaItemAliasesTask struct { + backfiller MediaItemAliasBackfiller +} + +func NewBackfillMediaItemAliasesTask(backfiller MediaItemAliasBackfiller) *BackfillMediaItemAliasesTask { + return &BackfillMediaItemAliasesTask{backfiller: backfiller} +} + +func (t *BackfillMediaItemAliasesTask) Key() string { return "backfill_media_item_aliases" } +func (t *BackfillMediaItemAliasesTask) Name() string { + return "Backfill Media Item Aliases" +} +func (t *BackfillMediaItemAliasesTask) Description() string { + return "Seeds searchable aliases from existing original and localized media titles in resumable batches." +} +func (t *BackfillMediaItemAliasesTask) Category() taskmanager.TaskCategory { + return taskmanager.TaskCategoryMetadata +} +func (t *BackfillMediaItemAliasesTask) IsHidden() bool { return false } +func (t *BackfillMediaItemAliasesTask) DefaultTriggers() []taskmanager.TriggerConfig { + return nil +} + +func (t *BackfillMediaItemAliasesTask) Execute(ctx context.Context, progress taskmanager.ProgressReporter) error { + if t == nil || t.backfiller == nil { + progress.Report(100, "Media item alias backfill is not configured") + return nil + } + + if err := t.backfiller.ResetCompletedBackfill(ctx); err != nil { + return fmt.Errorf("reset completed media item alias backfill: %w", err) + } + + cursor := "" + total := 0 + progress.Report(0, "Backfilling media item aliases") + for { + next, processed, err := t.backfiller.BackfillBatch(ctx, cursor, mediaItemAliasBackfillBatchSize) + if err != nil { + return fmt.Errorf("backfill media item aliases after %q: %w", cursor, err) + } + total += processed + if processed == 0 || next == cursor { + break + } + cursor = next + progress.Report(0, fmt.Sprintf("Backfilled aliases for %d media items", total)) + if processed < mediaItemAliasBackfillBatchSize { + break + } + } + + result, _ := json.Marshal(map[string]any{"processed_items": total, "last_content_id": cursor}) + progress.SetResultData(result) + progress.Report(100, fmt.Sprintf("Backfilled aliases for %d media items", total)) + return nil +} diff --git a/internal/taskmanager/tasks/backfill_media_item_aliases_test.go b/internal/taskmanager/tasks/backfill_media_item_aliases_test.go new file mode 100644 index 00000000..0a607204 --- /dev/null +++ b/internal/taskmanager/tasks/backfill_media_item_aliases_test.go @@ -0,0 +1,68 @@ +package tasks + +import ( + "context" + "encoding/json" + "testing" +) + +type aliasBackfillPage struct { + next string + count int +} + +type fakeAliasBackfiller struct { + pages []aliasBackfillPage + cursors []string + resets int +} + +func (f *fakeAliasBackfiller) ResetCompletedBackfill(context.Context) error { + f.resets++ + return nil +} + +func (f *fakeAliasBackfiller) BackfillBatch(_ context.Context, cursor string, limit int) (string, int, error) { + f.cursors = append(f.cursors, cursor) + if limit != mediaItemAliasBackfillBatchSize { + panic("unexpected batch size") + } + page := f.pages[0] + f.pages = f.pages[1:] + return page.next, page.count, nil +} + +type aliasBackfillProgress struct { + result json.RawMessage +} + +func (*aliasBackfillProgress) Report(float64, string) {} +func (p *aliasBackfillProgress) SetResultData(data json.RawMessage) { + p.result = append(p.result[:0], data...) +} + +func TestBackfillMediaItemAliasesTaskAdvancesCursorUntilShortBatch(t *testing.T) { + t.Parallel() + + backfiller := &fakeAliasBackfiller{pages: []aliasBackfillPage{ + {next: "item-0500", count: mediaItemAliasBackfillBatchSize}, + {next: "item-0502", count: 2}, + }} + progress := &aliasBackfillProgress{} + if err := NewBackfillMediaItemAliasesTask(backfiller).Execute(context.Background(), progress); err != nil { + t.Fatalf("Execute() error = %v", err) + } + if len(backfiller.cursors) != 2 || backfiller.cursors[0] != "" || backfiller.cursors[1] != "item-0500" { + t.Fatalf("cursors = %#v", backfiller.cursors) + } + if backfiller.resets != 1 { + t.Fatalf("completed-state resets = %d, want exactly 1 before batching", backfiller.resets) + } + var result map[string]any + if err := json.Unmarshal(progress.result, &result); err != nil { + t.Fatalf("result JSON = %q: %v", progress.result, err) + } + if result["processed_items"] != float64(502) || result["last_content_id"] != "item-0502" { + t.Fatalf("result = %#v", result) + } +} diff --git a/migrations/sql/20260721211249_media_item_aliases.sql b/migrations/sql/20260721211249_media_item_aliases.sql new file mode 100644 index 00000000..29468a0a --- /dev/null +++ b/migrations/sql/20260721211249_media_item_aliases.sql @@ -0,0 +1,36 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE media_item_aliases ( + id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + content_id text NOT NULL REFERENCES media_items(content_id) ON DELETE CASCADE, + title text NOT NULL CHECK (btrim(title) <> ''), + normalized_title text GENERATED ALWAYS AS (public.normalize_search_text(title)) STORED, + language text NOT NULL DEFAULT '', + kind text NOT NULL CHECK (kind IN ('original', 'localized', 'alternate')), + provider text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT now(), + updated_at timestamptz NOT NULL DEFAULT now() +); + +CREATE UNIQUE INDEX idx_media_item_aliases_unique + ON media_item_aliases (content_id, normalized_title, language, kind, provider); +CREATE INDEX idx_media_item_aliases_content_provider + ON media_item_aliases (content_id, provider); +CREATE INDEX idx_media_item_aliases_normalized_trgm + ON media_item_aliases USING gin (normalized_title public.gin_trgm_ops); +CREATE INDEX idx_media_item_aliases_search_vector + ON media_item_aliases USING gin (to_tsvector('simple', normalized_title)); + +-- Persist the batch cursor so an interrupted backfill resumes at its last +-- committed item instead of rescanning the whole catalog on every run. +CREATE TABLE media_item_alias_backfill_state ( + task_key text PRIMARY KEY, + last_content_id text NOT NULL DEFAULT '', + completed boolean NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL DEFAULT now() +); +-- +goose StatementEnd + +-- +goose Down +DROP TABLE IF EXISTS media_item_alias_backfill_state; +DROP TABLE IF EXISTS media_item_aliases; diff --git a/migrations/sql/20260724135806_add_item_alias_snapshot_language.sql b/migrations/sql/20260724135806_add_item_alias_snapshot_language.sql new file mode 100644 index 00000000..d6e1b64b --- /dev/null +++ b/migrations/sql/20260724135806_add_item_alias_snapshot_language.sql @@ -0,0 +1,38 @@ +-- +goose Up +-- An alias can describe one language while belonging to a snapshot requested +-- in another language. Track the request scope separately so replacing a +-- complete snapshot removes every alias that snapshot previously supplied. +-- NULL is reserved for rows that predate request-scope tracking; every new +-- writer supplies either a normalized language or '' for a provider-wide +-- snapshot. +ALTER TABLE media_item_aliases + ADD COLUMN snapshot_language text; + +DROP INDEX idx_media_item_aliases_unique; +CREATE UNIQUE INDEX idx_media_item_aliases_unique + ON media_item_aliases ( + content_id, + normalized_title, + language, + kind, + provider, + snapshot_language + ); + +-- +goose Down +-- Multiple request scopes may contain the same public alias. Keep one before +-- restoring the original uniqueness definition. +DELETE FROM media_item_aliases duplicate +USING media_item_aliases keeper +WHERE duplicate.id > keeper.id + AND duplicate.content_id = keeper.content_id + AND duplicate.normalized_title = keeper.normalized_title + AND duplicate.language = keeper.language + AND duplicate.kind = keeper.kind + AND duplicate.provider = keeper.provider; + +DROP INDEX idx_media_item_aliases_unique; +ALTER TABLE media_item_aliases + DROP COLUMN snapshot_language; +CREATE UNIQUE INDEX idx_media_item_aliases_unique + ON media_item_aliases (content_id, normalized_title, language, kind, provider); diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 0030453c..0b7f9d17 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -4362,6 +4362,13 @@ export interface TaskInfo { // Match dialog types export interface MatchCandidate { title: string; + original_title?: string; + aliases?: Array<{ title: string; language?: string; kind: string; provider?: string }>; + title_language?: string; + title_is_fallback?: boolean; + matched_title?: string; + match_score?: number; + match_reasons?: string[]; year: number; content_type: string; provider_ids: Record; diff --git a/web/src/components/MatchItemDialog.test.tsx b/web/src/components/MatchItemDialog.test.tsx index 4030a6eb..e780538b 100644 --- a/web/src/components/MatchItemDialog.test.tsx +++ b/web/src/components/MatchItemDialog.test.tsx @@ -91,6 +91,8 @@ describe("MatchItemDialog", () => { provider_ids: { tmdb: "27205", imdb: "tt1375666" }, sources: ["tmdb", "tvdb"], agreement_hints: ["agreed_by_tmdb_and_tvdb"], + match_score: 87.25, + match_reasons: ["title_exact", "provider_id_consensus"], }, { title: "Inception (TV)", @@ -114,9 +116,82 @@ describe("MatchItemDialog", () => { expect(markup).toContain("Inception (TV)"); expect(markup).toContain("tmdb"); expect(markup).toContain("2 sources agree"); + expect(markup).toContain("Score 87.3"); + expect(markup).toContain("Match reasons:"); + expect(markup).toContain("title exact, provider id consensus"); expect(markup).toContain('data-testid="match-candidate"'); }); + it("shows a matched library-language alias before a native fallback title", () => { + mocks.useSearchItemMatchCandidates.mockReturnValue({ + mutate: vi.fn(), + isPending: false, + isSuccess: true, + data: { + candidates: [ + { + title: "倒凶十将伝", + original_title: "倒凶十将伝", + title_language: "ja", + title_is_fallback: true, + matched_title: "10 Tokyo Warriors", + year: 1999, + content_type: "series", + image_url: "", + overview: "", + provider_ids: { tvdb: "123" }, + sources: ["tvdb"], + agreement_hints: [], + }, + ], + }, + }); + + const markup = renderToStaticMarkup( + {}} />, + ); + + expect(markup).toContain(">10 Tokyo Warriors { + mocks.useSearchItemMatchCandidates.mockReturnValue({ + mutate: vi.fn(), + isPending: false, + isSuccess: true, + data: { + candidates: [ + { + title: "10 Tokyo Warriors", + original_title: "倒凶十将伝", + title_language: "en", + matched_title: "倒凶十将伝", + year: 1999, + content_type: "series", + image_url: "", + overview: "", + provider_ids: { tvdb: "123" }, + sources: ["tvdb"], + agreement_hints: [], + }, + ], + }, + }); + + const markup = renderToStaticMarkup( + {}} />, + ); + + expect(markup).toContain(">10 Tokyo Warriors { mocks.useSearchItemMatchCandidates.mockReturnValue({ mutate: vi.fn(), diff --git a/web/src/components/MatchItemDialog.tsx b/web/src/components/MatchItemDialog.tsx index 99d3bab4..63803c98 100644 --- a/web/src/components/MatchItemDialog.tsx +++ b/web/src/components/MatchItemDialog.tsx @@ -313,6 +313,13 @@ export default function MatchItemDialog({ item, open, onOpenChange }: MatchItemD const candidateKey = Object.entries(candidate.provider_ids) .map(([k, v]) => `${k}-${v}`) .join("_"); + const matchedFallbackTitle = + candidate.title_is_fallback && + candidate.matched_title && + candidate.matched_title !== candidate.title + ? candidate.matched_title + : undefined; + const displayTitle = matchedFallbackTitle ?? candidate.title; return ( );