feat(metadata): improve match accuracy and localized titles (#461)

* feat(metadata): improve match accuracy and localized titles

* fix(metadata): address matching review findings

* test(catalog): align empty alias snapshot scope

---------

Co-authored-by: Quick104 <31828688+Quick104@users.noreply.github.com>
This commit is contained in:
Suspense
2026-07-24 12:02:52 -04:00
committed by GitHub
co-authored by Quick104
parent 2bc6ebb013
commit 383973ec22
38 changed files with 4992 additions and 217 deletions
+1
View File
@@ -1939,6 +1939,7 @@ func main() {
taskMgr.Register(tasks.NewScanLibrariesTask(deps.FolderRepo, deps.LibraryScanQueue, deps.EventBus))
}
taskMgr.Register(tasks.NewCleanupOrphanedMediaItemsTask(catalog.NewOrphanedProvisionalCleaner(deps.DB)))
taskMgr.Register(tasks.NewBackfillMediaItemAliasesTask(catalog.NewItemAliasRepository(deps.DB)))
if deps.S3Public != nil {
taskMgr.Register(tasks.NewCleanupArtworkRevisionsTask(
metadata.NewArtworkRevisionGarbageCollector(deps.DB, deps.S3Public),
+1 -1
View File
@@ -109,7 +109,7 @@ require (
)
require (
github.com/Silo-Server/silo-plugin-sdk v0.10.0
github.com/Silo-Server/silo-plugin-sdk v0.12.0
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect
+2 -2
View File
@@ -4,8 +4,8 @@ github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0g
github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI=
github.com/SherClockHolmes/webpush-go v1.4.0 h1:ocnzNKWN23T9nvHi6IfyrQjkIc0oJWv1B1pULsf9i3s=
github.com/SherClockHolmes/webpush-go v1.4.0/go.mod h1:XSq8pKX11vNV8MJEMwjrlTkxhAj1zKfxmyhdV7Pd6UA=
github.com/Silo-Server/silo-plugin-sdk v0.10.0 h1:OU2PfQwnUQAnf8FEZSTfWI+3x0OBiVfjKHHaVWhD/E0=
github.com/Silo-Server/silo-plugin-sdk v0.10.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
github.com/Silo-Server/silo-plugin-sdk v0.12.0 h1:7MnaSIJBVdRE8FhfLXWx7Pe+Q2apJiaPMotfhb1VHoE=
github.com/Silo-Server/silo-plugin-sdk v0.12.0/go.mod h1:etqmxLTwjxpFH9goAjBDfNDoqHMv2/sqUXu8yx3hNfA=
github.com/abadojack/whatlanggo v1.0.1 h1:19N6YogDnf71CTHm3Mp2qhYfkRdyvbgwWdd2EPxJRG4=
github.com/abadojack/whatlanggo v1.0.1/go.mod h1:66WiQbSbJBIlOZMsvbKe5m6pzQovxCH9B/K8tQB2uoc=
github.com/agnivade/levenshtein v1.2.1 h1:EHBY3UOn1gwdy/VbFwgo4cxecRznFk7fKWN1KOX7eoM=
+324
View File
@@ -0,0 +1,324 @@
package catalog
import (
"context"
"fmt"
"strings"
"github.com/Silo-Server/silo-server/internal/lang"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/jackc/pgx/v5/pgxpool"
)
const (
itemAliasKindOriginal = "original"
itemAliasKindLocalized = "localized"
itemAliasKindAlternate = "alternate"
)
// ItemAliasRepository persists and backfills provider-confirmed item titles.
type ItemAliasRepository struct {
pool *pgxpool.Pool
events *SearchIndexEventRepository
}
func NewItemAliasRepository(pool *pgxpool.Pool) *ItemAliasRepository {
return &ItemAliasRepository{pool: pool, events: NewSearchIndexEventRepository(pool)}
}
// ReplaceProvider replaces one provider's aliases atomically. Other providers'
// aliases are deliberately untouched so a partial provider outage cannot erase
// titles learned from successful sources.
func (r *ItemAliasRepository) ReplaceProvider(ctx context.Context, contentID, provider string, aliases []models.MediaItemAlias) error {
return r.writeProviderAliases(ctx, contentID, provider, "", true, aliases, true)
}
// ReplaceProviderLanguage refreshes one provider response for a particular
// requested language. The request scope is independent of each alias's own
// language, so a multilingual response can be replaced without erasing aliases
// learned from another library-language request.
func (r *ItemAliasRepository) ReplaceProviderLanguage(ctx context.Context, contentID, provider, language string, aliases []models.MediaItemAlias) error {
language = normalizeAliasLanguage(language)
if language == "" {
return r.ReplaceProvider(ctx, contentID, provider, aliases)
}
return r.writeProviderAliases(ctx, contentID, provider, language, false, aliases, true)
}
// RefreshProviderLanguage persists one provider response. Complete snapshots
// replace the request-language/provider scope; legacy or partial responses
// merge only, preserving aliases that the response cannot authoritatively
// declare stale.
func (r *ItemAliasRepository) RefreshProviderLanguage(ctx context.Context, contentID, provider, language string, aliases []models.MediaItemAlias, complete bool) error {
language = normalizeAliasLanguage(language)
if language == "" {
return r.writeProviderAliases(ctx, contentID, provider, "", true, aliases, complete)
}
return r.writeProviderAliases(ctx, contentID, provider, language, false, aliases, complete)
}
func (r *ItemAliasRepository) writeProviderAliases(
ctx context.Context,
contentID string,
provider string,
snapshotLanguage string,
replaceProvider bool,
aliases []models.MediaItemAlias,
replace bool,
) error {
contentID = strings.TrimSpace(contentID)
provider = strings.ToLower(strings.TrimSpace(provider))
snapshotLanguage = normalizeAliasLanguage(snapshotLanguage)
if r == nil || r.pool == nil || contentID == "" || provider == "" {
return nil
}
// An empty legacy/partial response carries no deletion authority. An empty
// complete snapshot is authoritative and must clear the refreshed provider
// scope so stale aliases do not remain searchable indefinitely.
if len(aliases) == 0 && !replace {
return nil
}
tx, err := r.pool.Begin(ctx)
if err != nil {
return fmt.Errorf("begin alias replacement: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
if replace {
deleteSQL := `DELETE FROM media_item_aliases WHERE content_id = $1 AND provider = $2`
deleteArgs := []any{contentID, provider}
if !replaceProvider {
// NULL is legacy provenance from before request-scope tracking.
// Adopt only the rows the old scoped replacement contract could
// authoritatively replace; other localized legacy aliases may
// belong to an independent library-language refresh.
deleteSQL += ` AND (
snapshot_language = $3 OR
(
snapshot_language IS NULL AND
(language = '' OR kind = 'original' OR language = $3)
)
)`
deleteArgs = append(deleteArgs, snapshotLanguage)
}
if _, err := tx.Exec(ctx, deleteSQL, deleteArgs...); err != nil {
return fmt.Errorf("delete provider aliases: %w", err)
}
}
for _, alias := range aliases {
title := strings.TrimSpace(alias.Title)
if title == "" {
continue
}
kind := strings.ToLower(strings.TrimSpace(alias.Kind))
switch kind {
case itemAliasKindOriginal, itemAliasKindLocalized, itemAliasKindAlternate:
default:
kind = itemAliasKindAlternate
}
language := normalizeAliasLanguage(alias.Language)
if _, err := tx.Exec(ctx, `
INSERT INTO media_item_aliases (content_id, title, language, kind, provider, snapshot_language)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (content_id, normalized_title, language, kind, provider, snapshot_language)
DO UPDATE SET title = EXCLUDED.title, updated_at = now()
`, contentID, title, language, kind, provider, snapshotLanguage); err != nil {
return fmt.Errorf("insert provider alias: %w", err)
}
}
if r.events != nil {
if err := r.events.EnqueueUpsert(ctx, tx, contentID); err != nil {
return fmt.Errorf("enqueue alias search index update: %w", err)
}
}
if err := tx.Commit(ctx); err != nil {
return fmt.Errorf("commit alias replacement: %w", err)
}
return nil
}
// BackfillBatch seeds aliases from existing original and localized titles.
// Its cursor is persisted in the same transaction as every batch, making an
// interrupted run resumable without relying on task-manager process state.
func (r *ItemAliasRepository) BackfillBatch(ctx context.Context, afterContentID string, limit int) (string, int, error) {
if r == nil || r.pool == nil || limit <= 0 {
return afterContentID, 0, nil
}
tx, err := r.pool.Begin(ctx)
if err != nil {
return afterContentID, 0, fmt.Errorf("begin item alias backfill: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
const taskKey = "media_item_aliases_v1"
if _, err := tx.Exec(ctx, `
INSERT INTO media_item_alias_backfill_state (task_key)
VALUES ($1)
ON CONFLICT (task_key) DO NOTHING
`, taskKey); err != nil {
return afterContentID, 0, fmt.Errorf("initialize item alias backfill state: %w", err)
}
var persistedCursor string
var completed bool
if err := tx.QueryRow(ctx, `
SELECT last_content_id, completed
FROM media_item_alias_backfill_state
WHERE task_key = $1
FOR UPDATE
`, taskKey).Scan(&persistedCursor, &completed); err != nil {
return afterContentID, 0, fmt.Errorf("lock item alias backfill state: %w", err)
}
if completed {
if err := tx.Commit(ctx); err != nil {
return persistedCursor, 0, fmt.Errorf("commit completed item alias backfill check: %w", err)
}
return persistedCursor, 0, nil
}
if persistedCursor > afterContentID {
afterContentID = persistedCursor
}
rows, err := tx.Query(ctx, `
WITH batch AS (
SELECT content_id, original_title, original_language
FROM media_items
WHERE content_id > $1
ORDER BY content_id
LIMIT $2
), inserted_originals AS (
INSERT INTO media_item_aliases (content_id, title, language, kind, provider, snapshot_language)
SELECT content_id, original_title,
public.canonical_language_code(split_part(replace(original_language, '_', '-'), '-', 1)),
'original', 'silo.backfill', ''
FROM batch
WHERE btrim(COALESCE(original_title, '')) <> ''
ON CONFLICT (content_id, normalized_title, language, kind, provider, snapshot_language) DO NOTHING
), inserted_localizations AS (
INSERT INTO media_item_aliases (content_id, title, language, kind, provider, snapshot_language)
SELECT loc.content_id, loc.title,
public.canonical_language_code(split_part(replace(loc.language, '_', '-'), '-', 1)),
'localized', 'silo.backfill', ''
FROM media_item_localizations loc
JOIN batch USING (content_id)
WHERE btrim(COALESCE(loc.title, '')) <> ''
ON CONFLICT (content_id, normalized_title, language, kind, provider, snapshot_language) DO NOTHING
)
SELECT content_id FROM batch ORDER BY content_id
`, afterContentID, limit)
if err != nil {
return afterContentID, 0, fmt.Errorf("backfill item aliases: %w", err)
}
defer rows.Close()
last := afterContentID
count := 0
for rows.Next() {
if err := rows.Scan(&last); err != nil {
return afterContentID, count, err
}
count++
}
if err := rows.Err(); err != nil {
return afterContentID, count, err
}
rows.Close()
if count > 0 && r.events != nil {
rows, err := tx.Query(ctx, `
SELECT content_id
FROM media_items
WHERE content_id > $1 AND content_id <= $2
ORDER BY content_id
`, afterContentID, last)
if err != nil {
return afterContentID, count, fmt.Errorf("load alias backfill search index ids: %w", err)
}
contentIDs := make([]string, 0, count)
for rows.Next() {
var contentID string
if err := rows.Scan(&contentID); err != nil {
rows.Close()
return afterContentID, count, fmt.Errorf("scan alias backfill search index id: %w", err)
}
contentIDs = append(contentIDs, contentID)
}
if err := rows.Err(); err != nil {
rows.Close()
return afterContentID, count, fmt.Errorf("iterate alias backfill search index ids: %w", err)
}
rows.Close()
if err := r.events.EnqueueUpserts(ctx, tx, contentIDs); err != nil {
return afterContentID, count, fmt.Errorf("enqueue alias backfill search index updates: %w", err)
}
}
batchCompleted := count < limit
if _, err := tx.Exec(ctx, `
UPDATE media_item_alias_backfill_state
SET last_content_id = $2, completed = $3, updated_at = now()
WHERE task_key = $1
`, taskKey, last, batchCompleted); err != nil {
return afterContentID, count, fmt.Errorf("save item alias backfill state: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return afterContentID, count, fmt.Errorf("commit item alias backfill: %w", err)
}
return last, count, nil
}
// ResetCompletedBackfill re-arms the backfill for a fresh full pass when a
// prior run finished. An interrupted run (completed = false) keeps its cursor
// so it resumes instead of rescanning; without this reset a manually
// re-triggered task would report success while processing zero items forever.
func (r *ItemAliasRepository) ResetCompletedBackfill(ctx context.Context) error {
if r == nil || r.pool == nil {
return nil
}
if _, err := r.pool.Exec(ctx, `
UPDATE media_item_alias_backfill_state
SET last_content_id = '', completed = false, updated_at = now()
WHERE task_key = 'media_item_aliases_v1' AND completed = true
`); err != nil {
return fmt.Errorf("reset item alias backfill state: %w", err)
}
return nil
}
func normalizeAliasLanguage(language string) string {
return lang.Canonical(strings.ReplaceAll(language, "_", "-"))
}
func (r *ItemAliasRepository) ListByContentIDs(ctx context.Context, contentIDs []string) (map[string][]models.MediaItemAlias, error) {
out := make(map[string][]models.MediaItemAlias)
if r == nil || r.pool == nil || len(contentIDs) == 0 {
return out, nil
}
rows, err := r.pool.Query(ctx, `
SELECT content_id, title, language, kind, provider
FROM (
SELECT DISTINCT ON (content_id, normalized_title, language, kind, provider)
content_id, title, language, kind, provider
FROM media_item_aliases
WHERE content_id = ANY($1)
ORDER BY
content_id,
normalized_title,
language,
kind,
provider,
updated_at DESC,
id DESC
) deduplicated
ORDER BY content_id, language, kind, title
`, contentIDs)
if err != nil {
return nil, err
}
defer rows.Close()
for rows.Next() {
var alias models.MediaItemAlias
if err := rows.Scan(&alias.ContentID, &alias.Title, &alias.Language, &alias.Kind, &alias.Provider); err != nil {
return nil, err
}
out[alias.ContentID] = append(out[alias.ContentID], alias)
}
return out, rows.Err()
}
+459
View File
@@ -0,0 +1,459 @@
package catalog
import (
"context"
"fmt"
"slices"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/models"
)
const (
catalogTestContentTypeSeries = "series"
catalogTestProviderTMDB = "tmdb"
catalogTestProviderTVDB = "tvdb"
)
func TestItemAliasRepositoryReplacesOnlySuccessfulProviderAndCascades(t *testing.T) {
pool := newSemanticCoverageTestPool(t)
ctx := context.Background()
var aliasTable *string
if err := pool.QueryRow(ctx, `SELECT to_regclass('public.media_item_aliases')::text`).Scan(&aliasTable); err != nil {
t.Fatalf("check media_item_aliases table: %v", err)
}
if aliasTable == nil || *aliasTable == "" {
t.Skip("test database has not applied media item aliases migration")
}
contentID := fmt.Sprintf("alias-repo-%d", time.Now().UnixNano())
seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched")
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) })
repo := NewItemAliasRepository(pool)
if err := repo.ReplaceProvider(ctx, contentID, catalogTestProviderTVDB, []models.MediaItemAlias{
{Title: "10 Tokyo Warriors", Language: "en", Kind: itemAliasKindAlternate},
}); err != nil {
t.Fatalf("seed TVDB aliases: %v", err)
}
if err := repo.ReplaceProvider(ctx, contentID, catalogTestProviderTMDB, []models.MediaItemAlias{
{Title: "Old TMDB Title", Language: "en", Kind: itemAliasKindLocalized},
}); err != nil {
t.Fatalf("seed TMDB aliases: %v", err)
}
if err := repo.ReplaceProvider(ctx, contentID, catalogTestProviderTMDB, []models.MediaItemAlias{
{Title: "New TMDB Title", Language: "en", Kind: itemAliasKindLocalized},
{Title: "New TMDB Title", Language: "en", Kind: itemAliasKindLocalized},
}); err != nil {
t.Fatalf("replace TMDB aliases: %v", err)
}
aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID})
if err != nil {
t.Fatalf("ListByContentIDs(): %v", err)
}
titles := make([]string, 0, len(aliasesByID[contentID]))
for _, alias := range aliasesByID[contentID] {
titles = append(titles, alias.Title)
}
if !slices.Contains(titles, "10 Tokyo Warriors") || !slices.Contains(titles, "New TMDB Title") || slices.Contains(titles, "Old TMDB Title") {
t.Fatalf("titles = %v", titles)
}
if len(titles) != 2 {
t.Fatalf("deduplicated titles = %v, want 2", titles)
}
if _, err := pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID); err != nil {
t.Fatalf("delete media item: %v", err)
}
var count int
if err := pool.QueryRow(ctx, `SELECT count(*) FROM media_item_aliases WHERE content_id = $1`, contentID).Scan(&count); err != nil {
t.Fatalf("count cascaded aliases: %v", err)
}
if count != 0 {
t.Fatalf("alias count after item delete = %d, want 0", count)
}
}
func TestItemAliasRepositoryLanguageRefreshPreservesOtherLibraryLanguages(t *testing.T) {
pool := newSemanticCoverageTestPool(t)
ctx := context.Background()
var aliasTable *string
if err := pool.QueryRow(ctx, `SELECT to_regclass('public.media_item_aliases')::text`).Scan(&aliasTable); err != nil {
t.Fatalf("check media_item_aliases table: %v", err)
}
if aliasTable == nil || *aliasTable == "" {
t.Skip("test database has not applied media item aliases migration")
}
contentID := fmt.Sprintf("alias-language-%d", time.Now().UnixNano())
seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched")
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) })
repo := NewItemAliasRepository(pool)
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "ja", []models.MediaItemAlias{
{Title: "日本語", Language: "ja", Kind: itemAliasKindLocalized},
}, true); err != nil {
t.Fatalf("seed Japanese aliases: %v", err)
}
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en", []models.MediaItemAlias{
{Title: "Old English", Language: "en", Kind: itemAliasKindLocalized},
{Title: "Old Untagged", Kind: itemAliasKindAlternate},
}, true); err != nil {
t.Fatalf("seed English aliases: %v", err)
}
if err := repo.ReplaceProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en-US", []models.MediaItemAlias{
{Title: "New English", Language: "en", Kind: itemAliasKindLocalized},
{Title: "日本語原題", Language: "ja", Kind: itemAliasKindOriginal},
{Title: "New Untagged", Kind: itemAliasKindAlternate},
}); err != nil {
t.Fatalf("refresh English aliases: %v", err)
}
aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID})
if err != nil {
t.Fatalf("ListByContentIDs(): %v", err)
}
titles := make([]string, 0, len(aliasesByID[contentID]))
for _, alias := range aliasesByID[contentID] {
titles = append(titles, alias.Title)
}
for _, want := range []string{"New English", "日本語", "日本語原題", "New Untagged"} {
if !slices.Contains(titles, want) {
t.Fatalf("titles = %v, want preserved/refreshed %q", titles, want)
}
}
for _, stale := range []string{"Old English", "Old Untagged"} {
if slices.Contains(titles, stale) {
t.Fatalf("titles = %v, stale alias %q was not replaced", titles, stale)
}
}
}
func TestItemAliasRepositoryCompleteSnapshotReplacesCrossLanguageAliasesInItsScope(t *testing.T) {
pool := newSemanticCoverageTestPool(t)
ctx := context.Background()
contentID := fmt.Sprintf("alias-cross-language-%d", time.Now().UnixNano())
seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched")
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) })
repo := NewItemAliasRepository(pool)
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "ja", []models.MediaItemAlias{
{Title: "日本語ライブラリ", Language: "ja", Kind: itemAliasKindLocalized},
}, true); err != nil {
t.Fatalf("seed Japanese snapshot: %v", err)
}
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en", []models.MediaItemAlias{
{Title: "English Title", Language: "en", Kind: itemAliasKindLocalized},
{Title: "Ancienne française", Language: "fr", Kind: itemAliasKindAlternate},
}, true); err != nil {
t.Fatalf("seed multilingual English snapshot: %v", err)
}
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en", []models.MediaItemAlias{
{Title: "Updated English Title", Language: "en", Kind: itemAliasKindLocalized},
}, true); err != nil {
t.Fatalf("replace English snapshot: %v", err)
}
aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID})
if err != nil {
t.Fatalf("ListByContentIDs(): %v", err)
}
titles := make([]string, 0, len(aliasesByID[contentID]))
for _, alias := range aliasesByID[contentID] {
titles = append(titles, alias.Title)
}
for _, want := range []string{"日本語ライブラリ", "Updated English Title"} {
if !slices.Contains(titles, want) {
t.Fatalf("titles = %v, want %q", titles, want)
}
}
for _, stale := range []string{"English Title", "Ancienne française"} {
if slices.Contains(titles, stale) {
t.Fatalf("titles = %v, stale cross-language alias %q survived", titles, stale)
}
}
}
func TestItemAliasRepositoryListDeduplicatesAliasesAcrossSnapshotScopes(t *testing.T) {
pool := newSemanticCoverageTestPool(t)
ctx := context.Background()
contentID := fmt.Sprintf("alias-snapshot-dedup-%d", time.Now().UnixNano())
seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched")
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) })
repo := NewItemAliasRepository(pool)
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en", []models.MediaItemAlias{
{Title: "The Office", Language: "en", Kind: itemAliasKindAlternate},
}, true); err != nil {
t.Fatalf("seed English snapshot: %v", err)
}
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "ja", []models.MediaItemAlias{
{Title: "the office", Language: "en", Kind: itemAliasKindAlternate},
}, true); err != nil {
t.Fatalf("seed Japanese snapshot: %v", err)
}
aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID})
if err != nil {
t.Fatalf("ListByContentIDs(): %v", err)
}
aliases := aliasesByID[contentID]
if len(aliases) != 1 {
t.Fatalf("aliases = %#v, want one normalized alias", aliases)
}
if aliases[0].Title != "the office" {
t.Fatalf("alias title = %q, want newest spelling", aliases[0].Title)
}
}
func TestItemAliasRepositoryScopedRefreshAdoptsLegacyWithoutErasingProviderWideSnapshot(t *testing.T) {
pool := newSemanticCoverageTestPool(t)
ctx := context.Background()
contentID := fmt.Sprintf("alias-scope-provenance-%d", time.Now().UnixNano())
seedSemanticCoverageMediaItem(t, pool, contentID, "movie", "matched")
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) })
repo := NewItemAliasRepository(pool)
if err := repo.ReplaceProvider(ctx, contentID, catalogTestProviderTMDB, []models.MediaItemAlias{
{Title: "Provider-wide title", Kind: itemAliasKindAlternate},
}); err != nil {
t.Fatalf("seed provider-wide snapshot: %v", err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO media_item_aliases (
content_id, title, language, kind, provider, snapshot_language
) VALUES
($1, 'Legacy unscoped title', '', 'alternate', $2, NULL),
($1, '従来の日本語タイトル', 'ja', 'localized', $2, NULL)
`, contentID, catalogTestProviderTMDB); err != nil {
t.Fatalf("seed legacy alias: %v", err)
}
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", []models.MediaItemAlias{
{Title: "Current English title", Language: "en", Kind: itemAliasKindLocalized},
}, true); err != nil {
t.Fatalf("refresh English snapshot: %v", err)
}
aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID})
if err != nil {
t.Fatalf("ListByContentIDs(): %v", err)
}
titles := make([]string, 0, len(aliasesByID[contentID]))
for _, alias := range aliasesByID[contentID] {
titles = append(titles, alias.Title)
}
for _, want := range []string{"Provider-wide title", "Current English title", "従来の日本語タイトル"} {
if !slices.Contains(titles, want) {
t.Fatalf("titles = %v, want preserved/refreshed %q", titles, want)
}
}
if slices.Contains(titles, "Legacy unscoped title") {
t.Fatalf("titles = %v, legacy alias was not adopted by complete refresh", titles)
}
}
func TestItemAliasRepositoryPartialRefreshMergesWithoutErasing(t *testing.T) {
pool := newSemanticCoverageTestPool(t)
ctx := context.Background()
contentID := fmt.Sprintf("alias-partial-%d", time.Now().UnixNano())
seedSemanticCoverageMediaItem(t, pool, contentID, "movie", "matched")
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) })
repo := NewItemAliasRepository(pool)
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", []models.MediaItemAlias{
{Title: "Authoritative English", Language: "en", Kind: itemAliasKindLocalized},
{Title: "Provider Alternate", Language: "en", Kind: itemAliasKindAlternate},
}, true); err != nil {
t.Fatalf("seed complete aliases: %v", err)
}
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", []models.MediaItemAlias{
{Title: "Legacy Primary Title", Language: "en", Kind: itemAliasKindLocalized},
}, false); err != nil {
t.Fatalf("merge partial aliases: %v", err)
}
aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID})
if err != nil {
t.Fatalf("ListByContentIDs(): %v", err)
}
titles := make([]string, 0, len(aliasesByID[contentID]))
for _, alias := range aliasesByID[contentID] {
titles = append(titles, alias.Title)
}
for _, want := range []string{"Authoritative English", "Provider Alternate", "Legacy Primary Title"} {
if !slices.Contains(titles, want) {
t.Fatalf("titles = %v, partial refresh erased %q", titles, want)
}
}
}
func TestItemAliasRepositoryEmptySnapshotHonorsCompleteness(t *testing.T) {
pool := newSemanticCoverageTestPool(t)
ctx := context.Background()
contentID := fmt.Sprintf("alias-empty-%d", time.Now().UnixNano())
seedSemanticCoverageMediaItem(t, pool, contentID, "movie", "matched")
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) })
repo := NewItemAliasRepository(pool)
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", []models.MediaItemAlias{
{Title: "Stale English", Language: "en", Kind: itemAliasKindLocalized},
}, true); err != nil {
t.Fatalf("seed TMDB aliases: %v", err)
}
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTVDB, "en", []models.MediaItemAlias{
{Title: "Keep TVDB", Language: "en", Kind: itemAliasKindAlternate},
}, true); err != nil {
t.Fatalf("seed TVDB aliases: %v", err)
}
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", nil, false); err != nil {
t.Fatalf("merge empty partial snapshot: %v", err)
}
aliasesByID, err := repo.ListByContentIDs(ctx, []string{contentID})
if err != nil {
t.Fatalf("list aliases after partial snapshot: %v", err)
}
if !slices.ContainsFunc(aliasesByID[contentID], func(alias models.MediaItemAlias) bool {
return alias.Title == "Stale English"
}) {
t.Fatalf("partial empty snapshot removed aliases: %+v", aliasesByID[contentID])
}
if err := repo.RefreshProviderLanguage(ctx, contentID, catalogTestProviderTMDB, "en", nil, true); err != nil {
t.Fatalf("replace with empty complete snapshot: %v", err)
}
aliasesByID, err = repo.ListByContentIDs(ctx, []string{contentID})
if err != nil {
t.Fatalf("list aliases after complete snapshot: %v", err)
}
for _, alias := range aliasesByID[contentID] {
if alias.Provider == catalogTestProviderTMDB {
t.Fatalf("complete empty snapshot retained TMDB alias: %+v", alias)
}
}
if !slices.ContainsFunc(aliasesByID[contentID], func(alias models.MediaItemAlias) bool {
return alias.Title == "Keep TVDB" && alias.Provider == catalogTestProviderTVDB
}) {
t.Fatalf("complete TMDB snapshot erased another provider: %+v", aliasesByID[contentID])
}
}
func TestItemAliasRepositoryBackfillResumesAndEnqueuesIndexEvents(t *testing.T) {
pool := newSemanticCoverageTestPool(t)
ctx := context.Background()
var stateTable *string
if err := pool.QueryRow(ctx, `SELECT to_regclass('public.media_item_alias_backfill_state')::text`).Scan(&stateTable); err != nil {
t.Fatalf("check alias backfill state table: %v", err)
}
if stateTable == nil || *stateTable == "" {
t.Skip("test database has not applied alias backfill state migration")
}
prefix := fmt.Sprintf("zz-alias-backfill-%d-", time.Now().UnixNano())
firstID, secondID := prefix+"a", prefix+"b"
for _, contentID := range []string{firstID, secondID} {
seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched")
if _, err := pool.Exec(ctx, `
UPDATE media_items SET original_title = '倒凶十将伝', original_language = 'jpn'
WHERE content_id = $1
`, contentID); err != nil {
t.Fatalf("seed original title: %v", err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO media_item_localizations (content_id, language, title)
VALUES ($1, 'en_US', '10 Tokyo Warriors')
`, contentID); err != nil {
t.Fatalf("seed localization: %v", err)
}
}
t.Cleanup(func() {
_, _ = pool.Exec(ctx, `DELETE FROM catalog_search_index_events WHERE content_id = ANY($1)`, []string{firstID, secondID})
_, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, []string{firstID, secondID})
_, _ = pool.Exec(ctx, `DELETE FROM media_item_alias_backfill_state WHERE task_key = 'media_item_aliases_v1'`)
})
if _, err := pool.Exec(ctx, `
INSERT INTO media_item_alias_backfill_state (task_key, last_content_id, completed)
VALUES ('media_item_aliases_v1', $1, false)
ON CONFLICT (task_key) DO UPDATE
SET last_content_id = EXCLUDED.last_content_id, completed = false
`, prefix); err != nil {
t.Fatalf("seed backfill cursor: %v", err)
}
repo := NewItemAliasRepository(pool)
repo.events.WithActiveProvider(SearchProviderMeilisearch)
firstCursor, processed, err := repo.BackfillBatch(ctx, "", 1)
if err != nil || processed != 1 || firstCursor != firstID {
t.Fatalf("first BackfillBatch() = (%q, %d, %v), want (%q, 1, nil)", firstCursor, processed, err, firstID)
}
secondCursor, processed, err := repo.BackfillBatch(ctx, "", 1)
if err != nil || processed != 1 || secondCursor != secondID {
t.Fatalf("resumed BackfillBatch() = (%q, %d, %v), want (%q, 1, nil)", secondCursor, processed, err, secondID)
}
for _, contentID := range []string{firstID, secondID} {
aliases, err := repo.ListByContentIDs(ctx, []string{contentID})
if err != nil {
t.Fatalf("list aliases for %s: %v", contentID, err)
}
languages := make([]string, 0, len(aliases[contentID]))
for _, alias := range aliases[contentID] {
languages = append(languages, alias.Language)
}
if !slices.Contains(languages, "ja") || !slices.Contains(languages, "en") {
t.Fatalf("alias languages for %s = %v, want canonical ja and en", contentID, languages)
}
var eventCount int
if err := pool.QueryRow(ctx, `
SELECT count(*) FROM catalog_search_index_events
WHERE content_id = $1 AND action = 'upsert'
`, contentID).Scan(&eventCount); err != nil {
t.Fatalf("count index events for %s: %v", contentID, err)
}
if eventCount == 0 {
t.Fatalf("no search-index upsert event for %s", contentID)
}
}
}
func TestItemRepositorySearchesPersistedAliasesAcrossExactFTSAndFuzzyPaths(t *testing.T) {
pool := newSemanticCoverageTestPool(t)
ctx := context.Background()
var aliasTable *string
if err := pool.QueryRow(ctx, `SELECT to_regclass('public.media_item_aliases')::text`).Scan(&aliasTable); err != nil {
t.Fatalf("check media_item_aliases table: %v", err)
}
if aliasTable == nil || *aliasTable == "" {
t.Skip("test database has not applied media item aliases migration")
}
contentID := fmt.Sprintf("alias-search-%d", time.Now().UnixNano())
seedSemanticCoverageMediaItem(t, pool, contentID, catalogTestContentTypeSeries, "matched")
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) })
if _, err := pool.Exec(ctx, `
UPDATE media_items SET title = '倒凶十将伝', original_title = '倒凶十将伝' WHERE content_id = $1
`, contentID); err != nil {
t.Fatalf("seed native title: %v", err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO media_item_aliases (content_id, title, language, kind, provider)
VALUES ($1, '10 Tokyo Warriors', 'en', 'alternate', 'tvdb')
`, contentID); err != nil {
t.Fatalf("seed searchable alias: %v", err)
}
repo := NewItemRepository(pool)
// The final query deliberately verifies fuzzy matching for a misspelling.
for _, query := range []string{"10 Tokyo Warriors", "Tokyo Warriors", "10 Tokyo Warriros"} { //nolint:misspell
items, _, err := repo.Search(ctx, query, []string{catalogTestContentTypeSeries}, 20, 0, AccessFilter{})
if err != nil {
t.Fatalf("Search(%q): %v", query, err)
}
found := false
for _, item := range items {
found = found || item.ContentID == contentID
}
if !found {
t.Fatalf("Search(%q) did not return alias-backed item %s", query, contentID)
}
}
}
+23 -4
View File
@@ -1325,13 +1325,26 @@ func (r *ItemRepository) buildFuzzySearchFromParsed(parsed parsedSearchQuery, it
// word-boundary extent of the title instead, and at equal thresholds it is
// a strict superset of % (the whole string is itself a valid extent). The
// same gin_trgm_ops index serves both operators.
conditions := []string{"public.normalize_search_text($1) <<% mi.title_normalized"}
// The alias arm is a scalar array subquery (InitPlan) for the same reason
// as the FTS builder's alias arms: `= ANY(param)` keeps the outer OR a
// BitmapOr over the two gin_trgm_ops indexes, where an `IN (subquery)` arm
// would force a seq scan evaluating <<% against every media_items row.
conditions := []string{`(
public.normalize_search_text($1) <<% mi.title_normalized OR
mi.content_id = ANY(COALESCE((
SELECT array_agg(DISTINCT mia.content_id) FROM media_item_aliases mia
WHERE public.normalize_search_text($1) <<% mia.normalized_title
), '{}'::text[]))
)`}
if minSimilarity > 0 {
// The floor is whole-title similarity, NOT word similarity: augmenting
// a query that already has hits must only admit near-identical titles,
// and word similarity rates embedded prefix words far too high (see
// fuzzyAugmentSimilarityFloor).
conditions = append(conditions, fmt.Sprintf("similarity(public.normalize_search_text($1), mi.title_normalized) >= $%d", argIdx))
conditions = append(conditions, fmt.Sprintf(`GREATEST(
similarity(public.normalize_search_text($1), mi.title_normalized),
COALESCE((SELECT MAX(similarity(public.normalize_search_text($1), mia.normalized_title)) FROM media_item_aliases mia WHERE mia.content_id = mi.content_id), 0)
) >= $%d`, argIdx))
args = append(args, minSimilarity)
argIdx++
}
@@ -1360,8 +1373,14 @@ func (r *ItemRepository) buildFuzzySearchFromParsed(parsed parsedSearchQuery, it
WITH scored AS (
SELECT
%s,
MAX(strict_word_similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_rank,
MAX(similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_full_rank
MAX(GREATEST(
strict_word_similarity(public.normalize_search_text($1), mi.title_normalized),
COALESCE((SELECT MAX(strict_word_similarity(public.normalize_search_text($1), mia.normalized_title)) FROM media_item_aliases mia WHERE mia.content_id = mi.content_id), 0)
)) AS fuzzy_rank,
MAX(GREATEST(
similarity(public.normalize_search_text($1), mi.title_normalized),
COALESCE((SELECT MAX(similarity(public.normalize_search_text($1), mia.normalized_title)) FROM media_item_aliases mia WHERE mia.content_id = mi.content_id), 0)
)) AS fuzzy_full_rank
FROM %s
%s
GROUP BY %s
+8 -5
View File
@@ -463,7 +463,7 @@ func TestItemRepo_Search_FTSQueryHasNoFuzzyArm(t *testing.T) {
}
// TestItemRepo_BuildFuzzySearchSQL asserts that the fuzzy fallback query scores
// only on the indexed title_normalized column (no title tsvector rebuild),
// only on indexed normalized title/alias columns (no title tsvector rebuild),
// matches via strict word similarity so long titles stay reachable, ranks by
// descending word similarity with whole-title closeness as tie-break, excludes
// already-seen content_ids, and applies the same scope filters (type, manga
@@ -477,12 +477,15 @@ func TestItemRepo_BuildFuzzySearchSQL(t *testing.T) {
if !strings.Contains(dataSQL, "public.normalize_search_text($1) <<% mi.title_normalized") {
t.Fatalf("expected strict-word-similarity arm against title_normalized; got:\n%s", dataSQL)
}
if !strings.Contains(dataSQL, "MAX(strict_word_similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_rank") {
if !strings.Contains(dataSQL, "strict_word_similarity(public.normalize_search_text($1), mi.title_normalized)") || !strings.Contains(dataSQL, "AS fuzzy_rank") {
t.Fatalf("expected strict word similarity ranking on title_normalized; got:\n%s", dataSQL)
}
if !strings.Contains(dataSQL, "MAX(similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_full_rank") {
if !strings.Contains(dataSQL, "similarity(public.normalize_search_text($1), mi.title_normalized)") || !strings.Contains(dataSQL, "AS fuzzy_full_rank") {
t.Fatalf("expected whole-title similarity tie-break rank; got:\n%s", dataSQL)
}
if !strings.Contains(dataSQL, "<<% mia.normalized_title") {
t.Fatalf("expected alias trigram candidates; got:\n%s", dataSQL)
}
// The whole point of the separate query: it must never rebuild the title
// tsvectors that made the fused query slow.
if strings.Contains(dataSQL, "to_tsvector") || strings.Contains(dataSQL, "ts_rank_cd") {
@@ -539,7 +542,7 @@ func TestItemRepo_BuildFuzzySearchSQL_SimilarityFloor(t *testing.T) {
// Whole-title similarity, not word similarity: the augment floor must not
// admit embedded prefix words ("coral" scores 0.5 word-similarity to
// "coraline").
if !strings.Contains(floorSQL, "AND similarity(public.normalize_search_text($1), mi.title_normalized) >= $2") {
if !strings.Contains(floorSQL, ") >= $2") || !strings.Contains(floorSQL, "mia.normalized_title") {
t.Fatalf("expected explicit whole-title similarity floor predicate as $2; got:\n%s", floorSQL)
}
if len(floorArgs) < 2 || floorArgs[1] != fuzzyAugmentSimilarityFloor {
@@ -551,7 +554,7 @@ func TestItemRepo_BuildFuzzySearchSQL_SimilarityFloor(t *testing.T) {
}
baseSQL, _, baseArgs := repo.buildFuzzySearchSQL("avegners", []string{"movie"}, 20, 0, AccessFilter{}, true, nil, 0)
if strings.Contains(baseSQL, "AND similarity(public.normalize_search_text($1), mi.title_normalized) >= $2") {
if strings.Contains(baseSQL, ") >= $2") {
t.Fatalf("zero floor must not add a similarity predicate; got:\n%s", baseSQL)
}
if len(baseArgs) != len(floorArgs)-1 {
+15 -2
View File
@@ -613,6 +613,7 @@ type catalogSearchDocument struct {
Title string `json:"title"`
SortTitle string `json:"sort_title,omitempty"`
OriginalTitle string `json:"original_title,omitempty"`
Aliases []string `json:"-"`
TitleVariants []string `json:"title_variants,omitempty"`
Year int `json:"year,omitempty"`
Overview string `json:"overview,omitempty"`
@@ -718,6 +719,7 @@ func mixedCatalogSearchDocumentSQL(mediaPredicate, episodePredicate string, type
COALESCE(mi.title, ''),
COALESCE(mi.sort_title, ''),
COALESCE(mi.original_title, ''),
COALESCE(aliases.titles, ARRAY[]::text[]),
COALESCE(mi.year, 0),
COALESCE(mi.overview, ''),
COALESCE(mi.tagline, ''),
@@ -730,6 +732,11 @@ func mixedCatalogSearchDocumentSQL(mediaPredicate, episodePredicate string, type
COALESCE(libraries.ids, ARRAY[]::integer[])
FROM candidates c
JOIN media_items mi ON c.type <> 'episode' AND mi.content_id = c.content_id
LEFT JOIN LATERAL (
SELECT array_agg(DISTINCT mia.title ORDER BY mia.title) FILTER (WHERE btrim(mia.title) <> '') AS titles
FROM media_item_aliases mia
WHERE mia.content_id = mi.content_id
) aliases ON true
LEFT JOIN LATERAL (
SELECT array_agg(DISTINCT p.name) FILTER (WHERE p.name IS NOT NULL AND p.name <> '') AS names
FROM item_people ip
@@ -748,6 +755,7 @@ func mixedCatalogSearchDocumentSQL(mediaPredicate, episodePredicate string, type
COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text),
COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text),
''::text,
ARRAY[]::text[],
COALESCE(EXTRACT(YEAR FROM e.air_date)::integer, 0),
COALESCE(e.overview, ''),
''::text,
@@ -776,6 +784,7 @@ func scanCatalogSearchDocuments(rows pgx.Rows) ([]catalogSearchDocument, error)
&doc.Title,
&doc.SortTitle,
&doc.OriginalTitle,
&doc.Aliases,
&doc.Year,
&doc.Overview,
&doc.Tagline,
@@ -965,12 +974,16 @@ func countCatalogSearchEligibleDocuments(ctx context.Context, q coverageQuerier,
}
func catalogSearchTitleVariants(doc catalogSearchDocument) []string {
return compactNonEmptyStrings([]string{
variants := []string{
doc.Title,
doc.SortTitle,
doc.OriginalTitle,
normalizeTitleForComparison(doc.Title),
normalizeTitleForComparison(doc.SortTitle),
normalizeTitleForComparison(doc.OriginalTitle),
})
}
for _, alias := range doc.Aliases {
variants = append(variants, alias, normalizeTitleForComparison(alias))
}
return compactNonEmptyStrings(variants)
}
+31
View File
@@ -137,6 +137,37 @@ func TestAttachDocumentVectorsSkipsWhenSemanticDisabled(t *testing.T) {
}
}
func TestCatalogSearchDocumentIncludesPersistedAliases(t *testing.T) {
pool := newSemanticCoverageTestPool(t)
ctx := context.Background()
var aliasTable *string
if err := pool.QueryRow(ctx, `SELECT to_regclass('public.media_item_aliases')::text`).Scan(&aliasTable); err != nil {
t.Fatalf("check media_item_aliases table: %v", err)
}
if aliasTable == nil || *aliasTable == "" {
t.Skip("test database has not applied media item aliases migration")
}
contentID := fmt.Sprintf("alias-doc-%d", time.Now().UnixNano())
seedSemanticCoverageMediaItem(t, pool, contentID, "series", "matched")
t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) })
if _, err := pool.Exec(ctx, `UPDATE media_items SET title = '倒凶十将伝', original_title = '倒凶十将伝' WHERE content_id = $1`, contentID); err != nil {
t.Fatalf("seed native title: %v", err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO media_item_aliases (content_id, title, language, kind, provider)
VALUES ($1, '10 Tokyo Warriors', 'en', 'alternate', 'tvdb')`, contentID); err != nil {
t.Fatalf("seed alias: %v", err)
}
docs, err := NewCatalogSearchIndexer(pool, nil).LoadDocumentsByIDs(ctx, []string{contentID}, []string{catalogTestContentTypeSeries}, "", false, false)
if err != nil {
t.Fatalf("LoadDocumentsByIDs(): %v", err)
}
if len(docs) != 1 || !slices.Contains(docs[0].TitleVariants, "10 Tokyo Warriors") {
t.Fatalf("documents = %#v, want English alias in title variants", docs)
}
}
// TestCountCatalogSearchVectorDocumentsModelCoverage verifies case (a): an
// embedding stored under a different model is excluded when a model is
// requested, but counted when the model filter is empty.
+87 -10
View File
@@ -103,7 +103,7 @@ func (r *ItemRepository) buildMixedSearchSQLFromParsed(
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B')
)`
mediaOverviewVector := `to_tsvector('english', COALESCE(mi.overview, ''))`
mediaConditions = append(mediaConditions, searchMatchCondition(mediaTitleVector, mediaOverviewVector))
mediaConditions = append(mediaConditions, searchMatchCondition(mediaTitleVector, mediaOverviewVector, mediaSearchAliasMatchArms()...))
if len(itemTypes) > 0 {
mediaConditions = append(mediaConditions, fmt.Sprintf("mi.type = ANY($%d)", argIdx))
args = append(args, mediaTypes)
@@ -157,6 +157,7 @@ func (r *ItemRepository) buildMixedSearchSQLFromParsed(
argIdx++
if includeMediaItems {
mediaAliasArms := mediaSearchAliasArms(exactIdx)
branches = append(branches, buildMixedSearchCandidateBranch(
"mi.content_id", "mi.type", "mi.title", "mi.year",
`(
@@ -167,6 +168,7 @@ func (r *ItemRepository) buildMixedSearchSQLFromParsed(
`to_tsvector('english', COALESCE(mi.overview, ''))`,
[]string{`mi.title_normalized`, `public.normalize_search_text(mi.original_title)`, `public.normalize_search_text(mi.sort_title)`},
"media_items mi", mediaConditions, exactIdx, yearIdx, phraseIdx,
&mediaAliasArms,
))
}
if includeEpisodes {
@@ -178,6 +180,7 @@ func (r *ItemRepository) buildMixedSearchSQLFromParsed(
[]string{episodeNormalizedTitle},
"episodes e JOIN media_items si ON si.content_id = e.series_id",
episodeConditions, exactIdx, yearIdx, phraseIdx,
nil,
))
}
@@ -252,11 +255,73 @@ func splitSearchItemTypes(itemTypes []string) (mediaTypes []string, includeEpiso
return mediaTypes, includeEpisodes
}
func searchMatchCondition(titleVector, overviewVector string) string {
func searchMatchCondition(titleVector, overviewVector string, extraArms ...string) string {
titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))`
prefixQuery := `to_tsquery('simple', $2)`
return fmt.Sprintf(`((%s) @@ %s OR ($2 <> '' AND (%s) @@ %s) OR (%s) @@ websearch_to_tsquery('english', $1))`,
titleVector, titleQuery, titleVector, prefixQuery, overviewVector)
arms := []string{
fmt.Sprintf(`(%s) @@ %s`, titleVector, titleQuery),
fmt.Sprintf(`($2 <> '' AND (%s) @@ %s)`, titleVector, prefixQuery),
fmt.Sprintf(`(%s) @@ websearch_to_tsquery('english', $1)`, overviewVector),
}
arms = append(arms, extraArms...)
return "(" + strings.Join(arms, " OR ") + ")"
}
// mediaSearchAliasMatchArms are the alias arms OR'd into the media branch's
// match condition. They are scalar array subqueries (InitPlans), NOT `IN
// (subquery)` arms: an uncorrelated subquery result is a plain parameter, so
// `content_id = ANY(...)` stays index-served and participates in the BitmapOr
// with the title/overview GIN arms. An `IN (SELECT ...)` arm inside an OR has
// no index path and forces a seq scan of media_items that rebuilds every
// tsvector per row.
func mediaSearchAliasMatchArms() []string {
return []string{
`mi.content_id = ANY(COALESCE((
SELECT array_agg(DISTINCT mia.content_id) FROM media_item_aliases mia
WHERE to_tsvector('simple', mia.normalized_title) @@ websearch_to_tsquery('simple', public.normalize_search_text($1))
), '{}'::text[]))`,
`($2 <> '' AND mi.content_id = ANY(COALESCE((
SELECT array_agg(DISTINCT mia.content_id) FROM media_item_aliases mia
WHERE to_tsvector('simple', mia.normalized_title) @@ to_tsquery('simple', $2)
), '{}'::text[])))`,
}
}
// mixedSearchAliasArms carries the media branch's provider-alias extensions to
// the per-branch ranking SELECT. Episodes have no aliases and pass nil.
type mixedSearchAliasArms struct {
exactArm string // OR'd into exact_title_match
contiguousArm string // OR'd into contiguous_title_match
titleRank string // GREATEST'd with title_rank
prefixRank string // GREATEST'd with title_prefix_rank
}
// mediaSearchAliasArms builds the alias ranking arms for the media branch.
// The correlated rank subqueries run per matched row only (content_id
// lookups), and setweight(..., 'A') makes an alias hit rank like a real title
// hit — an unweighted tsvector defaults to weight D (0.1), which buried exact
// alias matches ~10x below partial real-title matches. The WHERE arms above
// stay unweighted to keep matching idx_media_item_aliases_search_vector's
// indexed expression.
func mediaSearchAliasArms(exactIdx int) mixedSearchAliasArms {
return mixedSearchAliasArms{
exactArm: fmt.Sprintf(`mi.content_id IN (
SELECT mia.content_id FROM media_item_aliases mia
WHERE mia.normalized_title = $%d
)`, exactIdx),
contiguousArm: fmt.Sprintf(`mi.content_id IN (
SELECT mia.content_id FROM media_item_aliases mia
WHERE mia.normalized_title LIKE '%%' || $%d || '%%'
)`, exactIdx),
titleRank: `COALESCE((
SELECT MAX(ts_rank_cd(setweight(to_tsvector('simple', mia.normalized_title), 'A'), websearch_to_tsquery('simple', public.normalize_search_text($1))))
FROM media_item_aliases mia WHERE mia.content_id = mi.content_id
), 0)`,
prefixRank: `COALESCE((
SELECT MAX(ts_rank_cd(setweight(to_tsvector('simple', mia.normalized_title), 'A'), to_tsquery('simple', $2)))
FROM media_item_aliases mia WHERE mia.content_id = mi.content_id
), 0)`,
}
}
func buildMixedSearchCandidateBranch(
@@ -265,15 +330,27 @@ func buildMixedSearchCandidateBranch(
fromClause string,
conditions []string,
exactIdx, yearIdx, phraseIdx int,
aliasArms *mixedSearchAliasArms,
) string {
exactArms := make([]string, 0, len(exactTitleExprs))
contiguousArms := make([]string, 0, len(exactTitleExprs))
exactArms := make([]string, 0, len(exactTitleExprs)+1)
contiguousArms := make([]string, 0, len(exactTitleExprs)+1)
for _, expr := range exactTitleExprs {
exactArms = append(exactArms, fmt.Sprintf("%s = $%d", expr, exactIdx))
contiguousArms = append(contiguousArms, fmt.Sprintf("%s LIKE '%%' || $%d || '%%'", expr, exactIdx))
}
titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))`
prefixQuery := `to_tsquery('simple', $2)`
titleRankExpr := fmt.Sprintf("ts_rank_cd(%s, %s)", titleVector, titleQuery)
prefixRankExpr := fmt.Sprintf("ts_rank_cd(%s, %s)", titleVector, prefixQuery)
if aliasArms != nil {
// Alias exact/contiguous arms are hashed subplans in the CASE select
// list (evaluated per grouped row, hash built once); the rank arms are
// per-row correlated content_id lookups over the matched set only.
exactArms = append(exactArms, aliasArms.exactArm)
contiguousArms = append(contiguousArms, aliasArms.contiguousArm)
titleRankExpr = fmt.Sprintf("GREATEST(%s, %s)", titleRankExpr, aliasArms.titleRank)
prefixRankExpr = fmt.Sprintf("GREATEST(%s, %s)", prefixRankExpr, aliasArms.prefixRank)
}
return fmt.Sprintf(`
SELECT
%s AS content_id,
@@ -282,8 +359,8 @@ func buildMixedSearchCandidateBranch(
CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS exact_title_match,
CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS contiguous_title_match,
CASE WHEN $%d::int IS NOT NULL AND (%s) = $%d::int THEN 1 ELSE 0 END AS year_match,
ts_rank_cd(%s, %s) AS title_rank,
CASE WHEN $2 <> '' THEN ts_rank_cd(%s, %s) ELSE 0 END AS title_prefix_rank,
%s AS title_rank,
CASE WHEN $2 <> '' THEN %s ELSE 0 END AS title_prefix_rank,
ts_rank_cd(%s, websearch_to_tsquery('english', $1)) AS overview_rank,
CASE WHEN $%d <> '' THEN ts_rank_cd(%s, phraseto_tsquery('simple', public.normalize_search_text($%d))) ELSE 0 END AS phrase_rank
FROM %s
@@ -292,8 +369,8 @@ func buildMixedSearchCandidateBranch(
exactIdx, strings.Join(exactArms, " OR "),
exactIdx, strings.Join(contiguousArms, " OR "),
yearIdx, yearExpr, yearIdx,
titleVector, titleQuery,
titleVector, prefixQuery,
titleRankExpr,
prefixRankExpr,
overviewVector,
phraseIdx, titleVector, phraseIdx,
fromClause, strings.Join(conditions, " AND "))
@@ -0,0 +1,577 @@
package metadata
import (
"context"
"fmt"
"path/filepath"
"regexp"
"slices"
"sort"
"strings"
"sync"
"time"
"unicode"
"unicode/utf8"
"github.com/Silo-Server/silo-server/internal/naming"
"golang.org/x/text/unicode/norm"
)
const (
// Search APIs often return many same-title remakes. Three candidates was too
// shallow for real libraries (the correct "Memories" result was below it),
// while eight keeps the worst-case provider work bounded.
maxEpisodeValidationCandidates = 8
maxEpisodeValidationProviders = 2
maxEpisodeValidationSamples = 3
maxEpisodeValidationSeasons = 3
episodeValidationTimeout = 10 * time.Second
)
var (
episodeCodePattern = regexp.MustCompile(`(?i)s\d{1,4}e\d{1,3}(?:e\d{1,3})?`)
episodeReleaseSuffixPattern = regexp.MustCompile(
`(?i)(?:^|[ ._-])(?:2160p|1080p|720p|576p|480p|webrip|web[ ._-]?dl|bluray|blu[ ._-]?ray|bdrip|hdtv|dvdrip|remux|x26[45]|h26[45]|hevc|av1|aac|ac3|eac3|dts(?:[ ._-]?hd)?|flac|hdr10?|dolby[ ._-]?vision)(?:$|[ ._-])`,
)
episodeGenericTitlePattern = regexp.MustCompile(`(?i)^(?:tba|tbd|pilot|episode|episode\s+\d+|ep\.?\s*\d+|part\s+\d+|chapter\s+\d+|\d+)$`)
episodeTechnicalTitlePattern = regexp.MustCompile(`(?i)^(?:\d{3,4}p|web(?:rip|dl)?|bluray|hdtv|remux|x26[45]|h26[45]|hevc)(?:\b|[ ._-])`)
)
type localEpisodeMatchHint struct {
SeasonNumber int
EpisodeNumber int
Title string
}
type episodeCandidateValidation struct {
candidate MatchCandidate
provider string
coordinates int
compared int
matches int
exactMatches int
similarityTotal float64
evaluated bool
err error
}
type episodeValidationProvider struct {
provider EpisodeProvider
slug string
}
// validateSeriesMatchByEpisodes uses a small amount of episode-title evidence
// to resolve otherwise ambiguous exact-title series searches. It is deliberately
// positive-only: unavailable, generic, or disagreeing episode data leaves the
// item unmatched instead of turning absence of evidence into a rejection.
func validateSeriesMatchByEpisodes(
ctx context.Context,
hints *MatchHints,
candidates []MatchCandidate,
providers []Provider,
language string,
) (*MatchCandidate, []error) {
if hints == nil || hints.Type != matchContentTypeSeries || trustedHintIDsPresent(hints) {
return nil, nil
}
// Coordinate selection keeps useful episode titles when present, but its
// primary purpose is to build a bounded series-shape fingerprint: the last
// locally present episode from up to three spread-out seasons.
localEpisodes := selectLocalEpisodeCoordinateHints(hints.AllGroupFilePaths)
if len(localEpisodes) == 0 {
return nil, nil
}
contenders, contendersExhaustive := episodeValidationContenders(hints, candidates)
if len(contenders) == 0 {
return nil, nil
}
validationCtx, cancel := context.WithTimeout(ctx, episodeValidationTimeout)
defer cancel()
validations := make([]episodeCandidateValidation, len(contenders))
var wg sync.WaitGroup
for i := range contenders {
candidateProviders := selectEpisodeValidationProviders(contenders[i], providers)
validations[i].candidate = contenders[i]
if len(candidateProviders) == 0 {
continue
}
wg.Add(1)
go func(index int, episodeProviders []episodeValidationProvider) {
defer wg.Done()
validations[index] = validateEpisodeCandidateAcrossProviders(
validationCtx,
validations[index].candidate,
episodeProviders,
localEpisodes,
language,
)
}(i, candidateProviders)
}
wg.Wait()
errs := make([]error, 0, len(validations))
for _, validation := range validations {
if validation.err != nil {
errs = append(errs, validation.err)
}
// Every contender must have been checked successfully. Otherwise a
// temporarily unavailable rival could be incorrectly treated as a zero.
if !validation.evaluated || validation.err != nil {
return nil, errs
}
}
titleValidations := append([]episodeCandidateValidation(nil), validations...)
sort.SliceStable(titleValidations, func(i, j int) bool {
if titleValidations[i].matches != titleValidations[j].matches {
return titleValidations[i].matches > titleValidations[j].matches
}
if titleValidations[i].exactMatches != titleValidations[j].exactMatches {
return titleValidations[i].exactMatches > titleValidations[j].exactMatches
}
return titleValidations[i].similarityTotal > titleValidations[j].similarityTotal
})
best := titleValidations[0]
if episodeEvidenceIsSufficient(best, localEpisodes) &&
(len(titleValidations) == 1 || episodeEvidenceClearlyWins(best, titleValidations[1])) {
winner := best.candidate
winner.MatchReasons = append(winner.MatchReasons,
fmt.Sprintf("episode_title_corroboration:%d_of_%d", best.matches, len(localEpisodes)))
return &winner, errs
}
coordinateValidations := append([]episodeCandidateValidation(nil), validations...)
sort.SliceStable(coordinateValidations, func(i, j int) bool {
if coordinateValidations[i].coordinates != coordinateValidations[j].coordinates {
return coordinateValidations[i].coordinates > coordinateValidations[j].coordinates
}
return episodeValidationEvidenceBetter(coordinateValidations[i], coordinateValidations[j])
})
best = coordinateValidations[0]
var next episodeCandidateValidation
if len(coordinateValidations) > 1 {
next = coordinateValidations[1]
}
if !episodeCoordinateEvidenceIsSufficient(
best, next, localEpisodes, len(coordinateValidations), contendersExhaustive,
) {
return nil, errs
}
winner := best.candidate
winner.MatchReasons = append(winner.MatchReasons,
fmt.Sprintf("episode_coordinate_corroboration:%d_of_%d", best.coordinates, len(localEpisodes)))
return &winner, errs
}
func episodeValidationContenders(hints *MatchHints, candidates []MatchCandidate) ([]MatchCandidate, bool) {
scored := make([]scoredMatchCandidate, 0, len(candidates))
for _, candidate := range candidates {
annotateCandidateMatch(&candidate, hints)
titleSimilarity, _ := bestCandidateTitleSimilarity(hints.Title, candidate, 0)
exactTitleCandidate := candidate.MatchScore >= automaticMatchAcceptanceFloor && titleSimilarity == 1
crossProviderYearCandidate := hints.Year != 0 && candidate.Year == hints.Year &&
candidateCorroboratingSourceCount(candidate) >= 2
if (!exactTitleCandidate && !crossProviderYearCandidate) ||
!candidateTypeMatchesHint(hints.Type, candidate.ContentType) || providerIDRichness(candidate.ProviderIDs) == 0 {
continue
}
scored = append(scored, scoredMatchCandidate{candidate: candidate, score: candidate.MatchScore})
}
sort.SliceStable(scored, func(i, j int) bool { return scored[i].score > scored[j].score })
exhaustive := len(scored) <= maxEpisodeValidationCandidates
if len(scored) > maxEpisodeValidationCandidates {
scored = scored[:maxEpisodeValidationCandidates]
}
out := make([]MatchCandidate, 0, len(scored))
for _, candidate := range scored {
out = append(out, candidate.candidate)
}
return out, exhaustive
}
func selectEpisodeValidationProviders(candidate MatchCandidate, providers []Provider) []episodeValidationProvider {
// TMDB fetches exactly one requested season, making it the cheapest
// validator when both first-party IDs are present. The configured chain
// order remains the fallback for TVDB-only and third-party candidates.
ordered := make([]Provider, 0, len(providers))
for _, provider := range providers {
if strings.EqualFold(provider.Slug(), "tmdb") {
ordered = append(ordered, provider)
}
}
for _, provider := range providers {
if !strings.EqualFold(provider.Slug(), "tmdb") {
ordered = append(ordered, provider)
}
}
selected := make([]episodeValidationProvider, 0, maxEpisodeValidationProviders)
for _, provider := range ordered {
if _, local := provider.(IdentityHintProvider); local {
continue
}
episodeProvider, ok := provider.(EpisodeProvider)
if !ok {
continue
}
providerSlug := strings.ToLower(strings.TrimSpace(provider.Slug()))
if strings.TrimSpace(candidate.ProviderIDs[providerSlug]) == "" {
continue
}
selected = append(selected, episodeValidationProvider{provider: episodeProvider, slug: providerSlug})
if len(selected) == maxEpisodeValidationProviders {
break
}
}
return selected
}
func validateEpisodeCandidateAcrossProviders(
ctx context.Context,
candidate MatchCandidate,
providers []episodeValidationProvider,
localEpisodes []localEpisodeMatchHint,
language string,
) episodeCandidateValidation {
best := episodeCandidateValidation{candidate: candidate}
var lastErr error
for _, provider := range providers {
validation := validateEpisodeCandidate(
ctx, candidate, provider.slug, provider.provider, localEpisodes, language,
)
if validation.err != nil {
lastErr = validation.err
continue
}
if !best.evaluated || episodeValidationEvidenceBetter(validation, best) {
best = validation
}
if episodeEvidenceIsSufficient(validation, localEpisodes) {
return validation
}
}
if best.evaluated {
return best
}
best.err = lastErr
return best
}
func episodeValidationEvidenceBetter(left, right episodeCandidateValidation) bool {
if left.matches != right.matches {
return left.matches > right.matches
}
if left.exactMatches != right.exactMatches {
return left.exactMatches > right.exactMatches
}
if left.similarityTotal != right.similarityTotal {
return left.similarityTotal > right.similarityTotal
}
if left.compared != right.compared {
return left.compared > right.compared
}
return left.coordinates > right.coordinates
}
func validateEpisodeCandidate(
ctx context.Context,
candidate MatchCandidate,
providerSlug string,
provider EpisodeProvider,
localEpisodes []localEpisodeMatchHint,
language string,
) episodeCandidateValidation {
validation := episodeCandidateValidation{candidate: candidate, provider: providerSlug}
validation.evaluated = true
localBySeason := make(map[int][]localEpisodeMatchHint)
seasonNumbers := make([]int, 0, maxEpisodeValidationSeasons)
for _, episode := range localEpisodes {
if _, exists := localBySeason[episode.SeasonNumber]; !exists {
seasonNumbers = append(seasonNumbers, episode.SeasonNumber)
}
localBySeason[episode.SeasonNumber] = append(localBySeason[episode.SeasonNumber], episode)
}
sort.Ints(seasonNumbers)
for _, seasonNumber := range seasonNumbers {
episodes, err := provider.GetEpisodes(ctx, EpisodesRequest{
ProviderIDs: copyMap(candidate.ProviderIDs),
SeasonNumber: seasonNumber,
Language: language,
})
if err != nil {
if isProvider404(err) {
// A missing season is valid negative evidence for this candidate.
continue
}
validation.err = fmt.Errorf("episode validation via %s for %s season %d: %w", providerSlug, candidate.Title, seasonNumber, err)
return validation
}
providerEpisodes := make(map[int]EpisodeResult, len(episodes))
for _, episode := range episodes {
if episode.SeasonNumber == seasonNumber {
providerEpisodes[episode.EpisodeNumber] = episode
}
}
for _, localEpisode := range localBySeason[seasonNumber] {
providerEpisode, ok := providerEpisodes[localEpisode.EpisodeNumber]
if !ok {
continue
}
validation.coordinates++
if isGenericEpisodeMatchTitle(localEpisode.Title) || isGenericEpisodeMatchTitle(providerEpisode.Title) {
continue
}
validation.compared++
similarity := episodeTitleSimilarity(localEpisode.Title, providerEpisode.Title)
if similarity < 0.8 {
continue
}
validation.matches++
validation.similarityTotal += similarity
if similarity == 1 {
validation.exactMatches++
}
}
}
return validation
}
func episodeEvidenceIsSufficient(validation episodeCandidateValidation, localEpisodes []localEpisodeMatchHint) bool {
if len(localEpisodes) >= 2 {
return validation.matches >= 2
}
return len(localEpisodes) == 1 && validation.matches == 1 && validation.exactMatches == 1 &&
isDistinctiveSingleEpisodeTitle(localEpisodes[0].Title)
}
func episodeEvidenceClearlyWins(best, next episodeCandidateValidation) bool {
if best.matches != next.matches {
return best.matches > next.matches
}
if best.exactMatches != next.exactMatches {
return best.exactMatches > next.exactMatches
}
return best.similarityTotal-next.similarityTotal >= 0.2
}
func episodeCoordinateEvidenceIsSufficient(
validation episodeCandidateValidation,
next episodeCandidateValidation,
localEpisodes []localEpisodeMatchHint,
contenderCount int,
contendersExhaustive bool,
) bool {
if len(localEpisodes) < 2 || validation.coordinates != len(localEpisodes) {
return false
}
// Two coordinates spread across two seasons carry meaningful series-shape
// evidence. Within one season, require all three spread-out samples; merely
// finding two early episodes is common across unrelated same-title series.
if distinctEpisodeSeasons(localEpisodes) < 2 && len(localEpisodes) < maxEpisodeValidationSamples {
return false
}
// Coordinates cannot prove that a differently named candidate is an alias.
// Cross-provider exact-year title variants are eligible for validation, but
// they must win through matching episode titles. Coordinate-only promotion
// remains limited to an exact normalized series title.
if !slices.Contains(validation.candidate.MatchReasons, "exact_title") {
return false
}
if contenderCount > 1 {
// Shape evidence can only choose between multiple exact-title candidates
// when the bounded candidate set was exhaustive and exactly one candidate
// contains every sampled coordinate. A truncated search result remains
// unmatched because an unchecked rival may have the same shape.
if !contendersExhaustive || next.coordinates == len(localEpisodes) {
return false
}
if distinctEpisodeSeasons(localEpisodes) < 2 {
return false
}
}
discriminating := episodeCoordinatesAreDiscriminating(localEpisodes)
return discriminating || candidateCorroboratingSourceCount(validation.candidate) >= 2
}
func distinctEpisodeSeasons(episodes []localEpisodeMatchHint) int {
seasons := make(map[int]struct{})
for _, episode := range episodes {
seasons[episode.SeasonNumber] = struct{}{}
}
return len(seasons)
}
func episodeCoordinatesAreDiscriminating(episodes []localEpisodeMatchHint) bool {
for _, episode := range episodes {
if episode.SeasonNumber > 1 || episode.EpisodeNumber >= 6 {
return true
}
}
return false
}
func candidateCorroboratingSourceCount(candidate MatchCandidate) int {
seen := make(map[string]struct{}, len(candidate.Sources))
for _, source := range candidate.Sources {
source = strings.ToLower(strings.TrimSpace(source))
if source == "" || nonCorroboratingSources[source] {
continue
}
seen[source] = struct{}{}
}
return len(seen)
}
func selectLocalEpisodeCoordinateHints(paths []string) []localEpisodeMatchHint {
bySeason := make(map[int]map[int]localEpisodeMatchHint)
for _, path := range paths {
parsed := naming.ParseFilename(path, "series")
if parsed == nil || parsed.EpisodeNum <= 0 {
continue
}
if bySeason[parsed.SeasonNum] == nil {
bySeason[parsed.SeasonNum] = make(map[int]localEpisodeMatchHint)
}
title := extractEpisodeMatchTitle(path)
if isGenericEpisodeMatchTitle(title) {
title = ""
}
existing, exists := bySeason[parsed.SeasonNum][parsed.EpisodeNum]
if exists && utf8.RuneCountInString(existing.Title) >= utf8.RuneCountInString(title) {
continue
}
bySeason[parsed.SeasonNum][parsed.EpisodeNum] = localEpisodeMatchHint{
SeasonNumber: parsed.SeasonNum, EpisodeNumber: parsed.EpisodeNum, Title: title,
}
}
seasonNumbers := make([]int, 0, len(bySeason))
for season, episodes := range bySeason {
if len(episodes) > 0 {
seasonNumbers = append(seasonNumbers, season)
}
}
if len(seasonNumbers) == 0 {
return nil
}
sort.Ints(seasonNumbers)
// Specials are weak shape evidence. Prefer numbered seasons whenever they
// are available, retaining season zero only for specials-only libraries.
numberedSeasons := make([]int, 0, len(seasonNumbers))
for _, season := range seasonNumbers {
if season > 0 {
numberedSeasons = append(numberedSeasons, season)
}
}
if len(numberedSeasons) > 0 {
seasonNumbers = numberedSeasons
}
if len(seasonNumbers) == 1 {
selectedSeason := seasonNumbers[0]
episodeNumbers := sortedEpisodeNumbers(bySeason[selectedSeason])
if len(episodeNumbers) > maxEpisodeValidationSamples {
episodeNumbers = []int{
episodeNumbers[0],
episodeNumbers[len(episodeNumbers)/2],
episodeNumbers[len(episodeNumbers)-1],
}
}
out := make([]localEpisodeMatchHint, 0, len(episodeNumbers))
for _, episodeNumber := range episodeNumbers {
out = append(out, bySeason[selectedSeason][episodeNumber])
}
return out
}
if len(seasonNumbers) > maxEpisodeValidationSeasons {
seasonNumbers = []int{
seasonNumbers[0],
seasonNumbers[len(seasonNumbers)/2],
seasonNumbers[len(seasonNumbers)-1],
}
}
// The last locally present episode of several seasons is far more
// discriminating than S01E01-style existence checks: same-name remakes
// rarely share both season count and episode count per season.
out := make([]localEpisodeMatchHint, 0, len(seasonNumbers))
for _, seasonNumber := range seasonNumbers {
episodeNumbers := sortedEpisodeNumbers(bySeason[seasonNumber])
out = append(out, bySeason[seasonNumber][episodeNumbers[len(episodeNumbers)-1]])
}
return out
}
func sortedEpisodeNumbers(episodes map[int]localEpisodeMatchHint) []int {
numbers := make([]int, 0, len(episodes))
for episodeNumber := range episodes {
numbers = append(numbers, episodeNumber)
}
sort.Ints(numbers)
return numbers
}
func extractEpisodeMatchTitle(path string) string {
base := filepath.Base(path)
stem := strings.TrimSuffix(base, filepath.Ext(base))
location := episodeCodePattern.FindStringIndex(stem)
if location == nil {
return ""
}
title := strings.TrimLeft(stem[location[1]:], " ._-")
if cut := strings.IndexAny(title, "[{"); cut >= 0 {
title = title[:cut]
}
if location := episodeReleaseSuffixPattern.FindStringIndex(title); location != nil {
title = title[:location[0]]
}
title = strings.NewReplacer(".", " ", "_", " ").Replace(title)
return strings.Join(strings.Fields(strings.Trim(title, " -")), " ")
}
func isGenericEpisodeMatchTitle(title string) bool {
normalized := strings.Join(strings.Fields(strings.TrimSpace(title)), " ")
return normalized == "" || episodeGenericTitlePattern.MatchString(normalized) || episodeTechnicalTitlePattern.MatchString(normalized)
}
func isDistinctiveSingleEpisodeTitle(title string) bool {
normalized := normalizeTitleForScoring(title)
return !isGenericEpisodeMatchTitle(normalized) && utf8.RuneCountInString(normalized) >= 10 && len(strings.Fields(normalized)) >= 2
}
func episodeTitleSimilarity(left, right string) float64 {
leftComparable := foldEpisodeTitleDiacritics(normalizeTitleForScoring(left))
rightComparable := foldEpisodeTitleDiacritics(normalizeTitleForScoring(right))
if leftComparable == "" || rightComparable == "" {
return 0
}
if leftComparable == rightComparable {
return 1
}
if naming.InferTitlesCoherent(leftComparable, rightComparable) {
return 0.8
}
return 0
}
func foldEpisodeTitleDiacritics(value string) string {
var builder strings.Builder
for _, r := range norm.NFD.String(value) {
if !unicode.Is(unicode.Mn, r) {
builder.WriteRune(r)
}
}
return norm.NFC.String(builder.String())
}
@@ -0,0 +1,745 @@
//nolint:goconst // Repeated provider IDs and episode titles make these fixtures readable.
package metadata
import (
"context"
"errors"
"sync"
"testing"
)
type episodeValidationStubProvider struct {
mu sync.Mutex
slug string
episodes map[string][]EpisodeResult
errors map[string]error
calls []string
}
type episodeValidationPipelineProvider struct {
*episodeValidationStubProvider
searchResults []SearchResult
}
func (p *episodeValidationPipelineProvider) Search(_ context.Context, _ SearchQuery) ([]SearchResult, error) {
return append([]SearchResult(nil), p.searchResults...), nil
}
func (p *episodeValidationPipelineProvider) GetMetadata(_ context.Context, req MetadataRequest) (*MetadataResult, error) {
id := req.ProviderIDs["tmdb"]
year := 0
switch id {
case "2015":
year = 2015
case "2020":
year = 2020
}
return &MetadataResult{
HasMetadata: true,
Title: "Crims",
Year: year,
ProviderIDs: map[string]string{"tmdb": id},
}, nil
}
func (p *episodeValidationStubProvider) Slug() string {
if p.slug == "" {
return "tmdb"
}
return p.slug
}
func (p *episodeValidationStubProvider) Name() string { return p.Slug() }
func (p *episodeValidationStubProvider) ForTypes() []string { return []string{"series"} }
func (p *episodeValidationStubProvider) GetSeasons(context.Context, SeasonsRequest) ([]SeasonResult, error) {
return nil, nil
}
func (p *episodeValidationStubProvider) GetEpisodes(_ context.Context, req EpisodesRequest) ([]EpisodeResult, error) {
p.mu.Lock()
defer p.mu.Unlock()
id := req.ProviderIDs[p.Slug()]
p.calls = append(p.calls, id)
if err := p.errors[id]; err != nil {
return nil, err
}
return append([]EpisodeResult(nil), p.episodes[id]...), nil
}
func (p *episodeValidationStubProvider) callCount() int {
p.mu.Lock()
defer p.mu.Unlock()
return len(p.calls)
}
func TestExtractEpisodeMatchTitle(t *testing.T) {
t.Parallel()
tests := map[string]string{
"standard radarr style": "/tv/Crims/Season 1/Crims - S01E01 - Day One[WEBDL-1080p.AAC.h264.NTb].mp4",
"dotted release": "/tv/Crims/Season 1/Crims.S01E01.Day.One.1080p.WEB-DL.x264-GROUP.mkv",
"no episode title": "/tv/Crims/Season 1/Crims.S01E01.1080p.WEB.x264-GROUP.mkv",
}
wants := map[string]string{
"standard radarr style": "Day One",
"dotted release": "Day One",
"no episode title": "",
}
for name, path := range tests {
name, path := name, path
t.Run(name, func(t *testing.T) {
t.Parallel()
if got := extractEpisodeMatchTitle(path); got != wants[name] {
t.Fatalf("extractEpisodeMatchTitle() = %q, want %q", got, wants[name])
}
})
}
}
func TestValidateSeriesMatchByEpisodes_SelectsSecondExactTitleCandidate(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"2020": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Arrival"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Departure"},
},
"2015": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Crime and Punishment"},
},
}}
hints := &MatchHints{
Title: "Crims", Type: "series",
AllGroupFilePaths: []string{
"/tv/Crims/Season 1/Crims - S01E01 - Day One[WEBDL-1080p].mkv",
"/tv/Crims/Season 1/Crims - S01E02 - Crime and Punishment[WEBDL-1080p].mkv",
},
}
candidates := []MatchCandidate{
{Title: "Crims", Year: 2020, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "2020"}},
{Title: "Crims", Year: 2015, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "2015"}},
}
winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if len(errs) != 0 {
t.Fatalf("validation errors = %v", errs)
}
if winner == nil || winner.ProviderIDs["tmdb"] != "2015" {
t.Fatalf("winner = %+v, want 2015 candidate", winner)
}
if !containsString(winner.MatchReasons, "episode_title_corroboration:2_of_2") {
t.Fatalf("winner reasons = %v", winner.MatchReasons)
}
}
func TestValidateSeriesMatchByEpisodes_LeavesMemoriesUnmatched(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"2018": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "A New Beginning"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Home"},
},
"1998": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Old Friends"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Reunion"},
},
}}
hints := &MatchHints{
Title: "Memories", Type: "series",
AllGroupFilePaths: []string{
"/anime/Memories/Season 1/Memories - S01E01 - Magnetic Rose[Bluray-1080p].mkv",
"/anime/Memories/Season 1/Memories - S01E02 - Stink Bomb[Bluray-1080p].mkv",
},
}
candidates := []MatchCandidate{
{Title: "Memories", Year: 2018, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "2018"}},
{Title: "Memories", Year: 1998, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "1998"}},
}
winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if winner != nil {
t.Fatalf("winner = %+v, want unmatched type conflict", winner)
}
}
func TestValidateSeriesMatchByEpisodes_SearchesPastFormerTopThree(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"1": {{SeasonNumber: 1, EpisodeNumber: 1, Title: "Wrong One"}, {SeasonNumber: 1, EpisodeNumber: 2, Title: "Wrong Two"}},
"2": {{SeasonNumber: 1, EpisodeNumber: 1, Title: "Wrong One"}, {SeasonNumber: 1, EpisodeNumber: 2, Title: "Wrong Two"}},
"3": {{SeasonNumber: 1, EpisodeNumber: 1, Title: "Wrong One"}, {SeasonNumber: 1, EpisodeNumber: 2, Title: "Wrong Two"}},
"4": {{SeasonNumber: 1, EpisodeNumber: 1, Title: "Right One"}, {SeasonNumber: 1, EpisodeNumber: 2, Title: "Right Two"}},
}}
hints := &MatchHints{
Title: "Shared", Type: "series",
AllGroupFilePaths: []string{
"/tv/Shared/Season 1/Shared - S01E01 - Right One[WEBDL-1080p].mkv",
"/tv/Shared/Season 1/Shared - S01E02 - Right Two[WEBDL-1080p].mkv",
},
}
candidates := make([]MatchCandidate, 0, 4)
for _, id := range []string{"1", "2", "3", "4"} {
candidates = append(candidates, MatchCandidate{
Title: "Shared", Year: 2000, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": id},
})
}
winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if winner == nil || winner.ProviderIDs["tmdb"] != "4" {
t.Fatalf("winner = %+v, want corroborated fourth candidate", winner)
}
if got := provider.callCount(); got != len(candidates) {
t.Fatalf("provider calls = %d, want %d", got, len(candidates))
}
}
func TestValidateSeriesMatchByEpisodes_BoundsCandidateFetches(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{}}
hints := &MatchHints{
Title: "Shared", Type: "series",
AllGroupFilePaths: []string{
"/tv/Shared/Shared - S01E01 - Right One.mkv",
"/tv/Shared/Shared - S01E02 - Right Two.mkv",
},
}
candidates := make([]MatchCandidate, 0, maxEpisodeValidationCandidates+4)
for i := 0; i < maxEpisodeValidationCandidates+4; i++ {
id := string(rune('a' + i))
provider.episodes[id] = []EpisodeResult{
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Wrong One"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Wrong Two"},
}
candidates = append(candidates, MatchCandidate{
Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": id},
})
}
validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if got := provider.callCount(); got != maxEpisodeValidationCandidates {
t.Fatalf("provider calls = %d, want bounded %d", got, maxEpisodeValidationCandidates)
}
}
func TestValidateSeriesMatchByEpisodes_ValidatesCrossProviderExactYearAlias(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"show": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "The Beginning"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "The Journey"},
},
}}
hints := &MatchHints{
Title: "Bunny Drop", Year: 2011, Type: "series",
AllGroupFilePaths: []string{
"/tv/Bunny Drop/Bunny Drop - S01E01 - The Beginning.mkv",
"/tv/Bunny Drop/Bunny Drop - S01E02 - The Journey.mkv",
},
}
candidates := []MatchCandidate{{
Title: "Usagi Drop", Year: 2011, ContentType: "series", Sources: []string{"tmdb", "tvdb"},
ProviderIDs: map[string]string{"tmdb": "show", "tvdb": "123"},
}}
winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if len(errs) != 0 || winner == nil || winner.ProviderIDs["tmdb"] != "show" {
t.Fatalf("winner = %+v errors = %v, want exact-year alias corroborated", winner, errs)
}
}
func TestValidateSeriesMatchByEpisodes_AcceptsDistinctiveSoleEpisode(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"232926": {{SeasonNumber: 1, EpisodeNumber: 1, Title: "The Fiancé Who Killed Me"}},
}}
hints := &MatchHints{
Title: "7th Time Loop", Type: "series",
AllGroupFilePaths: []string{
"/tv/7th Time Loop/Season 1/7th Time Loop - S01E01 - The Fiance Who Killed Me[WEBDL-1080p].mkv",
},
}
candidates := []MatchCandidate{{
Title: "7th Time Loop", Year: 2024, ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "232926"},
}}
winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if winner == nil {
t.Fatal("distinctive exact episode title should corroborate the sole candidate")
}
}
func TestValidateSeriesMatchByEpisodes_FallsBackWhenPreferredProviderHasGenericTitles(t *testing.T) {
t.Parallel()
tmdb := &episodeValidationStubProvider{slug: "tmdb", episodes: map[string][]EpisodeResult{
"61834": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Episode 1"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Episode 2"},
},
}}
tvdb := &episodeValidationStubProvider{slug: "tvdb", episodes: map[string][]EpisodeResult{
"289456": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Day Thirty-Six"},
},
}}
hints := &MatchHints{
Title: "Crims", Type: "series",
AllGroupFilePaths: []string{
"/tv/Crims/Season 1/Crims - S01E01 - Day One.WEBDL-1080p.mkv",
"/tv/Crims/Season 1/Crims - S01E02 - Day Thirty-Six.WEBDL-1080p.mkv",
},
}
candidates := []MatchCandidate{{
Title: "Crims", Year: 2015, ContentType: "series", Sources: []string{"tmdb", "tvdb"},
ProviderIDs: map[string]string{"tmdb": "61834", "tvdb": "289456"},
}}
winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{tvdb, tmdb}, "en")
if len(errs) != 0 {
t.Fatalf("validation errors = %v", errs)
}
if winner == nil || winner.ProviderIDs["tvdb"] != "289456" {
t.Fatalf("winner = %+v, want TVDB-corroborated Crims", winner)
}
if tmdb.callCount() != 1 || tvdb.callCount() != 1 {
t.Fatalf("provider calls tmdb=%d tvdb=%d, want one each", tmdb.callCount(), tvdb.callCount())
}
}
func TestValidateSeriesMatchByEpisodes_TreatsMissingRivalSeasonAsZeroEvidence(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{
episodes: map[string][]EpisodeResult{
"correct": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Day Thirty-Six"},
},
},
errors: map[string]error{"rival": errors.New("tmdb: HTTP 404: season not found")},
}
hints := &MatchHints{
Title: "Crims", Type: "series",
AllGroupFilePaths: []string{
"/tv/Crims/Crims - S01E01 - Day One.mkv",
"/tv/Crims/Crims - S01E02 - Day Thirty-Six.mkv",
},
}
candidates := []MatchCandidate{
{Title: "Crims", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "rival"}},
{Title: "Crims", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "correct"}},
}
winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if len(errs) != 0 {
t.Fatalf("validation errors = %v, want missing season treated as evaluated evidence", errs)
}
if winner == nil || winner.ProviderIDs["tmdb"] != "correct" {
t.Fatalf("winner = %+v, want corroborated candidate", winner)
}
}
func TestValidateSeriesMatchByEpisodes_OperationalRivalErrorBlocksPromotion(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{
episodes: map[string][]EpisodeResult{
"correct": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Day Thirty-Six"},
},
},
errors: map[string]error{"rival": errors.New("tmdb: HTTP 503: unavailable")},
}
hints := &MatchHints{
Title: "Crims", Type: "series",
AllGroupFilePaths: []string{
"/tv/Crims/Crims - S01E01 - Day One.mkv",
"/tv/Crims/Crims - S01E02 - Day Thirty-Six.mkv",
},
}
candidates := []MatchCandidate{
{Title: "Crims", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "rival"}},
{Title: "Crims", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "correct"}},
}
winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if winner != nil {
t.Fatalf("winner = %+v, want operationally unavailable rival to block promotion", winner)
}
if len(errs) != 1 {
t.Fatalf("validation errors = %v, want one operational error", errs)
}
}
func TestValidateSeriesMatchByEpisodes_AcceptsMultiSourceCoordinateCorroboration(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"show": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Episode 1"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Episode 2"},
{SeasonNumber: 1, EpisodeNumber: 3, Title: "Episode 3"},
},
}}
hints := &MatchHints{
Title: "Shared", Type: "series",
AllGroupFilePaths: []string{
"/tv/Shared/Shared - S01E01.mkv",
"/tv/Shared/Shared - S01E02.mkv",
"/tv/Shared/Shared - S01E03.mkv",
},
}
candidates := []MatchCandidate{{
Title: "Shared", ContentType: "series", Sources: []string{"tmdb", "tvdb"},
ProviderIDs: map[string]string{"tmdb": "show", "tvdb": "other"},
}}
winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if len(errs) != 0 {
t.Fatalf("validation errors = %v", errs)
}
if winner == nil || !containsString(winner.MatchReasons, "episode_coordinate_corroboration:3_of_3") {
t.Fatalf("winner = %+v, want coordinate corroboration", winner)
}
}
func TestValidateSeriesMatchByEpisodes_RejectsWeakSingleSourceCoordinates(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"show": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Episode 1"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Episode 2"},
{SeasonNumber: 1, EpisodeNumber: 3, Title: "Episode 3"},
},
}}
hints := &MatchHints{
Title: "Shared", Type: "series",
AllGroupFilePaths: []string{
"/tv/Shared/Shared - S01E01.mkv",
"/tv/Shared/Shared - S01E02.mkv",
"/tv/Shared/Shared - S01E03.mkv",
},
}
candidates := []MatchCandidate{{
Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "show"},
}}
winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if winner != nil {
t.Fatalf("winner = %+v, weak S01E01-E03 coordinates from one source must stay unmatched", winner)
}
}
func TestValidateSeriesMatchByEpisodes_AcceptsDistinctiveSingleSourceCoordinates(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"show": {
{SeasonNumber: 2, EpisodeNumber: 1, Title: "Episode 1"},
{SeasonNumber: 2, EpisodeNumber: 4, Title: "Episode 4"},
{SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"},
},
}}
hints := &MatchHints{
Title: "Shared", Type: "series",
AllGroupFilePaths: []string{
"/tv/Shared/Shared - S02E01.mkv",
"/tv/Shared/Shared - S02E04.mkv",
"/tv/Shared/Shared - S02E08.mkv",
},
}
candidates := []MatchCandidate{{
Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "show"},
}}
winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if winner == nil || !containsString(winner.MatchReasons, "episode_coordinate_corroboration:3_of_3") {
t.Fatalf("winner = %+v, want distinctive coordinate corroboration", winner)
}
}
func TestValidateSeriesMatchByEpisodes_RejectsCoordinateOnlyChoiceBetweenExactTitles(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"first": {
{SeasonNumber: 2, EpisodeNumber: 1, Title: "Episode 1"},
{SeasonNumber: 2, EpisodeNumber: 4, Title: "Episode 4"},
{SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"},
},
"second": {
{SeasonNumber: 2, EpisodeNumber: 1, Title: "Episode 1"},
},
}}
hints := &MatchHints{
Title: "Shared", Type: "series",
AllGroupFilePaths: []string{
"/tv/Shared/Shared - S02E01.mkv",
"/tv/Shared/Shared - S02E04.mkv",
"/tv/Shared/Shared - S02E08.mkv",
},
}
candidates := []MatchCandidate{
{Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "first"}},
{Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "second"}},
}
winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if winner != nil {
t.Fatalf("winner = %+v, coordinates alone must not choose between exact-title candidates", winner)
}
}
func TestSelectLocalEpisodeCoordinateHints_SamplesLastEpisodeAcrossSeasons(t *testing.T) {
t.Parallel()
hints := selectLocalEpisodeCoordinateHints([]string{
"/tv/Shared/Season 1/Shared - S01E01.mkv",
"/tv/Shared/Season 1/Shared - S01E10.mkv",
"/tv/Shared/Season 2/Shared - S02E08.mkv",
"/tv/Shared/Season 3/Shared - S03E12.mkv",
"/tv/Shared/Season 4/Shared - S04E04.mkv",
})
want := []localEpisodeMatchHint{
{SeasonNumber: 1, EpisodeNumber: 10},
{SeasonNumber: 3, EpisodeNumber: 12},
{SeasonNumber: 4, EpisodeNumber: 4},
}
if len(hints) != len(want) {
t.Fatalf("hints = %+v, want %+v", hints, want)
}
for i := range want {
if hints[i].SeasonNumber != want[i].SeasonNumber || hints[i].EpisodeNumber != want[i].EpisodeNumber {
t.Fatalf("hints = %+v, want %+v", hints, want)
}
}
}
func TestValidateSeriesMatchByEpisodes_AcceptsUniqueMultiSeasonShape(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"original": {
{SeasonNumber: 1, EpisodeNumber: 10, Title: "Episode 10"},
{SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"},
{SeasonNumber: 4, EpisodeNumber: 6, Title: "Episode 6"},
},
"remake": {
{SeasonNumber: 1, EpisodeNumber: 10, Title: "Episode 10"},
{SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"},
},
}}
hints := &MatchHints{
Title: "Shared", Type: "series",
AllGroupFilePaths: []string{
"/tv/Shared/Season 1/Shared - S01E10.mkv",
"/tv/Shared/Season 2/Shared - S02E08.mkv",
"/tv/Shared/Season 4/Shared - S04E06.mkv",
},
}
candidates := []MatchCandidate{
{Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "remake"}},
{Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": "original"}},
}
winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if len(errs) != 0 {
t.Fatalf("validation errors = %v", errs)
}
if winner == nil || winner.ProviderIDs["tmdb"] != "original" {
t.Fatalf("winner = %+v, want original multi-season shape", winner)
}
if !containsString(winner.MatchReasons, "episode_coordinate_corroboration:3_of_3") {
t.Fatalf("winner reasons = %v", winner.MatchReasons)
}
}
func TestValidateSeriesMatchByEpisodes_RejectsTwoCoordinateSingleSeasonShape(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"candidate": {
{SeasonNumber: 2, EpisodeNumber: 1, Title: "Episode 1"},
{SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"},
},
}}
hints := &MatchHints{
Title: "Shared", Type: "series",
AllGroupFilePaths: []string{
"/tv/Shared/Season 2/Shared - S02E01.mkv",
"/tv/Shared/Season 2/Shared - S02E08.mkv",
},
}
candidates := []MatchCandidate{{
Title: "Shared", ContentType: "series", Sources: []string{"tmdb"},
ProviderIDs: map[string]string{"tmdb": "candidate"},
}}
winner, errs := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if len(errs) != 0 {
t.Fatalf("validation errors = %v", errs)
}
if winner != nil {
t.Fatalf("winner = %+v, two coordinates from one season are insufficient shape evidence", winner)
}
}
func TestValidateSeriesMatchByEpisodes_RejectsShapeWhenCandidatesAreTruncated(t *testing.T) {
t.Parallel()
provider := &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{}}
hints := &MatchHints{
Title: "Shared", Type: "series",
AllGroupFilePaths: []string{
"/tv/Shared/Season 1/Shared - S01E10.mkv",
"/tv/Shared/Season 2/Shared - S02E08.mkv",
},
}
candidates := make([]MatchCandidate, 0, maxEpisodeValidationCandidates+1)
for i := 0; i < maxEpisodeValidationCandidates+1; i++ {
id := string(rune('a' + i))
provider.episodes[id] = []EpisodeResult{{SeasonNumber: 1, EpisodeNumber: 10, Title: "Episode 10"}}
if i == 0 {
provider.episodes[id] = append(provider.episodes[id], EpisodeResult{SeasonNumber: 2, EpisodeNumber: 8, Title: "Episode 8"})
}
candidates = append(candidates, MatchCandidate{
Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}, ProviderIDs: map[string]string{"tmdb": id},
})
}
winner, _ := validateSeriesMatchByEpisodes(context.Background(), hints, candidates, []Provider{provider}, "en")
if winner != nil {
t.Fatalf("winner = %+v, truncated candidate set must stay unmatched", winner)
}
}
func TestInitialMatchPipelineUsesEpisodeCorroboration(t *testing.T) {
t.Parallel()
harness := newTestHarness()
harness.service.seasonRepo = newFakeSeasonRepo()
harness.service.episodeRepo = newFakeEpisodeRepo()
provider := &episodeValidationPipelineProvider{
episodeValidationStubProvider: &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"2020": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Arrival"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Departure"},
},
"2015": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Crime and Punishment"},
},
}},
searchResults: []SearchResult{
{Name: "Crims", Year: 2020, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "2020"}},
{Name: "Crims", Year: 2015, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "2015"}},
},
}
result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{
Hints: &MatchHints{
Title: "Crims", Type: "series",
AllGroupFilePaths: []string{
"/tv/Crims/Season 1/Crims - S01E01 - Day One[WEBDL-1080p].mkv",
"/tv/Crims/Season 1/Crims - S01E02 - Crime and Punishment[WEBDL-1080p].mkv",
},
},
Language: "en",
Mode: ModeInitialMatch,
}, []Provider{provider})
if err != nil {
t.Fatalf("ProcessWithProviders() error = %v", err)
}
if result == nil || !result.Updated || result.Decision == nil || result.Decision.Outcome != "matched" {
t.Fatalf("result = %#v, want matched update", result)
}
item, err := harness.itemRepo.GetByID(context.Background(), result.ContentID)
if err != nil {
t.Fatalf("load matched item: %v", err)
}
if item.TmdbID != "2015" || item.Year != 2015 {
t.Fatalf("matched item = tmdb:%q year:%d, want tmdb:2015 year:2015", item.TmdbID, item.Year)
}
foundReason := false
for _, candidate := range result.Decision.TopCandidates {
if candidate.ProviderIDs["tmdb"] == "2015" && containsString(candidate.Reasons, "episode_title_corroboration:2_of_2") {
foundReason = true
}
}
if !foundReason {
t.Fatalf("decision = %+v, want episode corroboration reason on selected candidate", result.Decision)
}
}
func TestInitialMatchPipelineDoesNotAddEpisodeValidationFetchForDecisiveMatch(t *testing.T) {
t.Parallel()
harness := newTestHarness()
harness.service.seasonRepo = newFakeSeasonRepo()
harness.service.episodeRepo = newFakeEpisodeRepo()
provider := &episodeValidationPipelineProvider{
episodeValidationStubProvider: &episodeValidationStubProvider{episodes: map[string][]EpisodeResult{
"2015": {
{SeasonNumber: 1, EpisodeNumber: 1, Title: "Day One"},
{SeasonNumber: 1, EpisodeNumber: 2, Title: "Crime and Punishment"},
},
}},
searchResults: []SearchResult{
{Name: "Crims", Year: 2015, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "2015"}},
},
}
result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{
Hints: &MatchHints{
Title: "Crims", Year: 2015, Type: "series",
AllGroupFilePaths: []string{
"/tv/Crims/Season 1/Crims - S01E01 - Day One.mkv",
"/tv/Crims/Season 1/Crims - S01E02 - Crime and Punishment.mkv",
},
},
Language: "en",
Mode: ModeInitialMatch,
}, []Provider{provider})
if err != nil {
t.Fatalf("ProcessWithProviders() error = %v", err)
}
if result == nil || result.Decision == nil || result.Decision.Outcome != "matched" {
t.Fatalf("result = %#v, want decisive match", result)
}
// The single call is the normal post-match episode enrichment. An episode
// validation fallback would add a second provider fetch before enrichment.
if calls := provider.callCount(); calls != 1 {
t.Fatalf("episode provider calls = %d, want only the normal enrichment fetch", calls)
}
}
func TestInitialMatchPipelinePreservesCandidateRejectionForPermanentEpisodeValidationError(t *testing.T) {
t.Parallel()
harness := newTestHarness()
provider := &episodeValidationPipelineProvider{
episodeValidationStubProvider: &episodeValidationStubProvider{errors: map[string]error{
"first": errors.New("tmdb: HTTP 400: bad request"),
"second": errors.New("tmdb: HTTP 400: bad request"),
}},
searchResults: []SearchResult{
{Name: "Shared", Year: 2020, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "first"}},
{Name: "Shared", Year: 2015, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "second"}},
},
}
result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{
Hints: &MatchHints{
Title: "Shared", Type: "series",
AllGroupFilePaths: []string{
"/tv/Shared/Shared - S01E01 - First Episode.mkv",
"/tv/Shared/Shared - S01E02 - Second Episode.mkv",
},
},
Language: "en",
Mode: ModeInitialMatch,
}, []Provider{provider})
if err != nil {
t.Fatalf("ProcessWithProviders() error = %v", err)
}
if result == nil || result.Decision == nil || result.Decision.Outcome != "candidate_rejected" {
t.Fatalf("result = %#v decision = %+v, want deterministic candidate rejection", result, result.Decision)
}
}
func containsString(values []string, want string) bool {
for _, value := range values {
if value == want {
return true
}
}
return false
}
+132
View File
@@ -0,0 +1,132 @@
//nolint:goconst // Repeated canonical IDs are the evidence under test.
package metadata
import (
"context"
"testing"
"github.com/Silo-Server/silo-server/internal/models"
)
type idConsensusPipelineProvider struct{}
func (p *idConsensusPipelineProvider) Slug() string { return "tmdb" }
func (p *idConsensusPipelineProvider) Name() string { return "TMDB" }
func (p *idConsensusPipelineProvider) ForTypes() []string { return []string{"series"} }
func (p *idConsensusPipelineProvider) Search(context.Context, SearchQuery) ([]SearchResult, error) {
return []SearchResult{
{
Name: "A Teacher", Year: 2020, Provider: "tvdb",
ProviderIDs: map[string]string{
"imdb": "tt10680614", "tmdb": "103992", "tvdb": "352440",
},
},
{
Name: "A Teacher", Year: 2020, Provider: "tmdb",
ProviderIDs: map[string]string{
"imdb": "tt10680614", "tmdb": "103992", "tvdb": "473725",
},
},
}, nil
}
func (p *idConsensusPipelineProvider) GetMetadata(_ context.Context, req MetadataRequest) (*MetadataResult, error) {
if req.ProviderIDs["tvdb"] != "" {
return nil, errUnexpectedQuarantinedProviderID
}
return &MetadataResult{
HasMetadata: true,
Title: "A Teacher",
Year: 2020,
// A provider detail response may repeat its stale cross-reference. The
// pipeline must not let it undo the search-phase quarantine.
ProviderIDs: map[string]string{
"imdb": "tt10680614", "tmdb": "103992", "tvdb": "473725",
},
}, nil
}
var errUnexpectedQuarantinedProviderID = &unexpectedQuarantinedProviderIDError{}
type unexpectedQuarantinedProviderIDError struct{}
func (*unexpectedQuarantinedProviderIDError) Error() string {
return "quarantined provider ID reached metadata phase"
}
func TestInitialMatchPipelineQuarantinesConflictingConsensusID(t *testing.T) {
t.Parallel()
harness := newTestHarness()
provider := &idConsensusPipelineProvider{}
result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{
Hints: &MatchHints{Title: "A Teacher", Type: "series"},
Mode: ModeInitialMatch,
}, []Provider{provider})
if err != nil {
t.Fatalf("ProcessWithProviders() error = %v", err)
}
if result == nil || !result.Updated || result.Decision == nil || result.Decision.Outcome != "matched" {
t.Fatalf("result = %#v, want matched consensus", result)
}
item, err := harness.itemRepo.GetByID(context.Background(), result.ContentID)
if err != nil {
t.Fatalf("load matched item: %v", err)
}
if item.ImdbID != "tt10680614" || item.TmdbID != "103992" {
t.Fatalf("persisted consensus IDs = imdb:%q tmdb:%q", item.ImdbID, item.TmdbID)
}
if item.TvdbID != "" {
t.Fatalf("conflicting TVDB ID persisted as %q", item.TvdbID)
}
}
func TestRefreshPipelinesQuarantineStoredConflictingConsensusID(t *testing.T) {
for name, mode := range map[string]RefreshMode{"scheduled": ModeScheduledRefresh, "manual": ModeManualRefresh} {
mode := mode
t.Run(name, func(t *testing.T) {
t.Parallel()
harness := newTestHarness()
const contentID = "series-refresh-consensus"
if err := harness.itemRepo.Upsert(context.Background(), &models.MediaItem{
ContentID: contentID, Type: "series", Title: "A Teacher", Year: 2020, Status: "matched",
ImdbID: "tt10680614", TmdbID: "103992", TvdbID: "473725",
Studios: []string{}, Networks: []string{}, Countries: []string{}, Genres: []string{},
}); err != nil {
t.Fatalf("seed item: %v", err)
}
result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{
ContentID: contentID,
Language: "en",
Mode: mode,
}, []Provider{&idConsensusPipelineProvider{}})
if err != nil {
t.Fatalf("ProcessWithProviders() error = %v", err)
}
if result == nil || !result.Updated {
t.Fatalf("result = %#v, want updated refresh", result)
}
item, err := harness.itemRepo.GetByID(context.Background(), contentID)
if err != nil {
t.Fatalf("load refreshed item: %v", err)
}
if item.ImdbID != "tt10680614" || item.TmdbID != "103992" || item.TvdbID != "" {
t.Fatalf("refreshed IDs = imdb:%q tmdb:%q tvdb:%q, want disputed TVDB ID quarantined", item.ImdbID, item.TmdbID, item.TvdbID)
}
})
}
}
func TestApplyCandidateProviderIDConsensusRemovesDisputedStoredID(t *testing.T) {
ids := map[string]string{"imdb": "tt10680614", "tmdb": "103992", "tvdb": "473725"}
winner := &MatchCandidate{
ProviderIDs: map[string]string{"imdb": "tt10680614", "tmdb": "103992", "tvdb": "473725"},
ConflictingProviderIDKeys: []string{"tvdb"},
}
applyCandidateProviderIDConsensus(ids, winner, nil)
if ids["tvdb"] != "" || ids["imdb"] != "tt10680614" || ids["tmdb"] != "103992" {
t.Fatalf("consensus IDs = %#v", ids)
}
}
+456 -48
View File
@@ -5,34 +5,49 @@ import (
"fmt"
"log/slog"
"math"
"slices"
"sort"
"strconv"
"strings"
"unicode"
"github.com/Silo-Server/silo-server/internal/lang"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/naming"
"github.com/Silo-Server/silo-server/internal/providerid"
)
// MatchCandidate represents a deduplicated search result grouped by normalized
// provider IDs. Multiple raw SearchResult rows from different providers that
// share the same TMDB/TVDB/IMDB IDs are collapsed into a single candidate.
type MatchCandidate struct {
Title string `json:"title"`
Year int `json:"year"`
ContentType string `json:"content_type"`
ProviderIDs map[string]string `json:"provider_ids"`
ImageURL string `json:"image_url,omitempty"`
Overview string `json:"overview,omitempty"`
Sources []string `json:"sources"`
AgreementHints []string `json:"agreement_hints"`
DetailScore int `json:"-"`
Title string `json:"title"`
OriginalTitle string `json:"original_title,omitempty"`
TitleAliases []TitleAlias `json:"aliases,omitempty"`
TitleLanguage string `json:"title_language,omitempty"`
TitleIsFallback bool `json:"title_is_fallback,omitempty"`
MatchedTitle string `json:"matched_title,omitempty"`
MatchScore float64 `json:"match_score,omitempty"`
MatchReasons []string `json:"match_reasons,omitempty"`
Year int `json:"year"`
ContentType string `json:"content_type"`
ProviderIDs map[string]string `json:"provider_ids"`
ImageURL string `json:"image_url,omitempty"`
Overview string `json:"overview,omitempty"`
Sources []string `json:"sources"`
AgreementHints []string `json:"agreement_hints"`
DetailScore int `json:"-"`
// ConflictingProviderIDKeys contains canonical provider IDs deliberately
// excluded after two other canonical IDs proved that provider results refer
// to the same work. These keys remain quarantined for the rest of the match
// pipeline so a later detail response cannot silently reintroduce them.
ConflictingProviderIDKeys []string `json:"-"`
titleRank int
}
var canonicalCandidateIDKeys = []string{"tmdb", "tvdb", "imdb"}
func compatibleProviderIDs(left, right map[string]string) bool {
overlap := false
func providerIDMergeEvidence(left, right map[string]string) (matches int, conflicts []string) {
for _, key := range canonicalCandidateIDKeys {
lv := strings.TrimSpace(left[key])
rv := strings.TrimSpace(right[key])
@@ -40,11 +55,23 @@ func compatibleProviderIDs(left, right map[string]string) bool {
continue
}
if lv != rv {
return false
conflicts = append(conflicts, key)
continue
}
overlap = true
matches++
}
return overlap
return matches, conflicts
}
func compatibleProviderIDs(left, right map[string]string) bool {
matches, conflicts := providerIDMergeEvidence(left, right)
if matches == 0 {
return false
}
// One agreeing ID plus one conflict is not consensus. Two independently
// agreeing canonical IDs are enough to prove identity while quarantining a
// stale third-party cross-reference.
return len(conflicts) == 0 || matches >= 2
}
func providerIDRichness(ids map[string]string) int {
@@ -57,9 +84,82 @@ func providerIDRichness(ids map[string]string) int {
return score
}
func sanitizeCandidateProviderIDs(ids map[string]string) map[string]string {
if len(ids) == 0 {
return nil
}
sanitized := make(map[string]string, len(ids))
for key, value := range ids {
key = strings.ToLower(strings.TrimSpace(key))
value, valid := sanitizeProviderIDValue(key, value)
if !valid {
continue
}
sanitized[key] = value
}
return sanitized
}
func sanitizeCanonicalProviderIDsInPlace(ids map[string]string) {
for _, key := range canonicalCandidateIDKeys {
value := strings.TrimSpace(ids[key])
if value == "" {
delete(ids, key)
continue
}
sanitized, valid := sanitizeProviderIDValue(key, value)
if !valid {
delete(ids, key)
continue
}
ids[key] = sanitized
}
}
func sanitizeProviderIDValue(key, value string) (string, bool) {
key = strings.ToLower(strings.TrimSpace(key))
value = strings.TrimSpace(value)
if key == "" || value == "" {
return "", false
}
switch key {
case "tmdb", "tvdb":
if !providerid.IsPositiveDecimal(value) {
return "", false
}
case "imdb":
value = strings.ToLower(value)
if !isValidIMDbProviderID(value) {
return "", false
}
}
return value, true
}
func sanitizedMatchHintProviderIDs(hints *MatchHints) *MatchHints {
if hints == nil {
return nil
}
sanitized := *hints
sanitized.TmdbID, _ = sanitizeProviderIDValue("tmdb", hints.TmdbID)
sanitized.TvdbID, _ = sanitizeProviderIDValue("tvdb", hints.TvdbID)
sanitized.ImdbID, _ = sanitizeProviderIDValue("imdb", hints.ImdbID)
return &sanitized
}
func isValidIMDbProviderID(value string) bool {
value = strings.ToLower(strings.TrimSpace(value))
if !strings.HasPrefix(value, "tt") {
return false
}
digits := strings.TrimPrefix(value, "tt")
return len(digits) >= 7 && len(digits) <= 10 && providerid.IsPositiveDecimal(digits)
}
const (
minimumDetailTieBreakScore = 20
minimumDetailTieBreakGap = 12
automaticMatchAcceptanceFloor = 55
minimumDetailTieBreakScore = 20
minimumDetailTieBreakGap = 12
)
func duplicateTieBreakWinner(hints *MatchHints, scoredCandidates []scoredMatchCandidate) (*MatchCandidate, bool) {
@@ -113,13 +213,13 @@ func duplicateTieBreakComparable(hints *MatchHints, left, right MatchCandidate)
!strings.EqualFold(left.ContentType, right.ContentType) {
return false
}
if inferTitleSimilarity(left.Title, right.Title, hints.Year) != 1 {
if candidateToCandidateTitleSimilarity(left, right, hints.Year) != 1 {
return false
}
if inferTitleSimilarity(hints.Title, left.Title, hints.Year) != 1 {
if similarity, _ := bestCandidateTitleSimilarity(hints.Title, left, hints.Year); similarity != 1 {
return false
}
if inferTitleSimilarity(hints.Title, right.Title, hints.Year) != 1 {
if similarity, _ := bestCandidateTitleSimilarity(hints.Title, right, hints.Year); similarity != 1 {
return false
}
return samePrimaryProvider(left.ProviderIDs, right.ProviderIDs)
@@ -182,15 +282,23 @@ func normalizedKey(ids map[string]string) string {
// provider IDs are unioned, sources list every provider slug that returned
// the result, and agreement_hints notes when multiple providers agree.
func NormalizeCandidates(results []SearchResult, contentType string) []MatchCandidate {
return NormalizeCandidatesForLanguage(results, contentType, "")
}
// NormalizeCandidatesForLanguage deduplicates results while preferring an
// explicitly localized provider title over a provider-marked fallback.
func NormalizeCandidatesForLanguage(results []SearchResult, contentType, language string) []MatchCandidate {
type bucket struct {
candidate MatchCandidate
sources map[string]bool
candidate MatchCandidate
sources map[string]bool
conflictingIDKeys map[string]bool
}
ordered := make([]string, 0)
buckets := make(map[string]*bucket)
for _, sr := range results {
sr.ProviderIDs = sanitizeCandidateProviderIDs(sr.ProviderIDs)
key := ""
for _, existingKey := range ordered {
if compatibleProviderIDs(buckets[existingKey].candidate.ProviderIDs, sr.ProviderIDs) {
@@ -208,26 +316,43 @@ func NormalizeCandidates(results []SearchResult, contentType string) []MatchCand
b, exists := buckets[key]
if !exists {
title, titleLanguage, fallback, titleRank := preferredSearchResultTitle(sr, language)
b = &bucket{
candidate: MatchCandidate{
Title: sr.Name,
Year: sr.Year,
ContentType: contentType,
ProviderIDs: make(map[string]string),
ImageURL: sr.ImageURL,
Overview: sr.Overview,
Title: title,
OriginalTitle: sr.OriginalTitle,
TitleAliases: copyTitleAliases(sr.TitleAliases, sr.Provider),
TitleLanguage: titleLanguage,
TitleIsFallback: fallback,
titleRank: titleRank,
Year: sr.Year,
ContentType: contentType,
ProviderIDs: make(map[string]string),
ImageURL: sr.ImageURL,
Overview: sr.Overview,
},
sources: make(map[string]bool),
sources: make(map[string]bool),
conflictingIDKeys: make(map[string]bool),
}
buckets[key] = b
ordered = append(ordered, key)
}
mergeCandidateTitles(&b.candidate, sr, language)
// Merge provider IDs.
// Merge provider IDs. When two canonical IDs agree but a third
// conflicts, the candidates are the same work and the disputed key is
// quarantined instead of allowing provider iteration order to choose it.
for k, v := range sr.ProviderIDs {
if v != "" {
b.candidate.ProviderIDs[k] = v
v = strings.TrimSpace(v)
if v == "" || b.conflictingIDKeys[k] {
continue
}
if existing := strings.TrimSpace(b.candidate.ProviderIDs[k]); existing != "" && existing != v && slices.Contains(canonicalCandidateIDKeys, k) {
delete(b.candidate.ProviderIDs, k)
b.conflictingIDKeys[k] = true
continue
}
b.candidate.ProviderIDs[k] = v
}
// Track source providers.
@@ -242,6 +367,7 @@ func NormalizeCandidates(results []SearchResult, contentType string) []MatchCand
if b.candidate.ImageURL == "" && sr.ImageURL != "" {
b.candidate.ImageURL = sr.ImageURL
}
b.candidate.DetailScore = candidateDetailScore(b.candidate)
}
// Build final list preserving insertion order.
@@ -255,6 +381,12 @@ func NormalizeCandidates(results []SearchResult, contentType string) []MatchCand
}
sort.Strings(sources)
b.candidate.Sources = sources
for _, key := range canonicalCandidateIDKeys {
if b.conflictingIDKeys[key] {
b.candidate.ConflictingProviderIDKeys = append(b.candidate.ConflictingProviderIDKeys, key)
b.candidate.AgreementHints = append(b.candidate.AgreementHints, "quarantined_"+key+"_id")
}
}
// Compute agreement hints from corroborating sources only: a local
// sidecar (nfo) echoing a remote result is not provider agreement.
@@ -275,15 +407,122 @@ func NormalizeCandidates(results []SearchResult, contentType string) []MatchCand
return candidates
}
func preferredSearchResultTitle(result SearchResult, language string) (string, string, bool, int) {
requested := baseMetadataLanguage(language)
resultLanguage := baseMetadataLanguage(result.TitleLanguage)
if strings.TrimSpace(result.Name) != "" && requested != "" && resultLanguage == requested && !result.TitleIsFallback {
return result.Name, resultLanguage, false, 3
}
for _, alias := range result.TitleAliases {
if strings.TrimSpace(alias.Title) != "" && requested != "" && baseMetadataLanguage(alias.Language) == requested {
return alias.Title, requested, false, 2
}
}
// Older plugins predate the language contract. Their primary title keeps
// normal first-provider priority; its original_title must not silently
// replace a title that may already be localized.
if strings.TrimSpace(result.Name) != "" && resultLanguage == "" && !result.TitleIsFallback {
return result.Name, "", false, 3
}
if requested != "" && strings.TrimSpace(result.OriginalTitle) != "" {
return result.OriginalTitle, baseMetadataLanguage(result.OriginalLanguage), true, 1
}
if strings.TrimSpace(result.Name) != "" {
return result.Name, resultLanguage, result.TitleIsFallback, 1
}
return result.OriginalTitle, baseMetadataLanguage(result.OriginalLanguage), true, 1
}
func baseMetadataLanguage(language string) string {
return lang.Canonical(strings.ReplaceAll(language, "_", "-"))
}
func copyTitleAliases(aliases []TitleAlias, provider string) []TitleAlias {
out := make([]TitleAlias, 0, len(aliases))
for _, alias := range aliases {
if alias.Provider == "" {
alias.Provider = provider
}
out = appendUniqueTitleAlias(out, alias)
}
return out
}
func mergeCandidateTitles(candidate *MatchCandidate, result SearchResult, language string) {
if candidate == nil {
return
}
if candidate.OriginalTitle == "" && strings.TrimSpace(result.OriginalTitle) != "" {
candidate.OriginalTitle = result.OriginalTitle
}
for _, alias := range copyTitleAliases(result.TitleAliases, result.Provider) {
candidate.TitleAliases = appendUniqueTitleAlias(candidate.TitleAliases, alias)
}
if strings.TrimSpace(result.OriginalTitle) != "" && !strings.EqualFold(result.OriginalTitle, candidate.Title) {
candidate.TitleAliases = appendUniqueTitleAlias(candidate.TitleAliases, TitleAlias{
Title: result.OriginalTitle, Language: baseMetadataLanguage(result.OriginalLanguage), Kind: titleAliasKindOriginal, Provider: result.Provider,
})
}
if strings.TrimSpace(result.Name) != "" && !strings.EqualFold(result.Name, candidate.Title) {
candidate.TitleAliases = appendUniqueTitleAlias(candidate.TitleAliases, TitleAlias{
Title: result.Name, Language: baseMetadataLanguage(result.TitleLanguage), Kind: titleAliasKindLocalized, Provider: result.Provider,
})
}
title, titleLanguage, fallback, titleRank := preferredSearchResultTitle(result, language)
if strings.TrimSpace(title) != "" && titleRank > candidate.titleRank {
candidate.Title = title
candidate.TitleLanguage = titleLanguage
candidate.TitleIsFallback = fallback
candidate.titleRank = titleRank
}
}
func appendUniqueTitleAlias(aliases []TitleAlias, alias TitleAlias) []TitleAlias {
alias.Title = strings.TrimSpace(alias.Title)
if alias.Title == "" {
return aliases
}
for _, existing := range aliases {
if strings.EqualFold(existing.Title, alias.Title) &&
baseMetadataLanguage(existing.Language) == baseMetadataLanguage(alias.Language) &&
strings.EqualFold(existing.Kind, alias.Kind) &&
strings.EqualFold(existing.Provider, alias.Provider) {
return aliases
}
}
return append(aliases, alias)
}
func candidateDetailScore(candidate MatchCandidate) int {
score := providerIDRichness(candidate.ProviderIDs) * 10
if candidate.Year != 0 {
score += 15
}
if strings.TrimSpace(candidate.Overview) != "" {
score += 20
}
if strings.TrimSpace(candidate.ImageURL) != "" {
score += 15
}
return score
}
// SearchAndNormalize is a convenience method that calls SearchProviders and
// normalizes the results into MatchCandidates. Plugin-prefixed image URLs
// (e.g. "metadb://...") are resolved to presigned HTTP URLs before returning.
func (s *MetadataService) SearchAndNormalize(ctx context.Context, query SearchQuery, folderID int) ([]MatchCandidate, error) {
if strings.TrimSpace(query.Language) == "" {
query.Language = s.resolveFolderLanguage(ctx, folderID)
}
results, err := s.SearchProviders(ctx, query, folderID)
if err != nil {
return nil, err
}
candidates := NormalizeCandidates(results, query.ContentType)
candidates := NormalizeCandidatesForLanguage(results, query.ContentType, query.Language)
for i := range candidates {
annotateCandidateMatch(&candidates[i], &MatchHints{Title: query.Title, Year: query.Year, Type: query.ContentType})
}
if s.imageResolver != nil {
for i, c := range candidates {
@@ -300,11 +539,17 @@ func (s *MetadataService) SearchAndNormalize(ctx context.Context, query SearchQu
}
func scoreMatchCandidate(hints *MatchHints, candidate MatchCandidate) float64 {
score, _, _ := scoreMatchCandidateDetailed(hints, candidate)
return score
}
func scoreMatchCandidateDetailed(hints *MatchHints, candidate MatchCandidate) (float64, string, []string) {
if hints == nil {
return 0
return 0, "", nil
}
score := 0.0
reasons := make([]string, 0, 5)
trustedIDMatches := 0
for _, key := range trustedSearchIDKeys {
hintValue := trustedIDValue(hints, key)
@@ -314,26 +559,37 @@ func scoreMatchCandidate(hints *MatchHints, candidate MatchCandidate) float64 {
if candidate.ProviderIDs[key] == hintValue {
score += 100
trustedIDMatches++
reasons = append(reasons, "trusted_"+key+"_id")
}
}
if trustedIDMatches > 0 {
score += float64(trustedIDMatches * 10)
}
score += float64(len(candidate.Sources) * 12)
if sourceCount := candidateCorroboratingSourceCount(candidate); sourceCount > 0 {
score += float64(sourceCount * 12)
reasons = append(reasons, "provider_sources")
}
if strings.TrimSpace(hints.Title) != "" && strings.TrimSpace(candidate.Title) != "" {
titleSimilarity := inferTitleSimilarity(hints.Title, candidate.Title, hints.Year)
matchedTitle := ""
if strings.TrimSpace(hints.Title) != "" {
titleSimilarity, title := bestCandidateTitleSimilarity(hints.Title, candidate, hints.Year)
matchedTitle = title
if titleSimilarity == 1 {
score += 45
reasons = append(reasons, "exact_title")
} else {
score += titleSimilarity * 35
if titleSimilarity > 0 {
reasons = append(reasons, "coherent_title")
}
}
}
switch {
case hints.Year != 0 && candidate.Year == hints.Year:
score += 20
reasons = append(reasons, "exact_year")
case hints.Year != 0 && candidate.Year != 0 && math.Abs(float64(candidate.Year-hints.Year)) == 1:
score += 5
}
@@ -343,7 +599,41 @@ func scoreMatchCandidate(hints *MatchHints, candidate MatchCandidate) float64 {
score += float64(providerIDRichness(candidate.ProviderIDs))
}
return score
return score, matchedTitle, reasons
}
func candidateTitles(candidate MatchCandidate) []string {
titles := []string{candidate.Title, candidate.OriginalTitle}
for _, alias := range candidate.TitleAliases {
titles = append(titles, alias.Title)
}
return titles
}
func bestCandidateTitleSimilarity(hint string, candidate MatchCandidate, year int) (float64, string) {
bestScore := 0.0
bestTitle := ""
for _, title := range candidateTitles(candidate) {
score := inferTitleSimilarity(hint, title, year)
if score > bestScore {
bestScore = score
bestTitle = title
}
}
return bestScore, bestTitle
}
func annotateCandidateMatch(candidate *MatchCandidate, hints *MatchHints) {
if candidate == nil {
return
}
candidate.MatchScore, candidate.MatchedTitle, candidate.MatchReasons = scoreMatchCandidateDetailed(hints, *candidate)
if len(candidate.ConflictingProviderIDKeys) > 0 {
candidate.MatchReasons = append(candidate.MatchReasons, "provider_id_consensus")
for _, key := range candidate.ConflictingProviderIDKeys {
candidate.MatchReasons = append(candidate.MatchReasons, "quarantined_"+key+"_id")
}
}
}
type scoredMatchCandidate struct {
@@ -384,7 +674,7 @@ func candidatesAreSingleDistinctShow(best MatchCandidate, scored []scoredMatchCa
if c.candidate.Year == 0 || c.candidate.Year != best.Year {
return false
}
if inferTitleSimilarity(best.Title, c.candidate.Title, best.Year) != 1 {
if candidateToCandidateTitleSimilarity(best, c.candidate, best.Year) != 1 {
return false
}
for _, key := range canonicalCandidateIDKeys {
@@ -521,9 +811,10 @@ func selectInitialMatchCandidate(hints *MatchHints, candidates []MatchCandidate,
scoredCandidates := make([]scoredMatchCandidate, 0, len(candidates))
for _, candidate := range candidates {
annotateCandidateMatch(&candidate, hints)
scoredCandidates = append(scoredCandidates, scoredMatchCandidate{
candidate: candidate,
score: scoreMatchCandidate(hints, candidate),
score: candidate.MatchScore,
})
}
sort.SliceStable(scoredCandidates, func(i, j int) bool {
@@ -556,15 +847,49 @@ func selectInitialMatchCandidate(hints *MatchHints, candidates []MatchCandidate,
}
}
best := scoredCandidates[0]
if trustedHintIDsPresent(hints) {
if candidateMatchesTrustedIDs(hints, best.candidate) {
return &best.candidate, true
// Provider searches can return noisy title-ranked results even for an ID
// query. A trusted local ID is decisive, so choose the highest-scored
// candidate that actually carries the matching ID instead of requiring
// the provider's first result to be correct.
for i := range scoredCandidates {
if candidateMatchesTrustedIDs(hints, scoredCandidates[i].candidate) {
return &scoredCandidates[i].candidate, true
}
}
return nil, false
}
if best.score < 55 {
// A title-only local sidecar is useful metadata, but it is not an
// independent search result. Once a remote candidate with canonical IDs is
// available, remove ID-less candidates from the automatic selection set so
// their presence cannot turn the remote result into a misleading
// multi-candidate score-gap win. They remain in the original candidate list
// for diagnostics.
if anyCandidateHasProviderIDs(scoredCandidates) {
eligible := make([]scoredMatchCandidate, 0, len(scoredCandidates))
for _, scored := range scoredCandidates {
if providerIDRichness(scored.candidate.ProviderIDs) > 0 {
eligible = append(eligible, scored)
}
}
scoredCandidates = eligible
}
if len(scoredCandidates) == 0 {
return nil, false
}
best := scoredCandidates[0]
if best.score < automaticMatchAcceptanceFloor {
return nil, false
}
// A known local year that conflicts with the provider by more than the
// tolerated release-date window is negative evidence, not something a high
// source-count score may erase. Trusted external IDs were handled above and
// remain decisive; title-only matches must respect this guard even when two
// providers return the same canonical item.
if hints != nil && hints.Year != 0 && best.candidate.Year != 0 &&
absYearDelta(best.candidate.Year, hints.Year) > 2 {
return nil, false
}
// A search that resolves to a single distinct show (one candidate, or the
@@ -588,7 +913,7 @@ func selectInitialMatchCandidate(hints *MatchHints, candidates []MatchCandidate,
// strong independent corroboration — it stands in for a missing hint year
// (folders without a "(YYYY)"). A lone single-source no-year result is NOT
// accepted here and stays subject to the single-candidate >=70 gate.
multiSourceCorroborated := distinctSourceCount(topGroup) >= 2
multiSourceCorroborated := hints.Year == 0 && distinctSourceCount(topGroup) >= 2
// An exact normalized-title match on a sole distinct show is strong
// corroboration on its own, even when the folder year is off by a year
// or two (festival vs wide-release date, regional release) — e.g.
@@ -596,13 +921,14 @@ func selectInitialMatchCandidate(hints *MatchHints, candidates []MatchCandidate,
// Bounded to ±2 years so same-title remakes decades apart still require
// a year or multi-source match. Uses the same normalizer as title scoring
// so "exact" here means a perfect title-similarity component.
titleSimilarity, _ := bestCandidateTitleSimilarity(hints.Title, best.candidate, hints.Year)
titleCorroborated := hints.Year != 0 && best.candidate.Year != 0 &&
absYearDelta(best.candidate.Year, hints.Year) <= 2 &&
normalizeTitleForScoring(best.candidate.Title) == normalizeTitleForScoring(hints.Title)
titleSimilarity == 1
// Year or source-count corroboration is only meaningful when the winning
// candidate is at least title-coherent with the scanner hint. Otherwise a
// high source/provider score can auto-accept an unrelated same-year result.
hintTitleCoherent := inferTitleSimilarity(hints.Title, best.candidate.Title, hints.Year) > 0
hintTitleCoherent := titleSimilarity > 0
if (hintTitleCoherent && (yearCorroborated || multiSourceCorroborated)) || titleCorroborated {
return pickByProviderPriority(topGroup, providerPriority), true
}
@@ -670,7 +996,20 @@ func exactTitleYearTypeMatch(hints *MatchHints, candidate MatchCandidate) bool {
if !candidateTypeMatchesHint(hints.Type, candidate.ContentType) {
return false
}
return inferTitleSimilarity(hints.Title, candidate.Title, hints.Year) == 1
similarity, _ := bestCandidateTitleSimilarity(hints.Title, candidate, hints.Year)
return similarity == 1
}
func candidateToCandidateTitleSimilarity(left, right MatchCandidate, year int) float64 {
best := 0.0
for _, leftTitle := range candidateTitles(left) {
for _, rightTitle := range candidateTitles(right) {
if similarity := inferTitleSimilarity(leftTitle, rightTitle, year); similarity > best {
best = similarity
}
}
}
return best
}
func candidatePrimaryProvider(candidate MatchCandidate) string {
@@ -715,6 +1054,22 @@ func trustedHintIDsPresent(hints *MatchHints) bool {
return false
}
func candidatesConflictWithTrustedIDs(hints *MatchHints, candidates []MatchCandidate) bool {
if hints == nil {
return false
}
for _, candidate := range candidates {
for _, key := range trustedSearchIDKeys {
hintValue := strings.TrimSpace(trustedIDValue(hints, key))
candidateValue := strings.TrimSpace(candidate.ProviderIDs[key])
if hintValue != "" && candidateValue != "" && hintValue != candidateValue {
return true
}
}
}
return false
}
func candidateMatchesTrustedIDs(hints *MatchHints, candidate MatchCandidate) bool {
matched := false
for _, key := range trustedSearchIDKeys {
@@ -825,7 +1180,60 @@ func normalizeTitleForScoring(title string) string {
}
}
return strings.Join(strings.Fields(builder.String()), " ")
fields := strings.Fields(builder.String())
for i, field := range fields {
fields[i] = normalizeNumberWord(field)
}
return strings.Join(fields, " ")
}
var normalizedNumberWords = map[string]string{
"zero": "0", "zeroth": "0",
//nolint:goconst // This is a declarative number/ordinal lookup, not a domain label.
"one": "1", "first": "1",
"two": "2", "second": "2", //nolint:goconst // Ordinal word mapping, not a reusable domain value.
"three": "3", "third": "3",
"four": "4", "fourth": "4",
"five": "5", "fifth": "5",
"six": "6", "sixth": "6",
"seven": "7", "seventh": "7",
"eight": "8", "eighth": "8",
"nine": "9", "ninth": "9",
"ten": "10", "tenth": "10",
"eleven": "11", "eleventh": "11",
"twelve": "12", "twelfth": "12",
"thirteen": "13", "thirteenth": "13",
"fourteen": "14", "fourteenth": "14",
"fifteen": "15", "fifteenth": "15",
"sixteen": "16", "sixteenth": "16",
"seventeen": "17", "seventeenth": "17",
"eighteen": "18", "eighteenth": "18",
"nineteen": "19", "nineteenth": "19",
"twenty": "20", "twentieth": "20",
}
func normalizeNumberWord(value string) string {
if normalized, ok := normalizedNumberWords[value]; ok {
return normalized
}
for _, suffix := range []string{"st", "nd", "rd", "th"} {
if stem, ok := strings.CutSuffix(value, suffix); ok && isASCIIDigits(stem) {
return stem
}
}
return value
}
func isASCIIDigits(value string) bool {
if value == "" {
return false
}
for _, r := range value {
if r < '0' || r > '9' {
return false
}
}
return true
}
func normalizeNumericRune(r rune) (rune, bool) {
+406
View File
@@ -1,6 +1,8 @@
//nolint:goconst // Repeated titles and provider keys keep scoring fixtures explicit.
package metadata
import (
"errors"
"testing"
"github.com/Silo-Server/silo-server/internal/models"
@@ -32,6 +34,186 @@ func TestSelectInitialMatchCandidate_IgnoresLocalContentIDForTrustedSelection(t
}
}
func TestSelectInitialMatchCandidate_TrustedIDCanWinBelowNoisyTopResult(t *testing.T) {
t.Parallel()
winner, ok := selectInitialMatchCandidate(
&MatchHints{Title: "10 Tricks", Year: 2022, Type: "movie", ImdbID: "tt0473100"},
[]MatchCandidate{
{
Title: "10 Tricks",
Year: 2022,
ContentType: "movie",
ProviderIDs: map[string]string{"imdb": "tt9999999", "tmdb": "1"},
Sources: []string{"tmdb", "tvdb"},
},
{
Title: "Ten Tricks",
Year: 2021,
ContentType: "movie",
ProviderIDs: map[string]string{"imdb": "tt0473100", "tmdb": "2"},
Sources: []string{"tmdb"},
},
},
nil,
)
if !ok || winner == nil {
t.Fatal("expected candidate carrying the trusted IMDb ID to win")
}
if got := winner.ProviderIDs["imdb"]; got != "tt0473100" {
t.Fatalf("winner IMDb id = %q, want tt0473100", got)
}
}
func TestBuildMatchDecisionClassifiesProviderFailures(t *testing.T) {
t.Parallel()
tests := []struct {
name string
err error
outcome string
}{
{name: "rate limit is transient", err: errors.New("provider returned HTTP 429"), outcome: "provider_transient"},
{name: "bad request is permanent", err: errors.New("provider returned HTTP 400"), outcome: "provider_permanent"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
decision := buildMatchDecision(&MatchHints{Title: "Example"}, nil, nil, false, []error{tt.err})
if string(decision.Outcome) != tt.outcome {
t.Fatalf("outcome = %q, want %q", decision.Outcome, tt.outcome)
}
})
}
}
func TestBuildMatchDecisionOnlyReportsTrustedIDConflictForExplicitConflict(t *testing.T) {
t.Parallel()
hints := &MatchHints{Title: "10 Tricks", ImdbID: "tt0473100"}
missingID := buildMatchDecision(hints, []MatchCandidate{{
Title: "10 Tricks", ProviderIDs: map[string]string{"tmdb": "123"},
}}, nil, false, nil)
if missingID.Outcome != "candidate_rejected" {
t.Fatalf("candidate with no IMDb id outcome = %q, want candidate_rejected", missingID.Outcome)
}
conflictingID := buildMatchDecision(hints, []MatchCandidate{{
Title: "10 Tricks", ProviderIDs: map[string]string{"imdb": "tt9999999"},
}}, nil, false, nil)
if conflictingID.Outcome != "trusted_id_conflict" {
t.Fatalf("conflicting IMDb id outcome = %q, want trusted_id_conflict", conflictingID.Outcome)
}
}
func TestBuildMatchDecisionAttachesWinnerReasonsOnlyToIDLessWinner(t *testing.T) {
t.Parallel()
hints := &MatchHints{Title: "Shared", Year: 2020, Type: "series"}
candidates := []MatchCandidate{
{Title: "Shared", Year: 2020, ContentType: "series", Sources: []string{"first"}},
{Title: "Shared", Year: 2020, ContentType: "series", Sources: []string{"second"}},
}
winner := candidates[1]
winner.MatchReasons = append(winner.MatchReasons, "episode_title_corroboration:2_of_2")
decision := buildMatchDecision(hints, candidates, &winner, true, nil)
if len(decision.TopCandidates) != 2 {
t.Fatalf("top candidates = %d, want 2", len(decision.TopCandidates))
}
for _, candidate := range decision.TopCandidates {
hasWinnerReason := containsString(candidate.Reasons, "episode_title_corroboration:2_of_2")
if candidate.Sources[0] == "second" && !hasWinnerReason {
t.Fatalf("winner reasons = %v, want episode corroboration", candidate.Reasons)
}
if candidate.Sources[0] == "first" && hasWinnerReason {
t.Fatalf("non-winner reasons = %v, must not contain winner-only evidence", candidate.Reasons)
}
}
}
func TestScoreMatchCandidate_DoesNotCountLocalNFOAsProviderCorroboration(t *testing.T) {
t.Parallel()
hints := &MatchHints{Title: "Shared", Type: "series"}
remote := MatchCandidate{Title: "Shared", ContentType: "series", Sources: []string{"tmdb"}}
remoteAndNFO := remote
remoteAndNFO.Sources = []string{"tmdb", "nfo"}
remoteScore := scoreMatchCandidate(hints, remote)
if got := scoreMatchCandidate(hints, remoteAndNFO); got != remoteScore {
t.Fatalf("remote+nfo score = %.1f, want remote-only score %.1f", got, remoteScore)
}
}
func TestMergePreferredTitleMetadataReplacesExplicitFallback(t *testing.T) {
t.Parallel()
accumulator := &MetadataResult{
Title: "倒凶十将伝", TitleLanguage: "ja", TitleIsFallback: true,
}
mergePreferredTitleMetadata(accumulator, &MetadataResult{
Title: "10 Tokyo Warriors", TitleLanguage: "en",
OriginalTitle: "倒凶十将伝", OriginalLanguage: "ja",
}, "en", "tmdb", true)
if accumulator.Title != "10 Tokyo Warriors" || accumulator.TitleLanguage != "en" || accumulator.TitleIsFallback {
t.Fatalf("localized title = (%q, %q, %v)", accumulator.Title, accumulator.TitleLanguage, accumulator.TitleIsFallback)
}
}
func TestMergePreferredTitleMetadataPreservesUnclassifiedFirstProviderTitle(t *testing.T) {
t.Parallel()
accumulator := &MetadataResult{Title: "Sidecar Title"}
mergePreferredTitleMetadata(accumulator, &MetadataResult{
Title: "Localized Provider Title", TitleLanguage: "en",
}, "en", "tmdb", true)
if accumulator.Title != "Sidecar Title" {
t.Fatalf("title = %q, want first-provider sidecar title", accumulator.Title)
}
}
func TestMergePreferredTitleMetadataRequiresEveryProviderResponseToBeComplete(t *testing.T) {
t.Parallel()
accumulator := &MetadataResult{}
mergePreferredTitleMetadata(accumulator, &MetadataResult{
Title: "Complete Title", TitleLanguage: "en", TitleAliasesComplete: true,
}, "en", "tmdb", true)
mergePreferredTitleMetadata(accumulator, &MetadataResult{
Title: "Legacy Title", TitleLanguage: "en", TitleAliasesComplete: false,
}, "en", "tmdb", true)
if complete := accumulator.titleAliasProviders["tmdb"]; complete {
t.Fatal("mixed complete and partial responses granted alias deletion authority")
}
}
func TestPreferredTitlesPreserveOldPluginPrimaryWithoutLanguageMetadata(t *testing.T) {
t.Parallel()
search := SearchResult{Name: "Localized Legacy Title", OriginalTitle: "Native Title"}
title, language, fallback, rank := preferredSearchResultTitle(search, "en")
if title != "Localized Legacy Title" || language != "" || fallback || rank != 3 {
t.Fatalf("legacy search title = (%q, %q, %t, %d)", title, language, fallback, rank)
}
metadata := &MetadataResult{Title: "Localized Legacy Title", OriginalTitle: "Native Title"}
title, language, fallback, rank = preferredMetadataResultTitle(metadata, "en")
if title != "Localized Legacy Title" || language != "" || fallback || rank != 3 {
t.Fatalf("legacy metadata title = (%q, %q, %t, %d)", title, language, fallback, rank)
}
}
func TestSanitizeCandidateProviderIDsRequiresStrictPositiveIMDbID(t *testing.T) {
t.Parallel()
for _, invalid := range []string{"tt1", "tt0000000", "tt12345678901", "nm1234567", "1234567"} {
if got := sanitizeCandidateProviderIDs(map[string]string{"imdb": invalid}); len(got) != 0 {
t.Fatalf("sanitize imdb %q = %#v, want empty", invalid, got)
}
}
for _, valid := range []string{"tt0000001", "tt12345678", "tt123456789", "tt1234567890"} {
if got := sanitizeCandidateProviderIDs(map[string]string{"imdb": valid}); got["imdb"] != valid {
t.Fatalf("sanitize imdb %q = %#v", valid, got)
}
}
}
func TestSelectInitialMatchCandidate_SoleExactTitleYearOffByTwoMatches(t *testing.T) {
t.Parallel()
@@ -80,6 +262,31 @@ func TestSelectInitialMatchCandidate_SoleExactTitleYearOffByThreeRejected(t *tes
}
}
func TestSelectInitialMatchCandidate_CrossProviderAgreementDoesNotOverrideYearConflict(t *testing.T) {
t.Parallel()
// Both providers know the same show by the local alias, but the five-year
// conflict is evidence that this is a different work. Source agreement may
// replace a missing local year; it must not override a known conflicting one.
winner, ok := selectInitialMatchCandidate(
&MatchHints{Title: "The Piano Forest", Year: 2007, Type: "series"},
[]MatchCandidate{
{
Title: "Five Fingers",
TitleAliases: []TitleAlias{{Title: "Piano Forest", Language: "en", Kind: "alternate"}},
Year: 2012,
ContentType: "series",
ProviderIDs: map[string]string{"imdb": "tt2242048", "tvdb": "261129"},
Sources: []string{"tmdb", "tvdb"},
},
},
nil,
)
if ok || winner != nil {
t.Fatalf("expected cross-provider year conflict to be rejected, got ok=%v winner=%+v", ok, winner)
}
}
func TestSelectInitialMatchCandidate_SoleDifferentTitleExactYearStillFloored(t *testing.T) {
t.Parallel()
@@ -220,6 +427,67 @@ func TestNormalizeCandidates(t *testing.T) {
}
},
},
{
name: "merge two-ID consensus and quarantine conflicting third ID",
results: []SearchResult{
{
Name: "A Teacher",
Year: 2020,
Provider: "tvdb",
ProviderIDs: map[string]string{
"imdb": "tt10680614", "tmdb": "103992", "tvdb": "352440",
},
},
{
Name: "A Teacher",
Year: 2020,
Provider: "tmdb",
ProviderIDs: map[string]string{
"imdb": "tt10680614", "tmdb": "103992", "tvdb": "473725",
},
},
},
content: "series",
wantLen: 1,
check: func(t *testing.T, candidates []MatchCandidate) {
candidate := candidates[0]
if candidate.ProviderIDs["imdb"] != "tt10680614" || candidate.ProviderIDs["tmdb"] != "103992" {
t.Fatalf("agreed provider IDs = %+v", candidate.ProviderIDs)
}
if candidate.ProviderIDs["tvdb"] != "" {
t.Fatalf("conflicting TVDB ID was retained: %+v", candidate.ProviderIDs)
}
if len(candidate.ConflictingProviderIDKeys) != 1 || candidate.ConflictingProviderIDKeys[0] != "tvdb" {
t.Fatalf("quarantined keys = %v, want [tvdb]", candidate.ConflictingProviderIDKeys)
}
annotateCandidateMatch(&candidate, &MatchHints{Title: "A Teacher", Type: "series"})
if !containsString(candidate.MatchReasons, "provider_id_consensus") ||
!containsString(candidate.MatchReasons, "quarantined_tvdb_id") {
t.Fatalf("match reasons = %v", candidate.MatchReasons)
}
},
},
{
name: "discard malformed canonical provider cross references",
results: []SearchResult{
{
Name: "Fast & Furious: Spy Racers",
Year: 2019,
Provider: "tvdb",
ProviderIDs: map[string]string{
"imdb": "TT8322592", "tmdb": "95594-fast-furious-spy-racers", "tvdb": "362429",
},
},
},
content: "series",
wantLen: 1,
check: func(t *testing.T, candidates []MatchCandidate) {
ids := candidates[0].ProviderIDs
if ids["tmdb"] != "" || ids["tvdb"] != "362429" || ids["imdb"] != "tt8322592" {
t.Fatalf("sanitized provider IDs = %+v", ids)
}
},
},
{
name: "no recognized provider IDs gets synthetic key and stays separate",
results: []SearchResult{
@@ -388,6 +656,144 @@ func TestNormalizeCandidates(t *testing.T) {
}
}
func TestNormalizeCandidatesForLanguage_PrefersKnownLibraryAlias(t *testing.T) {
results := []SearchResult{
{
Name: "倒凶十将伝", OriginalTitle: "倒凶十将伝", OriginalLanguage: "ja",
TitleLanguage: "ja", TitleIsFallback: true,
TitleAliases: []TitleAlias{{Title: "10 Tokyo Warriors", Language: "en", Kind: "alternate"}},
Year: 1999, Provider: "tvdb", ProviderIDs: map[string]string{"tvdb": "123"},
},
{
Name: "10 Tokyo Warriors", OriginalTitle: "倒凶十将伝", OriginalLanguage: "ja",
TitleLanguage: "en",
Year: 1999, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "456", "tvdb": "123"},
},
}
candidates := NormalizeCandidatesForLanguage(results, "series", "en")
if len(candidates) != 1 {
t.Fatalf("candidates = %d, want 1: %+v", len(candidates), candidates)
}
if candidates[0].Title != "10 Tokyo Warriors" || candidates[0].OriginalTitle != "倒凶十将伝" {
t.Fatalf("localized candidate = %+v", candidates[0])
}
winner, ok := selectInitialMatchCandidate(&MatchHints{Title: "10 Tokyo Warriors", Type: "series"}, candidates, []string{"tvdb", "tmdb"})
if !ok || winner == nil {
t.Fatal("expected alias-coherent multi-source candidate to be selected")
}
if winner.MatchedTitle != "10 Tokyo Warriors" || winner.MatchScore < 70 {
t.Fatalf("winner diagnostics = %+v", winner)
}
}
func TestNormalizeCandidates_UnknownLanguageAliasMatchesButDoesNotBecomePrimary(t *testing.T) {
candidates := NormalizeCandidatesForLanguage([]SearchResult{{
Name: "倒凶十将伝", OriginalTitle: "倒凶十将伝", OriginalLanguage: "ja",
TitleLanguage: "ja", TitleIsFallback: true,
TitleAliases: []TitleAlias{{Title: "10 Tokyo Warriors", Kind: "alternate"}},
Year: 1999, Provider: "tvdb", ProviderIDs: map[string]string{"tvdb": "123"},
}}, "series", "en")
if got := candidates[0].Title; got != "倒凶十将伝" {
t.Fatalf("primary title = %q, want native fallback", got)
}
annotateCandidateMatch(&candidates[0], &MatchHints{Title: "10 Tokyo Warriors", Type: "series"})
if got := candidates[0].MatchedTitle; got != "10 Tokyo Warriors" {
t.Fatalf("matched title = %q", got)
}
}
func TestNormalizeCandidatesForLanguage_PrefersNativeTitleOverNonNativeFallback(t *testing.T) {
candidates := NormalizeCandidatesForLanguage([]SearchResult{{
Name: "Titre de secours", OriginalTitle: "Native Title", OriginalLanguage: "ja",
TitleLanguage: "fr", TitleIsFallback: true,
Provider: "tvdb", ProviderIDs: map[string]string{"tvdb": "123"},
}}, "series", "en")
if len(candidates) != 1 {
t.Fatalf("candidates = %d, want 1", len(candidates))
}
if got := candidates[0].Title; got != "Native Title" {
t.Fatalf("primary title = %q, want provider-confirmed native title", got)
}
if got := candidates[0].TitleLanguage; got != "ja" {
t.Fatalf("title language = %q, want ja", got)
}
}
func TestPreferredMetadataResultTitle_PrefersNativeTitleOverNonNativeFallback(t *testing.T) {
title, language, fallback, rank := preferredMetadataResultTitle(&MetadataResult{
Title: "Titre de secours", OriginalTitle: "Native Title", OriginalLanguage: "jpn",
TitleLanguage: "fr", TitleIsFallback: true,
}, "en")
if title != "Native Title" || language != "ja" || !fallback || rank != 1 {
t.Fatalf("preferred title = %q, %q, %t, %d", title, language, fallback, rank)
}
}
func TestMergePreferredTitleMetadataClassifiesNativeFallbackAsOriginalAlias(t *testing.T) {
accumulator := &MetadataResult{}
mergePreferredTitleMetadata(accumulator, &MetadataResult{
Title: "倒凶十将伝", OriginalTitle: "倒凶十将伝", OriginalLanguage: "ja",
TitleLanguage: "ja", TitleIsFallback: true,
}, "en", "tvdb", true)
if len(accumulator.TitleAliases) != 1 || accumulator.TitleAliases[0].Kind != "original" || accumulator.TitleAliases[0].Language != "ja" {
t.Fatalf("native aliases = %#v", accumulator.TitleAliases)
}
}
func TestMergePreferredTitleMetadataDoesNotAttributeIdentityHintAliases(t *testing.T) {
t.Parallel()
accumulator := &MetadataResult{}
mergePreferredTitleMetadata(accumulator, &MetadataResult{
Title: "Curated Sidecar Title",
TitleLanguage: "en",
OriginalTitle: "Native Sidecar Title",
OriginalLanguage: "ja",
TitleAliases: []TitleAlias{{Title: "Sidecar Alternate", Language: "en", Kind: "alternate"}},
TitleAliasesComplete: true,
}, "en", "nfo", false)
if accumulator.Title != "Curated Sidecar Title" || accumulator.OriginalTitle != "Native Sidecar Title" {
t.Fatalf("metadata fields were not retained: %#v", accumulator)
}
if len(accumulator.TitleAliases) != 0 {
t.Fatalf("identity-hint aliases = %#v, want none", accumulator.TitleAliases)
}
if _, attributed := accumulator.titleAliasProviders["nfo"]; attributed {
t.Fatal("identity-hint provider received alias persistence authority")
}
}
func TestSelectInitialMatchCandidateDirectIMDbIDFor10Tricks(t *testing.T) {
t.Parallel()
candidates := NormalizeCandidatesForLanguage([]SearchResult{{
Name: "Ten Tricks", Year: 2022, Provider: "tmdb",
ProviderIDs: map[string]string{"imdb": "tt0473100", "tmdb": "12345"},
}}, "movie", "en")
winner, ok := selectInitialMatchCandidate(&MatchHints{
Title: "10 Tricks", Year: 2022, Type: "movie", ImdbID: "tt0473100",
}, candidates, []string{"tmdb"})
if !ok || winner == nil {
t.Fatal("trusted IMDb ID did not produce a decisive match")
}
if winner.ProviderIDs["imdb"] != "tt0473100" || winner.MatchScore < 100 {
t.Fatalf("winner = %#v", winner)
}
}
func TestNormalizeTitleForScoring_NumberWordsAndOrdinals(t *testing.T) {
for _, pair := range [][2]string{
{"10 Tricks", "Ten Tricks"},
{"Dune Part Two", "Dune Part 2"},
{"The 10th Kingdom", "The Tenth Kingdom"},
} {
if got, want := normalizeTitleForScoring(pair[0]), normalizeTitleForScoring(pair[1]); got != want {
t.Errorf("normalize %q = %q, %q = %q", pair[0], got, pair[1], want)
}
}
}
func TestSelectInitialMatchCandidate_AcceptsSinglePunctuationEquivalentCandidate(t *testing.T) {
tests := []struct {
name string
@@ -0,0 +1,54 @@
package metadata
import (
"strconv"
"strings"
)
const maxProviderIdentityFallbacks = 3
func compactAlternateMatchIdentities(hints *MatchHints) []MatchIdentityHint {
if hints == nil || len(hints.AlternateIdentities) == 0 {
return nil
}
seen := map[string]struct{}{
matchIdentityComparisonKey(hints.Title, hints.Year): {},
}
out := make([]MatchIdentityHint, 0, min(len(hints.AlternateIdentities), maxProviderIdentityFallbacks))
for _, alternate := range hints.AlternateIdentities {
alternate.Title = strings.Join(strings.Fields(strings.TrimSpace(alternate.Title)), " ")
key := matchIdentityComparisonKey(alternate.Title, alternate.Year)
if alternate.Title == "" || key == "" {
continue
}
if _, exists := seen[key]; exists {
continue
}
seen[key] = struct{}{}
out = append(out, alternate)
if len(out) == maxProviderIdentityFallbacks {
break
}
}
return out
}
func matchIdentityComparisonKey(title string, year int) string {
normalized := strings.Join(strings.Fields(normalizeTitleForScoring(title)), " ")
if normalized == "" {
return ""
}
return normalized + "\x00" + strconv.Itoa(year)
}
func bestCandidateScore(hints *MatchHints, candidates []MatchCandidate) float64 {
best := 0.0
for _, candidate := range candidates {
score, _, _ := scoreMatchCandidateDetailed(hints, candidate)
if score > best {
best = score
}
}
return best
}
@@ -0,0 +1,126 @@
//nolint:goconst // Repeated titles and provider IDs document each fallback scenario.
package metadata
import (
"context"
"sync"
"testing"
)
type identityFallbackProvider struct {
mu sync.Mutex
queries []SearchQuery
}
type typeMismatchProvider struct{}
func (p *typeMismatchProvider) Slug() string { return "tmdb" }
func (p *typeMismatchProvider) Name() string { return "TMDB" }
func (p *typeMismatchProvider) ForTypes() []string { return []string{"movie", "series"} }
func (p *typeMismatchProvider) Search(_ context.Context, query SearchQuery) ([]SearchResult, error) {
if query.ContentType != "series" || query.ProviderIDs["imdb"] != "tt10473306" {
return nil, nil
}
return []SearchResult{{
Name: "Are You Afraid of the Dark?", Year: 2019, Provider: "tmdb",
ProviderIDs: map[string]string{"imdb": "tt10473306", "tmdb": "83755"},
}}, nil
}
func (p *typeMismatchProvider) GetMetadata(context.Context, MetadataRequest) (*MetadataResult, error) {
return &MetadataResult{}, nil
}
func (p *identityFallbackProvider) Slug() string { return "tmdb" }
func (p *identityFallbackProvider) Name() string { return "TMDB" }
func (p *identityFallbackProvider) ForTypes() []string { return []string{"movie"} }
func (p *identityFallbackProvider) Search(_ context.Context, query SearchQuery) ([]SearchResult, error) {
p.mu.Lock()
p.queries = append(p.queries, query)
p.mu.Unlock()
if query.Title != "Alien Invasion" || query.Year != 2018 {
return nil, nil
}
return []SearchResult{{
Name: "Alien Invasion", Year: 2018, Provider: "tmdb",
ProviderIDs: map[string]string{"tmdb": "4242"},
}}, nil
}
func (p *identityFallbackProvider) GetMetadata(_ context.Context, req MetadataRequest) (*MetadataResult, error) {
return &MetadataResult{
HasMetadata: true, Title: "Alien Invasion", Year: 2018,
ProviderIDs: map[string]string{"tmdb": req.ProviderIDs["tmdb"]},
}, nil
}
func TestInitialMatchUsesAlternatePathIdentityAfterPrimaryMiss(t *testing.T) {
t.Parallel()
harness := newTestHarness()
provider := &identityFallbackProvider{}
result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{
Hints: &MatchHints{
Title: "After the Lethargy", Year: 2018, Type: "movie",
AlternateIdentities: []MatchIdentityHint{{Title: "Alien Invasion", Year: 2018, Source: "filename"}},
},
Mode: ModeInitialMatch,
}, []Provider{provider})
if err != nil {
t.Fatalf("ProcessWithProviders() error = %v", err)
}
if result == nil || !result.Updated || result.Decision == nil || result.Decision.Outcome != "matched" {
t.Fatalf("result = %#v, want alternate-identity match", result)
}
item, err := harness.itemRepo.GetByID(context.Background(), result.ContentID)
if err != nil {
t.Fatalf("load item: %v", err)
}
if item.Title != "Alien Invasion" || item.TmdbID != "4242" {
t.Fatalf("item = title %q tmdb %q", item.Title, item.TmdbID)
}
provider.mu.Lock()
defer provider.mu.Unlock()
if len(provider.queries) != 2 || provider.queries[1].Title != "Alien Invasion" {
t.Fatalf("queries = %#v, want primary then filename fallback", provider.queries)
}
}
func TestCompactAlternateMatchIdentitiesDeduplicatesAndBounds(t *testing.T) {
t.Parallel()
hints := &MatchHints{
Title: "Primary", Year: 2020,
AlternateIdentities: []MatchIdentityHint{
{Title: "Primary", Year: 2020},
{Title: "A", Year: 2020},
{Title: "A", Year: 2020},
{Title: "B", Year: 2021},
{Title: "C", Year: 2022},
{Title: "D", Year: 2023},
},
}
got := compactAlternateMatchIdentities(hints)
if len(got) != maxProviderIdentityFallbacks || got[0].Title != "A" || got[2].Title != "C" {
t.Fatalf("compact identities = %#v", got)
}
}
func TestInitialMatchDiagnosesTrustedIDInOppositeContentType(t *testing.T) {
t.Parallel()
harness := newTestHarness()
result, err := harness.service.ProcessWithProviders(context.Background(), ProcessRequest{
Hints: &MatchHints{
Title: "Are You Afraid of the Dark!", Year: 2019, Type: "movie", ImdbID: "tt10473306",
},
Mode: ModeInitialMatch,
}, []Provider{&typeMismatchProvider{}})
if err != nil {
t.Fatalf("ProcessWithProviders() error = %v", err)
}
if result == nil || result.Decision == nil || result.Decision.Outcome != "trusted_id_type_mismatch" {
t.Fatalf("result = %#v, want trusted-ID type mismatch", result)
}
if result.Decision.CandidateCount != 1 {
t.Fatalf("decision = %+v, want one opposite-type candidate", result.Decision)
}
}
@@ -0,0 +1,64 @@
package metadata
import (
"context"
"strings"
)
func (s *MetadataService) trustedIDTypeMismatchCandidates(
ctx context.Context,
hints *MatchHints,
providerIDs map[string]string,
language string,
resolveChain chainResolver,
) []MatchCandidate {
if hints == nil || !trustedHintIDsPresent(hints) || resolveChain == nil {
return nil
}
var oppositeType string
switch strings.ToLower(strings.TrimSpace(hints.Type)) {
case matchContentTypeMovie:
oppositeType = matchContentTypeSeries
case matchContentTypeSeries:
oppositeType = matchContentTypeMovie
default:
return nil
}
chain, err := resolveChain(providerChainContentLevel(oppositeType))
if err != nil {
return nil
}
query := SearchQuery{
ContentType: oppositeType,
ProviderIDs: copyMap(providerIDs),
Language: language,
FilePath: hints.FilePath,
RepresentativeFilePath: hints.RepresentativeFilePath,
ObservedRootPath: hints.ObservedRootPath,
AllGroupFilePaths: append([]string(nil), hints.AllGroupFilePaths...),
PrimarySidecarSearchPaths: append([]string(nil), hints.PrimarySidecarSearchPaths...),
}
results := make([]SearchResult, 0)
for _, provider := range chain {
searchProvider, ok := provider.(SearchProvider)
if !ok {
continue
}
providerResults, searchErr := searchProvider.Search(ctx, query)
if searchErr != nil {
// This is optional diagnosis. A provider outage must not turn a
// deterministic same-type miss into a transient queue failure.
continue
}
for _, result := range providerResults {
candidate := MatchCandidate{ProviderIDs: result.ProviderIDs}
if candidateMatchesTrustedIDs(hints, candidate) {
results = append(results, result)
}
}
}
return NormalizeCandidatesForLanguage(results, oppositeType, language)
}
+68
View File
@@ -1,3 +1,4 @@
//nolint:goconst // Repeated titles and provider IDs keep NFO scenarios recognizable.
package metadata
import (
@@ -99,6 +100,73 @@ func (p *remoteStubProvider) lastMetadataIDs() map[string]string {
return copyMap(p.lastMetaIDs)
}
func (p *remoteStubProvider) lastSearchProviderIDs() map[string]string {
p.mu.Lock()
defer p.mu.Unlock()
return copyMap(p.lastSearchIDs)
}
func TestInitialMatch_MalformedNFOIdentityDoesNotSuppressTitleSearch(t *testing.T) {
t.Parallel()
h := newTestHarness()
nfo := &localHintStubProvider{
hints: map[string]string{"tmdb": "not-a-number", "imdb": "nm1234567"},
searchResults: []SearchResult{{Name: "The Matrix", Year: 1999, Provider: "nfo", ProviderIDs: map[string]string{"tmdb": "not-a-number"}}},
metadata: &MetadataResult{HasMetadata: true, Title: "The Matrix"},
}
remote := &remoteStubProvider{
slug: "tmdb",
searchResults: []SearchResult{{
Name: "The Matrix", Year: 1999, Provider: "tmdb", ProviderIDs: map[string]string{"tmdb": "603", "imdb": "tt0133093"},
}},
metadata: &MetadataResult{HasMetadata: true, Title: "The Matrix", Year: 1999, ProviderIDs: map[string]string{"tmdb": "603", "imdb": "tt0133093"}},
}
result, err := h.service.ProcessWithProviders(context.Background(), ProcessRequest{
Hints: &MatchHints{Title: "The Matrix", Year: 1999, Type: "movie"},
Mode: ModeInitialMatch,
}, []Provider{nfo, remote})
if err != nil {
t.Fatalf("ProcessWithProviders() error = %v", err)
}
if result == nil || !result.Updated {
t.Fatalf("result = %#v, want title-based remote match", result)
}
if ids := remote.lastSearchProviderIDs(); ids["tmdb"] != "" || ids["imdb"] != "" {
t.Fatalf("remote search received malformed trusted IDs: %#v", ids)
}
}
func TestInitialMatch_MalformedScannerIdentityDoesNotSuppressTitleSearch(t *testing.T) {
t.Parallel()
h := newTestHarness()
remote := &remoteStubProvider{
slug: "tmdb",
searchResults: []SearchResult{{
Name: "The Matrix", Year: 1999, Provider: "tmdb",
ProviderIDs: map[string]string{"tmdb": "603", "imdb": "tt0133093"},
}},
metadata: &MetadataResult{HasMetadata: true, Title: "The Matrix", Year: 1999, ProviderIDs: map[string]string{"tmdb": "603", "imdb": "tt0133093"}},
}
result, err := h.service.ProcessWithProviders(context.Background(), ProcessRequest{
Hints: &MatchHints{
Title: "The Matrix", Year: 1999, Type: "movie",
TmdbID: "not-a-number", ImdbID: "nm1234567",
},
Mode: ModeInitialMatch,
}, []Provider{remote})
if err != nil {
t.Fatalf("ProcessWithProviders() error = %v", err)
}
if result == nil || !result.Updated {
t.Fatalf("result = %#v, want title-based remote match", result)
}
if ids := remote.lastSearchProviderIDs(); ids["tmdb"] != "" || ids["imdb"] != "" {
t.Fatalf("remote search received malformed scanner IDs: %#v", ids)
}
}
func seedMovieItem(t *testing.T, h *testHarness, contentID, title string, year int) {
t.Helper()
if err := h.itemRepo.Upsert(context.Background(), &models.MediaItem{
+58 -36
View File
@@ -202,12 +202,17 @@ func (p *PluginProvider) Search(ctx context.Context, query SearchQuery) ([]Searc
results := make([]SearchResult, 0, len(response.GetResults()))
for _, result := range response.GetResults() {
results = append(results, SearchResult{
Name: result.GetTitle(),
Year: int(result.GetYear()),
ProviderIDs: mergePluginProviderIDs(p.capabilityID, result.GetProviderId(), result.GetProviderIds()),
ImageURL: result.GetImageUrl(),
Overview: result.GetOverview(),
Provider: p.Slug(),
Name: result.GetTitle(),
OriginalTitle: result.GetOriginalTitle(),
TitleAliases: titleAliasesFromPlugin(result.GetTitleAliases(), p.Slug()),
TitleLanguage: result.GetTitleLanguage(),
TitleIsFallback: result.GetTitleIsFallback(),
OriginalLanguage: result.GetOriginalLanguage(),
Year: int(result.GetYear()),
ProviderIDs: mergePluginProviderIDs(p.capabilityID, result.GetProviderId(), result.GetProviderIds()),
ImageURL: result.GetImageUrl(),
Overview: result.GetOverview(),
Provider: p.Slug(),
})
}
return results, nil
@@ -244,39 +249,56 @@ func (p *PluginProvider) GetMetadata(ctx context.Context, req MetadataRequest) (
}
return &MetadataResult{
HasMetadata: true,
ProviderIDs: mergePluginProviderIDs(p.capabilityID, response.GetItem().GetProviderId(), response.GetItem().GetProviderIds()),
Title: response.GetItem().GetTitle(),
OriginalTitle: response.GetItem().GetOriginalTitle(),
SortTitle: response.GetItem().GetSortTitle(),
Overview: response.GetItem().GetOverview(),
Tagline: response.GetItem().GetTagline(),
Year: int(response.GetItem().GetYear()),
Runtime: int(response.GetItem().GetRuntime()),
Genres: append([]string(nil), response.GetItem().GetGenres()...),
Keywords: keywordsFromPluginMetadata(response.GetItem().GetMetadata()),
Studios: append([]string(nil), response.GetItem().GetStudios()...),
Networks: append([]string(nil), response.GetItem().GetNetworks()...),
Countries: append([]string(nil), response.GetItem().GetCountries()...),
OriginalLanguage: response.GetItem().GetOriginalLanguage(),
ContentRating: response.GetItem().GetContentRating(),
Ratings: ratingsFromStruct(response.GetItem().GetRatings()),
People: peopleFromRecords(response.GetItem().GetPeople()),
Videos: videosFromRecords(p.Slug(), response.GetItem().GetVideos()),
PosterPath: response.GetItem().GetPosterPath(),
PosterThumbhash: response.GetItem().GetPosterThumbhash(),
BackdropPath: response.GetItem().GetBackdropPath(),
ShowStatus: response.GetItem().GetStatus(),
BackdropThumbhash: response.GetItem().GetBackdropThumbhash(),
LogoPath: response.GetItem().GetLogoPath(),
SeasonCount: int(response.GetItem().GetSeasonCount()),
FirstAirDate: response.GetItem().GetFirstAirDate(),
LastAirDate: response.GetItem().GetLastAirDate(),
AirTime: response.GetItem().GetAirTime(),
ReleaseDate: response.GetItem().GetReleaseDate(),
HasMetadata: true,
ProviderIDs: mergePluginProviderIDs(p.capabilityID, response.GetItem().GetProviderId(), response.GetItem().GetProviderIds()),
Title: response.GetItem().GetTitle(),
OriginalTitle: response.GetItem().GetOriginalTitle(),
TitleAliases: titleAliasesFromPlugin(response.GetItem().GetTitleAliases(), p.Slug()),
TitleAliasesComplete: response.GetItem().GetTitleAliasesComplete(),
TitleLanguage: response.GetItem().GetTitleLanguage(),
TitleIsFallback: response.GetItem().GetTitleIsFallback(),
SortTitle: response.GetItem().GetSortTitle(),
Overview: response.GetItem().GetOverview(),
Tagline: response.GetItem().GetTagline(),
Year: int(response.GetItem().GetYear()),
Runtime: int(response.GetItem().GetRuntime()),
Genres: append([]string(nil), response.GetItem().GetGenres()...),
Keywords: keywordsFromPluginMetadata(response.GetItem().GetMetadata()),
Studios: append([]string(nil), response.GetItem().GetStudios()...),
Networks: append([]string(nil), response.GetItem().GetNetworks()...),
Countries: append([]string(nil), response.GetItem().GetCountries()...),
OriginalLanguage: response.GetItem().GetOriginalLanguage(),
ContentRating: response.GetItem().GetContentRating(),
Ratings: ratingsFromStruct(response.GetItem().GetRatings()),
People: peopleFromRecords(response.GetItem().GetPeople()),
Videos: videosFromRecords(p.Slug(), response.GetItem().GetVideos()),
PosterPath: response.GetItem().GetPosterPath(),
PosterThumbhash: response.GetItem().GetPosterThumbhash(),
BackdropPath: response.GetItem().GetBackdropPath(),
ShowStatus: response.GetItem().GetStatus(),
BackdropThumbhash: response.GetItem().GetBackdropThumbhash(),
LogoPath: response.GetItem().GetLogoPath(),
SeasonCount: int(response.GetItem().GetSeasonCount()),
FirstAirDate: response.GetItem().GetFirstAirDate(),
LastAirDate: response.GetItem().GetLastAirDate(),
AirTime: response.GetItem().GetAirTime(),
ReleaseDate: response.GetItem().GetReleaseDate(),
}, nil
}
func titleAliasesFromPlugin(aliases []*pluginv1.TitleAlias, provider string) []TitleAlias {
out := make([]TitleAlias, 0, len(aliases))
for _, alias := range aliases {
if alias == nil || strings.TrimSpace(alias.GetTitle()) == "" {
continue
}
out = append(out, TitleAlias{
Title: alias.GetTitle(), Language: alias.GetLanguage(), Kind: alias.GetKind(), Provider: provider,
})
}
return out
}
func (p *PluginProvider) GetPersonDetail(ctx context.Context, req PersonDetailRequest) (*PersonDetailResult, error) {
client, err := p.clientFactory(ctx, p.installationID, p.capabilityID)
if err != nil {
+426 -64
View File
@@ -25,6 +25,8 @@ import (
"github.com/Silo-Server/silo-server/internal/scanner"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
)
// FileContentUpdater updates content_id on media_files.
@@ -297,6 +299,7 @@ type MetadataService struct {
libraryRepo metadataLibraryRepo
folderRepo metadataFolderRepo
itemLocalizationRepo *catalog.MediaItemLocalizationRepository
itemAliasRepo *catalog.ItemAliasRepository
seasonLocalizationRepo *catalog.SeasonLocalizationRepository
episodeLocalizationRepo *catalog.EpisodeLocalizationRepository
autoTranslator AutoTranslator // optional; set via SetAutoTranslator
@@ -387,6 +390,7 @@ func NewMetadataService(
rootClaimRepo *catalog.RootClaimRepository,
) *MetadataService {
var itemLocalizationRepo *catalog.MediaItemLocalizationRepository
var itemAliasRepo *catalog.ItemAliasRepository
var seasonLocalizationRepo *catalog.SeasonLocalizationRepository
var episodeLocalizationRepo *catalog.EpisodeLocalizationRepository
var scannedRootRepo metadataScannedRootRepo
@@ -401,6 +405,7 @@ func NewMetadataService(
dbPool = pool
videoRepo = catalog.NewVideoRepository(pool)
itemLocalizationRepo = catalog.NewMediaItemLocalizationRepository(pool)
itemAliasRepo = catalog.NewItemAliasRepository(pool)
seasonLocalizationRepo = catalog.NewSeasonLocalizationRepository(pool)
episodeLocalizationRepo = catalog.NewEpisodeLocalizationRepository(pool)
scannedRootRepo = scanner.NewScannedRootRepository(pool)
@@ -420,6 +425,7 @@ func NewMetadataService(
libraryRepo: libraryRepo,
folderRepo: folderRepo,
itemLocalizationRepo: itemLocalizationRepo,
itemAliasRepo: itemAliasRepo,
seasonLocalizationRepo: seasonLocalizationRepo,
episodeLocalizationRepo: episodeLocalizationRepo,
personRepo: personRepo,
@@ -569,6 +575,9 @@ func (s *MetadataService) Process(ctx context.Context, req ProcessRequest) (*Pro
}
final.IsNew = final.IsNew || result.IsNew
final.Updated = final.Updated || result.Updated
if result.Decision != nil {
final.Decision = result.Decision
}
}
s.maybeAutoTranslate(ctx, folderID, final.ContentID)
@@ -1140,6 +1149,7 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
// Phase 1: Search — find provider IDs.
accumulatedIDs := make(map[string]string)
maps.Copy(accumulatedIDs, req.ProviderIDs)
sanitizeCanonicalProviderIDsInPlace(accumulatedIDs)
// Track provider 404s as stale external IDs. This applies to initial match
// as well so bad embedded folder/file IDs can be recorded and dropped
@@ -1148,25 +1158,29 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
if req.ContentID != "" {
provider404s = make(map[string]string)
}
var matchDecision *MatchDecision
var providerMatchErrors []error
quarantinedProviderIDKeys := make(map[string]struct{})
switch req.Mode {
case ModeInitialMatch:
if req.Hints == nil {
return nil, fmt.Errorf("initial match requires hints")
}
selectionHints := sanitizedMatchHintProviderIDs(req.Hints)
// Seed external IDs from hints. Hints.ContentID is Silo's local
// skeleton item ID, not a searchable provider ID.
if req.Hints.FileHash != "" {
accumulatedIDs["oshash"] = req.Hints.FileHash
if selectionHints.FileHash != "" {
accumulatedIDs["oshash"] = selectionHints.FileHash
}
if req.Hints.TmdbID != "" {
accumulatedIDs["tmdb"] = req.Hints.TmdbID
if selectionHints.TmdbID != "" {
accumulatedIDs["tmdb"] = selectionHints.TmdbID
}
if req.Hints.TvdbID != "" {
accumulatedIDs["tvdb"] = req.Hints.TvdbID
if selectionHints.TvdbID != "" {
accumulatedIDs["tvdb"] = selectionHints.TvdbID
}
if req.Hints.ImdbID != "" {
accumulatedIDs["imdb"] = req.Hints.ImdbID
if selectionHints.ImdbID != "" {
accumulatedIDs["imdb"] = selectionHints.ImdbID
}
// Add filepath for local providers.
if req.Hints.RepresentativeFilePath != "" {
@@ -1200,53 +1214,58 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
protectedKeys[key] = true
}
wonHints := applyBuiltinIdentityHints(ctx, itemChain, searchQuery, accumulatedIDs, protectedKeys)
selectionHints := req.Hints
if len(wonHints) > 0 {
hintsCopy := *req.Hints
hintsCopy := *selectionHints
overrideHintIDs(&hintsCopy, wonHints)
selectionHints = &hintsCopy
}
searchQuery = suppressTitleYearFallbackForTrustedIDs(searchQuery)
allResults := make([]SearchResult, 0)
for _, p := range itemChain {
sp, ok := p.(SearchProvider)
if !ok {
continue
}
results, err := sp.Search(ctx, searchQuery)
if err != nil {
if handleProvider404(provider404s, accumulatedIDs, p.Slug(), err,
"title", req.Hints.Title,
"year", req.Hints.Year,
) {
searchProviders := func(query SearchQuery) []SearchResult {
resultsForQuery := make([]SearchResult, 0)
for _, p := range itemChain {
sp, ok := p.(SearchProvider)
if !ok {
continue
}
slog.WarnContext(ctx, "metadata: search provider error", "component", "metadata",
"provider", p.Slug(), "error", err)
continue
}
slog.DebugContext(ctx, "metadata: provider search result", "component", "metadata",
"provider", p.Slug(),
"query_title", searchQuery.Title,
"query_year", searchQuery.Year,
"result_count", len(results),
)
for _, result := range results {
if searchResultConflictsWithTrustedIDs(accumulatedIDs, result.ProviderIDs) {
slog.WarnContext(ctx, "metadata: skipping conflicting search result", "component", "metadata",
"provider", p.Slug(),
"title", req.Hints.Title,
"year", req.Hints.Year,
"hinted_ids", accumulatedIDs,
"candidate_ids", result.ProviderIDs,
)
results, searchErr := sp.Search(ctx, query)
if searchErr != nil {
if handleProvider404(provider404s, accumulatedIDs, p.Slug(), searchErr,
"title", query.Title,
"year", query.Year,
) {
providerMatchErrors = append(providerMatchErrors, searchErr)
continue
}
slog.WarnContext(ctx, "metadata: search provider error", "component", "metadata",
"provider", p.Slug(), "error", searchErr)
providerMatchErrors = append(providerMatchErrors, searchErr)
continue
}
allResults = append(allResults, result)
slog.DebugContext(ctx, "metadata: provider search result", "component", "metadata",
"provider", p.Slug(),
"query_title", query.Title,
"query_year", query.Year,
"result_count", len(results),
)
for _, result := range results {
if searchResultConflictsWithTrustedIDs(accumulatedIDs, result.ProviderIDs) {
slog.WarnContext(ctx, "metadata: retaining conflicting search result for rejection diagnostics", "component", "metadata",
"provider", p.Slug(),
"title", query.Title,
"year", query.Year,
"hinted_ids", accumulatedIDs,
"candidate_ids", result.ProviderIDs,
)
}
resultsForQuery = append(resultsForQuery, result)
}
}
return resultsForQuery
}
candidates := NormalizeCandidates(allResults, contentType)
allResults := searchProviders(searchQuery)
candidates := NormalizeCandidatesForLanguage(allResults, contentType, searchQuery.Language)
slog.DebugContext(ctx, "metadata: search candidates assembled", "component", "metadata",
"query_title", searchQuery.Title,
"query_year", searchQuery.Year,
@@ -1257,17 +1276,74 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
for _, p := range itemChain {
providerPriority = append(providerPriority, p.Slug())
}
if winner, ok := selectInitialMatchCandidate(selectionHints, candidates, providerPriority); ok && winner != nil {
for k, v := range winner.ProviderIDs {
if v != "" {
accumulatedIDs[k] = v
trySelection := func(hints *MatchHints, selectionCandidates []MatchCandidate) (*MatchCandidate, bool) {
selected, ok := selectInitialMatchCandidate(hints, selectionCandidates, providerPriority)
if ok {
return selected, true
}
episodeWinner, validationErrors := validateSeriesMatchByEpisodes(
ctx, hints, selectionCandidates, itemChain, searchQuery.Language,
)
providerMatchErrors = append(providerMatchErrors, validationErrors...)
return episodeWinner, episodeWinner != nil
}
effectiveHints := selectionHints
winner, matched := trySelection(selectionHints, candidates)
// Structured IDs remain decisive and never fall back to a title search.
// For title/year identities, try independently parsed path hypotheses only
// after the primary identity fails. Each attempt is bounded and scored
// against the identity that produced its provider query.
if !matched && !trustedHintIDsPresent(selectionHints) {
for _, alternate := range compactAlternateMatchIdentities(selectionHints) {
alternateHints := *selectionHints
alternateHints.Title = alternate.Title
alternateHints.Year = alternate.Year
alternateHints.HintSource = alternate.Source
alternateHints.AlternateIdentities = nil
alternateQuery := searchQuery
alternateQuery.Title = alternate.Title
alternateQuery.Year = alternate.Year
alternateResults := searchProviders(alternateQuery)
alternateCandidates := NormalizeCandidatesForLanguage(alternateResults, contentType, alternateQuery.Language)
if len(alternateCandidates) == 0 {
continue
}
alternateWinner, alternateMatched := trySelection(&alternateHints, alternateCandidates)
if len(candidates) == 0 || alternateMatched || bestCandidateScore(&alternateHints, alternateCandidates) > bestCandidateScore(effectiveHints, candidates) {
candidates = alternateCandidates
effectiveHints = &alternateHints
}
if alternateMatched {
winner = alternateWinner
matched = true
break
}
}
}
trustedIDTypeMismatch := false
if !matched && len(candidates) == 0 && trustedHintIDsPresent(selectionHints) {
if mismatchCandidates := s.trustedIDTypeMismatchCandidates(
ctx, selectionHints, accumulatedIDs, searchQuery.Language, resolveChain,
); len(mismatchCandidates) > 0 {
candidates = mismatchCandidates
effectiveHints = selectionHints
trustedIDTypeMismatch = true
}
}
matchDecision = buildMatchDecision(effectiveHints, candidates, winner, matched, providerMatchErrors)
if trustedIDTypeMismatch {
matchDecision.Outcome = MatchOutcomeTrustedIDTypeMismatch
}
if matched && winner != nil {
applyCandidateProviderIDConsensus(accumulatedIDs, winner, quarantinedProviderIDKeys)
}
case ModeIdentify:
// Use user-provided IDs directly.
maps.Copy(accumulatedIDs, req.ProviderIDs)
sanitizeCanonicalProviderIDsInPlace(accumulatedIDs)
if contentType == "" && req.ContentID != "" {
if existing, err := s.itemRepo.GetByID(ctx, req.ContentID); err == nil {
contentType = existing.Type
@@ -1297,6 +1373,7 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
if existing.ImdbID != "" {
accumulatedIDs["imdb"] = existing.ImdbID
}
sanitizeCanonicalProviderIDsInPlace(accumulatedIDs)
contentType = existing.Type
if err := s.suppressRecordedStaleProviderIDs(ctx, req.ContentID, accumulatedIDs); err != nil {
return nil, err
@@ -1355,18 +1432,16 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
}
for _, result := range results {
if searchResultConflictsWithTrustedIDs(accumulatedIDs, result.ProviderIDs) {
continue
slog.WarnContext(ctx, "metadata: retaining conflicting refresh search result for trusted-id rejection", "component", "metadata",
"provider", p.Slug(), "content_id", req.ContentID,
"hinted_ids", accumulatedIDs, "candidate_ids", result.ProviderIDs)
}
allResults = append(allResults, result)
}
}
candidates := NormalizeCandidates(allResults, contentType)
candidates := NormalizeCandidatesForLanguage(allResults, contentType, searchQuery.Language)
if winner, ok := selectRefreshMatchCandidate(existing, wonHints, candidates); ok && winner != nil {
for k, v := range winner.ProviderIDs {
if v != "" {
accumulatedIDs[k] = v
}
}
applyCandidateProviderIDConsensus(accumulatedIDs, winner, quarantinedProviderIDKeys)
}
}
if req.Mode != ModeIdentify {
@@ -1429,15 +1504,22 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
if handleProvider404(provider404s, accumulatedIDs, p.Slug(), err,
"content_id", req.ContentID,
) {
if req.Mode == ModeInitialMatch {
providerMatchErrors = append(providerMatchErrors, err)
}
continue
}
slog.WarnContext(ctx, "metadata: provider error", "component", "metadata",
"provider", p.Slug(), "error", err)
if req.Mode == ModeInitialMatch {
providerMatchErrors = append(providerMatchErrors, err)
}
continue
}
if result == nil || !result.HasMetadata {
continue
}
result.ProviderIDs = sanitizeCandidateProviderIDs(result.ProviderIDs)
// Identity-hint providers contribute IDs exclusively through the
// trusted-hint phase: their Phase-2 results merge metadata fields but
// never inject provider-id keys that conflict with or extend the
@@ -1445,12 +1527,19 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
if isIdentityHinter {
result.ProviderIDs = nil
}
for key := range quarantinedProviderIDKeys {
delete(result.ProviderIDs, key)
}
mergePreferredTitleMetadata(accumulator, result, req.Language, p.Slug(), !isIdentityHinter)
// Bootstrap: feed new IDs to subsequent providers.
mergeProviderIDs(accumulator, result)
accumulatedIDs = accumulator.ProviderIDs
// Merge fields into accumulator (FillEmpty — first provider wins).
MergeMetadata(result, accumulator, nil, MergeFillEmpty)
}
if len(quarantinedProviderIDKeys) > 0 {
accumulator.quarantinedProviderIDKeys = maps.Clone(quarantinedProviderIDKeys)
}
// Phase 3: Images — all ImageProviders run, collect all available images.
var allImages []RemoteImage
for _, p := range itemChain {
@@ -1603,13 +1692,32 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
}
}
}
return &ProcessResult{Updated: false}, nil
if matchDecision == nil {
matchDecision = &MatchDecision{Outcome: MatchOutcomeMetadataEmpty, Threshold: automaticMatchAcceptanceFloor}
} else if matchDecision.Outcome == MatchOutcomeMatched {
matchDecision.Outcome = MatchOutcomeMetadataEmpty
}
if hasTransientMatchError(providerMatchErrors) {
matchDecision.Outcome = MatchOutcomeProviderTransient
} else if len(providerMatchErrors) > 0 &&
(matchDecision.Outcome == MatchOutcomeNoCandidates || matchDecision.Outcome == MatchOutcomeMetadataEmpty) {
// Optional corroboration failures must not turn a successfully
// completed search with rejected candidates into a permanent provider
// failure. The candidate decision remains actionable and deterministic.
matchDecision.Outcome = MatchOutcomeProviderPermanent
}
return &ProcessResult{Updated: false, Decision: matchDecision}, nil
}
result, err := s.mergeAndPersist(ctx, req, accumulator, allImages, allSeasons, allEpisodes, contentType)
if err != nil {
return nil, err
}
if result != nil && strings.TrimSpace(result.ContentID) != "" {
if err := s.persistItemAliases(ctx, result.ContentID, req.Language, accumulator); err != nil {
return nil, err
}
}
// Refresh stale ID records on successful refresh: clear anything resolved,
// then keep only the providers that still 404ed during this run.
@@ -1649,9 +1757,211 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques
}
}
if result != nil {
result.Decision = matchDecision
}
return result, nil
}
func buildMatchDecision(hints *MatchHints, candidates []MatchCandidate, winner *MatchCandidate, matched bool, providerErrors []error) *MatchDecision {
decision := &MatchDecision{CandidateCount: len(candidates), Threshold: automaticMatchAcceptanceFloor}
switch {
case matched && winner != nil:
decision.Outcome = MatchOutcomeMatched
case hasTransientMatchError(providerErrors):
decision.Outcome = MatchOutcomeProviderTransient
case len(providerErrors) > 0 && len(candidates) == 0:
decision.Outcome = MatchOutcomeProviderPermanent
case len(candidates) == 0:
decision.Outcome = MatchOutcomeNoCandidates
case candidatesConflictWithTrustedIDs(hints, candidates):
decision.Outcome = MatchOutcomeTrustedIDConflict
default:
decision.Outcome = MatchOutcomeCandidateRejected
}
scored := make([]MatchCandidate, len(candidates))
copy(scored, candidates)
for i := range scored {
annotateCandidateMatch(&scored[i], hints)
}
sort.SliceStable(scored, func(i, j int) bool { return scored[i].MatchScore > scored[j].MatchScore })
if len(scored) > 3 {
scored = scored[:3]
}
for _, candidate := range scored {
reasons := append([]string(nil), candidate.MatchReasons...)
if matched && winner != nil && sameMatchCandidate(candidate, *winner) {
for _, reason := range winner.MatchReasons {
if !slices.Contains(reasons, reason) {
reasons = append(reasons, reason)
}
}
}
decision.TopCandidates = append(decision.TopCandidates, MatchDecisionCandidate{
Title: candidate.Title, MatchedTitle: candidate.MatchedTitle, Year: candidate.Year,
ProviderIDs: copyMap(candidate.ProviderIDs), Sources: append([]string(nil), candidate.Sources...),
Score: candidate.MatchScore, Reasons: reasons,
})
}
return decision
}
func sameMatchCandidate(left, right MatchCandidate) bool {
leftKey, rightKey := normalizedKey(left.ProviderIDs), normalizedKey(right.ProviderIDs)
if leftKey != "" || rightKey != "" {
return leftKey != "" && leftKey == rightKey
}
if left.Title != right.Title || left.OriginalTitle != right.OriginalTitle ||
left.Year != right.Year || left.ContentType != right.ContentType ||
!slices.Equal(left.Sources, right.Sources) || len(left.ProviderIDs) != len(right.ProviderIDs) {
return false
}
for key, value := range left.ProviderIDs {
if right.ProviderIDs[key] != value {
return false
}
}
return true
}
func hasTransientMatchError(errs []error) bool {
for _, err := range errs {
if err == nil {
continue
}
switch status.Code(err) {
case codes.DeadlineExceeded, codes.ResourceExhausted, codes.Aborted, codes.Unavailable:
return true
}
message := strings.ToLower(err.Error())
for _, marker := range []string{"timeout", "deadline exceeded", "connection reset", "connection refused", "temporarily unavailable", "unavailable", "rate limit", "too many requests", "http 429", "http 500", "http 502", "http 503", "http 504"} {
if strings.Contains(message, marker) {
return true
}
}
}
return false
}
func mergePreferredTitleMetadata(accumulator, result *MetadataResult, language, provider string, attributeAliases bool) {
if accumulator == nil || result == nil {
return
}
if attributeAliases {
if accumulator.titleAliasProviders == nil {
accumulator.titleAliasProviders = make(map[string]bool)
}
providerComplete, providerSeen := accumulator.titleAliasProviders[provider]
if !providerSeen {
accumulator.titleAliasProviders[provider] = result.TitleAliasesComplete
} else {
// Multiple installations can share a provider slug. The aggregate is
// authoritative only when every contributing response says its alias
// snapshot is complete; one legacy/partial response removes deletion
// authority for the shared provider scope.
accumulator.titleAliasProviders[provider] = providerComplete && result.TitleAliasesComplete
}
if strings.TrimSpace(result.Title) != "" {
kind := titleAliasKindLocalized
if strings.TrimSpace(result.OriginalTitle) != "" && strings.EqualFold(strings.TrimSpace(result.Title), strings.TrimSpace(result.OriginalTitle)) &&
(baseMetadataLanguage(result.OriginalLanguage) == "" || baseMetadataLanguage(result.TitleLanguage) == baseMetadataLanguage(result.OriginalLanguage)) {
kind = titleAliasKindOriginal
}
accumulator.TitleAliases = appendUniqueTitleAlias(accumulator.TitleAliases, TitleAlias{
Title: result.Title, Language: baseMetadataLanguage(result.TitleLanguage), Kind: kind, Provider: provider,
})
}
for _, alias := range result.TitleAliases {
if alias.Provider == "" {
alias.Provider = provider
}
accumulator.TitleAliases = appendUniqueTitleAlias(accumulator.TitleAliases, alias)
}
if result.OriginalTitle != "" && !strings.EqualFold(result.OriginalTitle, result.Title) {
accumulator.TitleAliases = appendUniqueTitleAlias(accumulator.TitleAliases, TitleAlias{
Title: result.OriginalTitle, Language: baseMetadataLanguage(result.OriginalLanguage), Kind: titleAliasKindOriginal, Provider: provider,
})
}
}
if accumulator.OriginalTitle == "" && strings.TrimSpace(result.OriginalTitle) != "" {
accumulator.OriginalTitle = result.OriginalTitle
}
title, titleLanguage, fallback, rank := preferredMetadataResultTitle(result, language)
currentRank := metadataTitlePreferenceRank(accumulator.TitleLanguage, accumulator.TitleIsFallback, language)
if strings.TrimSpace(accumulator.Title) != "" && currentRank == 0 {
currentRank = 1
}
if strings.TrimSpace(accumulator.Title) == "" || rank > currentRank {
accumulator.Title = title
accumulator.TitleLanguage = titleLanguage
accumulator.TitleIsFallback = fallback
}
}
func (s *MetadataService) persistItemAliases(ctx context.Context, contentID, language string, result *MetadataResult) error {
if s == nil || s.itemAliasRepo == nil || result == nil {
return nil
}
byProvider := make(map[string][]models.MediaItemAlias)
for _, alias := range result.TitleAliases {
provider := strings.TrimSpace(alias.Provider)
if provider == "" {
continue
}
byProvider[provider] = append(byProvider[provider], models.MediaItemAlias{
ContentID: contentID, Title: alias.Title, Language: alias.Language, Kind: alias.Kind, Provider: provider,
})
}
for provider, complete := range result.titleAliasProviders {
if err := s.itemAliasRepo.RefreshProviderLanguage(ctx, contentID, provider, language, byProvider[provider], complete); err != nil {
return fmt.Errorf("persisting %s title aliases: %w", provider, err)
}
}
return nil
}
func preferredMetadataResultTitle(result *MetadataResult, language string) (string, string, bool, int) {
requested := baseMetadataLanguage(language)
titleLanguage := baseMetadataLanguage(result.TitleLanguage)
if strings.TrimSpace(result.Title) != "" && requested != "" && titleLanguage == requested && !result.TitleIsFallback {
return result.Title, titleLanguage, false, 3
}
for _, alias := range result.TitleAliases {
if strings.TrimSpace(alias.Title) != "" && requested != "" && baseMetadataLanguage(alias.Language) == requested {
return alias.Title, requested, false, 2
}
}
if strings.TrimSpace(result.Title) != "" && titleLanguage == "" && !result.TitleIsFallback {
return result.Title, "", false, 3
}
if requested != "" && strings.TrimSpace(result.OriginalTitle) != "" {
return result.OriginalTitle, baseMetadataLanguage(result.OriginalLanguage), true, 1
}
if strings.TrimSpace(result.Title) != "" {
return result.Title, titleLanguage, result.TitleIsFallback, 1
}
return result.OriginalTitle, baseMetadataLanguage(result.OriginalLanguage), true, 1
}
func metadataTitlePreferenceRank(titleLanguage string, fallback bool, language string) int {
if strings.TrimSpace(titleLanguage) == "" {
// A provider that does not participate in the language contract (notably
// the local NFO provider and older plugins) retains normal first-provider
// priority. Only an explicitly marked fallback may be displaced by a
// later requested-language result.
if fallback {
return 1
}
return 3
}
if baseMetadataLanguage(titleLanguage) == baseMetadataLanguage(language) && !fallback {
return 3
}
return 1
}
// refreshFollowUpContentID returns the content ID that a completed refresh's
// follow-up writes (stale-ID clear/record, debt sync) should target: the
// canonical ID the item was persisted or merged into when available, falling
@@ -1675,6 +1985,12 @@ func (s *MetadataService) mergeAndPersist(
episodes []EpisodeResult,
contentType string,
) (*ProcessResult, error) {
// Quarantined IDs are unsafe for deduplication, canonical ID derivation, and
// persistence regardless of whether this is a new item or a refresh.
for key := range accumulator.quarantinedProviderIDKeys {
delete(accumulator.ProviderIDs, key)
}
// Determine merge mode.
var mergeMode MergeMode
switch req.Mode {
@@ -1805,6 +2121,9 @@ func (s *MetadataService) mergeAndPersist(
accumulator.ProviderIDs = make(map[string]string, len(durableIDs))
}
for key, value := range durableIDs {
if _, quarantined := accumulator.quarantinedProviderIDKeys[key]; quarantined {
continue
}
if _, exists := accumulator.ProviderIDs[key]; !exists {
accumulator.ProviderIDs[key] = value
}
@@ -1845,6 +2164,10 @@ func (s *MetadataService) mergeAndPersist(
} else {
MergeGlobalMetadata(accumulator, existingResult, locked, mergeMode)
}
for key := range accumulator.quarantinedProviderIDKeys {
delete(existingResult.ProviderIDs, key)
}
existingResult.quarantinedProviderIDKeys = accumulator.quarantinedProviderIDKeys
accumulator = existingResult
}
@@ -2749,6 +3072,7 @@ func (s *MetadataService) resolveSeriesRefreshProviderIDs(ctx context.Context, s
return nil, err
}
maps.Copy(accumulatedIDs, durableIDs)
sanitizeCanonicalProviderIDsInPlace(accumulatedIDs)
if err := s.suppressRecordedStaleProviderIDs(ctx, series.ContentID, accumulatedIDs); err != nil {
return nil, err
}
@@ -2792,18 +3116,16 @@ func (s *MetadataService) resolveSeriesRefreshProviderIDs(ctx context.Context, s
}
for _, result := range results {
if searchResultConflictsWithTrustedIDs(accumulatedIDs, result.ProviderIDs) {
continue
slog.WarnContext(ctx, "metadata: retaining conflicting target-refresh result for consensus", "component", "metadata",
"provider", p.Slug(), "content_id", series.ContentID,
"hinted_ids", accumulatedIDs, "candidate_ids", result.ProviderIDs)
}
allResults = append(allResults, result)
}
}
candidates := NormalizeCandidates(allResults, series.Type)
candidates := NormalizeCandidatesForLanguage(allResults, series.Type, searchQuery.Language)
if winner, ok := selectRefreshMatchCandidate(series, nil, candidates); ok && winner != nil {
for k, v := range winner.ProviderIDs {
if v != "" {
accumulatedIDs[k] = v
}
}
applyCandidateProviderIDConsensus(accumulatedIDs, winner, nil)
}
if err := s.suppressRecordedStaleProviderIDs(ctx, series.ContentID, accumulatedIDs); err != nil {
return nil, err
@@ -6357,6 +6679,42 @@ func searchResultConflictsWithTrustedIDs(hintedIDs, candidateIDs map[string]stri
return false
}
// applyCandidateProviderIDConsensus replaces accumulated canonical IDs with a
// normalized winner while removing any cross-provider IDs quarantined during
// candidate grouping. The optional quarantine map carries the decision into
// Phase 2 so detail responses cannot reintroduce the disputed ID.
func applyCandidateProviderIDConsensus(accumulatedIDs map[string]string, winner *MatchCandidate, quarantine map[string]struct{}) {
if winner == nil {
return
}
locallyQuarantined := make(map[string]struct{}, len(winner.ConflictingProviderIDKeys))
for _, key := range winner.ConflictingProviderIDKeys {
key = strings.ToLower(strings.TrimSpace(key))
if key == "" {
continue
}
delete(accumulatedIDs, key)
locallyQuarantined[key] = struct{}{}
if quarantine != nil {
quarantine[key] = struct{}{}
}
}
for key, value := range winner.ProviderIDs {
key = strings.ToLower(strings.TrimSpace(key))
value = strings.TrimSpace(value)
if key == "" || value == "" {
continue
}
if _, quarantined := locallyQuarantined[key]; quarantined {
continue
}
if _, quarantined := quarantine[key]; quarantined {
continue
}
accumulatedIDs[key] = value
}
}
// applyBuiltinIdentityHints consults the chain's IdentityHintProviders (the
// built-in NFO provider) and folds their curated tmdb/imdb/tvdb ids into
// accumulatedIDs ahead of Phase-1 search. Keys in protected (stored durable
@@ -6379,8 +6737,12 @@ func applyBuiltinIdentityHints(
}
hints := hinter.IdentityHints(ctx, query)
for _, key := range trustedSearchIDKeys {
value := strings.TrimSpace(hints[key])
if value == "" {
value, valid := sanitizeProviderIDValue(key, hints[key])
if !valid {
if strings.TrimSpace(hints[key]) != "" {
slog.WarnContext(ctx, "metadata: ignoring malformed local identity hint", "component", "metadata",
"provider", p.Slug(), "key", key, "value", strings.TrimSpace(hints[key]))
}
continue
}
current := strings.TrimSpace(accumulatedIDs[key])
+106 -24
View File
@@ -78,8 +78,50 @@ type ProcessResult struct {
ContentID string
IsNew bool // True if this was an initial match (new item created)
Updated bool // True if any fields were changed
Decision *MatchDecision
}
// MatchDecision is a bounded, persistence-safe explanation of an automatic
// matcher outcome. It intentionally excludes artwork and provider payloads.
type MatchDecision struct {
Outcome MatchOutcome `json:"outcome"`
CandidateCount int `json:"candidate_count"`
Threshold float64 `json:"threshold"`
TopCandidates []MatchDecisionCandidate `json:"top_candidates,omitempty"`
}
type MatchDecisionCandidate struct {
Title string `json:"title"`
MatchedTitle string `json:"matched_title,omitempty"`
Year int `json:"year,omitempty"`
ProviderIDs map[string]string `json:"provider_ids,omitempty"`
Sources []string `json:"sources,omitempty"`
Score float64 `json:"score"`
Reasons []string `json:"reasons,omitempty"`
}
type MatchFailure struct {
Kind MatchOutcome
Message string
Decision *MatchDecision
}
// MatchOutcome is the stable matcher/queue failure taxonomy exposed in
// bounded diagnostics. Keep additions backward-compatible with persisted
// failure_kind values and the admin API.
type MatchOutcome string
const (
MatchOutcomeMatched MatchOutcome = "matched"
MatchOutcomeNoCandidates MatchOutcome = "no_candidates"
MatchOutcomeCandidateRejected MatchOutcome = "candidate_rejected"
MatchOutcomeTrustedIDConflict MatchOutcome = "trusted_id_conflict"
MatchOutcomeTrustedIDTypeMismatch MatchOutcome = "trusted_id_type_mismatch"
MatchOutcomeMetadataEmpty MatchOutcome = "metadata_empty"
MatchOutcomeProviderTransient MatchOutcome = "provider_transient"
MatchOutcomeProviderPermanent MatchOutcome = "provider_permanent"
)
// MatchHints carries scanner-extracted data into the pipeline.
// Adapted from matcher.MatchHints with FilePath added for local providers.
type MatchHints struct {
@@ -99,6 +141,17 @@ type MatchHints struct {
ObservedRootPath string
AllGroupFilePaths []string
PrimarySidecarSearchPaths []string
// AlternateIdentities are independently parsed title/year hypotheses from
// the filename and surrounding directories. They are tried only after the
// primary scanner identity fails, keeping provider traffic bounded while
// allowing a good filename to recover from a stale or release-like folder.
AlternateIdentities []MatchIdentityHint
}
type MatchIdentityHint struct {
Title string
Year int
Source string
}
// SearchQuery is passed to SearchProvider.Search().
@@ -118,14 +171,34 @@ type SearchQuery struct {
// SearchResult is returned from SearchProvider.Search().
type SearchResult struct {
Name string
Year int
ProviderIDs map[string]string
ImageURL string
Overview string
Provider string // Slug of the provider that returned this
Name string
OriginalTitle string
TitleAliases []TitleAlias
TitleLanguage string
TitleIsFallback bool
OriginalLanguage string
Year int
ProviderIDs map[string]string
ImageURL string
Overview string
Provider string // Slug of the provider that returned this
}
// TitleAlias is a provider-confirmed title for the same work.
type TitleAlias struct {
Title string `json:"title"`
Language string `json:"language,omitempty"`
Kind string `json:"kind"`
Provider string `json:"provider,omitempty"`
}
const (
titleAliasKindOriginal = "original"
titleAliasKindLocalized = "localized"
matchContentTypeMovie = "movie"
matchContentTypeSeries = "series"
)
// MetadataRequest is passed to MetadataProvider.GetMetadata().
type MetadataRequest struct {
ProviderIDs map[string]string
@@ -163,24 +236,33 @@ type PersonDetailResult struct {
// MetadataResult carries structured metadata from a single provider.
type MetadataResult struct {
HasMetadata bool
ProviderIDs map[string]string
Title string
OriginalTitle string
SortTitle string
Overview string
Tagline string
Year int
Runtime int
Genres []string
Studios []string
Networks []string
Countries []string
Keywords []string
OriginalLanguage string
ContentRating string
Ratings Ratings
People []models.ItemPerson
HasMetadata bool
ProviderIDs map[string]string
Title string
OriginalTitle string
SortTitle string
Overview string
Tagline string
Year int
Runtime int
Genres []string
Studios []string
Networks []string
Countries []string
Keywords []string
OriginalLanguage string
TitleAliases []TitleAlias
TitleAliasesComplete bool
TitleLanguage string
TitleIsFallback bool
// titleAliasProviders records whether each contributing provider declared
// its aliases to be an authoritative snapshot. False means merge-only,
// which is the safe default for legacy plugins and partial responses.
titleAliasProviders map[string]bool
quarantinedProviderIDKeys map[string]struct{}
ContentRating string
Ratings Ratings
People []models.ItemPerson
// Images (S3 paths or URLs).
PosterPath string
PosterThumbhash string
+134 -6
View File
@@ -16,6 +16,7 @@ import (
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/librarykind"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/naming"
"github.com/Silo-Server/silo-server/internal/notifications"
)
@@ -332,6 +333,7 @@ func (w *MatchWorker) buildProcessRequestForGroup(ctx context.Context, represent
ImdbID: skeleton.ImdbID,
TvdbID: skeleton.TvdbID,
HintSource: "scanner",
AlternateIdentities: queuedMatchIdentityAlternates(representative, skeleton),
}
return ProcessRequest{
@@ -342,6 +344,62 @@ func (w *MatchWorker) buildProcessRequestForGroup(ctx context.Context, represent
}
}
const maxAlternateMatchIdentities = 3
func queuedMatchIdentityAlternates(file *models.MediaFile, skeleton *skeletonResult) []MatchIdentityHint {
if file == nil || skeleton == nil {
return nil
}
seen := map[string]struct{}{
matchIdentityKey(skeleton.Title, skeleton.Year): {},
}
out := make([]MatchIdentityHint, 0, maxAlternateMatchIdentities)
add := func(title string, year int, source string) {
title = strings.TrimSpace(naming.StripComparisonSafeEditionSuffix(title))
key := matchIdentityKey(title, year)
if title == "" || key == "" {
return
}
if _, exists := seen[key]; exists {
return
}
seen[key] = struct{}{}
out = append(out, MatchIdentityHint{Title: title, Year: year, Source: source})
}
itemType := strings.ToLower(strings.TrimSpace(skeleton.Type))
if parsed := naming.ParseFilename(file.FilePath, itemType); parsed != nil {
add(parsed.Title, parsed.Year, "current_path")
}
if itemType == matchContentTypeMovie {
base := strings.TrimSuffix(filepath.Base(file.FilePath), filepath.Ext(file.FilePath))
stem := naming.ParseInferMovieStem(base, skeleton.Title, skeleton.Year)
add(stem.Title, stem.Year, "filename")
observedRoot := strings.TrimSpace(skeleton.ObservedRootPath)
if observedRoot == "" {
observedRoot = filepath.Dir(file.FilePath)
}
rootStem := naming.ParseInferMovieStem(filepath.Base(observedRoot), "", 0)
add(rootStem.Title, rootStem.Year, "release_folder")
}
if len(out) > maxAlternateMatchIdentities {
out = out[:maxAlternateMatchIdentities]
}
return out
}
func matchIdentityKey(title string, year int) string {
title = strings.ToLower(strings.Join(strings.Fields(strings.TrimSpace(title)), " "))
if title == "" {
return ""
}
return title + "\x00" + strconv.Itoa(year)
}
func compactUniquePaths(paths []string) []string {
if len(paths) == 0 {
return nil
@@ -728,7 +786,8 @@ func (w *MatchWorker) queuedMovieSkeleton(ctx context.Context, file *models.Medi
return skeleton, true, nil
}
skeleton, err := w.service.createOrFindSkeleton(ctx, file, file.MediaFolderID)
currentFile := reparseQueuedFileIdentity(file, "movie")
skeleton, err := w.service.createOrFindSkeleton(ctx, currentFile, currentFile.MediaFolderID)
if err != nil {
return nil, false, err
}
@@ -776,6 +835,34 @@ func (w *MatchWorker) reusableQueuedMovieSkeleton(ctx context.Context, file *mod
if year == 0 {
year = file.BaseYear
}
currentFile := reparseQueuedFileIdentity(file, itemType)
if currentFile.BaseTitle != "" {
title = currentFile.BaseTitle
}
if currentFile.BaseYear != 0 {
year = currentFile.BaseYear
}
if currentFile.BaseType != "" {
itemType = currentFile.BaseType
}
refreshedIDs := trustedStructuredIDsForSkeleton(file.FilePath, observedRootPath, rootPath)
// This is an unmatched-style skeleton being re-evaluated. Structured path
// IDs are current input; IDs that disappeared from the path must not remain
// trusted forever merely because an older parser copied them onto the
// provisional item.
tmdbID, imdbID, tvdbID := "", "", ""
if refreshedIDs != nil {
if refreshedIDs.TmdbID != "" {
tmdbID = refreshedIDs.TmdbID
}
if refreshedIDs.ImdbID != "" {
imdbID = refreshedIDs.ImdbID
}
if refreshedIDs.TvdbID != "" {
tvdbID = refreshedIDs.TvdbID
}
}
return &skeletonResult{
ContentID: item.ContentID,
@@ -787,12 +874,37 @@ func (w *MatchWorker) reusableQueuedMovieSkeleton(ctx context.Context, file *mod
Title: title,
Year: year,
Type: itemType,
TmdbID: item.TmdbID,
ImdbID: item.ImdbID,
TvdbID: item.TvdbID,
TmdbID: tmdbID,
ImdbID: imdbID,
TvdbID: tvdbID,
}, true
}
// reparseQueuedFileIdentity refreshes the in-memory scanner identity from the
// current path without mutating the persisted scan row. Queue entries can live
// across parser releases, including entries that have not created a skeleton
// yet, so both fresh and reusable skeleton paths must use this view.
func reparseQueuedFileIdentity(file *models.MediaFile, fallbackType string) *models.MediaFile {
if file == nil {
return nil
}
current := *file
parseType := strings.TrimSpace(fallbackType)
if parseType == "" {
parseType = strings.TrimSpace(current.BaseType)
}
if parsed := naming.ParseFilename(current.FilePath, parseType); parsed != nil {
if parsed.Title != "" {
current.BaseTitle = parsed.Title
}
current.BaseYear = parsed.Year
if parsed.Type != "" {
current.BaseType = parsed.Type
}
}
return &current
}
func scopedMatcherPath(useSeriesQueue bool, useMovieQueue bool) string {
switch {
case useSeriesQueue && useMovieQueue:
@@ -946,7 +1058,8 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo
return len(groupFiles), nil
}
skeleton, err := w.service.createOrFindSkeleton(ctx, representative, job.MediaFolderID)
currentRepresentative := reparseQueuedFileIdentity(representative, "series")
skeleton, err := w.service.createOrFindSkeleton(ctx, currentRepresentative, job.MediaFolderID)
if err != nil {
queueErr := truncateSeriesQueueError(err.Error())
if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, queueErr); updateErr != nil {
@@ -971,7 +1084,22 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo
return 0, fmt.Errorf("relinking series root %d/%s: %w", job.MediaFolderID, job.ObservedRootPath, err)
}
if skeleton.IsNew && skeleton.ItemStatus != "ambiguous" {
needsInitialMatch := skeleton.IsNew
if !needsInitialMatch && strings.TrimSpace(skeleton.ContentID) != "" && w.service.itemRepo != nil {
item, loadErr := w.service.itemRepo.GetByID(ctx, skeleton.ContentID)
if loadErr != nil {
queueErr := truncateSeriesQueueError(loadErr.Error())
if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, queueErr); updateErr != nil {
return 0, updateErr
}
return 0, fmt.Errorf("loading linked series skeleton %s: %w", skeleton.ContentID, loadErr)
}
if item != nil {
skeleton.ItemStatus = item.Status
needsInitialMatch = isSkeletonLikeStatus(item.Status)
}
}
if needsInitialMatch && skeleton.ItemStatus != "ambiguous" {
req := w.buildProcessRequestForGroup(ctx, representative, skeleton, groupFiles)
result, processErr := w.service.Process(ctx, req)
if processErr != nil {
+131
View File
@@ -12,6 +12,8 @@ import (
"github.com/Silo-Server/silo-server/internal/models"
)
const queuedShowName = "Show Name"
type fakeWorkerFolderRepo struct {
folders map[int]*models.MediaFolder
}
@@ -1474,3 +1476,132 @@ func TestProcessSeriesRoot_Site2_NonNotFoundErrorStillFails(t *testing.T) {
t.Fatalf("unexpected error message: %v", err)
}
}
func TestReparseQueuedFileIdentityUsesCurrentPathWithoutMutatingScanRow(t *testing.T) {
t.Parallel()
const expectedTitle = "10 Tricks"
file := &models.MediaFile{
FilePath: "/movies/10 Tricks (2022)/10 Tricks (2022) (tt0473100) [WEBDL-1080p].mp4",
BaseTitle: "stale scanner title",
BaseYear: 0,
BaseType: "movie",
}
current := reparseQueuedFileIdentity(file, "movie")
if current == file {
t.Fatal("reparse returned the persisted model pointer")
}
if current.BaseTitle != expectedTitle || current.BaseYear != 2022 || current.BaseType != matchContentTypeMovie {
t.Fatalf("reparsed identity = %q/%d/%q", current.BaseTitle, current.BaseYear, current.BaseType)
}
if file.BaseTitle != "stale scanner title" || file.BaseYear != 0 {
t.Fatalf("persisted scan model was mutated: %#v", file)
}
}
func TestReparseQueuedFileIdentityClearsStaleYearAndUsesQueueType(t *testing.T) {
t.Parallel()
file := &models.MediaFile{
FilePath: "/movies/Show Name/Show Name.mkv",
BaseTitle: "stale",
BaseYear: 1999,
BaseType: "series",
}
current := reparseQueuedFileIdentity(file, "movie")
if current.BaseTitle != queuedShowName || current.BaseYear != 0 || current.BaseType != "movie" {
t.Fatalf("reparsed identity = %q/%d/%q, want Show Name/0/movie", current.BaseTitle, current.BaseYear, current.BaseType)
}
if file.BaseYear != 1999 || file.BaseType != "series" {
t.Fatalf("persisted scan model was mutated: %#v", file)
}
}
func TestReusableQueuedMovieSkeletonPreservesKnownYearWhenCurrentPathHasNone(t *testing.T) {
t.Parallel()
h := newTestHarness()
ctx := context.Background()
const contentID = "local-year-fallback"
if err := h.itemRepo.Upsert(ctx, &models.MediaItem{
ContentID: contentID,
Status: "pending",
Title: queuedShowName,
Year: 1999,
Type: "movie",
Studios: []string{},
Networks: []string{},
Countries: []string{},
Genres: []string{},
}); err != nil {
t.Fatalf("seed item: %v", err)
}
file := &models.MediaFile{
ContentID: contentID,
FilePath: "/movies/Show Name/Show Name.mkv",
BaseTitle: queuedShowName,
BaseYear: 1987,
BaseType: "movie",
}
worker := NewMatchWorker(h.service, h.fileRepo, 1, 1, 0)
skeleton, ok := worker.reusableQueuedMovieSkeleton(ctx, file)
if !ok || skeleton == nil {
t.Fatal("expected reusable skeleton")
}
if skeleton.Year != 1999 {
t.Fatalf("skeleton year = %d, want preserved item year 1999", skeleton.Year)
}
}
func TestQueuedMatchIdentityAlternatesRecoversSeriesParentShow(t *testing.T) {
file := &models.MediaFile{
FilePath: "/shows/Becoming You/Becoming.You.S01.HDR.2160p.WEB-DL-GROUP/Becoming.You.S01E01.2160p.WEB-DL.mkv",
}
skeleton := &skeletonResult{
Title: "Becoming.You.S01.HDR.2160p.WEB-DL-GROUP",
Type: "series",
ObservedRootPath: "/shows/Becoming You/Becoming.You.S01.HDR.2160p.WEB-DL-GROUP",
}
got := queuedMatchIdentityAlternates(file, skeleton)
if len(got) == 0 || got[0].Title != "Becoming You" || got[0].Source != "current_path" {
t.Fatalf("alternate identities = %#v, want current-path Becoming You", got)
}
}
func TestQueuedMatchIdentityAlternatesRecoversMovieFilenameAndReleaseFolder(t *testing.T) {
const alienInvasionTitle = "Alien Invasion"
tests := []struct {
name string
file *models.MediaFile
skeleton *skeletonResult
wantTitle string
wantYear int
wantSource string
}{
{
name: "filename beats divergent parent",
file: &models.MediaFile{FilePath: "/movies/After the Lethargy (2018)/Alien Invasion (2018).mkv"},
skeleton: &skeletonResult{Title: "After the Lethargy", Year: 2018, Type: "movie",
ObservedRootPath: "/movies/After the Lethargy (2018)"},
wantTitle: alienInvasionTitle, wantYear: 2018, wantSource: "filename",
},
{
name: "hash filename uses release folder",
file: &models.MediaFile{FilePath: "/movies/Jurassic.World.2015.1080p.WEB-DL-GROUP/291684abcdef012345.mkv"},
skeleton: &skeletonResult{Title: "291684abcdef012345", Type: "movie",
ObservedRootPath: "/movies/Jurassic.World.2015.1080p.WEB-DL-GROUP"},
wantTitle: "Jurassic World", wantYear: 2015, wantSource: "release_folder",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := queuedMatchIdentityAlternates(tt.file, tt.skeleton)
for _, identity := range got {
if identity.Title == tt.wantTitle && identity.Year == tt.wantYear && identity.Source == tt.wantSource {
return
}
}
t.Fatalf("alternate identities = %#v, want %q/%d from %s", got, tt.wantTitle, tt.wantYear, tt.wantSource)
})
}
}
+9
View File
@@ -418,6 +418,15 @@ type MediaItem struct {
AddedAt *time.Time // populated by browse queries (MIN(mil.first_seen_at))
}
// MediaItemAlias is a provider-confirmed searchable title for a media item.
type MediaItemAlias struct {
ContentID string
Title string
Language string
Kind string
Provider string
}
// Season represents a row in the seasons table.
type Season struct {
ContentID string
+37 -4
View File
@@ -10,9 +10,9 @@ import (
)
var (
// folderTagRe matches bracketed or braced provider-ID tags like
// [tmdbid-27205], {tvdb-81189}, [imdbid-tt1375666], etc.
folderTagRe = regexp.MustCompile(`\s*[{\[](tmdb|tmdbid|imdb|imdbid|tvdb|tvdbid)-[\w]+[}\]]`)
// folderTagRe matches strict provider-ID tags in square, curly, or round
// brackets, including Plex-style bare IMDb tags such as (tt0473100).
folderTagRe = regexp.MustCompile(`(?i)\s*(?:\((?:(?:tmdb|tmdbid|tvdb|tvdbid)-\d+|(?:imdb|imdbid)-tt\d{7,10}|tt\d{7,10})\)|\[(?:(?:tmdb|tmdbid|tvdb|tvdbid)-\d+|(?:imdb|imdbid)-tt\d{7,10}|tt\d{7,10})\]|\{(?:(?:tmdb|tmdbid|tvdb|tvdbid)-\d+|(?:imdb|imdbid)-tt\d{7,10}|tt\d{7,10})\})`)
// titleYearRe matches "Title (Year)" with optional trailing content.
titleYearRe = regexp.MustCompile(`^(.+?)\s*\((\d{4})\)`)
@@ -34,6 +34,13 @@ var (
// specialsDirRe matches common specials/extras folders.
specialsDirRe = regexp.MustCompile(`(?i)^(?:specials?|extras?)$`)
// seasonReleaseDirRe recognizes release-pack directories such as
// "Show.Name.S01.2160p.WEB-DL-GROUP". These are season containers, not
// show roots; treating them as roots fragments one show into one metadata
// item per release directory and searches providers with the release name.
seasonReleaseDirRe = regexp.MustCompile(`(?i)(?:^|[ ._-])s\d{1,4}(?:[ ._-]|$)`)
seasonReleaseTechRe = regexp.MustCompile(`(?i)(?:^|[ ._-])(?:2160p|1080p|720p|576p|480p|web(?:[ ._-]?dl|rip)?|blu[ ._-]?ray|bluray|hdtv|remux|x26[45]|h26[45]|hevc)(?:[ ._-]|$)`)
)
// ResolvePathContext classifies a media path using both naming heuristics and
@@ -272,7 +279,7 @@ func detectMovieFolderEvidence(parentBase string, nameNoExt string, hasSeasonStr
}
func hasExplicitFolderIDs(name string) bool {
return folderIDPattern.MatchString(name)
return ParseStructuredFolderIDs(name) != nil
}
func parseTitleYearCandidate(name string) (string, int) {
@@ -307,6 +314,12 @@ func collapseWhitespace(value string) string {
func deriveSeriesRoot(filePath string, hasEpisodePattern bool, forceParent bool) (*SeriesRoot, bool) {
parentDir := path.Dir(filePath)
if isSeasonReleaseDirectoryForParent(path.Base(parentDir), path.Base(path.Dir(parentDir))) {
rootPath := path.Dir(parentDir)
if rootPath != "." && rootPath != "/" && rootPath != "" {
return &SeriesRoot{RootPath: rootPath, FolderName: path.Base(rootPath)}, true
}
}
for current := parentDir; current != "." && current != "/" && current != ""; current = path.Dir(current) {
segment := path.Base(current)
if isSeasonDirSegment(segment, hasEpisodePattern || forceParent) || specialsDirRe.MatchString(segment) {
@@ -334,6 +347,26 @@ func deriveSeriesRoot(filePath string, hasEpisodePattern bool, forceParent bool)
return nil, false
}
func isSeasonReleaseDirectory(segment string) bool {
return seasonReleaseDirRe.MatchString(segment) && seasonReleaseTechRe.MatchString(segment)
}
func isSeasonReleaseDirectoryForParent(segment, parentSegment string) bool {
if !isSeasonReleaseDirectory(segment) {
return false
}
location := seasonReleaseDirRe.FindStringIndex(segment)
if location == nil || location[0] == 0 {
return false
}
releaseTitle := strings.Trim(segment[:location[0]], " ._-")
parentTitle, _ := parseTitleYearCandidate(parentSegment)
if parentTitle == "" {
parentTitle = parentSegment
}
return inferTitlesCoherent(releaseTitle, parentTitle)
}
func deriveMovieRoot(filePath string) (*CanonicalRoot, bool) {
baseName := path.Base(filePath)
nameNoExt := strings.TrimSuffix(baseName, filepath.Ext(baseName))
+54
View File
@@ -282,6 +282,60 @@ func TestParseFilename(t *testing.T) {
}
}
func TestParseFilename_StripsStructuredProviderIDsFromTitles(t *testing.T) {
t.Parallel()
for _, test := range []struct {
path string
wantTitle string
wantYear int
}{
{
path: "/movies/10 Tricks (2022)/10 Tricks (2022) (tt0473100) [WEBDL-1080p.AAC.h264.GNOME].mp4",
wantTitle: "10 Tricks",
wantYear: 2022,
},
{path: "/movies/Some Movie [tt1234567]/Some Movie.mkv", wantTitle: "Some Movie"},
{path: "/movies/Some Movie (tmdb-12345)/Some Movie.mkv", wantTitle: "Some Movie"},
} {
hints := ParseFilename(test.path, "movie")
if hints.Title != test.wantTitle || hints.Year != test.wantYear {
t.Errorf("ParseFilename(%q) = title %q year %d, want %q/%d", test.path, hints.Title, hints.Year, test.wantTitle, test.wantYear)
}
}
}
func TestParseFilename_SeriesReleasePackDirectoryUsesParentShowRoot(t *testing.T) {
path := "/library/Becoming You/Becoming.You.S01.HDR.2160p.WEB-DL.DDP5.1.H.265-ROCCaT/Becoming.You.S01E01.HDR.2160p.WEB-DL.mkv"
hints := ParseFilename(path, "series")
if hints == nil {
t.Fatal("ParseFilename returned nil")
return
}
if hints.Title != "Becoming You" || hints.Year != 0 || hints.Type != "series" {
t.Fatalf("ParseFilename() = title %q year %d type %q, want Becoming You/0/series", hints.Title, hints.Year, hints.Type)
}
root, ok := DetectSeriesRoot(path, "series")
if !ok || root.RootPath != "/library/Becoming You" {
t.Fatalf("DetectSeriesRoot() = %+v, %v", root, ok)
}
}
func TestParseFilename_SeriesReleasePackDirectoryRequiresCoherentParent(t *testing.T) {
t.Parallel()
flatPath := "/library/Becoming.You.S01.HDR.2160p.WEB-DL-GROUP/Becoming.You.S01E01.mkv"
root, ok := DetectSeriesRoot(flatPath, "series")
if !ok || root.RootPath != "/library/Becoming.You.S01.HDR.2160p.WEB-DL-GROUP" {
t.Fatalf("flat release root = %+v, %v; must not collapse to the library root", root, ok)
}
nestedPath := "/library/Becoming You (2020)/Becoming.You.S01.1080p.WEB-DL/Becoming.You.S01E01.mkv"
root, ok = DetectSeriesRoot(nestedPath, "series")
if !ok || root.RootPath != "/library/Becoming You (2020)" {
t.Fatalf("nested release root = %+v, %v; want coherent parent show folder", root, ok)
}
}
func TestResolvePathContext(t *testing.T) {
tests := []struct {
name string
+35 -9
View File
@@ -3,18 +3,20 @@ package naming
import (
"regexp"
"strings"
"github.com/Silo-Server/silo-server/internal/providerid"
)
// folderIDPattern matches patterns like [tmdbid-27205], {tmdb-27205},
// [imdbid-tt1375666], {imdb-tt1375666}, [tvdbid-81189], {tvdb-81189}, etc.
// The regex captures the provider prefix and the ID value.
var folderIDPattern = regexp.MustCompile(`[{\[](tmdb|tmdbid|imdb|imdbid|tvdb|tvdbid)-([\w]+)[}\]]`)
var trailingImdbIDPattern = regexp.MustCompile(`(?i)(?:^|\s)(tt\d{7,8})$`)
var folderIDPattern = regexp.MustCompile(`(?i)(?:\((tmdb|tmdbid|imdb|imdbid|tvdb|tvdbid)-([a-z0-9]+)\)|\[(tmdb|tmdbid|imdb|imdbid|tvdb|tvdbid)-([a-z0-9]+)\]|\{(tmdb|tmdbid|imdb|imdbid|tvdb|tvdbid)-([a-z0-9]+)\})`)
var trailingImdbIDPattern = regexp.MustCompile(`(?i)(?:^|\s)(tt\d{7,10})$`)
// bracketedBareImdbPattern matches a bare IMDb id wrapped in brackets without a
// provider prefix, e.g. [tt10011226] or {tt0095016} (Plex/Kodi-style tags). A
// tt-prefixed number is unambiguously IMDb.
var bracketedBareImdbPattern = regexp.MustCompile(`(?i)[{\[](tt\d{7,8})[}\]]`)
var bracketedBareImdbPattern = regexp.MustCompile(`(?i)(?:\((tt\d{7,10})\)|\[(tt\d{7,10})\]|\{(tt\d{7,10})\})`)
// ParseStructuredFolderIDs extracts only explicit provider IDs from a folder or
// file name, such as {tmdb-27205}, [imdbid-tt1375666], or [tt1375666]. It does
@@ -23,21 +25,37 @@ func ParseStructuredFolderIDs(name string) *FolderIDHints {
matches := folderIDPattern.FindAllStringSubmatch(name, -1)
hints := &FolderIDHints{}
for _, m := range matches {
provider := strings.ToLower(m[1])
id := m[2]
provider, id := "", ""
for i := 1; i+1 < len(m); i += 2 {
if m[i] != "" {
provider, id = strings.ToLower(m[i]), strings.ToLower(m[i+1])
break
}
}
switch provider {
case "tmdb", "tmdbid":
hints.TmdbID = id
if providerid.IsPositiveDecimal(id) {
hints.TmdbID = id
}
case "imdb", "imdbid":
hints.ImdbID = id
if isIMDbProviderID(id) {
hints.ImdbID = id
}
case "tvdb", "tvdbid":
hints.TvdbID = id
if providerid.IsPositiveDecimal(id) {
hints.TvdbID = id
}
}
}
if m := bracketedBareImdbPattern.FindStringSubmatch(name); m != nil && hints.ImdbID == "" {
hints.ImdbID = strings.ToLower(m[1])
for _, value := range m[1:] {
if isIMDbProviderID(value) {
hints.ImdbID = strings.ToLower(value)
break
}
}
}
if hints.TmdbID == "" && hints.ImdbID == "" && hints.TvdbID == "" {
@@ -46,6 +64,14 @@ func ParseStructuredFolderIDs(name string) *FolderIDHints {
return hints
}
func isIMDbProviderID(value string) bool {
value = strings.ToLower(strings.TrimSpace(value))
if !trailingImdbIDPattern.MatchString(value) {
return false
}
return providerid.IsPositiveDecimal(strings.TrimPrefix(value, "tt"))
}
// ParseFolderIDs extracts external provider IDs from a folder name. Mirroring
// Jellyfin's path-attribute model, only explicit evidence is honored: bracket
// tags with provider prefixes ({tmdb-27205}, [tvdbid-81189], [imdbid-tt1375666]),
+31
View File
@@ -9,6 +9,9 @@ func TestParseFolderIDs_BracketedBareImdb(t *testing.T) {
{"square brackets", "17 Blocks (2021) [tt10011226]", "tt10011226"},
{"curly braces", "Some Show {tt1234567}", "tt1234567"},
{"8-digit tt", "Movie (2024) [tt12345678]", "tt12345678"},
{"9-digit tt", "Movie (2024) [tt123456789]", "tt123456789"},
{"10-digit tt", "Movie (2024) [tt1234567890]", "tt1234567890"},
{"parentheses production pattern", "10 Tricks (2022) (tt0473100) [WEBDL-1080p.AAC.h264.GNOME].mp4", "tt0473100"},
}
for _, c := range cases {
got := ParseFolderIDs(c.folder)
@@ -28,6 +31,34 @@ func TestParseFolderIDs_BracketedBareImdb(t *testing.T) {
}
}
func TestParseStructuredFolderIDs_ParenthesizedProviderIDs(t *testing.T) {
tests := []struct {
name string
want FolderIDHints
}{
{"Movie (imdb-tt1375666)", FolderIDHints{ImdbID: "tt1375666"}},
{"Movie (tmdb-27205)", FolderIDHints{TmdbID: "27205"}},
{"Show (tvdbid-81189)", FolderIDHints{TvdbID: "81189"}},
}
for _, tt := range tests {
got := ParseStructuredFolderIDs(tt.name)
if got == nil || *got != tt.want {
t.Errorf("ParseStructuredFolderIDs(%q) = %+v, want %+v", tt.name, got, tt.want)
}
}
for _, invalid := range []string{
"Movie (tt123)", "Movie (tt12345678901)", "Movie (tmdb-12x)",
"Movie (imdb-1234567)", "Movie [tmdb-123)", "Movie {tt1234567]",
"Movie (tmdb-0)", "Show (tvdb-000)", "Movie (tt0000000)",
"Movie (imdb-tt0000000)", "District 9", "Beverly Hills 90210",
} {
if got := ParseStructuredFolderIDs(invalid); got != nil {
t.Errorf("ParseStructuredFolderIDs(%q) = %+v, want nil", invalid, got)
}
}
}
func TestParseStructuredFolderIDs_BracketedBareImdb(t *testing.T) {
got := ParseStructuredFolderIDs("17 Blocks (2021) [tt10011226]")
if got == nil || got.ImdbID != "tt10011226" {
+23
View File
@@ -0,0 +1,23 @@
// Package providerid contains validation shared by provider-ID ingestion
// boundaries.
package providerid
// IsPositiveDecimal reports whether value is a non-zero base-10 integer
// written using ASCII digits only. Leading zeroes are accepted because
// providers may treat the original text as their canonical identifier.
func IsPositiveDecimal(value string) bool {
if value == "" {
return false
}
nonZero := false
for _, r := range value {
if r < '0' || r > '9' {
return false
}
if r != '0' {
nonZero = true
}
}
return nonZero
}
+25
View File
@@ -0,0 +1,25 @@
package providerid
import "testing"
func TestIsPositiveDecimal(t *testing.T) {
t.Parallel()
tests := map[string]bool{
"": false,
"0": false,
"000": false,
"1": true,
"001": true,
"123456": true,
"-1": false,
"1.0": false,
"123": false,
"12a": false,
}
for value, want := range tests {
if got := IsPositiveDecimal(value); got != want {
t.Errorf("IsPositiveDecimal(%q) = %v, want %v", value, got, want)
}
}
}
@@ -0,0 +1,80 @@
package tasks
import (
"context"
"encoding/json"
"fmt"
"github.com/Silo-Server/silo-server/internal/taskmanager"
)
const mediaItemAliasBackfillBatchSize = 500
type MediaItemAliasBackfiller interface {
BackfillBatch(ctx context.Context, afterContentID string, limit int) (nextContentID string, processed int, err error)
// ResetCompletedBackfill re-arms a finished backfill so a manual re-run
// performs a fresh pass instead of a permanent no-op; an interrupted run
// keeps its persisted cursor and resumes.
ResetCompletedBackfill(ctx context.Context) error
}
// BackfillMediaItemAliasesTask seeds the alias table from metadata Silo
// already owns. It is manual and idempotent; provider refreshes subsequently
// replace only their own authoritative aliases.
type BackfillMediaItemAliasesTask struct {
backfiller MediaItemAliasBackfiller
}
func NewBackfillMediaItemAliasesTask(backfiller MediaItemAliasBackfiller) *BackfillMediaItemAliasesTask {
return &BackfillMediaItemAliasesTask{backfiller: backfiller}
}
func (t *BackfillMediaItemAliasesTask) Key() string { return "backfill_media_item_aliases" }
func (t *BackfillMediaItemAliasesTask) Name() string {
return "Backfill Media Item Aliases"
}
func (t *BackfillMediaItemAliasesTask) Description() string {
return "Seeds searchable aliases from existing original and localized media titles in resumable batches."
}
func (t *BackfillMediaItemAliasesTask) Category() taskmanager.TaskCategory {
return taskmanager.TaskCategoryMetadata
}
func (t *BackfillMediaItemAliasesTask) IsHidden() bool { return false }
func (t *BackfillMediaItemAliasesTask) DefaultTriggers() []taskmanager.TriggerConfig {
return nil
}
func (t *BackfillMediaItemAliasesTask) Execute(ctx context.Context, progress taskmanager.ProgressReporter) error {
if t == nil || t.backfiller == nil {
progress.Report(100, "Media item alias backfill is not configured")
return nil
}
if err := t.backfiller.ResetCompletedBackfill(ctx); err != nil {
return fmt.Errorf("reset completed media item alias backfill: %w", err)
}
cursor := ""
total := 0
progress.Report(0, "Backfilling media item aliases")
for {
next, processed, err := t.backfiller.BackfillBatch(ctx, cursor, mediaItemAliasBackfillBatchSize)
if err != nil {
return fmt.Errorf("backfill media item aliases after %q: %w", cursor, err)
}
total += processed
if processed == 0 || next == cursor {
break
}
cursor = next
progress.Report(0, fmt.Sprintf("Backfilled aliases for %d media items", total))
if processed < mediaItemAliasBackfillBatchSize {
break
}
}
result, _ := json.Marshal(map[string]any{"processed_items": total, "last_content_id": cursor})
progress.SetResultData(result)
progress.Report(100, fmt.Sprintf("Backfilled aliases for %d media items", total))
return nil
}
@@ -0,0 +1,68 @@
package tasks
import (
"context"
"encoding/json"
"testing"
)
type aliasBackfillPage struct {
next string
count int
}
type fakeAliasBackfiller struct {
pages []aliasBackfillPage
cursors []string
resets int
}
func (f *fakeAliasBackfiller) ResetCompletedBackfill(context.Context) error {
f.resets++
return nil
}
func (f *fakeAliasBackfiller) BackfillBatch(_ context.Context, cursor string, limit int) (string, int, error) {
f.cursors = append(f.cursors, cursor)
if limit != mediaItemAliasBackfillBatchSize {
panic("unexpected batch size")
}
page := f.pages[0]
f.pages = f.pages[1:]
return page.next, page.count, nil
}
type aliasBackfillProgress struct {
result json.RawMessage
}
func (*aliasBackfillProgress) Report(float64, string) {}
func (p *aliasBackfillProgress) SetResultData(data json.RawMessage) {
p.result = append(p.result[:0], data...)
}
func TestBackfillMediaItemAliasesTaskAdvancesCursorUntilShortBatch(t *testing.T) {
t.Parallel()
backfiller := &fakeAliasBackfiller{pages: []aliasBackfillPage{
{next: "item-0500", count: mediaItemAliasBackfillBatchSize},
{next: "item-0502", count: 2},
}}
progress := &aliasBackfillProgress{}
if err := NewBackfillMediaItemAliasesTask(backfiller).Execute(context.Background(), progress); err != nil {
t.Fatalf("Execute() error = %v", err)
}
if len(backfiller.cursors) != 2 || backfiller.cursors[0] != "" || backfiller.cursors[1] != "item-0500" {
t.Fatalf("cursors = %#v", backfiller.cursors)
}
if backfiller.resets != 1 {
t.Fatalf("completed-state resets = %d, want exactly 1 before batching", backfiller.resets)
}
var result map[string]any
if err := json.Unmarshal(progress.result, &result); err != nil {
t.Fatalf("result JSON = %q: %v", progress.result, err)
}
if result["processed_items"] != float64(502) || result["last_content_id"] != "item-0502" {
t.Fatalf("result = %#v", result)
}
}
@@ -0,0 +1,36 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE media_item_aliases (
id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
content_id text NOT NULL REFERENCES media_items(content_id) ON DELETE CASCADE,
title text NOT NULL CHECK (btrim(title) <> ''),
normalized_title text GENERATED ALWAYS AS (public.normalize_search_text(title)) STORED,
language text NOT NULL DEFAULT '',
kind text NOT NULL CHECK (kind IN ('original', 'localized', 'alternate')),
provider text NOT NULL DEFAULT '',
created_at timestamptz NOT NULL DEFAULT now(),
updated_at timestamptz NOT NULL DEFAULT now()
);
CREATE UNIQUE INDEX idx_media_item_aliases_unique
ON media_item_aliases (content_id, normalized_title, language, kind, provider);
CREATE INDEX idx_media_item_aliases_content_provider
ON media_item_aliases (content_id, provider);
CREATE INDEX idx_media_item_aliases_normalized_trgm
ON media_item_aliases USING gin (normalized_title public.gin_trgm_ops);
CREATE INDEX idx_media_item_aliases_search_vector
ON media_item_aliases USING gin (to_tsvector('simple', normalized_title));
-- Persist the batch cursor so an interrupted backfill resumes at its last
-- committed item instead of rescanning the whole catalog on every run.
CREATE TABLE media_item_alias_backfill_state (
task_key text PRIMARY KEY,
last_content_id text NOT NULL DEFAULT '',
completed boolean NOT NULL DEFAULT false,
updated_at timestamptz NOT NULL DEFAULT now()
);
-- +goose StatementEnd
-- +goose Down
DROP TABLE IF EXISTS media_item_alias_backfill_state;
DROP TABLE IF EXISTS media_item_aliases;
@@ -0,0 +1,38 @@
-- +goose Up
-- An alias can describe one language while belonging to a snapshot requested
-- in another language. Track the request scope separately so replacing a
-- complete snapshot removes every alias that snapshot previously supplied.
-- NULL is reserved for rows that predate request-scope tracking; every new
-- writer supplies either a normalized language or '' for a provider-wide
-- snapshot.
ALTER TABLE media_item_aliases
ADD COLUMN snapshot_language text;
DROP INDEX idx_media_item_aliases_unique;
CREATE UNIQUE INDEX idx_media_item_aliases_unique
ON media_item_aliases (
content_id,
normalized_title,
language,
kind,
provider,
snapshot_language
);
-- +goose Down
-- Multiple request scopes may contain the same public alias. Keep one before
-- restoring the original uniqueness definition.
DELETE FROM media_item_aliases duplicate
USING media_item_aliases keeper
WHERE duplicate.id > keeper.id
AND duplicate.content_id = keeper.content_id
AND duplicate.normalized_title = keeper.normalized_title
AND duplicate.language = keeper.language
AND duplicate.kind = keeper.kind
AND duplicate.provider = keeper.provider;
DROP INDEX idx_media_item_aliases_unique;
ALTER TABLE media_item_aliases
DROP COLUMN snapshot_language;
CREATE UNIQUE INDEX idx_media_item_aliases_unique
ON media_item_aliases (content_id, normalized_title, language, kind, provider);
+7
View File
@@ -4362,6 +4362,13 @@ export interface TaskInfo {
// Match dialog types
export interface MatchCandidate {
title: string;
original_title?: string;
aliases?: Array<{ title: string; language?: string; kind: string; provider?: string }>;
title_language?: string;
title_is_fallback?: boolean;
matched_title?: string;
match_score?: number;
match_reasons?: string[];
year: number;
content_type: string;
provider_ids: Record<string, string>;
@@ -91,6 +91,8 @@ describe("MatchItemDialog", () => {
provider_ids: { tmdb: "27205", imdb: "tt1375666" },
sources: ["tmdb", "tvdb"],
agreement_hints: ["agreed_by_tmdb_and_tvdb"],
match_score: 87.25,
match_reasons: ["title_exact", "provider_id_consensus"],
},
{
title: "Inception (TV)",
@@ -114,9 +116,82 @@ describe("MatchItemDialog", () => {
expect(markup).toContain("Inception (TV)");
expect(markup).toContain("tmdb");
expect(markup).toContain("2 sources agree");
expect(markup).toContain("Score 87.3");
expect(markup).toContain("Match reasons:");
expect(markup).toContain("title exact, provider id consensus");
expect(markup).toContain('data-testid="match-candidate"');
});
it("shows a matched library-language alias before a native fallback title", () => {
mocks.useSearchItemMatchCandidates.mockReturnValue({
mutate: vi.fn(),
isPending: false,
isSuccess: true,
data: {
candidates: [
{
title: "倒凶十将伝",
original_title: "倒凶十将伝",
title_language: "ja",
title_is_fallback: true,
matched_title: "10 Tokyo Warriors",
year: 1999,
content_type: "series",
image_url: "",
overview: "",
provider_ids: { tvdb: "123" },
sources: ["tvdb"],
agreement_hints: [],
},
],
},
});
const markup = renderToStaticMarkup(
<MatchItemDialog item={baseItem} open={true} onOpenChange={() => {}} />,
);
expect(markup).toContain(">10 Tokyo Warriors</");
expect(markup).toContain("Native title: 倒凶十将伝");
expect(markup).not.toContain("Matched alias:");
});
it("keeps a confirmed localized title ahead of a native matched alias", () => {
mocks.useSearchItemMatchCandidates.mockReturnValue({
mutate: vi.fn(),
isPending: false,
isSuccess: true,
data: {
candidates: [
{
title: "10 Tokyo Warriors",
original_title: "倒凶十将伝",
title_language: "en",
matched_title: "倒凶十将伝",
year: 1999,
content_type: "series",
image_url: "",
overview: "",
provider_ids: { tvdb: "123" },
sources: ["tvdb"],
agreement_hints: [],
},
],
},
});
const markup = renderToStaticMarkup(
<MatchItemDialog item={baseItem} open={true} onOpenChange={() => {}} />,
);
expect(markup).toContain(">10 Tokyo Warriors</");
expect(markup).toContain("Original: 倒凶十将伝");
// The matched alias equals the original title shown directly above it;
// repeating the same string as a second line is noise, so it is suppressed.
expect(markup).not.toContain("Matched alias: 倒凶十将伝");
expect(markup).not.toContain("Native title: 10 Tokyo Warriors");
});
it("shows no results message when search returns empty", () => {
mocks.useSearchItemMatchCandidates.mockReturnValue({
mutate: vi.fn(),
+40 -2
View File
@@ -313,6 +313,13 @@ export default function MatchItemDialog({ item, open, onOpenChange }: MatchItemD
const candidateKey = Object.entries(candidate.provider_ids)
.map(([k, v]) => `${k}-${v}`)
.join("_");
const matchedFallbackTitle =
candidate.title_is_fallback &&
candidate.matched_title &&
candidate.matched_title !== candidate.title
? candidate.matched_title
: undefined;
const displayTitle = matchedFallbackTitle ?? candidate.title;
return (
<button
key={`${candidateKey}-${index}`}
@@ -341,7 +348,7 @@ export default function MatchItemDialog({ item, open, onOpenChange }: MatchItemD
>
<img
src={candidate.image_url}
alt={candidate.title}
alt={displayTitle}
className="h-72 w-48 rounded-md object-cover"
/>
</TooltipContent>
@@ -350,11 +357,34 @@ export default function MatchItemDialog({ item, open, onOpenChange }: MatchItemD
<div className="bg-muted h-24 w-16 shrink-0 rounded" />
)}
<div className="min-w-0 flex-1">
<div className="truncate text-sm font-medium">{candidate.title}</div>
<div className="truncate text-sm font-medium">{displayTitle}</div>
{matchedFallbackTitle ? (
<div className="text-muted-foreground truncate text-xs">
Native title: {candidate.title}
</div>
) : candidate.original_title &&
candidate.original_title !== candidate.title ? (
<div className="text-muted-foreground truncate text-xs">
Original: {candidate.original_title}
</div>
) : null}
{!matchedFallbackTitle &&
candidate.matched_title &&
candidate.matched_title !== candidate.title &&
candidate.matched_title !== candidate.original_title ? (
<div className="text-muted-foreground truncate text-xs">
Matched alias: {candidate.matched_title}
</div>
) : null}
<div className="text-muted-foreground text-xs">
{candidate.year ? candidate.year : ""}
</div>
<div className="mt-1 flex min-w-0 flex-wrap gap-1">
{candidate.match_score !== undefined ? (
<Badge variant="secondary" className="text-[10px] tabular-nums">
Score {candidate.match_score.toFixed(1)}
</Badge>
) : null}
{candidate.sources.map((source) => (
<Badge key={source} variant="outline" className="text-[10px]">
{source}
@@ -366,6 +396,14 @@ export default function MatchItemDialog({ item, open, onOpenChange }: MatchItemD
</Badge>
)}
</div>
{candidate.match_reasons?.length ? (
<div className="text-muted-foreground mt-1 text-xs">
Match reasons:{" "}
{candidate.match_reasons
.map((reason) => reason.replace(/_/g, " "))
.join(", ")}
</div>
) : null}
</div>
</button>
);