diff --git a/internal/api/handlers/legacy_read_routes.go b/internal/api/handlers/legacy_read_routes.go index 57019c07..8e7c293b 100644 --- a/internal/api/handlers/legacy_read_routes.go +++ b/internal/api/handlers/legacy_read_routes.go @@ -54,7 +54,7 @@ func buildLegacySearchCatalogValues(values url.Values) (url.Values, bool) { if len(types) > 1 { return nil, false } - if len(types) == 1 && types[0] != "movie" && types[0] != "series" { + if len(types) == 1 && types[0] != "movie" && types[0] != "series" && types[0] != "episode" { return nil, false } diff --git a/internal/api/handlers/legacy_read_routes_test.go b/internal/api/handlers/legacy_read_routes_test.go new file mode 100644 index 00000000..db33872e --- /dev/null +++ b/internal/api/handlers/legacy_read_routes_test.go @@ -0,0 +1,33 @@ +package handlers + +import ( + "net/url" + "testing" +) + +func TestBuildLegacySearchCatalogValuesAcceptsEpisode(t *testing.T) { + values := url.Values{ + "q": {"Who Are You?"}, + "type": {"episode"}, + "limit": {"25"}, + "offset": {"50"}, + } + + translated, ok := buildLegacySearchCatalogValues(values) + if !ok { + t.Fatal("episode search was not routed through the catalog resolver") + } + for key, want := range map[string]string{ + "source": "query", + "q": "Who Are You?", + "type": "episode", + "sort": "relevance", + "order": "desc", + "limit": "25", + "offset": "50", + } { + if got := translated.Get(key); got != want { + t.Fatalf("translated %s = %q, want %q", key, got, want) + } + } +} diff --git a/internal/catalog/item_repo.go b/internal/catalog/item_repo.go index 8a058a5a..efa41285 100644 --- a/internal/catalog/item_repo.go +++ b/internal/catalog/item_repo.go @@ -132,9 +132,9 @@ func (r *ItemRepository) GetPosterPath(ctx context.Context, contentID string) (s const overviewMatchFloor = 0.15 // itemColumnNames lists, in scan order, every column selected by media_items -// item queries. Shared by itemColumns, qualifiedItemColumns, -// qualifiedListItemColumns, and qualifiedItemColumnRefs so the select lists -// can never drift from each other or from scanItem. +// item queries. Shared by itemColumns, qualifiedItemColumns, and +// qualifiedListItemColumns so the select lists can never drift from each +// other or from scanItem. var itemColumnNames = []string{ "content_id", "type", "title", "sort_title", "default_metadata_language", "original_title", "year", "genres", "content_rating", "runtime", "overview", "tagline", @@ -194,17 +194,6 @@ func qualifiedItemColumns(alias string) string { return joinItemColumns(alias) } -// qualifiedItemColumnRefs renders plain alias-qualified column references -// without COALESCE or AS aliases, for contexts like GROUP BY where output -// aliases are invalid. -func qualifiedItemColumnRefs(alias string) string { - refs := make([]string, len(itemColumnNames)) - for i, col := range itemColumnNames { - refs[i] = alias + "." + col - } - return strings.Join(refs, ", ") -} - func qualifiedListItemColumns(alias string) string { exprs := make([]string, len(itemColumnNames)) for i, col := range itemColumnNames { @@ -965,224 +954,7 @@ func (r *ItemRepository) buildSearchSQL(query string, itemTypes []string, limit, } func (r *ItemRepository) buildSearchSQLWithTotal(query string, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool) (dataSQL, countSQL string, args []any) { - parsed := parseSearchQuery(query) - searchText := parsed.Text - if searchText == "" { - searchText = collapseSearchWhitespace(strings.ReplaceAll(strings.TrimSpace(query), "\"", " ")) - } - if searchText == "" { - return "", "", nil - } - var conditions []string - titlePrefixIdx := 2 - args = []any{searchText, buildTitlePrefixTsQuery(searchText)} - argIdx := 3 - - // All title-side text on both sides of @@ flows through - // public.normalize_search_text() (migrations 127 / 138), which strips - // non-alphanumeric chars, drops standalone "and" tokens, and normalizes - // common title numbers. The expression must match - // idx_media_items_search_title_fields exactly for the GIN index to be used. - // - // Overview uses the 'english' config which natively treats "and" as a - // stop word, so it does not need explicit normalization. - titleVector := `( - setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.title, ''))), 'A') || - setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.original_title, ''))), 'A') || - setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B') - )` - overviewVector := `to_tsvector('english', COALESCE(mi.overview, ''))` - normalizedTitleExpr := `public.normalize_search_text(%s)` - titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))` - titlePrefixQuery := fmt.Sprintf("to_tsquery('simple', $%d)", titlePrefixIdx) - titleMatch := fmt.Sprintf("%s @@ %s", titleVector, titleQuery) - titlePrefixMatch := fmt.Sprintf("($%d <> '' AND %s @@ %s)", titlePrefixIdx, titleVector, titlePrefixQuery) - overviewMatch := fmt.Sprintf("%s @@ websearch_to_tsquery('english', $1)", overviewVector) - - // Keep the base match condition index-friendly; exact-title logic is used as - // a ranking boost later, not as an additional scan predicate. - conditions = append(conditions, fmt.Sprintf("(%s OR %s OR %s)", titleMatch, titlePrefixMatch, overviewMatch)) - - if len(itemTypes) > 0 { - placeholders := make([]string, 0, len(itemTypes)) - for _, itemType := range itemTypes { - if strings.TrimSpace(itemType) == "" { - continue - } - placeholders = append(placeholders, fmt.Sprintf("$%d", argIdx)) - args = append(args, strings.ToLower(strings.TrimSpace(itemType))) - argIdx++ - } - if len(placeholders) > 0 { - conditions = append(conditions, fmt.Sprintf("mi.type IN (%s)", strings.Join(placeholders, ", "))) - } - } - - needsLibJoin := filter.AllowedLibraryIDs != nil || len(filter.DisabledLibraryIDs) > 0 - fromClause := "media_items mi" - if filter.AllowedLibraryIDs != nil { - // Caller (Search) is expected to short-circuit when len == 0; we still - // guard here so the builder is safe to invoke from tests. - if len(filter.AllowedLibraryIDs) > 0 { - conditions = append(conditions, fmt.Sprintf("mil.media_folder_id = ANY($%d)", argIdx)) - args = append(args, filter.AllowedLibraryIDs) - argIdx++ - } - } - if len(filter.DisabledLibraryIDs) > 0 { - conditions = append(conditions, fmt.Sprintf("NOT (mil.media_folder_id = ANY($%d))", argIdx)) - args = append(args, filter.DisabledLibraryIDs) - argIdx++ - } - if needsLibJoin { - fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id" - } - applyAccessFilter("mi", AccessFilter{MaxContentRating: filter.MaxContentRating, ExcludedMediaTypes: filter.ExcludedMediaTypes}, &conditions, &args, &argIdx) - - // Manga chapters (type='ebook' rows linked into a manga series) are internal - // sub-units and must never surface as standalone search results. - conditions = append(conditions, MangaChapterExclusionWhere("mi")) - - whereClause := "WHERE " + strings.Join(conditions, " AND ") - - // Bind ExactTitleHint exactly once. The same arg index is referenced by - // both contiguous_title_match and exact_title_match (used as ranking - // signals in the ORDER BY). The post-CTE WHERE itself gates on title_rank - // > 0 (true title FTS match), not on contiguous_title_match. - exactIdx := argIdx - args = append(args, parsed.ExactTitleHint) - argIdx++ - - // mi.title uses the title_normalized stored generated column (migrations - // 105 / 127), so the LIKE '%pattern%' arm hits the gin_trgm_ops index - // instead of recomputing normalization per row. original_title and - // sort_title do not have a trigram index of their own, so their LIKE - // arms call public.normalize_search_text() per row. The tsvector @@ - // path covers all three via idx_media_items_search_title_fields. - contiguousTitleMatch := fmt.Sprintf(`( - $%d <> '' AND ( - %s LIKE '%%' || $%d || '%%' OR - %s LIKE '%%' || $%d || '%%' OR - %s LIKE '%%' || $%d || '%%' - ) - )`, - exactIdx, - "mi.title_normalized", exactIdx, - fmt.Sprintf(normalizedTitleExpr, "mi.original_title"), exactIdx, - fmt.Sprintf(normalizedTitleExpr, "mi.sort_title"), exactIdx, - ) - - var yearArg any - if parsed.Year != nil { - yearArg = *parsed.Year - } - yearIdx := argIdx - args = append(args, yearArg) - argIdx++ - phraseIdx := argIdx - args = append(args, parsed.Phrase) - argIdx++ - - exactTitleMatch := fmt.Sprintf(`( - $%d <> '' AND ( - %s = $%d OR - %s = $%d OR - %s = $%d - ) - )`, - exactIdx, - "mi.title_normalized", exactIdx, - fmt.Sprintf(normalizedTitleExpr, "mi.original_title"), exactIdx, - fmt.Sprintf(normalizedTitleExpr, "mi.sort_title"), exactIdx, - ) - - // Use qualified column names inside the CTE to avoid ambiguity when - // the FROM clause includes a JOIN to media_item_libraries. The select - // list aliases coalesced columns back to their own names (poster_path - // etc.) so the outer query can re-reference them; GROUP BY needs the - // raw references because output aliases are invalid there. - qualifiedCols := qualifiedItemColumns("mi") - groupByCols := qualifiedItemColumnRefs("mi") - scoredCTE := fmt.Sprintf(` - WITH scored AS ( - SELECT - %s, - MAX(CASE - WHEN %s THEN 1 - ELSE 0 - END) AS exact_title_match, - MAX(CASE - WHEN %s THEN 1 - ELSE 0 - END) AS contiguous_title_match, - MAX(CASE - WHEN $%d::int IS NOT NULL AND mi.year = $%d::int THEN 1 - ELSE 0 - END) AS year_match, - MAX(ts_rank_cd(%s, %s)) AS title_rank, - MAX(CASE - WHEN $%d <> '' THEN ts_rank_cd(%s, %s) - ELSE 0 - END) AS title_prefix_rank, - MAX(ts_rank_cd(%s, websearch_to_tsquery('english', $1))) AS overview_rank, - MAX(CASE - WHEN $%d <> '' THEN ts_rank_cd(%s, phraseto_tsquery('simple', public.normalize_search_text($%d))) - ELSE 0 - END) AS phrase_rank - FROM %s - %s - GROUP BY %s - ) - `, qualifiedCols, exactTitleMatch, contiguousTitleMatch, yearIdx, yearIdx, titleVector, titleQuery, titlePrefixIdx, titleVector, titlePrefixQuery, overviewVector, phraseIdx, titleVector, phraseIdx, fromClause, whereClause, groupByCols) - - // COUNT(*) OVER () runs after the GROUP BY in the scored CTE collapses - // duplicates from the library JOIN, so the window count preserves the - // COUNT(DISTINCT mi.content_id) semantics of the prior 2-query path. - // The stats CTE + CROSS JOIN below means the window count reflects the - // post-filter row count automatically. - // - // The stats CTE gates on title_rank > 0 (true title FTS match) rather - // than contiguous_title_match (LIKE substring), so reordered-token title - // queries like "order law" → "Law and Order" aren't wrongly demoted to - // the overview-fallback bucket. The post-CTE WHERE then keeps every title - // FTS hit, and admits overview-only rows only when no title hit exists - // anywhere in the candidate set AND the overview rank clears - // overviewMatchFloor. This suppresses single-occurrence body matches - // (which score at PostgreSQL's default D weight of 0.1) for common - // single-word queries like "obsession" that would otherwise flood - // results with description-only hits. - // - // countSQL is a count-only sibling that omits LIMIT/OFFSET/ORDER BY. It is - // invoked only as a fallback when the data SELECT returns an empty page - // past offset 0 — COUNT(*) OVER () emits no rows in that case so total - // would otherwise default to 0. - statsCTE := ` - , stats AS ( - SELECT MAX(CASE WHEN title_rank > 0 OR title_prefix_rank > 0 THEN 1 ELSE 0 END) AS has_title_match - FROM scored - )` - // postFilter is the FROM + CROSS JOIN + WHERE shared by both dataSQL and - // countSQL. Keeping it in one string ensures the empty-page fallback count - // can never drift from the data query's filter. - postFilter := fmt.Sprintf(`FROM scored - CROSS JOIN stats - WHERE scored.title_rank > 0 - OR scored.title_prefix_rank > 0 - OR (COALESCE(stats.has_title_match, 0) = 0 AND scored.overview_rank >= %g)`, overviewMatchFloor) - totalColumn := "" - if includeTotal { - totalColumn = ", COUNT(*) OVER () AS total_count" - } - dataSQL = scoredCTE + statsCTE + fmt.Sprintf(` - SELECT %s%s - %s - ORDER BY exact_title_match DESC, contiguous_title_match DESC, year_match DESC, phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC, LOWER(title) ASC, content_id ASC - LIMIT $%d OFFSET $%d`, itemColumns, totalColumn, postFilter, argIdx, argIdx+1) - countSQL = scoredCTE + statsCTE + fmt.Sprintf(` - SELECT COUNT(*) - %s`, postFilter) - args = append(args, limit, offset) - return dataSQL, countSQL, args + return r.buildMixedSearchSQLWithTotal(query, itemTypes, limit, offset, filter, includeTotal) } // ListUnmatchedByFolderAndPathPrefix returns content IDs for unmatched-style diff --git a/internal/catalog/item_repo_test.go b/internal/catalog/item_repo_test.go index cbedbe48..527a3621 100644 --- a/internal/catalog/item_repo_test.go +++ b/internal/catalog/item_repo_test.go @@ -490,32 +490,73 @@ func TestItemRepo_Search_ScoredCTEExposesItemColumnNames(t *testing.T) { } } -// TestItemRepo_Search_GroupByHasNoOutputAliases asserts the scored CTE's -// GROUP BY uses raw column references: AS aliases are valid in a select list -// but are a syntax error inside GROUP BY, so the CTE must not reuse the -// aliased projection there. -func TestItemRepo_Search_GroupByHasNoOutputAliases(t *testing.T) { +// TestItemRepo_Search_ScoredCTEIsLean asserts that relevance ranking carries +// only candidate identity/rank fields. Wide MediaItem columns are hydrated +// from the page CTE after LIMIT/OFFSET, which keeps broad episode searches +// from sorting metadata arrays and image fields for every candidate. +func TestItemRepo_Search_ScoredCTEIsLean(t *testing.T) { repo := &ItemRepository{} dataSQL, countSQL, _ := repo.buildSearchSQL("avatar", []string{"movie"}, 20, 0, AccessFilter{}) for _, sql := range []string{dataSQL, countSQL} { - idx := strings.Index(sql, "GROUP BY") - if idx < 0 { - t.Fatalf("expected GROUP BY in scored CTE; got:\n%s", sql) + end := strings.Index(sql, "), stats AS") + if end < 0 { + t.Fatalf("expected scored CTE boundary; got:\n%s", sql) } - clause := sql[idx:] - if end := strings.Index(clause, ")"); end >= 0 { - clause = clause[:end] + scored := sql[:end] + for _, wideColumn := range []string{"poster_path", "backdrop_path", "metadata_s3_path", "genres"} { + if strings.Contains(scored, wideColumn) { + t.Fatalf("scored CTE must not carry %s; got:\n%s", wideColumn, scored) + } } - if strings.Contains(clause, " AS ") { - t.Fatalf("GROUP BY must not contain output aliases; got:\n%s", clause) - } - if strings.Contains(clause, "COALESCE") { - t.Fatalf("GROUP BY should group by raw columns, not COALESCE expressions; got:\n%s", clause) + } +} + +func TestItemRepo_Search_UnscopedIncludesEpisodeCandidateBranch(t *testing.T) { + repo := &ItemRepository{} + sql, _, _ := repo.buildSearchSQL("Who Are You?", nil, 20, 0, AccessFilter{}) + for _, want := range []string{ + "FROM episodes e JOIN media_items si", + "si.type = 'series'", + "FROM episode_libraries available_el", + "UNION ALL", + } { + if !strings.Contains(sql, want) { + t.Fatalf("mixed search SQL missing %q:\n%s", want, sql) } } } +func TestItemRepo_Search_EpisodeScopeOmitsMediaItemCandidateBranch(t *testing.T) { + repo := &ItemRepository{} + sql, _, _ := repo.buildSearchSQL("Who Are You?", []string{"episode"}, 20, 0, AccessFilter{}) + scoredEnd := strings.Index(sql, "), stats AS") + if scoredEnd < 0 { + t.Fatalf("expected scored CTE boundary; got:\n%s", sql) + } + scored := sql[:scoredEnd] + if strings.Contains(scored, "FROM media_items mi") { + t.Fatalf("episode-only candidate set must not scan media_items directly:\n%s", scored) + } + if !strings.Contains(scored, "FROM episodes e JOIN media_items si") { + t.Fatalf("episode-only candidate set missing episode branch:\n%s", scored) + } +} + +func TestItemRepo_Search_EpisodeAccessUsesIndependentMembershipPredicates(t *testing.T) { + repo := &ItemRepository{} + sql, _, _ := repo.buildSearchSQL("Who Are You?", []string{"episode"}, 20, 0, AccessFilter{ + AllowedLibraryIDs: []int{1}, + DisabledLibraryIDs: []int{2}, + }) + if !strings.Contains(sql, "FROM episode_libraries allowed_el") { + t.Fatalf("episode search missing allowed-library EXISTS:\n%s", sql) + } + if !strings.Contains(sql, "NOT EXISTS (SELECT 1 FROM episode_libraries disabled_el") { + t.Fatalf("episode search missing independent disabled-library NOT EXISTS:\n%s", sql) + } +} + // TestItemRepo_ListUnmatchedByFolderAndPathPrefix_ExcludesMangaChapters pins // the manga-chapter exclusion in the unmatched-item lister. Manga chapters are // type='ebook' items that stay status='pending' by design (the type='manga' diff --git a/internal/catalog/search_episode_db_test.go b/internal/catalog/search_episode_db_test.go new file mode 100644 index 00000000..da33eb19 --- /dev/null +++ b/internal/catalog/search_episode_db_test.go @@ -0,0 +1,177 @@ +package catalog + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestEpisodeSearchPostgresAndDocumentSource(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + + suffix := time.Now().UnixNano() + seriesID := fmt.Sprintf("episode-search-series-%d", suffix) + episodeID := fmt.Sprintf("episode-search-episode-%d", suffix) + unavailableID := fmt.Sprintf("episode-search-unavailable-%d", suffix) + podcastID := fmt.Sprintf("episode-search-podcast-%d", suffix) + podcastEpisodeID := fmt.Sprintf("episode-search-podcast-episode-%d", suffix) + var folderID int + if err := pool.QueryRow(ctx, + `INSERT INTO media_folders (type, name, enabled) VALUES ('series', $1, true) RETURNING id`, + seriesID, + ).Scan(&folderID); err != nil { + t.Fatalf("seed folder: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, []string{seriesID, podcastID}) + _, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, folderID) + }) + + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres) + VALUES ($1, 'series', 'Silo Search Fixture', 'matched', '{}'::text[]), + ($2, 'podcast', 'Podcast Search Fixture', 'matched', '{}'::text[]) + `, seriesID, podcastID); err != nil { + t.Fatalf("seed parent items: %v", err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO episodes (content_id, series_id, season_number, episode_number, title, overview) + VALUES ($1, $2, 3, 1, 'Who Are You?', 'A buried signal returns. A buried signal changes everything.'), + ($3, $2, 3, 2, 'Who Are You?', 'Unavailable metadata-only episode.'), + ($4, $5, 1, 1, 'Who Are You?', 'Podcast episode with the same title.') + `, episodeID, seriesID, unavailableID, podcastEpisodeID, podcastID); err != nil { + t.Fatalf("seed episodes: %v", err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO episode_libraries (episode_id, media_folder_id, first_seen_at) + VALUES ($1, $3, NOW()), ($2, $3, NOW()) + `, episodeID, podcastEpisodeID, folderID); err != nil { + t.Fatalf("seed episode memberships: %v", err) + } + + repo := NewItemRepository(pool) + for _, types := range [][]string{nil, {"episode"}} { + items, total, _, exact, err := repo.SearchPage(ctx, "Who Are You?", types, 20, 0, AccessFilter{}, true) + if err != nil { + t.Fatalf("search types %v: %v", types, err) + } + found := false + for _, item := range items { + if item.ContentID == episodeID && item.Type == "episode" { + found = true + } + if item.ContentID == unavailableID || item.ContentID == podcastEpisodeID { + t.Fatalf("search types %v leaked ineligible episode %s", types, item.ContentID) + } + } + if !exact || total < 1 || !found { + t.Fatalf("search types %v = total %d exact %v items %+v", types, total, exact, items) + } + } + + overviewItems, _, _, _, err := repo.SearchPage(ctx, "buried signal", []string{"episode"}, 20, 0, AccessFilter{}, true) + if err != nil || len(overviewItems) != 1 || overviewItems[0].ContentID != episodeID { + t.Fatalf("overview search = items %+v err %v", overviewItems, err) + } + + indexer := NewCatalogSearchIndexer(pool, nil) + docs, err := indexer.LoadDocumentsByIDs(ctx, []string{episodeID, unavailableID, podcastEpisodeID}, nil, DefaultMeilisearchEmbedder, true, false) + if err != nil { + t.Fatalf("load episode documents: %v", err) + } + if len(docs) != 1 || docs[0].ContentID != episodeID || docs[0].Type != "episode" { + t.Fatalf("episode documents = %+v", docs) + } + if len(docs[0].LibraryIDs) != 1 || int(docs[0].LibraryIDs[0]) != folderID { + t.Fatalf("episode library ids = %v, want [%d]", docs[0].LibraryIDs, folderID) + } + if docs[0].Vectors != nil { + t.Fatalf("episode document unexpectedly has vectors: %#v", docs[0].Vectors) + } +} + +func TestEpisodeSearchOutboxStatementTriggers(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + var captureFunction *string + if err := pool.QueryRow(ctx, `SELECT to_regprocedure('public.catalog_search_capture_enabled()')::text`).Scan(&captureFunction); err != nil || captureFunction == nil { + t.Skip("episode search outbox trigger migration is not applied") + } + + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatalf("begin: %v", err) + } + defer tx.Rollback(ctx) //nolint:errcheck + if _, err := tx.Exec(ctx, `UPDATE server_settings SET value = 'meilisearch' WHERE key = 'catalog.search.provider'`); err != nil { + t.Fatalf("enable trigger capture: %v", err) + } + suffix := time.Now().UnixNano() + seriesID := fmt.Sprintf("episode-trigger-series-%d", suffix) + episodeID := fmt.Sprintf("episode-trigger-episode-%d", suffix) + var folderID int + if err := tx.QueryRow(ctx, + `INSERT INTO media_folders (type, name, enabled) VALUES ('series', $1, true) RETURNING id`, seriesID, + ).Scan(&folderID); err != nil { + t.Fatalf("seed folder: %v", err) + } + if _, err := tx.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres) + VALUES ($1, 'series', 'Trigger Series', 'matched', '{}'::text[]) + `, seriesID); err != nil { + t.Fatalf("seed series: %v", err) + } + if _, err := tx.Exec(ctx, `INSERT INTO media_item_libraries (content_id, media_folder_id) VALUES ($1, $2)`, seriesID, folderID); err != nil { + t.Fatalf("seed item membership: %v", err) + } + if _, err := tx.Exec(ctx, ` + INSERT INTO episodes (content_id, series_id, season_number, episode_number, title) + VALUES ($1, $2, 1, 1, 'Trigger Episode') + `, episodeID, seriesID); err != nil { + t.Fatalf("seed episode: %v", err) + } + if _, err := tx.Exec(ctx, `INSERT INTO episode_libraries (episode_id, media_folder_id) VALUES ($1, $2)`, episodeID, folderID); err != nil { + t.Fatalf("seed episode membership: %v", err) + } + + var upserts int + if err := tx.QueryRow(ctx, ` + SELECT COUNT(*) + FROM catalog_search_index_events + WHERE action = 'upsert' AND content_id = $1 + `, episodeID).Scan(&upserts); err != nil || upserts < 2 { + t.Fatalf("episode insert/membership upserts = %d, err %v", upserts, err) + } + if _, err := tx.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, seriesID); err != nil { + t.Fatalf("delete series: %v", err) + } + var deletes int + if err := tx.QueryRow(ctx, ` + SELECT COUNT(*) + FROM catalog_search_index_events + WHERE action = 'delete' AND content_id = $1 + `, episodeID).Scan(&deletes); err != nil || deletes == 0 { + t.Fatalf("episode cascade deletes = %d, err %v", deletes, err) + } +} diff --git a/internal/catalog/search_indexer.go b/internal/catalog/search_indexer.go index fae05318..9b91707e 100644 --- a/internal/catalog/search_indexer.go +++ b/internal/catalog/search_indexer.go @@ -535,7 +535,7 @@ func (i *CatalogSearchIndexer) CheckConnection(ctx context.Context, settings Cat func catalogSearchMeilisearchSettings(embedder string, semanticEnabled, binaryQuantized bool) map[string]any { settings := map[string]any{ "displayedAttributes": []string{"content_id", "type"}, - "filterableAttributes": []string{"type"}, + "filterableAttributes": []string{"type", "library_ids"}, "searchableAttributes": []string{ "title", "original_title", @@ -623,6 +623,7 @@ type catalogSearchDocument struct { Countries []string `json:"countries,omitempty"` Keywords []string `json:"keywords,omitempty"` People []string `json:"people,omitempty"` + LibraryIDs []int32 `json:"library_ids,omitempty"` SchemaVersion int `json:"schema_version"` Vectors map[string][]float32 `json:"_vectors,omitempty"` } @@ -632,19 +633,14 @@ func (i *CatalogSearchIndexer) LoadDocumentsAfter(ctx context.Context, afterCont return nil, nil } typeFilter := normalizeCatalogSearchItemTypes(itemTypes) - whereClause := ` - WHERE ($1::text = '' OR mi.content_id > $1) - AND ` + catalogSearchExcludeMangaChaptersSQL args := []any{afterContentID, limit} + typeArg := 0 if len(typeFilter) > 0 { - whereClause += ` - AND mi.type = ANY($3)` + typeArg = 3 args = append(args, typeFilter) } - rows, err := i.pool.Query(ctx, catalogSearchDocumentSelectSQL(whereClause, ` - ORDER BY mi.content_id ASC - LIMIT $2 - `), args...) + rows, err := i.pool.Query(ctx, mixedCatalogSearchDocumentSQL( + `mi.content_id > $1`, `e.content_id > $1`, typeArg, "LIMIT $2"), args...) if err != nil { return nil, err } @@ -666,18 +662,14 @@ func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentID return nil, nil } typeFilter := normalizeCatalogSearchItemTypes(itemTypes) - whereClause := ` - WHERE mi.content_id = ANY($1) - AND ` + catalogSearchExcludeMangaChaptersSQL args := []any{contentIDs} + typeArg := 0 if len(typeFilter) > 0 { - whereClause += ` - AND mi.type = ANY($2)` + typeArg = 2 args = append(args, typeFilter) } - rows, err := i.pool.Query(ctx, catalogSearchDocumentSelectSQL(whereClause, ` - ORDER BY mi.content_id ASC - `), args...) + rows, err := i.pool.Query(ctx, mixedCatalogSearchDocumentSQL( + `mi.content_id = ANY($1)`, `e.content_id = ANY($1)`, typeArg, ""), args...) if err != nil { return nil, err } @@ -693,31 +685,85 @@ func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentID return docs, nil } -func catalogSearchDocumentSelectSQL(whereClause string, tailClause string) string { - return ` - SELECT - mi.content_id, - mi.type, - COALESCE(mi.title, ''), - COALESCE(mi.sort_title, ''), - COALESCE(mi.original_title, ''), - COALESCE(mi.year, 0), - COALESCE(mi.overview, ''), - COALESCE(mi.tagline, ''), - COALESCE(mi.genres, ARRAY[]::text[]), - COALESCE(mi.studios, ARRAY[]::text[]), - COALESCE(mi.networks, ARRAY[]::text[]), - COALESCE(mi.countries, ARRAY[]::text[]), - COALESCE(mi.keywords, ARRAY[]::text[]), - COALESCE(array_agg(DISTINCT p.name) FILTER (WHERE p.name IS NOT NULL AND p.name <> ''), ARRAY[]::text[]) AS people - FROM media_items mi - LEFT JOIN item_people ip ON ip.content_id = mi.content_id - LEFT JOIN people p ON p.id = ip.person_id - ` + whereClause + ` - GROUP BY - mi.content_id, mi.type, mi.title, mi.sort_title, mi.original_title, mi.year, - mi.overview, mi.tagline, mi.genres, mi.studios, mi.networks, mi.countries, mi.keywords - ` + tailClause +// mixedCatalogSearchDocumentSQL first pages narrow IDs across both physical +// sources, then aggregates people and library memberships only for that batch. +// This keeps rebuild work proportional to RebuildBatchSize rather than the +// total number of catalog rows. +func mixedCatalogSearchDocumentSQL(mediaPredicate, episodePredicate string, typeArg int, limitClause string) string { + mediaTypePredicate := "" + episodeTypePredicate := "" + if typeArg > 0 { + mediaTypePredicate = fmt.Sprintf(" AND mi.type = ANY($%d)", typeArg) + episodeTypePredicate = fmt.Sprintf(" AND 'episode' = ANY($%d)", typeArg) + } + return fmt.Sprintf(` + WITH candidates AS ( + SELECT mi.content_id, mi.type + FROM media_items mi + WHERE %s + AND %s%s + UNION ALL + SELECT e.content_id, 'episode'::text AS type + FROM episodes e + JOIN media_items si ON si.content_id = e.series_id AND si.type = 'series' + WHERE %s%s + AND EXISTS (SELECT 1 FROM episode_libraries el WHERE el.episode_id = e.content_id) + ORDER BY content_id ASC + %s + ) + SELECT * FROM ( + SELECT + mi.content_id, + mi.type, + COALESCE(mi.title, ''), + COALESCE(mi.sort_title, ''), + COALESCE(mi.original_title, ''), + COALESCE(mi.year, 0), + COALESCE(mi.overview, ''), + COALESCE(mi.tagline, ''), + COALESCE(mi.genres, ARRAY[]::text[]), + COALESCE(mi.studios, ARRAY[]::text[]), + COALESCE(mi.networks, ARRAY[]::text[]), + COALESCE(mi.countries, ARRAY[]::text[]), + COALESCE(mi.keywords, ARRAY[]::text[]), + COALESCE(people.names, ARRAY[]::text[]), + COALESCE(libraries.ids, ARRAY[]::integer[]) + FROM candidates c + JOIN media_items mi ON c.type <> 'episode' AND mi.content_id = c.content_id + LEFT JOIN LATERAL ( + SELECT array_agg(DISTINCT p.name) FILTER (WHERE p.name IS NOT NULL AND p.name <> '') AS names + FROM item_people ip + JOIN people p ON p.id = ip.person_id + WHERE ip.content_id = mi.content_id + ) people ON true + LEFT JOIN LATERAL ( + SELECT array_agg(DISTINCT mil.media_folder_id ORDER BY mil.media_folder_id) AS ids + FROM media_item_libraries mil + WHERE mil.content_id = mi.content_id + ) libraries ON true + UNION ALL + SELECT + e.content_id, + 'episode'::text, + COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text), + COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text), + ''::text, + COALESCE(EXTRACT(YEAR FROM e.air_date)::integer, 0), + COALESCE(e.overview, ''), + ''::text, + ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[], + COALESCE(libraries.ids, ARRAY[]::integer[]) + FROM candidates c + JOIN episodes e ON c.type = 'episode' AND e.content_id = c.content_id + LEFT JOIN LATERAL ( + SELECT array_agg(DISTINCT el.media_folder_id ORDER BY el.media_folder_id) AS ids + FROM episode_libraries el + WHERE el.episode_id = e.content_id + ) libraries ON true + ) documents + ORDER BY content_id ASC`, + mediaPredicate, catalogSearchExcludeMangaChaptersSQL, mediaTypePredicate, + episodePredicate, episodeTypePredicate, limitClause) } func scanCatalogSearchDocuments(rows pgx.Rows) ([]catalogSearchDocument, error) { @@ -739,6 +785,7 @@ func scanCatalogSearchDocuments(rows pgx.Rows) ([]catalogSearchDocument, error) &doc.Countries, &doc.Keywords, &doc.People, + &doc.LibraryIDs, ); err != nil { return nil, err } @@ -759,7 +806,7 @@ func (i *CatalogSearchIndexer) attachDocumentVectors(ctx context.Context, docs [ } ids := make([]string, 0, len(docs)) for _, doc := range docs { - if strings.TrimSpace(doc.ContentID) != "" { + if doc.Type != "episode" && strings.TrimSpace(doc.ContentID) != "" { ids = append(ids, doc.ContentID) } } @@ -814,6 +861,10 @@ func setCatalogSearchDocumentVectors(docs []catalogSearchDocument, vectors map[s } count := 0 for idx := range docs { + if docs[idx].Type == "episode" { + docs[idx].Vectors = nil + continue + } vector := vectors[docs[idx].ContentID] docs[idx].Vectors = map[string][]float32{embedder: nil} if len(vector) > 0 { @@ -891,10 +942,17 @@ func countCatalogSearchEligibleDocuments(ctx context.Context, q coverageQuerier, } var count int if err := q.QueryRow(ctx, ` - SELECT COUNT(*) - FROM media_items mi - WHERE `+catalogSearchExcludeMangaChaptersSQL+` - AND ($1::text[] IS NULL OR mi.type = ANY($1)) + SELECT + (SELECT COUNT(*) + FROM media_items mi + WHERE `+catalogSearchExcludeMangaChaptersSQL+` + AND ($1::text[] IS NULL OR mi.type = ANY($1))) + + + (SELECT COUNT(*) + FROM episodes e + JOIN media_items si ON si.content_id = e.series_id AND si.type = 'series' + WHERE ($1::text[] IS NULL OR 'episode' = ANY($1)) + AND EXISTS (SELECT 1 FROM episode_libraries el WHERE el.episode_id = e.content_id)) `, typeArg).Scan(&count); err != nil { return 0, fmt.Errorf("count catalog search eligible documents: %w", err) } diff --git a/internal/catalog/search_meilisearch_client.go b/internal/catalog/search_meilisearch_client.go index 43a98db4..407bd1cb 100644 --- a/internal/catalog/search_meilisearch_client.go +++ b/internal/catalog/search_meilisearch_client.go @@ -106,6 +106,30 @@ type meilisearchSearchResponse struct { Query string `json:"query"` } +type meilisearchFederatedSearchRequest struct { + Federation meilisearchFederationOptions `json:"federation"` + Queries []meilisearchFederatedQuery `json:"queries"` +} + +type meilisearchFederationOptions struct { + Offset int `json:"offset"` + Limit int `json:"limit"` +} + +type meilisearchFederatedQuery struct { + IndexUID string `json:"indexUid"` + Query string `json:"q"` + Filter string `json:"filter,omitempty"` + AttributesToRetrieve []string `json:"attributesToRetrieve"` + AttributesToSearchOn []string `json:"attributesToSearchOn,omitempty"` + MatchingStrategy string `json:"matchingStrategy,omitempty"` + Vector []float32 `json:"vector,omitempty"` + Hybrid *meilisearchHybridRequest `json:"hybrid,omitempty"` + FederationOptions struct { + Weight float64 `json:"weight"` + } `json:"federationOptions"` +} + type meilisearchStatsResponse struct { NumberOfDocuments int `json:"numberOfDocuments"` } @@ -217,6 +241,12 @@ func (c *meilisearchClient) Search(ctx context.Context, uid string, req meilisea return out, err } +func (c *meilisearchClient) FederatedSearch(ctx context.Context, req meilisearchFederatedSearchRequest) (meilisearchSearchResponse, error) { + var out meilisearchSearchResponse + err := c.do(ctx, http.MethodPost, "/multi-search", req, &out) + return out, err +} + func (c *meilisearchClient) AddDocuments(ctx context.Context, uid string, docs []catalogSearchDocument) (meilisearchTaskRef, error) { if len(docs) == 0 { return meilisearchTaskRef{}, nil diff --git a/internal/catalog/search_meilisearch_client_test.go b/internal/catalog/search_meilisearch_client_test.go index 97299552..ac0ddaee 100644 --- a/internal/catalog/search_meilisearch_client_test.go +++ b/internal/catalog/search_meilisearch_client_test.go @@ -2,6 +2,7 @@ package catalog import ( "context" + "encoding/json" "errors" "fmt" "net/http" @@ -11,6 +12,45 @@ import ( "time" ) +func TestMeilisearchFederatedSearchUsesMultiSearchPagination(t *testing.T) { + var gotPath string + var gotRequest meilisearchFederatedSearchRequest + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + if err := json.NewDecoder(r.Body).Decode(&gotRequest); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"hits":[{"content_id":"episode-1"}],"offset":20,"limit":10,"estimatedTotalHits":1}`)) + })) + defer server.Close() + + client, err := newMeilisearchClient(server.URL, "", time.Second) + if err != nil { + t.Fatalf("newMeilisearchClient: %v", err) + } + request := meilisearchFederatedSearchRequest{ + Federation: meilisearchFederationOptions{Offset: 20, Limit: 10}, + Queries: []meilisearchFederatedQuery{ + {IndexUID: "catalog-v3", Query: "Who Are You?"}, + }, + } + response, err := client.FederatedSearch(context.Background(), request) + if err != nil { + t.Fatalf("FederatedSearch: %v", err) + } + if gotPath != "/multi-search" { + t.Fatalf("request path = %q, want /multi-search", gotPath) + } + if gotRequest.Federation.Offset != 20 || gotRequest.Federation.Limit != 10 || len(gotRequest.Queries) != 1 { + t.Fatalf("decoded request = %#v", gotRequest) + } + if len(response.Hits) != 1 || response.Hits[0].ContentID != "episode-1" { + t.Fatalf("response = %#v", response) + } +} + func TestMeilisearchWaitTaskAcceptsZeroTaskUID(t *testing.T) { var gotPath string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/internal/catalog/search_meilisearch_provider.go b/internal/catalog/search_meilisearch_provider.go index 3b61c308..fc7b6a85 100644 --- a/internal/catalog/search_meilisearch_provider.go +++ b/internal/catalog/search_meilisearch_provider.go @@ -7,6 +7,7 @@ import ( "net" "net/http" "sort" + "strconv" "strings" "sync" "time" @@ -43,6 +44,7 @@ var meilisearchTitleSearchAttributes = []string{ "original_title", "sort_title", "title_variants", + "overview", } type meilisearchIndexStateStore interface { @@ -97,6 +99,9 @@ type MeilisearchSearchProvider struct { capMu sync.Mutex capCache CatalogSearchSemanticCapability capCachedAt time.Time + + federationMu sync.Mutex + federationUnsupported bool } type cachedCatalogSearchQueryVector struct { @@ -151,6 +156,14 @@ func NewMeilisearchSearchProvider( } func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearchRequest) (*CatalogSearchResult, error) { + if req.Access.AllowedLibraryIDs != nil && len(req.Access.AllowedLibraryIDs) == 0 { + return &CatalogSearchResult{ + Items: []*models.MediaItem{}, + TotalExact: !req.SkipTotal, + Provider: SearchProviderMeilisearch, + Mode: "keyword", + }, nil + } if p == nil || p.client == nil { return p.fallbackSearch(ctx, req, "meilisearch not configured") } @@ -270,6 +283,13 @@ func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req C estimatedTotalHits := 0 exhausted := false baseSearchReq, semanticFallback := p.buildMeilisearchSearchRequest(ctx, req) + useFederation := baseSearchReq.Hybrid != nil && searchRequestMixesEpisodeAndMedia(req.ItemTypes) + if useFederation && p.isFederationUnsupported() { + semanticFallback = "meilisearch federation unsupported; using keyword search" + baseSearchReq.Vector = nil + baseSearchReq.Hybrid = nil + useFederation = false + } for len(accessible) < target && !exhausted { if scanned >= meilisearchCandidateScanCap { @@ -282,8 +302,25 @@ func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req C searchReq := baseSearchReq searchReq.Offset = meiliOffset searchReq.Limit = nextLimit - resp, err := p.client.Search(ctx, indexUID, searchReq) - if err != nil && baseSearchReq.Hybrid != nil { + var resp meilisearchSearchResponse + var err error + if useFederation { + resp, err = p.client.FederatedSearch(ctx, p.buildFederatedSearchRequest(indexUID, req, baseSearchReq, meiliOffset, nextLimit)) + if err != nil && isMeilisearchFederationUnsupported(err) { + p.markFederationUnsupported() + semanticFallback = "meilisearch federation unsupported; using keyword search" + baseSearchReq.Vector = nil + baseSearchReq.Hybrid = nil + useFederation = false + searchReq = baseSearchReq + searchReq.Offset = meiliOffset + searchReq.Limit = nextLimit + resp, err = p.client.Search(ctx, indexUID, searchReq) + } + } else { + resp, err = p.client.Search(ctx, indexUID, searchReq) + } + if err != nil && baseSearchReq.Hybrid != nil && !useFederation { semanticFallback = "meilisearch hybrid search failed: " + err.Error() baseSearchReq.Vector = nil baseSearchReq.Hybrid = nil @@ -316,7 +353,7 @@ func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req C position[id] = i ids = append(ids, id) } - hydrated, err := p.itemRepo.GetByIDsWithAccess(ctx, ids, req.Access) + hydrated, err := p.itemRepo.GetSearchItemsByIDsWithAccess(ctx, ids, req.Access) if err != nil { return nil, err } @@ -371,13 +408,13 @@ func (p *MeilisearchSearchProvider) buildMeilisearchSearchRequest(ctx context.Co if p == nil { return meilisearchSearchRequest{ Query: strings.TrimSpace(req.Query), - Filter: meilisearchTypeFilter(req.ItemTypes), + Filter: meilisearchSearchFilter(req.ItemTypes, req.Access), AttributesToRetrieve: []string{"content_id"}, }, "" } searchReq := meilisearchSearchRequest{ Query: strings.TrimSpace(req.Query), - Filter: meilisearchTypeFilter(req.ItemTypes), + Filter: meilisearchSearchFilter(req.ItemTypes, req.Access), AttributesToRetrieve: []string{"content_id"}, AttributesToSearchOn: p.attributesToSearchOnForRequest(req), MatchingStrategy: p.matchingStrategyForRequest(req), @@ -412,9 +449,116 @@ func (p *MeilisearchSearchProvider) shouldUseSemanticSearch(req CatalogSearchReq if p == nil || !p.config.SemanticEnabled { return false } + mediaTypes, includesEpisodes := splitSearchItemTypes(req.ItemTypes) + if includesEpisodes && len(req.ItemTypes) > 0 && len(mediaTypes) == 0 { + return false + } return len(catalogSearchQueryTerms(req.Query)) >= meilisearchShortHybridMinTerms } +func searchRequestMixesEpisodeAndMedia(itemTypes []string) bool { + mediaTypes, includesEpisodes := splitSearchItemTypes(itemTypes) + return includesEpisodes && (len(itemTypes) == 0 || len(mediaTypes) > 0) +} + +func (p *MeilisearchSearchProvider) buildFederatedSearchRequest( + indexUID string, + req CatalogSearchRequest, + base meilisearchSearchRequest, + offset, limit int, +) meilisearchFederatedSearchRequest { + mediaTypes, _ := splitSearchItemTypes(req.ItemTypes) + mediaFilter := "" + if len(req.ItemTypes) == 0 { + mediaFilter = joinMeilisearchFilters(`type != "episode"`, meilisearchSearchFilter(nil, req.Access)) + } else { + mediaFilter = meilisearchSearchFilter(mediaTypes, req.Access) + } + episodeFilter := meilisearchSearchFilter([]string{"episode"}, req.Access) + mediaQuery := meilisearchFederatedQuery{ + IndexUID: indexUID, + Query: base.Query, + Filter: mediaFilter, + AttributesToRetrieve: base.AttributesToRetrieve, + AttributesToSearchOn: base.AttributesToSearchOn, + MatchingStrategy: base.MatchingStrategy, + Vector: base.Vector, + Hybrid: base.Hybrid, + } + mediaQuery.FederationOptions.Weight = 1 + episodeQuery := meilisearchFederatedQuery{ + IndexUID: indexUID, + Query: base.Query, + Filter: episodeFilter, + AttributesToRetrieve: base.AttributesToRetrieve, + AttributesToSearchOn: base.AttributesToSearchOn, + MatchingStrategy: base.MatchingStrategy, + } + episodeQuery.FederationOptions.Weight = 1 + return meilisearchFederatedSearchRequest{ + Federation: meilisearchFederationOptions{Offset: offset, Limit: limit}, + Queries: []meilisearchFederatedQuery{mediaQuery, episodeQuery}, + } +} + +func joinMeilisearchFilters(parts ...string) string { + cleaned := make([]string, 0, len(parts)) + for _, part := range parts { + if part = strings.TrimSpace(part); part != "" { + cleaned = append(cleaned, part) + } + } + return strings.Join(cleaned, " AND ") +} + +func (p *MeilisearchSearchProvider) isFederationUnsupported() bool { + p.federationMu.Lock() + defer p.federationMu.Unlock() + return p.federationUnsupported +} + +func (p *MeilisearchSearchProvider) markFederationUnsupported() { + p.federationMu.Lock() + p.federationUnsupported = true + p.federationMu.Unlock() +} + +func isMeilisearchFederationUnsupported(err error) bool { + var httpErr *meilisearchHTTPError + if !errors.As(err, &httpErr) { + return false + } + code := strings.ToLower(strings.TrimSpace(httpErr.Code)) + message := strings.ToLower(strings.TrimSpace(httpErr.Message)) + if httpErr.StatusCode == http.StatusNotFound { + // A cached active index can disappear during an atomic rebuild swap. + // That is a resource miss, not evidence that this Meilisearch process + // lacks federation; caching it would disable mixed hybrid search until + // Silo restarts even after the provider refreshes the active index UID. + if code == "index_not_found" || code == "document_not_found" { + return false + } + return code == "not_found" || + (code == "" && (strings.Contains(message, "route") || strings.Contains(message, "multi-search"))) + } + if httpErr.StatusCode != http.StatusBadRequest { + return false + } + + // Older Meilisearch versions expose /multi-search but reject the newer + // federation object as an unknown field. Cache only that capability-shaped + // failure; malformed or incompatible federated queries should keep falling + // through to PostgreSQL so a request bug is not hidden for the process life. + details := code + " " + message + mentionsFederation := strings.Contains(details, "federation") + unsupported := strings.Contains(details, "not supported") || + strings.Contains(details, "unsupported") || + strings.Contains(details, "unknown field") || + strings.Contains(details, "unrecognized field") || + strings.Contains(details, "feature not enabled") + return mentionsFederation && unsupported +} + func (p *MeilisearchSearchProvider) matchingStrategyForRequest(req CatalogSearchRequest) string { strategy := strings.TrimSpace(strings.ToLower(p.config.MatchingStrategy)) if strategy == "" { @@ -740,6 +884,31 @@ func meilisearchTypeFilter(itemTypes []string) string { return "(" + strings.Join(parts, " OR ") + ")" } +func meilisearchSearchFilter(itemTypes []string, access AccessFilter) string { + parts := make([]string, 0, 4) + if typeFilter := meilisearchTypeFilter(itemTypes); typeFilter != "" { + parts = append(parts, typeFilter) + } + if access.AllowedLibraryIDs != nil { + parts = append(parts, "library_ids IN ["+joinMeilisearchIntFilterValues(access.AllowedLibraryIDs)+"]") + } + if len(access.DisabledLibraryIDs) > 0 { + if access.AllowedLibraryIDs == nil { + parts = append(parts, "library_ids IS NOT EMPTY") + } + parts = append(parts, "library_ids NOT IN ["+joinMeilisearchIntFilterValues(access.DisabledLibraryIDs)+"]") + } + return strings.Join(parts, " AND ") +} + +func joinMeilisearchIntFilterValues(values []int) string { + parts := make([]string, 0, len(values)) + for _, value := range values { + parts = append(parts, strconv.Itoa(value)) + } + return strings.Join(parts, ", ") +} + func normalizeCatalogSearchQueryForVector(query string) string { return strings.Join(strings.Fields(query), " ") } diff --git a/internal/catalog/search_postgres_mixed.go b/internal/catalog/search_postgres_mixed.go new file mode 100644 index 00000000..5d7624d3 --- /dev/null +++ b/internal/catalog/search_postgres_mixed.go @@ -0,0 +1,327 @@ +package catalog + +import ( + "context" + "fmt" + "strings" + + "github.com/Silo-Server/silo-server/internal/access" + "github.com/Silo-Server/silo-server/internal/models" +) + +// GetSearchItemsByIDsWithAccess hydrates mixed search hits in one round trip. +// Meilisearch IDs are only candidates: both branches reapply the effective +// access filter, and callers restore provider order after this method returns. +func (r *ItemRepository) GetSearchItemsByIDsWithAccess( + ctx context.Context, + contentIDs []string, + filter AccessFilter, +) ([]*models.MediaItem, error) { + if len(contentIDs) == 0 || (filter.AllowedLibraryIDs != nil && len(filter.AllowedLibraryIDs) == 0) { + return []*models.MediaItem{}, nil + } + + args := []any{contentIDs} + argIdx := 2 + mediaConditions := []string{"hydrated_mi.content_id = ANY($1)"} + appendLibraryAccessConditions("hydrated_mi.content_id", filter, &mediaConditions, &args, &argIdx) + applyAccessFilter("hydrated_mi", AccessFilter{ + MaxContentRating: filter.MaxContentRating, + ExcludedMediaTypes: filter.ExcludedMediaTypes, + }, &mediaConditions, &args, &argIdx) + + episodeConditions := []string{"mi.content_id = ANY($1)"} + appendEpisodeLibrarySearchAccess("mi.content_id", filter, &episodeConditions, &args, &argIdx) + applyAccessFilter("mi", AccessFilter{ + MaxContentRating: filter.MaxContentRating, + ExcludedMediaTypes: filter.ExcludedMediaTypes, + }, &episodeConditions, &args, &argIdx) + + query := fmt.Sprintf(` + SELECT %s + FROM media_items hydrated_mi + WHERE %s + UNION ALL + SELECT %s + FROM %s + WHERE %s`, + qualifiedItemColumns("hydrated_mi"), strings.Join(mediaConditions, " AND "), + qualifiedItemColumns("mi"), episodeCatalogBaseRelation, strings.Join(episodeConditions, " AND ")) + + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("hydrating mixed search items: %w", err) + } + defer rows.Close() + return scanItems(rows) +} + +const episodeSearchTitleExpr = `COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text)` + +const episodeSearchTitleVector = `setweight( + to_tsvector('simple', public.normalize_search_text(COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text))), + 'A' +)` + +const episodeSearchOverviewVector = `to_tsvector('english', COALESCE(e.overview, ''))` + +const mixedSearchOrder = `exact_title_match DESC, contiguous_title_match DESC, year_match DESC, + phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC, + LOWER(title) ASC, content_id ASC` + +// buildMixedSearchSQLWithTotal builds one ranked candidate set from the two +// physical catalog sources. The scored CTE deliberately carries only ranking +// fields; the wide MediaItem projection is hydrated after LIMIT/OFFSET so a +// broad match never sorts posters, arrays, or metadata blobs for every hit. +func (r *ItemRepository) buildMixedSearchSQLWithTotal( + query string, + itemTypes []string, + limit, offset int, + filter AccessFilter, + includeTotal bool, +) (dataSQL, countSQL string, args []any) { + parsed := parseSearchQuery(query) + searchText := parsed.Text + if searchText == "" { + searchText = collapseSearchWhitespace(strings.ReplaceAll(strings.TrimSpace(query), "\"", " ")) + } + if searchText == "" { + return "", "", nil + } + + mediaTypes, includeEpisodes := splitSearchItemTypes(itemTypes) + includeMediaItems := len(itemTypes) == 0 || len(mediaTypes) > 0 + if !includeMediaItems && !includeEpisodes { + return "", "", nil + } + + args = []any{searchText, buildTitlePrefixTsQuery(searchText)} + argIdx := 3 + var branches []string + + mediaConditions := []string{} + if includeMediaItems { + mediaTitleVector := `( + setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.title, ''))), 'A') || + setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.original_title, ''))), 'A') || + setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B') + )` + mediaOverviewVector := `to_tsvector('english', COALESCE(mi.overview, ''))` + mediaConditions = append(mediaConditions, searchMatchCondition(mediaTitleVector, mediaOverviewVector)) + if len(itemTypes) > 0 { + mediaConditions = append(mediaConditions, fmt.Sprintf("mi.type = ANY($%d)", argIdx)) + args = append(args, mediaTypes) + argIdx++ + } + appendLibraryAccessConditions("mi.content_id", filter, &mediaConditions, &args, &argIdx) + applyAccessFilter("mi", AccessFilter{ + MaxContentRating: filter.MaxContentRating, + ExcludedMediaTypes: filter.ExcludedMediaTypes, + }, &mediaConditions, &args, &argIdx) + mediaConditions = append(mediaConditions, MangaChapterExclusionWhere("mi")) + } + + episodeConditions := []string{} + if includeEpisodes { + episodeConditions = append(episodeConditions, + "si.type = 'series'", + `EXISTS (SELECT 1 FROM episode_libraries available_el WHERE available_el.episode_id = e.content_id)`, + searchMatchCondition(episodeSearchTitleVector, episodeSearchOverviewVector), + ) + appendEpisodeLibrarySearchAccess("e.content_id", filter, &episodeConditions, &args, &argIdx) + if filter.MaxContentRating != "" { + allowedRatings := access.AllowedRatingsUpTo(filter.MaxContentRating) + if len(allowedRatings) == 0 { + episodeConditions = append(episodeConditions, "1 = 0") + } else { + episodeConditions = append(episodeConditions, fmt.Sprintf("si.content_rating = ANY($%d)", argIdx)) + args = append(args, allowedRatings) + argIdx++ + } + } + if len(filter.ExcludedMediaTypes) > 0 { + episodeConditions = append(episodeConditions, fmt.Sprintf("NOT ('episode' = ANY($%d))", argIdx)) + args = append(args, filter.ExcludedMediaTypes) + argIdx++ + } + } + + exactIdx := argIdx + args = append(args, parsed.ExactTitleHint) + argIdx++ + var yearArg any + if parsed.Year != nil { + yearArg = *parsed.Year + } + yearIdx := argIdx + args = append(args, yearArg) + argIdx++ + phraseIdx := argIdx + args = append(args, parsed.Phrase) + argIdx++ + + if includeMediaItems { + branches = append(branches, buildMixedSearchCandidateBranch( + "mi.content_id", "mi.type", "mi.title", "mi.year", + `( + setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.title, ''))), 'A') || + setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.original_title, ''))), 'A') || + setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B') + )`, + `to_tsvector('english', COALESCE(mi.overview, ''))`, + []string{`mi.title_normalized`, `public.normalize_search_text(mi.original_title)`, `public.normalize_search_text(mi.sort_title)`}, + "media_items mi", mediaConditions, exactIdx, yearIdx, phraseIdx, + )) + } + if includeEpisodes { + episodeNormalizedTitle := `public.normalize_search_text(` + episodeSearchTitleExpr + `)` + branches = append(branches, buildMixedSearchCandidateBranch( + "e.content_id", "'episode'::text", episodeSearchTitleExpr, + "COALESCE(si.year, EXTRACT(YEAR FROM e.air_date)::integer, 0)", + episodeSearchTitleVector, episodeSearchOverviewVector, + []string{episodeNormalizedTitle}, + "episodes e JOIN media_items si ON si.content_id = e.series_id", + episodeConditions, exactIdx, yearIdx, phraseIdx, + )) + } + + scoredCTE := "WITH scored AS (\n" + strings.Join(branches, "\nUNION ALL\n") + "\n)" + statsCTE := `, stats AS ( + SELECT MAX(CASE WHEN title_rank > 0 OR title_prefix_rank > 0 THEN 1 ELSE 0 END) AS has_title_match + FROM scored + )` + postFilter := fmt.Sprintf(`FROM scored + CROSS JOIN stats + WHERE scored.title_rank > 0 + OR scored.title_prefix_rank > 0 + OR (COALESCE(stats.has_title_match, 0) = 0 AND scored.overview_rank >= %g)`, overviewMatchFloor) + + pageTotalColumn := "" + finalTotalColumn := "" + if includeTotal { + pageTotalColumn = ", COUNT(*) OVER () AS total_count" + finalTotalColumn = ", page.total_count" + } + limitIdx, offsetIdx := argIdx, argIdx+1 + args = append(args, limit, offset) + + pageCTE := fmt.Sprintf(`, page AS ( + SELECT scored.*, ROW_NUMBER() OVER (ORDER BY %s) AS ordinal%s + %s + ORDER BY %s + LIMIT $%d OFFSET $%d + )`, mixedSearchOrder, pageTotalColumn, postFilter, mixedSearchOrder, limitIdx, offsetIdx) + + hydratedRelation := fmt.Sprintf(`LATERAL ( + SELECT %s + FROM media_items hydrated_mi + WHERE page.type <> 'episode' + AND hydrated_mi.content_id = page.content_id + UNION ALL + SELECT %s + FROM %s + WHERE page.type = 'episode' + AND mi.content_id = page.content_id + ) hydrated`, qualifiedItemColumns("hydrated_mi"), qualifiedItemColumns("mi"), episodeCatalogBaseRelation) + + dataSQL = scoredCTE + statsCTE + pageCTE + fmt.Sprintf(` + SELECT %s%s + FROM page + JOIN %s ON true + ORDER BY page.ordinal`, qualifiedItemColumns("hydrated"), finalTotalColumn, hydratedRelation) + countSQL = scoredCTE + statsCTE + fmt.Sprintf("\nSELECT COUNT(*)\n%s", postFilter) + return dataSQL, countSQL, args +} + +func splitSearchItemTypes(itemTypes []string) (mediaTypes []string, includeEpisodes bool) { + if len(itemTypes) == 0 { + return nil, true + } + seen := make(map[string]struct{}, len(itemTypes)) + for _, itemType := range itemTypes { + itemType = strings.ToLower(strings.TrimSpace(itemType)) + if itemType == "" { + continue + } + if itemType == "episode" { + includeEpisodes = true + continue + } + if _, ok := seen[itemType]; ok { + continue + } + seen[itemType] = struct{}{} + mediaTypes = append(mediaTypes, itemType) + } + return mediaTypes, includeEpisodes +} + +func searchMatchCondition(titleVector, overviewVector string) string { + titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))` + prefixQuery := `to_tsquery('simple', $2)` + return fmt.Sprintf(`((%s) @@ %s OR ($2 <> '' AND (%s) @@ %s) OR (%s) @@ websearch_to_tsquery('english', $1))`, + titleVector, titleQuery, titleVector, prefixQuery, overviewVector) +} + +func buildMixedSearchCandidateBranch( + contentIDExpr, typeExpr, titleExpr, yearExpr, titleVector, overviewVector string, + exactTitleExprs []string, + fromClause string, + conditions []string, + exactIdx, yearIdx, phraseIdx int, +) string { + exactArms := make([]string, 0, len(exactTitleExprs)) + contiguousArms := make([]string, 0, len(exactTitleExprs)) + for _, expr := range exactTitleExprs { + exactArms = append(exactArms, fmt.Sprintf("%s = $%d", expr, exactIdx)) + contiguousArms = append(contiguousArms, fmt.Sprintf("%s LIKE '%%' || $%d || '%%'", expr, exactIdx)) + } + titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))` + prefixQuery := `to_tsquery('simple', $2)` + return fmt.Sprintf(` + SELECT + %s AS content_id, + %s AS type, + %s AS title, + CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS exact_title_match, + CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS contiguous_title_match, + CASE WHEN $%d::int IS NOT NULL AND (%s) = $%d::int THEN 1 ELSE 0 END AS year_match, + ts_rank_cd(%s, %s) AS title_rank, + CASE WHEN $2 <> '' THEN ts_rank_cd(%s, %s) ELSE 0 END AS title_prefix_rank, + ts_rank_cd(%s, websearch_to_tsquery('english', $1)) AS overview_rank, + CASE WHEN $%d <> '' THEN ts_rank_cd(%s, phraseto_tsquery('simple', public.normalize_search_text($%d))) ELSE 0 END AS phrase_rank + FROM %s + WHERE %s`, + contentIDExpr, typeExpr, titleExpr, + exactIdx, strings.Join(exactArms, " OR "), + exactIdx, strings.Join(contiguousArms, " OR "), + yearIdx, yearExpr, yearIdx, + titleVector, titleQuery, + titleVector, prefixQuery, + overviewVector, + phraseIdx, titleVector, phraseIdx, + fromClause, strings.Join(conditions, " AND ")) +} + +func appendEpisodeLibrarySearchAccess( + episodeIDExpr string, + filter AccessFilter, + conditions *[]string, + args *[]any, + argIdx *int, +) { + if filter.AllowedLibraryIDs != nil { + *conditions = append(*conditions, fmt.Sprintf( + "EXISTS (SELECT 1 FROM episode_libraries allowed_el WHERE allowed_el.episode_id = %s AND allowed_el.media_folder_id = ANY($%d))", + episodeIDExpr, *argIdx)) + *args = append(*args, filter.AllowedLibraryIDs) + *argIdx++ + } + if len(filter.DisabledLibraryIDs) > 0 { + *conditions = append(*conditions, fmt.Sprintf( + "NOT EXISTS (SELECT 1 FROM episode_libraries disabled_el WHERE disabled_el.episode_id = %s AND disabled_el.media_folder_id = ANY($%d))", + episodeIDExpr, *argIdx)) + *args = append(*args, filter.DisabledLibraryIDs) + *argIdx++ + } +} diff --git a/internal/catalog/search_provider.go b/internal/catalog/search_provider.go index 97fbe11d..41c87e36 100644 --- a/internal/catalog/search_provider.go +++ b/internal/catalog/search_provider.go @@ -37,7 +37,7 @@ const ( DefaultMeilisearchIndex = "silo_media_items" DefaultMeilisearchTimeoutMS = 800 DefaultMeilisearchMatchingStrategy = "last" - SearchMeilisearchSchemaVersion = 2 + SearchMeilisearchSchemaVersion = 3 DefaultMeilisearchSyncBatchSize = 500 DefaultMeilisearchRebuildBatchSize = 5000 diff --git a/internal/catalog/search_provider_test.go b/internal/catalog/search_provider_test.go index ee74253e..16ffc976 100644 --- a/internal/catalog/search_provider_test.go +++ b/internal/catalog/search_provider_test.go @@ -150,6 +150,14 @@ func TestCatalogSearchMeilisearchSettingsIncludeEmbeddersWhenSemanticEnabled(t * } } +func TestCatalogSearchMeilisearchSettingsFilterLibraryIDs(t *testing.T) { + settings := catalogSearchMeilisearchSettings(DefaultMeilisearchEmbedder, false, false) + filterable, ok := settings["filterableAttributes"].([]string) + if !ok || !reflect.DeepEqual(filterable, []string{"type", "library_ids"}) { + t.Fatalf("filterable attributes = %#v", settings["filterableAttributes"]) + } +} + func TestCatalogSearchDocumentPayloadBatchesSplitVectorDocs(t *testing.T) { vector := make([]float32, 128) docs := []catalogSearchDocument{ @@ -296,6 +304,133 @@ func TestMeilisearchSearchRequestBuildsHybridWhenCoverageReady(t *testing.T) { } } +func TestMeilisearchEpisodeSearchRemainsKeywordOnly(t *testing.T) { + vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}} + provider := &MeilisearchSearchProvider{config: MeilisearchProviderConfig{ + SemanticEnabled: true, + SemanticRatio: 0.4, + Embedder: DefaultMeilisearchEmbedder, + Vectorizer: vectorizer, + Coverage: fakeCoverageGate{ready: true}, + }} + req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{ + Query: "Who Are You", + ItemTypes: []string{"episode"}, + Access: AccessFilter{AllowedLibraryIDs: []int{4, 9}, DisabledLibraryIDs: []int{12}}, + }) + if fallback != "" || req.Hybrid != nil || req.Vector != nil { + t.Fatalf("episode request should be keyword-only, fallback/request = %q/%#v", fallback, req) + } + for _, want := range []string{`type = "episode"`, "library_ids IN [4, 9]", "library_ids NOT IN [12]"} { + if !strings.Contains(req.Filter, want) { + t.Fatalf("episode filter missing %q: %s", want, req.Filter) + } + } + if vectorizer.calls != 0 { + t.Fatalf("episode request generated a vector, calls=%d", vectorizer.calls) + } +} + +func TestMeilisearchMixedSemanticSearchBuildsFederation(t *testing.T) { + provider := &MeilisearchSearchProvider{config: MeilisearchProviderConfig{}} + base := meilisearchSearchRequest{ + Query: "Who Are You", + AttributesToRetrieve: []string{"content_id"}, + MatchingStrategy: "all", + Vector: []float32{0.5, 0.25}, + Hybrid: &meilisearchHybridRequest{Embedder: DefaultMeilisearchEmbedder, SemanticRatio: 0.4}, + } + federated := provider.buildFederatedSearchRequest("catalog-v3", CatalogSearchRequest{ + Access: AccessFilter{AllowedLibraryIDs: []int{4}}, + }, base, 100, 25) + if federated.Federation.Offset != 100 || federated.Federation.Limit != 25 { + t.Fatalf("federation pagination = %#v", federated.Federation) + } + if len(federated.Queries) != 2 { + t.Fatalf("federated queries = %#v", federated.Queries) + } + media, episode := federated.Queries[0], federated.Queries[1] + if media.Hybrid == nil || len(media.Vector) == 0 || !strings.Contains(media.Filter, `type != "episode"`) { + t.Fatalf("media federation branch = %#v", media) + } + if episode.Hybrid != nil || episode.Vector != nil || !strings.Contains(episode.Filter, `type = "episode"`) { + t.Fatalf("episode federation branch = %#v", episode) + } + if media.FederationOptions.Weight != 1 || episode.FederationOptions.Weight != 1 { + t.Fatalf("federation weights = %v/%v", media.FederationOptions.Weight, episode.FederationOptions.Weight) + } +} + +func TestMeilisearchFederationUnsupportedClassification(t *testing.T) { + tests := []struct { + name string + err error + want bool + }{ + { + name: "missing index during rebuild swap is transient", + err: &meilisearchHTTPError{ + StatusCode: http.StatusNotFound, + Code: "index_not_found", + Message: "Index catalog_rebuild_old not found.", + }, + want: false, + }, + { + name: "missing multi-search route is unsupported", + err: &meilisearchHTTPError{ + StatusCode: http.StatusNotFound, + Code: "not_found", + Message: "The multi-search route does not exist.", + }, + want: true, + }, + { + name: "older server rejects federation field", + err: &meilisearchHTTPError{ + StatusCode: http.StatusBadRequest, + Code: "bad_request", + Message: "Unknown field federation: expected queries.", + }, + want: true, + }, + { + name: "invalid federated request is not a capability result", + err: &meilisearchHTTPError{ + StatusCode: http.StatusBadRequest, + Code: "invalid_multi_search_query_federated", + Message: "A query has federationOptions but federation is missing.", + }, + want: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := isMeilisearchFederationUnsupported(test.err); got != test.want { + t.Fatalf("isMeilisearchFederationUnsupported(%v) = %t, want %t", test.err, got, test.want) + } + }) + } +} + +func TestCatalogSearchEpisodeDocumentsNeverCarryVectors(t *testing.T) { + docs := []catalogSearchDocument{ + {ContentID: "movie-1", Type: "movie"}, + {ContentID: "episode-1", Type: "episode"}, + } + count := setCatalogSearchDocumentVectors(docs, map[string][]float32{ + "movie-1": {0.1, 0.2}, + "episode-1": {0.3, 0.4}, + }, DefaultMeilisearchEmbedder) + if count != 1 || docs[0].Vectors == nil { + t.Fatalf("media vector assignment = count %d, vectors %#v", count, docs[0].Vectors) + } + if docs[1].Vectors != nil { + t.Fatalf("episode vectors = %#v, want omitted", docs[1].Vectors) + } +} + func TestSemanticModelProviderUsesCommaOkSafety(t *testing.T) { if got := semanticModelProvider(nil); got != nil { t.Fatalf("nil vectorizer should yield nil model provider, got %#v", got) diff --git a/migrations/sql/20260713233000_episode_search_indexes.sql b/migrations/sql/20260713233000_episode_search_indexes.sql new file mode 100644 index 00000000..deb00152 --- /dev/null +++ b/migrations/sql/20260713233000_episode_search_indexes.sql @@ -0,0 +1,53 @@ +-- +goose NO TRANSACTION + +-- +goose Up +-- Search evaluates these exact expressions in the episode candidate branch. +-- Build concurrently so large episode catalogs remain readable and writable. +-- Remove INVALID remnants first: IF NOT EXISTS otherwise accepts a failed +-- concurrent build and Goose would record a search index that cannot be used. +-- +goose StatementBegin +DO $$ +DECLARE + index_name text; +BEGIN + FOREACH index_name IN ARRAY ARRAY[ + 'idx_episodes_search_title', + 'idx_episodes_search_overview' + ] LOOP + IF EXISTS ( + SELECT 1 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_index i ON i.indexrelid = c.oid + WHERE n.nspname = 'public' + AND c.relname = index_name + AND NOT i.indisvalid + ) THEN + EXECUTE format('DROP INDEX public.%I', index_name); + END IF; + END LOOP; +END; +$$; +-- +goose StatementEnd + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_episodes_search_title +ON public.episodes USING gin ( + setweight( + to_tsvector( + 'simple', + public.normalize_search_text( + COALESCE(NULLIF(BTRIM(title), ''), 'Episode ' || episode_number::text) + ) + ), + 'A' + ) +); + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_episodes_search_overview +ON public.episodes USING gin ( + to_tsvector('english', COALESCE(overview, '')) +); + +-- +goose Down +DROP INDEX CONCURRENTLY IF EXISTS public.idx_episodes_search_overview; +DROP INDEX CONCURRENTLY IF EXISTS public.idx_episodes_search_title; diff --git a/migrations/sql/20260713233100_episode_search_outbox_triggers.sql b/migrations/sql/20260713233100_episode_search_outbox_triggers.sql new file mode 100644 index 00000000..275c2280 --- /dev/null +++ b/migrations/sql/20260713233100_episode_search_outbox_triggers.sql @@ -0,0 +1,216 @@ +-- +goose Up +-- +goose StatementBegin + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_enabled() +RETURNS boolean +LANGUAGE sql +STABLE +AS $$ + SELECT lower(COALESCE( + (SELECT value FROM public.server_settings WHERE key = 'catalog.search.provider'), + 'postgres' + )) = 'meilisearch' +$$; + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_insert() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF; + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'upsert', content_id, '' FROM new_rows GROUP BY content_id; + RETURN NULL; +END; +$$; + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_update() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF; + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'delete', old_rows.content_id, '' + FROM old_rows + LEFT JOIN new_rows ON new_rows.content_id = old_rows.content_id + WHERE new_rows.content_id IS NULL + GROUP BY old_rows.content_id; + + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'upsert', content_id, '' FROM new_rows GROUP BY content_id; + RETURN NULL; +END; +$$; + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_delete() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF; + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'delete', content_id, '' FROM old_rows GROUP BY content_id; + RETURN NULL; +END; +$$; + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_library_insert() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF; + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'upsert', episode_id, '' FROM new_rows GROUP BY episode_id; + RETURN NULL; +END; +$$; + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_library_update() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF; + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'upsert', episode_id, '' + FROM ( + SELECT episode_id FROM old_rows + UNION + SELECT episode_id FROM new_rows + ) changed; + RETURN NULL; +END; +$$; + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_library_delete() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF; + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'upsert', episode_id, '' FROM old_rows GROUP BY episode_id; + RETURN NULL; +END; +$$; + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_item_library_insert() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF; + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'upsert', content_id, '' FROM new_rows GROUP BY content_id; + RETURN NULL; +END; +$$; + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_item_library_update() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF; + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'upsert', content_id, '' + FROM ( + SELECT content_id FROM old_rows + UNION + SELECT content_id FROM new_rows + ) changed; + RETURN NULL; +END; +$$; + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_item_library_delete() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF; + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'upsert', content_id, '' FROM old_rows GROUP BY content_id; + RETURN NULL; +END; +$$; + +CREATE OR REPLACE FUNCTION public.catalog_search_capture_series_type_update() +RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF; + INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id) + SELECT 'meilisearch', 'upsert', e.content_id, '' + FROM ( + SELECT new_rows.content_id + FROM new_rows + JOIN old_rows USING (content_id) + WHERE new_rows.type IS DISTINCT FROM old_rows.type + AND (new_rows.type = 'series' OR old_rows.type = 'series') + ) changed + JOIN public.episodes e ON e.series_id = changed.content_id + GROUP BY e.content_id; + RETURN NULL; +END; +$$; + +CREATE TRIGGER trg_catalog_search_episode_insert +AFTER INSERT ON public.episodes +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_insert(); + +CREATE TRIGGER trg_catalog_search_episode_update +AFTER UPDATE ON public.episodes +REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_update(); + +CREATE TRIGGER trg_catalog_search_episode_delete +AFTER DELETE ON public.episodes +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_delete(); + +CREATE TRIGGER trg_catalog_search_episode_library_insert +AFTER INSERT ON public.episode_libraries +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_library_insert(); + +CREATE TRIGGER trg_catalog_search_episode_library_update +AFTER UPDATE ON public.episode_libraries +REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_library_update(); + +CREATE TRIGGER trg_catalog_search_episode_library_delete +AFTER DELETE ON public.episode_libraries +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_library_delete(); + +CREATE TRIGGER trg_catalog_search_item_library_insert +AFTER INSERT ON public.media_item_libraries +REFERENCING NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_item_library_insert(); + +CREATE TRIGGER trg_catalog_search_item_library_update +AFTER UPDATE ON public.media_item_libraries +REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_item_library_update(); + +CREATE TRIGGER trg_catalog_search_item_library_delete +AFTER DELETE ON public.media_item_libraries +REFERENCING OLD TABLE AS old_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_item_library_delete(); + +CREATE TRIGGER trg_catalog_search_series_type_update +AFTER UPDATE ON public.media_items +REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows +FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_series_type_update(); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TRIGGER IF EXISTS trg_catalog_search_series_type_update ON public.media_items; +DROP TRIGGER IF EXISTS trg_catalog_search_item_library_delete ON public.media_item_libraries; +DROP TRIGGER IF EXISTS trg_catalog_search_item_library_update ON public.media_item_libraries; +DROP TRIGGER IF EXISTS trg_catalog_search_item_library_insert ON public.media_item_libraries; +DROP TRIGGER IF EXISTS trg_catalog_search_episode_library_delete ON public.episode_libraries; +DROP TRIGGER IF EXISTS trg_catalog_search_episode_library_update ON public.episode_libraries; +DROP TRIGGER IF EXISTS trg_catalog_search_episode_library_insert ON public.episode_libraries; +DROP TRIGGER IF EXISTS trg_catalog_search_episode_delete ON public.episodes; +DROP TRIGGER IF EXISTS trg_catalog_search_episode_update ON public.episodes; +DROP TRIGGER IF EXISTS trg_catalog_search_episode_insert ON public.episodes; + +DROP FUNCTION IF EXISTS public.catalog_search_capture_series_type_update(); +DROP FUNCTION IF EXISTS public.catalog_search_capture_item_library_delete(); +DROP FUNCTION IF EXISTS public.catalog_search_capture_item_library_update(); +DROP FUNCTION IF EXISTS public.catalog_search_capture_item_library_insert(); +DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_library_delete(); +DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_library_update(); +DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_library_insert(); +DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_delete(); +DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_update(); +DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_insert(); +DROP FUNCTION IF EXISTS public.catalog_search_capture_enabled(); +-- +goose StatementEnd diff --git a/web/src/components/SearchBar.test.tsx b/web/src/components/SearchBar.test.tsx new file mode 100644 index 00000000..1a555f10 --- /dev/null +++ b/web/src/components/SearchBar.test.tsx @@ -0,0 +1,49 @@ +import { act, fireEvent, render, screen } from "@testing-library/react"; +import { MemoryRouter, useLocation } from "react-router"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { buildCatalogQueryUpdateHref, parseCatalogSearchParams } from "@/pages/catalogSearchParams"; + +import SearchBar from "./SearchBar"; + +function LocationProbe() { + const location = useLocation(); + return {`${location.pathname}${location.search}`}; +} + +describe("SearchBar", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("keeps All Media selected when live search changes the query", () => { + vi.useFakeTimers(); + const state = parseCatalogSearchParams( + new URLSearchParams("source=query&q=heat&type=all&genre=Drama"), + ); + + render( + + buildCatalogQueryUpdateHref(state, query)} + /> + + , + ); + + fireEvent.change(screen.getByRole("textbox"), { target: { value: "heater" } }); + act(() => { + vi.advanceTimersByTime(101); + }); + + const location = new URL(`http://example.test${screen.getByLabelText("location").textContent}`); + expect(location.searchParams.get("q")).toBe("heater"); + expect(location.searchParams.get("type")).toBe("all"); + expect(parseCatalogSearchParams(location.searchParams).query_definition.groups).toContainEqual({ + match: "all", + rules: [{ field: "genre", op: "contains", value: "Drama" }], + }); + }); +}); diff --git a/web/src/components/SearchBar.tsx b/web/src/components/SearchBar.tsx index dc25e662..952a8b36 100644 --- a/web/src/components/SearchBar.tsx +++ b/web/src/components/SearchBar.tsx @@ -12,19 +12,26 @@ interface SearchBarProps { initialQuery?: string; autoFocus?: boolean; prominent?: boolean; + buildSearchHref?: (query: string) => string; } export default function SearchBar({ initialQuery = "", autoFocus = false, prominent = false, + buildSearchHref = buildQueryCatalogHref, }: SearchBarProps) { const [query, setQuery] = useState(initialQuery); const navigate = useViewTransitionNavigate(); const inputRef = useRef(null); const isInitialMount = useRef(true); + const buildSearchHrefRef = useRef(buildSearchHref); const debouncedQuery = useDebounce(query, SEARCH_NAVIGATION_DEBOUNCE_MS); + useEffect(() => { + buildSearchHrefRef.current = buildSearchHref; + }, [buildSearchHref]); + useEffect(() => { if (autoFocus && inputRef.current) { inputRef.current.focus(); @@ -39,14 +46,14 @@ export default function SearchBar({ return; } if (debouncedQuery.trim()) { - navigate(buildQueryCatalogHref(debouncedQuery.trim()), { replace: true }); + navigate(buildSearchHrefRef.current(debouncedQuery.trim()), { replace: true }); } }, [debouncedQuery, prominent, navigate]); function handleSubmit(e: FormEvent) { e.preventDefault(); if (query.trim()) { - navigate(buildQueryCatalogHref(query.trim())); + navigate(buildSearchHref(query.trim())); } } diff --git a/web/src/pages/Catalog.tsx b/web/src/pages/Catalog.tsx index 6cd07d11..dcde0c60 100644 --- a/web/src/pages/Catalog.tsx +++ b/web/src/pages/Catalog.tsx @@ -24,7 +24,8 @@ import { } from "@/lib/historyRemoval"; import { - buildCatalogApiSearchParams, + buildCatalogFilterSearchParams, + buildCatalogQueryUpdateHref, catalogSourceAllowsOverlay, parseCatalogSearchParams, } from "./catalogSearchParams"; @@ -132,6 +133,10 @@ function CatalogResults({ query_definition: { ...state.query_definition, media_scope: preferredScope }, }; }, [hasExplicitScope, preferredScope, state]); + const buildSearchHref = useCallback( + (query: string) => buildCatalogQueryUpdateHref(effectiveState, query), + [effectiveState], + ); const mediaScope = effectiveState.query_definition.media_scope; const activeChipScope: SearchMediaScope = @@ -257,7 +262,12 @@ function CatalogResults({ {state.source === "query" ? (
- +
) : null} @@ -265,7 +275,7 @@ function CatalogResults({ { - const nextSearchParams = buildCatalogApiSearchParams(nextState); + const nextSearchParams = buildCatalogFilterSearchParams(nextState); if (nextSearchParams.toString() !== searchParams.toString()) { setSearchParams(nextSearchParams); } diff --git a/web/src/pages/catalogSearchParams.test.ts b/web/src/pages/catalogSearchParams.test.ts index 1163eef1..767be019 100644 --- a/web/src/pages/catalogSearchParams.test.ts +++ b/web/src/pages/catalogSearchParams.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest"; import { buildCatalogApiSearchParams, + buildCatalogFilterSearchParams, + buildCatalogQueryUpdateHref, buildCatalogHref, buildPersonCatalogHref, buildPersonalCatalogHref, @@ -127,6 +129,37 @@ describe("catalogSourceAllowsOverlay", () => { }); describe("buildCatalogHref", () => { + it("preserves type=all when the query filter selects All Media", () => { + const state = parseCatalogSearchParams(params("source=query&q=heat&type=video")); + state.query_definition = { + ...state.query_definition, + media_scope: undefined, + }; + + const built = buildCatalogFilterSearchParams(state); + + expect(built.get("type")).toBe("all"); + expect(parseCatalogSearchParams(built).query_definition.media_scope).toBeUndefined(); + }); + + it("preserves All Media and other filters when the live query changes", () => { + const state = parseCatalogSearchParams( + params("source=query&q=heat&type=all&genre=Drama&sort=title&order=asc"), + ); + + const href = buildCatalogQueryUpdateHref(state, "heater"); + const built = new URL(`http://example.test${href}`); + + expect(built.searchParams.get("q")).toBe("heater"); + expect(built.searchParams.get("type")).toBe("all"); + expect(built.searchParams.get("sort")).toBe("title"); + expect(built.searchParams.get("order")).toBe("asc"); + expect(parseCatalogSearchParams(built.searchParams).query_definition.groups).toContainEqual({ + match: "all", + rules: [{ field: "genre", op: "contains", value: "Drama" }], + }); + }); + it("keeps explicit added-at sorting in query-source URLs", () => { expect( buildCatalogApiSearchParams({ diff --git a/web/src/pages/catalogSearchParams.ts b/web/src/pages/catalogSearchParams.ts index 7ef1ddaa..31e16748 100644 --- a/web/src/pages/catalogSearchParams.ts +++ b/web/src/pages/catalogSearchParams.ts @@ -181,6 +181,28 @@ export function buildCatalogHref(state: CatalogSearchState): string { return `/catalog?${params.toString()}`; } +// Interactive query filters need an explicit `all` sentinel. Internally an +// unscoped query is represented by an undefined media_scope, but omitting the +// URL type makes Catalog reapply the user's saved default (normally `video`). +// Keep this behavior scoped to filter updates so fresh search URLs can still +// inherit that preference. +export function buildCatalogFilterSearchParams(state: CatalogSearchState): URLSearchParams { + const params = buildCatalogApiSearchParams(state); + if (state.source === "query" && !state.type_override && !state.query_definition.media_scope) { + params.set("type", "all"); + } + return params; +} + +export function buildCatalogQueryUpdateHref(state: CatalogSearchState, q: string): string { + const params = buildCatalogFilterSearchParams({ + ...state, + source: "query", + q, + }); + return `/catalog?${params.toString()}`; +} + export function buildQueryCatalogHref(q?: string): string { return buildCatalogHref({ source: "query",