feat(search): add scalable TV episode search

Add mixed PostgreSQL and Meilisearch episode search with access-safe hydration, incremental indexing, and regression coverage for explicit all-media searches.\n\nPart of #396
This commit is contained in:
Suspense
2026-07-14 00:21:46 +02:00
parent 28c6ddc237
commit 808883cf8f
19 changed files with 1480 additions and 308 deletions
+1 -1
View File
@@ -54,7 +54,7 @@ func buildLegacySearchCatalogValues(values url.Values) (url.Values, bool) {
if len(types) > 1 {
return nil, false
}
if len(types) == 1 && types[0] != "movie" && types[0] != "series" {
if len(types) == 1 && types[0] != "movie" && types[0] != "series" && types[0] != "episode" {
return nil, false
}
@@ -0,0 +1,33 @@
package handlers
import (
"net/url"
"testing"
)
func TestBuildLegacySearchCatalogValuesAcceptsEpisode(t *testing.T) {
values := url.Values{
"q": {"Who Are You?"},
"type": {"episode"},
"limit": {"25"},
"offset": {"50"},
}
translated, ok := buildLegacySearchCatalogValues(values)
if !ok {
t.Fatal("episode search was not routed through the catalog resolver")
}
for key, want := range map[string]string{
"source": "query",
"q": "Who Are You?",
"type": "episode",
"sort": "relevance",
"order": "desc",
"limit": "25",
"offset": "50",
} {
if got := translated.Get(key); got != want {
t.Fatalf("translated %s = %q, want %q", key, got, want)
}
}
}
+4 -232
View File
@@ -132,9 +132,9 @@ func (r *ItemRepository) GetPosterPath(ctx context.Context, contentID string) (s
const overviewMatchFloor = 0.15
// itemColumnNames lists, in scan order, every column selected by media_items
// item queries. Shared by itemColumns, qualifiedItemColumns,
// qualifiedListItemColumns, and qualifiedItemColumnRefs so the select lists
// can never drift from each other or from scanItem.
// item queries. Shared by itemColumns, qualifiedItemColumns, and
// qualifiedListItemColumns so the select lists can never drift from each
// other or from scanItem.
var itemColumnNames = []string{
"content_id", "type", "title", "sort_title", "default_metadata_language", "original_title", "year", "genres",
"content_rating", "runtime", "overview", "tagline",
@@ -194,17 +194,6 @@ func qualifiedItemColumns(alias string) string {
return joinItemColumns(alias)
}
// qualifiedItemColumnRefs renders plain alias-qualified column references
// without COALESCE or AS aliases, for contexts like GROUP BY where output
// aliases are invalid.
func qualifiedItemColumnRefs(alias string) string {
refs := make([]string, len(itemColumnNames))
for i, col := range itemColumnNames {
refs[i] = alias + "." + col
}
return strings.Join(refs, ", ")
}
func qualifiedListItemColumns(alias string) string {
exprs := make([]string, len(itemColumnNames))
for i, col := range itemColumnNames {
@@ -965,224 +954,7 @@ func (r *ItemRepository) buildSearchSQL(query string, itemTypes []string, limit,
}
func (r *ItemRepository) buildSearchSQLWithTotal(query string, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool) (dataSQL, countSQL string, args []any) {
parsed := parseSearchQuery(query)
searchText := parsed.Text
if searchText == "" {
searchText = collapseSearchWhitespace(strings.ReplaceAll(strings.TrimSpace(query), "\"", " "))
}
if searchText == "" {
return "", "", nil
}
var conditions []string
titlePrefixIdx := 2
args = []any{searchText, buildTitlePrefixTsQuery(searchText)}
argIdx := 3
// All title-side text on both sides of @@ flows through
// public.normalize_search_text() (migrations 127 / 138), which strips
// non-alphanumeric chars, drops standalone "and" tokens, and normalizes
// common title numbers. The expression must match
// idx_media_items_search_title_fields exactly for the GIN index to be used.
//
// Overview uses the 'english' config which natively treats "and" as a
// stop word, so it does not need explicit normalization.
titleVector := `(
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.title, ''))), 'A') ||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.original_title, ''))), 'A') ||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B')
)`
overviewVector := `to_tsvector('english', COALESCE(mi.overview, ''))`
normalizedTitleExpr := `public.normalize_search_text(%s)`
titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))`
titlePrefixQuery := fmt.Sprintf("to_tsquery('simple', $%d)", titlePrefixIdx)
titleMatch := fmt.Sprintf("%s @@ %s", titleVector, titleQuery)
titlePrefixMatch := fmt.Sprintf("($%d <> '' AND %s @@ %s)", titlePrefixIdx, titleVector, titlePrefixQuery)
overviewMatch := fmt.Sprintf("%s @@ websearch_to_tsquery('english', $1)", overviewVector)
// Keep the base match condition index-friendly; exact-title logic is used as
// a ranking boost later, not as an additional scan predicate.
conditions = append(conditions, fmt.Sprintf("(%s OR %s OR %s)", titleMatch, titlePrefixMatch, overviewMatch))
if len(itemTypes) > 0 {
placeholders := make([]string, 0, len(itemTypes))
for _, itemType := range itemTypes {
if strings.TrimSpace(itemType) == "" {
continue
}
placeholders = append(placeholders, fmt.Sprintf("$%d", argIdx))
args = append(args, strings.ToLower(strings.TrimSpace(itemType)))
argIdx++
}
if len(placeholders) > 0 {
conditions = append(conditions, fmt.Sprintf("mi.type IN (%s)", strings.Join(placeholders, ", ")))
}
}
needsLibJoin := filter.AllowedLibraryIDs != nil || len(filter.DisabledLibraryIDs) > 0
fromClause := "media_items mi"
if filter.AllowedLibraryIDs != nil {
// Caller (Search) is expected to short-circuit when len == 0; we still
// guard here so the builder is safe to invoke from tests.
if len(filter.AllowedLibraryIDs) > 0 {
conditions = append(conditions, fmt.Sprintf("mil.media_folder_id = ANY($%d)", argIdx))
args = append(args, filter.AllowedLibraryIDs)
argIdx++
}
}
if len(filter.DisabledLibraryIDs) > 0 {
conditions = append(conditions, fmt.Sprintf("NOT (mil.media_folder_id = ANY($%d))", argIdx))
args = append(args, filter.DisabledLibraryIDs)
argIdx++
}
if needsLibJoin {
fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id"
}
applyAccessFilter("mi", AccessFilter{MaxContentRating: filter.MaxContentRating, ExcludedMediaTypes: filter.ExcludedMediaTypes}, &conditions, &args, &argIdx)
// Manga chapters (type='ebook' rows linked into a manga series) are internal
// sub-units and must never surface as standalone search results.
conditions = append(conditions, MangaChapterExclusionWhere("mi"))
whereClause := "WHERE " + strings.Join(conditions, " AND ")
// Bind ExactTitleHint exactly once. The same arg index is referenced by
// both contiguous_title_match and exact_title_match (used as ranking
// signals in the ORDER BY). The post-CTE WHERE itself gates on title_rank
// > 0 (true title FTS match), not on contiguous_title_match.
exactIdx := argIdx
args = append(args, parsed.ExactTitleHint)
argIdx++
// mi.title uses the title_normalized stored generated column (migrations
// 105 / 127), so the LIKE '%pattern%' arm hits the gin_trgm_ops index
// instead of recomputing normalization per row. original_title and
// sort_title do not have a trigram index of their own, so their LIKE
// arms call public.normalize_search_text() per row. The tsvector @@
// path covers all three via idx_media_items_search_title_fields.
contiguousTitleMatch := fmt.Sprintf(`(
$%d <> '' AND (
%s LIKE '%%' || $%d || '%%' OR
%s LIKE '%%' || $%d || '%%' OR
%s LIKE '%%' || $%d || '%%'
)
)`,
exactIdx,
"mi.title_normalized", exactIdx,
fmt.Sprintf(normalizedTitleExpr, "mi.original_title"), exactIdx,
fmt.Sprintf(normalizedTitleExpr, "mi.sort_title"), exactIdx,
)
var yearArg any
if parsed.Year != nil {
yearArg = *parsed.Year
}
yearIdx := argIdx
args = append(args, yearArg)
argIdx++
phraseIdx := argIdx
args = append(args, parsed.Phrase)
argIdx++
exactTitleMatch := fmt.Sprintf(`(
$%d <> '' AND (
%s = $%d OR
%s = $%d OR
%s = $%d
)
)`,
exactIdx,
"mi.title_normalized", exactIdx,
fmt.Sprintf(normalizedTitleExpr, "mi.original_title"), exactIdx,
fmt.Sprintf(normalizedTitleExpr, "mi.sort_title"), exactIdx,
)
// Use qualified column names inside the CTE to avoid ambiguity when
// the FROM clause includes a JOIN to media_item_libraries. The select
// list aliases coalesced columns back to their own names (poster_path
// etc.) so the outer query can re-reference them; GROUP BY needs the
// raw references because output aliases are invalid there.
qualifiedCols := qualifiedItemColumns("mi")
groupByCols := qualifiedItemColumnRefs("mi")
scoredCTE := fmt.Sprintf(`
WITH scored AS (
SELECT
%s,
MAX(CASE
WHEN %s THEN 1
ELSE 0
END) AS exact_title_match,
MAX(CASE
WHEN %s THEN 1
ELSE 0
END) AS contiguous_title_match,
MAX(CASE
WHEN $%d::int IS NOT NULL AND mi.year = $%d::int THEN 1
ELSE 0
END) AS year_match,
MAX(ts_rank_cd(%s, %s)) AS title_rank,
MAX(CASE
WHEN $%d <> '' THEN ts_rank_cd(%s, %s)
ELSE 0
END) AS title_prefix_rank,
MAX(ts_rank_cd(%s, websearch_to_tsquery('english', $1))) AS overview_rank,
MAX(CASE
WHEN $%d <> '' THEN ts_rank_cd(%s, phraseto_tsquery('simple', public.normalize_search_text($%d)))
ELSE 0
END) AS phrase_rank
FROM %s
%s
GROUP BY %s
)
`, qualifiedCols, exactTitleMatch, contiguousTitleMatch, yearIdx, yearIdx, titleVector, titleQuery, titlePrefixIdx, titleVector, titlePrefixQuery, overviewVector, phraseIdx, titleVector, phraseIdx, fromClause, whereClause, groupByCols)
// COUNT(*) OVER () runs after the GROUP BY in the scored CTE collapses
// duplicates from the library JOIN, so the window count preserves the
// COUNT(DISTINCT mi.content_id) semantics of the prior 2-query path.
// The stats CTE + CROSS JOIN below means the window count reflects the
// post-filter row count automatically.
//
// The stats CTE gates on title_rank > 0 (true title FTS match) rather
// than contiguous_title_match (LIKE substring), so reordered-token title
// queries like "order law" → "Law and Order" aren't wrongly demoted to
// the overview-fallback bucket. The post-CTE WHERE then keeps every title
// FTS hit, and admits overview-only rows only when no title hit exists
// anywhere in the candidate set AND the overview rank clears
// overviewMatchFloor. This suppresses single-occurrence body matches
// (which score at PostgreSQL's default D weight of 0.1) for common
// single-word queries like "obsession" that would otherwise flood
// results with description-only hits.
//
// countSQL is a count-only sibling that omits LIMIT/OFFSET/ORDER BY. It is
// invoked only as a fallback when the data SELECT returns an empty page
// past offset 0 — COUNT(*) OVER () emits no rows in that case so total
// would otherwise default to 0.
statsCTE := `
, stats AS (
SELECT MAX(CASE WHEN title_rank > 0 OR title_prefix_rank > 0 THEN 1 ELSE 0 END) AS has_title_match
FROM scored
)`
// postFilter is the FROM + CROSS JOIN + WHERE shared by both dataSQL and
// countSQL. Keeping it in one string ensures the empty-page fallback count
// can never drift from the data query's filter.
postFilter := fmt.Sprintf(`FROM scored
CROSS JOIN stats
WHERE scored.title_rank > 0
OR scored.title_prefix_rank > 0
OR (COALESCE(stats.has_title_match, 0) = 0 AND scored.overview_rank >= %g)`, overviewMatchFloor)
totalColumn := ""
if includeTotal {
totalColumn = ", COUNT(*) OVER () AS total_count"
}
dataSQL = scoredCTE + statsCTE + fmt.Sprintf(`
SELECT %s%s
%s
ORDER BY exact_title_match DESC, contiguous_title_match DESC, year_match DESC, phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC, LOWER(title) ASC, content_id ASC
LIMIT $%d OFFSET $%d`, itemColumns, totalColumn, postFilter, argIdx, argIdx+1)
countSQL = scoredCTE + statsCTE + fmt.Sprintf(`
SELECT COUNT(*)
%s`, postFilter)
args = append(args, limit, offset)
return dataSQL, countSQL, args
return r.buildMixedSearchSQLWithTotal(query, itemTypes, limit, offset, filter, includeTotal)
}
// ListUnmatchedByFolderAndPathPrefix returns content IDs for unmatched-style
+57 -16
View File
@@ -490,32 +490,73 @@ func TestItemRepo_Search_ScoredCTEExposesItemColumnNames(t *testing.T) {
}
}
// TestItemRepo_Search_GroupByHasNoOutputAliases asserts the scored CTE's
// GROUP BY uses raw column references: AS aliases are valid in a select list
// but are a syntax error inside GROUP BY, so the CTE must not reuse the
// aliased projection there.
func TestItemRepo_Search_GroupByHasNoOutputAliases(t *testing.T) {
// TestItemRepo_Search_ScoredCTEIsLean asserts that relevance ranking carries
// only candidate identity/rank fields. Wide MediaItem columns are hydrated
// from the page CTE after LIMIT/OFFSET, which keeps broad episode searches
// from sorting metadata arrays and image fields for every candidate.
func TestItemRepo_Search_ScoredCTEIsLean(t *testing.T) {
repo := &ItemRepository{}
dataSQL, countSQL, _ := repo.buildSearchSQL("avatar", []string{"movie"}, 20, 0, AccessFilter{})
for _, sql := range []string{dataSQL, countSQL} {
idx := strings.Index(sql, "GROUP BY")
if idx < 0 {
t.Fatalf("expected GROUP BY in scored CTE; got:\n%s", sql)
end := strings.Index(sql, "), stats AS")
if end < 0 {
t.Fatalf("expected scored CTE boundary; got:\n%s", sql)
}
clause := sql[idx:]
if end := strings.Index(clause, ")"); end >= 0 {
clause = clause[:end]
scored := sql[:end]
for _, wideColumn := range []string{"poster_path", "backdrop_path", "metadata_s3_path", "genres"} {
if strings.Contains(scored, wideColumn) {
t.Fatalf("scored CTE must not carry %s; got:\n%s", wideColumn, scored)
}
}
if strings.Contains(clause, " AS ") {
t.Fatalf("GROUP BY must not contain output aliases; got:\n%s", clause)
}
if strings.Contains(clause, "COALESCE") {
t.Fatalf("GROUP BY should group by raw columns, not COALESCE expressions; got:\n%s", clause)
}
}
func TestItemRepo_Search_UnscopedIncludesEpisodeCandidateBranch(t *testing.T) {
repo := &ItemRepository{}
sql, _, _ := repo.buildSearchSQL("Who Are You?", nil, 20, 0, AccessFilter{})
for _, want := range []string{
"FROM episodes e JOIN media_items si",
"si.type = 'series'",
"FROM episode_libraries available_el",
"UNION ALL",
} {
if !strings.Contains(sql, want) {
t.Fatalf("mixed search SQL missing %q:\n%s", want, sql)
}
}
}
func TestItemRepo_Search_EpisodeScopeOmitsMediaItemCandidateBranch(t *testing.T) {
repo := &ItemRepository{}
sql, _, _ := repo.buildSearchSQL("Who Are You?", []string{"episode"}, 20, 0, AccessFilter{})
scoredEnd := strings.Index(sql, "), stats AS")
if scoredEnd < 0 {
t.Fatalf("expected scored CTE boundary; got:\n%s", sql)
}
scored := sql[:scoredEnd]
if strings.Contains(scored, "FROM media_items mi") {
t.Fatalf("episode-only candidate set must not scan media_items directly:\n%s", scored)
}
if !strings.Contains(scored, "FROM episodes e JOIN media_items si") {
t.Fatalf("episode-only candidate set missing episode branch:\n%s", scored)
}
}
func TestItemRepo_Search_EpisodeAccessUsesIndependentMembershipPredicates(t *testing.T) {
repo := &ItemRepository{}
sql, _, _ := repo.buildSearchSQL("Who Are You?", []string{"episode"}, 20, 0, AccessFilter{
AllowedLibraryIDs: []int{1},
DisabledLibraryIDs: []int{2},
})
if !strings.Contains(sql, "FROM episode_libraries allowed_el") {
t.Fatalf("episode search missing allowed-library EXISTS:\n%s", sql)
}
if !strings.Contains(sql, "NOT EXISTS (SELECT 1 FROM episode_libraries disabled_el") {
t.Fatalf("episode search missing independent disabled-library NOT EXISTS:\n%s", sql)
}
}
// TestItemRepo_ListUnmatchedByFolderAndPathPrefix_ExcludesMangaChapters pins
// the manga-chapter exclusion in the unmatched-item lister. Manga chapters are
// type='ebook' items that stay status='pending' by design (the type='manga'
+177
View File
@@ -0,0 +1,177 @@
package catalog
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
)
func TestEpisodeSearchPostgresAndDocumentSource(t *testing.T) {
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("SILO_TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatalf("connect test database: %v", err)
}
t.Cleanup(pool.Close)
suffix := time.Now().UnixNano()
seriesID := fmt.Sprintf("episode-search-series-%d", suffix)
episodeID := fmt.Sprintf("episode-search-episode-%d", suffix)
unavailableID := fmt.Sprintf("episode-search-unavailable-%d", suffix)
podcastID := fmt.Sprintf("episode-search-podcast-%d", suffix)
podcastEpisodeID := fmt.Sprintf("episode-search-podcast-episode-%d", suffix)
var folderID int
if err := pool.QueryRow(ctx,
`INSERT INTO media_folders (type, name, enabled) VALUES ('series', $1, true) RETURNING id`,
seriesID,
).Scan(&folderID); err != nil {
t.Fatalf("seed folder: %v", err)
}
t.Cleanup(func() {
_, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, []string{seriesID, podcastID})
_, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, folderID)
})
if _, err := pool.Exec(ctx, `
INSERT INTO media_items (content_id, type, title, status, genres)
VALUES ($1, 'series', 'Silo Search Fixture', 'matched', '{}'::text[]),
($2, 'podcast', 'Podcast Search Fixture', 'matched', '{}'::text[])
`, seriesID, podcastID); err != nil {
t.Fatalf("seed parent items: %v", err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO episodes (content_id, series_id, season_number, episode_number, title, overview)
VALUES ($1, $2, 3, 1, 'Who Are You?', 'A buried signal returns. A buried signal changes everything.'),
($3, $2, 3, 2, 'Who Are You?', 'Unavailable metadata-only episode.'),
($4, $5, 1, 1, 'Who Are You?', 'Podcast episode with the same title.')
`, episodeID, seriesID, unavailableID, podcastEpisodeID, podcastID); err != nil {
t.Fatalf("seed episodes: %v", err)
}
if _, err := pool.Exec(ctx, `
INSERT INTO episode_libraries (episode_id, media_folder_id, first_seen_at)
VALUES ($1, $3, NOW()), ($2, $3, NOW())
`, episodeID, podcastEpisodeID, folderID); err != nil {
t.Fatalf("seed episode memberships: %v", err)
}
repo := NewItemRepository(pool)
for _, types := range [][]string{nil, {"episode"}} {
items, total, _, exact, err := repo.SearchPage(ctx, "Who Are You?", types, 20, 0, AccessFilter{}, true)
if err != nil {
t.Fatalf("search types %v: %v", types, err)
}
found := false
for _, item := range items {
if item.ContentID == episodeID && item.Type == "episode" {
found = true
}
if item.ContentID == unavailableID || item.ContentID == podcastEpisodeID {
t.Fatalf("search types %v leaked ineligible episode %s", types, item.ContentID)
}
}
if !exact || total < 1 || !found {
t.Fatalf("search types %v = total %d exact %v items %+v", types, total, exact, items)
}
}
overviewItems, _, _, _, err := repo.SearchPage(ctx, "buried signal", []string{"episode"}, 20, 0, AccessFilter{}, true)
if err != nil || len(overviewItems) != 1 || overviewItems[0].ContentID != episodeID {
t.Fatalf("overview search = items %+v err %v", overviewItems, err)
}
indexer := NewCatalogSearchIndexer(pool, nil)
docs, err := indexer.LoadDocumentsByIDs(ctx, []string{episodeID, unavailableID, podcastEpisodeID}, nil, DefaultMeilisearchEmbedder, true, false)
if err != nil {
t.Fatalf("load episode documents: %v", err)
}
if len(docs) != 1 || docs[0].ContentID != episodeID || docs[0].Type != "episode" {
t.Fatalf("episode documents = %+v", docs)
}
if len(docs[0].LibraryIDs) != 1 || int(docs[0].LibraryIDs[0]) != folderID {
t.Fatalf("episode library ids = %v, want [%d]", docs[0].LibraryIDs, folderID)
}
if docs[0].Vectors != nil {
t.Fatalf("episode document unexpectedly has vectors: %#v", docs[0].Vectors)
}
}
func TestEpisodeSearchOutboxStatementTriggers(t *testing.T) {
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("SILO_TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatalf("connect test database: %v", err)
}
t.Cleanup(pool.Close)
var captureFunction *string
if err := pool.QueryRow(ctx, `SELECT to_regprocedure('public.catalog_search_capture_enabled()')::text`).Scan(&captureFunction); err != nil || captureFunction == nil {
t.Skip("episode search outbox trigger migration is not applied")
}
tx, err := pool.Begin(ctx)
if err != nil {
t.Fatalf("begin: %v", err)
}
defer tx.Rollback(ctx) //nolint:errcheck
if _, err := tx.Exec(ctx, `UPDATE server_settings SET value = 'meilisearch' WHERE key = 'catalog.search.provider'`); err != nil {
t.Fatalf("enable trigger capture: %v", err)
}
suffix := time.Now().UnixNano()
seriesID := fmt.Sprintf("episode-trigger-series-%d", suffix)
episodeID := fmt.Sprintf("episode-trigger-episode-%d", suffix)
var folderID int
if err := tx.QueryRow(ctx,
`INSERT INTO media_folders (type, name, enabled) VALUES ('series', $1, true) RETURNING id`, seriesID,
).Scan(&folderID); err != nil {
t.Fatalf("seed folder: %v", err)
}
if _, err := tx.Exec(ctx, `
INSERT INTO media_items (content_id, type, title, status, genres)
VALUES ($1, 'series', 'Trigger Series', 'matched', '{}'::text[])
`, seriesID); err != nil {
t.Fatalf("seed series: %v", err)
}
if _, err := tx.Exec(ctx, `INSERT INTO media_item_libraries (content_id, media_folder_id) VALUES ($1, $2)`, seriesID, folderID); err != nil {
t.Fatalf("seed item membership: %v", err)
}
if _, err := tx.Exec(ctx, `
INSERT INTO episodes (content_id, series_id, season_number, episode_number, title)
VALUES ($1, $2, 1, 1, 'Trigger Episode')
`, episodeID, seriesID); err != nil {
t.Fatalf("seed episode: %v", err)
}
if _, err := tx.Exec(ctx, `INSERT INTO episode_libraries (episode_id, media_folder_id) VALUES ($1, $2)`, episodeID, folderID); err != nil {
t.Fatalf("seed episode membership: %v", err)
}
var upserts int
if err := tx.QueryRow(ctx, `
SELECT COUNT(*)
FROM catalog_search_index_events
WHERE action = 'upsert' AND content_id = $1
`, episodeID).Scan(&upserts); err != nil || upserts < 2 {
t.Fatalf("episode insert/membership upserts = %d, err %v", upserts, err)
}
if _, err := tx.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, seriesID); err != nil {
t.Fatalf("delete series: %v", err)
}
var deletes int
if err := tx.QueryRow(ctx, `
SELECT COUNT(*)
FROM catalog_search_index_events
WHERE action = 'delete' AND content_id = $1
`, episodeID).Scan(&deletes); err != nil || deletes == 0 {
t.Fatalf("episode cascade deletes = %d, err %v", deletes, err)
}
}
+106 -48
View File
@@ -535,7 +535,7 @@ func (i *CatalogSearchIndexer) CheckConnection(ctx context.Context, settings Cat
func catalogSearchMeilisearchSettings(embedder string, semanticEnabled, binaryQuantized bool) map[string]any {
settings := map[string]any{
"displayedAttributes": []string{"content_id", "type"},
"filterableAttributes": []string{"type"},
"filterableAttributes": []string{"type", "library_ids"},
"searchableAttributes": []string{
"title",
"original_title",
@@ -623,6 +623,7 @@ type catalogSearchDocument struct {
Countries []string `json:"countries,omitempty"`
Keywords []string `json:"keywords,omitempty"`
People []string `json:"people,omitempty"`
LibraryIDs []int32 `json:"library_ids,omitempty"`
SchemaVersion int `json:"schema_version"`
Vectors map[string][]float32 `json:"_vectors,omitempty"`
}
@@ -632,19 +633,14 @@ func (i *CatalogSearchIndexer) LoadDocumentsAfter(ctx context.Context, afterCont
return nil, nil
}
typeFilter := normalizeCatalogSearchItemTypes(itemTypes)
whereClause := `
WHERE ($1::text = '' OR mi.content_id > $1)
AND ` + catalogSearchExcludeMangaChaptersSQL
args := []any{afterContentID, limit}
typeArg := 0
if len(typeFilter) > 0 {
whereClause += `
AND mi.type = ANY($3)`
typeArg = 3
args = append(args, typeFilter)
}
rows, err := i.pool.Query(ctx, catalogSearchDocumentSelectSQL(whereClause, `
ORDER BY mi.content_id ASC
LIMIT $2
`), args...)
rows, err := i.pool.Query(ctx, mixedCatalogSearchDocumentSQL(
`mi.content_id > $1`, `e.content_id > $1`, typeArg, "LIMIT $2"), args...)
if err != nil {
return nil, err
}
@@ -666,18 +662,14 @@ func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentID
return nil, nil
}
typeFilter := normalizeCatalogSearchItemTypes(itemTypes)
whereClause := `
WHERE mi.content_id = ANY($1)
AND ` + catalogSearchExcludeMangaChaptersSQL
args := []any{contentIDs}
typeArg := 0
if len(typeFilter) > 0 {
whereClause += `
AND mi.type = ANY($2)`
typeArg = 2
args = append(args, typeFilter)
}
rows, err := i.pool.Query(ctx, catalogSearchDocumentSelectSQL(whereClause, `
ORDER BY mi.content_id ASC
`), args...)
rows, err := i.pool.Query(ctx, mixedCatalogSearchDocumentSQL(
`mi.content_id = ANY($1)`, `e.content_id = ANY($1)`, typeArg, ""), args...)
if err != nil {
return nil, err
}
@@ -693,31 +685,85 @@ func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentID
return docs, nil
}
func catalogSearchDocumentSelectSQL(whereClause string, tailClause string) string {
return `
SELECT
mi.content_id,
mi.type,
COALESCE(mi.title, ''),
COALESCE(mi.sort_title, ''),
COALESCE(mi.original_title, ''),
COALESCE(mi.year, 0),
COALESCE(mi.overview, ''),
COALESCE(mi.tagline, ''),
COALESCE(mi.genres, ARRAY[]::text[]),
COALESCE(mi.studios, ARRAY[]::text[]),
COALESCE(mi.networks, ARRAY[]::text[]),
COALESCE(mi.countries, ARRAY[]::text[]),
COALESCE(mi.keywords, ARRAY[]::text[]),
COALESCE(array_agg(DISTINCT p.name) FILTER (WHERE p.name IS NOT NULL AND p.name <> ''), ARRAY[]::text[]) AS people
FROM media_items mi
LEFT JOIN item_people ip ON ip.content_id = mi.content_id
LEFT JOIN people p ON p.id = ip.person_id
` + whereClause + `
GROUP BY
mi.content_id, mi.type, mi.title, mi.sort_title, mi.original_title, mi.year,
mi.overview, mi.tagline, mi.genres, mi.studios, mi.networks, mi.countries, mi.keywords
` + tailClause
// mixedCatalogSearchDocumentSQL first pages narrow IDs across both physical
// sources, then aggregates people and library memberships only for that batch.
// This keeps rebuild work proportional to RebuildBatchSize rather than the
// total number of catalog rows.
func mixedCatalogSearchDocumentSQL(mediaPredicate, episodePredicate string, typeArg int, limitClause string) string {
mediaTypePredicate := ""
episodeTypePredicate := ""
if typeArg > 0 {
mediaTypePredicate = fmt.Sprintf(" AND mi.type = ANY($%d)", typeArg)
episodeTypePredicate = fmt.Sprintf(" AND 'episode' = ANY($%d)", typeArg)
}
return fmt.Sprintf(`
WITH candidates AS (
SELECT mi.content_id, mi.type
FROM media_items mi
WHERE %s
AND %s%s
UNION ALL
SELECT e.content_id, 'episode'::text AS type
FROM episodes e
JOIN media_items si ON si.content_id = e.series_id AND si.type = 'series'
WHERE %s%s
AND EXISTS (SELECT 1 FROM episode_libraries el WHERE el.episode_id = e.content_id)
ORDER BY content_id ASC
%s
)
SELECT * FROM (
SELECT
mi.content_id,
mi.type,
COALESCE(mi.title, ''),
COALESCE(mi.sort_title, ''),
COALESCE(mi.original_title, ''),
COALESCE(mi.year, 0),
COALESCE(mi.overview, ''),
COALESCE(mi.tagline, ''),
COALESCE(mi.genres, ARRAY[]::text[]),
COALESCE(mi.studios, ARRAY[]::text[]),
COALESCE(mi.networks, ARRAY[]::text[]),
COALESCE(mi.countries, ARRAY[]::text[]),
COALESCE(mi.keywords, ARRAY[]::text[]),
COALESCE(people.names, ARRAY[]::text[]),
COALESCE(libraries.ids, ARRAY[]::integer[])
FROM candidates c
JOIN media_items mi ON c.type <> 'episode' AND mi.content_id = c.content_id
LEFT JOIN LATERAL (
SELECT array_agg(DISTINCT p.name) FILTER (WHERE p.name IS NOT NULL AND p.name <> '') AS names
FROM item_people ip
JOIN people p ON p.id = ip.person_id
WHERE ip.content_id = mi.content_id
) people ON true
LEFT JOIN LATERAL (
SELECT array_agg(DISTINCT mil.media_folder_id ORDER BY mil.media_folder_id) AS ids
FROM media_item_libraries mil
WHERE mil.content_id = mi.content_id
) libraries ON true
UNION ALL
SELECT
e.content_id,
'episode'::text,
COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text),
COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text),
''::text,
COALESCE(EXTRACT(YEAR FROM e.air_date)::integer, 0),
COALESCE(e.overview, ''),
''::text,
ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[],
COALESCE(libraries.ids, ARRAY[]::integer[])
FROM candidates c
JOIN episodes e ON c.type = 'episode' AND e.content_id = c.content_id
LEFT JOIN LATERAL (
SELECT array_agg(DISTINCT el.media_folder_id ORDER BY el.media_folder_id) AS ids
FROM episode_libraries el
WHERE el.episode_id = e.content_id
) libraries ON true
) documents
ORDER BY content_id ASC`,
mediaPredicate, catalogSearchExcludeMangaChaptersSQL, mediaTypePredicate,
episodePredicate, episodeTypePredicate, limitClause)
}
func scanCatalogSearchDocuments(rows pgx.Rows) ([]catalogSearchDocument, error) {
@@ -739,6 +785,7 @@ func scanCatalogSearchDocuments(rows pgx.Rows) ([]catalogSearchDocument, error)
&doc.Countries,
&doc.Keywords,
&doc.People,
&doc.LibraryIDs,
); err != nil {
return nil, err
}
@@ -759,7 +806,7 @@ func (i *CatalogSearchIndexer) attachDocumentVectors(ctx context.Context, docs [
}
ids := make([]string, 0, len(docs))
for _, doc := range docs {
if strings.TrimSpace(doc.ContentID) != "" {
if doc.Type != "episode" && strings.TrimSpace(doc.ContentID) != "" {
ids = append(ids, doc.ContentID)
}
}
@@ -814,6 +861,10 @@ func setCatalogSearchDocumentVectors(docs []catalogSearchDocument, vectors map[s
}
count := 0
for idx := range docs {
if docs[idx].Type == "episode" {
docs[idx].Vectors = nil
continue
}
vector := vectors[docs[idx].ContentID]
docs[idx].Vectors = map[string][]float32{embedder: nil}
if len(vector) > 0 {
@@ -891,10 +942,17 @@ func countCatalogSearchEligibleDocuments(ctx context.Context, q coverageQuerier,
}
var count int
if err := q.QueryRow(ctx, `
SELECT COUNT(*)
FROM media_items mi
WHERE `+catalogSearchExcludeMangaChaptersSQL+`
AND ($1::text[] IS NULL OR mi.type = ANY($1))
SELECT
(SELECT COUNT(*)
FROM media_items mi
WHERE `+catalogSearchExcludeMangaChaptersSQL+`
AND ($1::text[] IS NULL OR mi.type = ANY($1)))
+
(SELECT COUNT(*)
FROM episodes e
JOIN media_items si ON si.content_id = e.series_id AND si.type = 'series'
WHERE ($1::text[] IS NULL OR 'episode' = ANY($1))
AND EXISTS (SELECT 1 FROM episode_libraries el WHERE el.episode_id = e.content_id))
`, typeArg).Scan(&count); err != nil {
return 0, fmt.Errorf("count catalog search eligible documents: %w", err)
}
@@ -106,6 +106,30 @@ type meilisearchSearchResponse struct {
Query string `json:"query"`
}
type meilisearchFederatedSearchRequest struct {
Federation meilisearchFederationOptions `json:"federation"`
Queries []meilisearchFederatedQuery `json:"queries"`
}
type meilisearchFederationOptions struct {
Offset int `json:"offset"`
Limit int `json:"limit"`
}
type meilisearchFederatedQuery struct {
IndexUID string `json:"indexUid"`
Query string `json:"q"`
Filter string `json:"filter,omitempty"`
AttributesToRetrieve []string `json:"attributesToRetrieve"`
AttributesToSearchOn []string `json:"attributesToSearchOn,omitempty"`
MatchingStrategy string `json:"matchingStrategy,omitempty"`
Vector []float32 `json:"vector,omitempty"`
Hybrid *meilisearchHybridRequest `json:"hybrid,omitempty"`
FederationOptions struct {
Weight float64 `json:"weight"`
} `json:"federationOptions"`
}
type meilisearchStatsResponse struct {
NumberOfDocuments int `json:"numberOfDocuments"`
}
@@ -217,6 +241,12 @@ func (c *meilisearchClient) Search(ctx context.Context, uid string, req meilisea
return out, err
}
func (c *meilisearchClient) FederatedSearch(ctx context.Context, req meilisearchFederatedSearchRequest) (meilisearchSearchResponse, error) {
var out meilisearchSearchResponse
err := c.do(ctx, http.MethodPost, "/multi-search", req, &out)
return out, err
}
func (c *meilisearchClient) AddDocuments(ctx context.Context, uid string, docs []catalogSearchDocument) (meilisearchTaskRef, error) {
if len(docs) == 0 {
return meilisearchTaskRef{}, nil
@@ -2,6 +2,7 @@ package catalog
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
@@ -11,6 +12,45 @@ import (
"time"
)
func TestMeilisearchFederatedSearchUsesMultiSearchPagination(t *testing.T) {
var gotPath string
var gotRequest meilisearchFederatedSearchRequest
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotPath = r.URL.Path
if err := json.NewDecoder(r.Body).Decode(&gotRequest); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"hits":[{"content_id":"episode-1"}],"offset":20,"limit":10,"estimatedTotalHits":1}`))
}))
defer server.Close()
client, err := newMeilisearchClient(server.URL, "", time.Second)
if err != nil {
t.Fatalf("newMeilisearchClient: %v", err)
}
request := meilisearchFederatedSearchRequest{
Federation: meilisearchFederationOptions{Offset: 20, Limit: 10},
Queries: []meilisearchFederatedQuery{
{IndexUID: "catalog-v3", Query: "Who Are You?"},
},
}
response, err := client.FederatedSearch(context.Background(), request)
if err != nil {
t.Fatalf("FederatedSearch: %v", err)
}
if gotPath != "/multi-search" {
t.Fatalf("request path = %q, want /multi-search", gotPath)
}
if gotRequest.Federation.Offset != 20 || gotRequest.Federation.Limit != 10 || len(gotRequest.Queries) != 1 {
t.Fatalf("decoded request = %#v", gotRequest)
}
if len(response.Hits) != 1 || response.Hits[0].ContentID != "episode-1" {
t.Fatalf("response = %#v", response)
}
}
func TestMeilisearchWaitTaskAcceptsZeroTaskUID(t *testing.T) {
var gotPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+174 -5
View File
@@ -7,6 +7,7 @@ import (
"net"
"net/http"
"sort"
"strconv"
"strings"
"sync"
"time"
@@ -43,6 +44,7 @@ var meilisearchTitleSearchAttributes = []string{
"original_title",
"sort_title",
"title_variants",
"overview",
}
type meilisearchIndexStateStore interface {
@@ -97,6 +99,9 @@ type MeilisearchSearchProvider struct {
capMu sync.Mutex
capCache CatalogSearchSemanticCapability
capCachedAt time.Time
federationMu sync.Mutex
federationUnsupported bool
}
type cachedCatalogSearchQueryVector struct {
@@ -151,6 +156,14 @@ func NewMeilisearchSearchProvider(
}
func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearchRequest) (*CatalogSearchResult, error) {
if req.Access.AllowedLibraryIDs != nil && len(req.Access.AllowedLibraryIDs) == 0 {
return &CatalogSearchResult{
Items: []*models.MediaItem{},
TotalExact: !req.SkipTotal,
Provider: SearchProviderMeilisearch,
Mode: "keyword",
}, nil
}
if p == nil || p.client == nil {
return p.fallbackSearch(ctx, req, "meilisearch not configured")
}
@@ -270,6 +283,13 @@ func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req C
estimatedTotalHits := 0
exhausted := false
baseSearchReq, semanticFallback := p.buildMeilisearchSearchRequest(ctx, req)
useFederation := baseSearchReq.Hybrid != nil && searchRequestMixesEpisodeAndMedia(req.ItemTypes)
if useFederation && p.isFederationUnsupported() {
semanticFallback = "meilisearch federation unsupported; using keyword search"
baseSearchReq.Vector = nil
baseSearchReq.Hybrid = nil
useFederation = false
}
for len(accessible) < target && !exhausted {
if scanned >= meilisearchCandidateScanCap {
@@ -282,8 +302,25 @@ func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req C
searchReq := baseSearchReq
searchReq.Offset = meiliOffset
searchReq.Limit = nextLimit
resp, err := p.client.Search(ctx, indexUID, searchReq)
if err != nil && baseSearchReq.Hybrid != nil {
var resp meilisearchSearchResponse
var err error
if useFederation {
resp, err = p.client.FederatedSearch(ctx, p.buildFederatedSearchRequest(indexUID, req, baseSearchReq, meiliOffset, nextLimit))
if err != nil && isMeilisearchFederationUnsupported(err) {
p.markFederationUnsupported()
semanticFallback = "meilisearch federation unsupported; using keyword search"
baseSearchReq.Vector = nil
baseSearchReq.Hybrid = nil
useFederation = false
searchReq = baseSearchReq
searchReq.Offset = meiliOffset
searchReq.Limit = nextLimit
resp, err = p.client.Search(ctx, indexUID, searchReq)
}
} else {
resp, err = p.client.Search(ctx, indexUID, searchReq)
}
if err != nil && baseSearchReq.Hybrid != nil && !useFederation {
semanticFallback = "meilisearch hybrid search failed: " + err.Error()
baseSearchReq.Vector = nil
baseSearchReq.Hybrid = nil
@@ -316,7 +353,7 @@ func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req C
position[id] = i
ids = append(ids, id)
}
hydrated, err := p.itemRepo.GetByIDsWithAccess(ctx, ids, req.Access)
hydrated, err := p.itemRepo.GetSearchItemsByIDsWithAccess(ctx, ids, req.Access)
if err != nil {
return nil, err
}
@@ -371,13 +408,13 @@ func (p *MeilisearchSearchProvider) buildMeilisearchSearchRequest(ctx context.Co
if p == nil {
return meilisearchSearchRequest{
Query: strings.TrimSpace(req.Query),
Filter: meilisearchTypeFilter(req.ItemTypes),
Filter: meilisearchSearchFilter(req.ItemTypes, req.Access),
AttributesToRetrieve: []string{"content_id"},
}, ""
}
searchReq := meilisearchSearchRequest{
Query: strings.TrimSpace(req.Query),
Filter: meilisearchTypeFilter(req.ItemTypes),
Filter: meilisearchSearchFilter(req.ItemTypes, req.Access),
AttributesToRetrieve: []string{"content_id"},
AttributesToSearchOn: p.attributesToSearchOnForRequest(req),
MatchingStrategy: p.matchingStrategyForRequest(req),
@@ -412,9 +449,116 @@ func (p *MeilisearchSearchProvider) shouldUseSemanticSearch(req CatalogSearchReq
if p == nil || !p.config.SemanticEnabled {
return false
}
mediaTypes, includesEpisodes := splitSearchItemTypes(req.ItemTypes)
if includesEpisodes && len(req.ItemTypes) > 0 && len(mediaTypes) == 0 {
return false
}
return len(catalogSearchQueryTerms(req.Query)) >= meilisearchShortHybridMinTerms
}
func searchRequestMixesEpisodeAndMedia(itemTypes []string) bool {
mediaTypes, includesEpisodes := splitSearchItemTypes(itemTypes)
return includesEpisodes && (len(itemTypes) == 0 || len(mediaTypes) > 0)
}
func (p *MeilisearchSearchProvider) buildFederatedSearchRequest(
indexUID string,
req CatalogSearchRequest,
base meilisearchSearchRequest,
offset, limit int,
) meilisearchFederatedSearchRequest {
mediaTypes, _ := splitSearchItemTypes(req.ItemTypes)
mediaFilter := ""
if len(req.ItemTypes) == 0 {
mediaFilter = joinMeilisearchFilters(`type != "episode"`, meilisearchSearchFilter(nil, req.Access))
} else {
mediaFilter = meilisearchSearchFilter(mediaTypes, req.Access)
}
episodeFilter := meilisearchSearchFilter([]string{"episode"}, req.Access)
mediaQuery := meilisearchFederatedQuery{
IndexUID: indexUID,
Query: base.Query,
Filter: mediaFilter,
AttributesToRetrieve: base.AttributesToRetrieve,
AttributesToSearchOn: base.AttributesToSearchOn,
MatchingStrategy: base.MatchingStrategy,
Vector: base.Vector,
Hybrid: base.Hybrid,
}
mediaQuery.FederationOptions.Weight = 1
episodeQuery := meilisearchFederatedQuery{
IndexUID: indexUID,
Query: base.Query,
Filter: episodeFilter,
AttributesToRetrieve: base.AttributesToRetrieve,
AttributesToSearchOn: base.AttributesToSearchOn,
MatchingStrategy: base.MatchingStrategy,
}
episodeQuery.FederationOptions.Weight = 1
return meilisearchFederatedSearchRequest{
Federation: meilisearchFederationOptions{Offset: offset, Limit: limit},
Queries: []meilisearchFederatedQuery{mediaQuery, episodeQuery},
}
}
func joinMeilisearchFilters(parts ...string) string {
cleaned := make([]string, 0, len(parts))
for _, part := range parts {
if part = strings.TrimSpace(part); part != "" {
cleaned = append(cleaned, part)
}
}
return strings.Join(cleaned, " AND ")
}
func (p *MeilisearchSearchProvider) isFederationUnsupported() bool {
p.federationMu.Lock()
defer p.federationMu.Unlock()
return p.federationUnsupported
}
func (p *MeilisearchSearchProvider) markFederationUnsupported() {
p.federationMu.Lock()
p.federationUnsupported = true
p.federationMu.Unlock()
}
func isMeilisearchFederationUnsupported(err error) bool {
var httpErr *meilisearchHTTPError
if !errors.As(err, &httpErr) {
return false
}
code := strings.ToLower(strings.TrimSpace(httpErr.Code))
message := strings.ToLower(strings.TrimSpace(httpErr.Message))
if httpErr.StatusCode == http.StatusNotFound {
// A cached active index can disappear during an atomic rebuild swap.
// That is a resource miss, not evidence that this Meilisearch process
// lacks federation; caching it would disable mixed hybrid search until
// Silo restarts even after the provider refreshes the active index UID.
if code == "index_not_found" || code == "document_not_found" {
return false
}
return code == "not_found" ||
(code == "" && (strings.Contains(message, "route") || strings.Contains(message, "multi-search")))
}
if httpErr.StatusCode != http.StatusBadRequest {
return false
}
// Older Meilisearch versions expose /multi-search but reject the newer
// federation object as an unknown field. Cache only that capability-shaped
// failure; malformed or incompatible federated queries should keep falling
// through to PostgreSQL so a request bug is not hidden for the process life.
details := code + " " + message
mentionsFederation := strings.Contains(details, "federation")
unsupported := strings.Contains(details, "not supported") ||
strings.Contains(details, "unsupported") ||
strings.Contains(details, "unknown field") ||
strings.Contains(details, "unrecognized field") ||
strings.Contains(details, "feature not enabled")
return mentionsFederation && unsupported
}
func (p *MeilisearchSearchProvider) matchingStrategyForRequest(req CatalogSearchRequest) string {
strategy := strings.TrimSpace(strings.ToLower(p.config.MatchingStrategy))
if strategy == "" {
@@ -740,6 +884,31 @@ func meilisearchTypeFilter(itemTypes []string) string {
return "(" + strings.Join(parts, " OR ") + ")"
}
func meilisearchSearchFilter(itemTypes []string, access AccessFilter) string {
parts := make([]string, 0, 4)
if typeFilter := meilisearchTypeFilter(itemTypes); typeFilter != "" {
parts = append(parts, typeFilter)
}
if access.AllowedLibraryIDs != nil {
parts = append(parts, "library_ids IN ["+joinMeilisearchIntFilterValues(access.AllowedLibraryIDs)+"]")
}
if len(access.DisabledLibraryIDs) > 0 {
if access.AllowedLibraryIDs == nil {
parts = append(parts, "library_ids IS NOT EMPTY")
}
parts = append(parts, "library_ids NOT IN ["+joinMeilisearchIntFilterValues(access.DisabledLibraryIDs)+"]")
}
return strings.Join(parts, " AND ")
}
func joinMeilisearchIntFilterValues(values []int) string {
parts := make([]string, 0, len(values))
for _, value := range values {
parts = append(parts, strconv.Itoa(value))
}
return strings.Join(parts, ", ")
}
func normalizeCatalogSearchQueryForVector(query string) string {
return strings.Join(strings.Fields(query), " ")
}
+327
View File
@@ -0,0 +1,327 @@
package catalog
import (
"context"
"fmt"
"strings"
"github.com/Silo-Server/silo-server/internal/access"
"github.com/Silo-Server/silo-server/internal/models"
)
// GetSearchItemsByIDsWithAccess hydrates mixed search hits in one round trip.
// Meilisearch IDs are only candidates: both branches reapply the effective
// access filter, and callers restore provider order after this method returns.
func (r *ItemRepository) GetSearchItemsByIDsWithAccess(
ctx context.Context,
contentIDs []string,
filter AccessFilter,
) ([]*models.MediaItem, error) {
if len(contentIDs) == 0 || (filter.AllowedLibraryIDs != nil && len(filter.AllowedLibraryIDs) == 0) {
return []*models.MediaItem{}, nil
}
args := []any{contentIDs}
argIdx := 2
mediaConditions := []string{"hydrated_mi.content_id = ANY($1)"}
appendLibraryAccessConditions("hydrated_mi.content_id", filter, &mediaConditions, &args, &argIdx)
applyAccessFilter("hydrated_mi", AccessFilter{
MaxContentRating: filter.MaxContentRating,
ExcludedMediaTypes: filter.ExcludedMediaTypes,
}, &mediaConditions, &args, &argIdx)
episodeConditions := []string{"mi.content_id = ANY($1)"}
appendEpisodeLibrarySearchAccess("mi.content_id", filter, &episodeConditions, &args, &argIdx)
applyAccessFilter("mi", AccessFilter{
MaxContentRating: filter.MaxContentRating,
ExcludedMediaTypes: filter.ExcludedMediaTypes,
}, &episodeConditions, &args, &argIdx)
query := fmt.Sprintf(`
SELECT %s
FROM media_items hydrated_mi
WHERE %s
UNION ALL
SELECT %s
FROM %s
WHERE %s`,
qualifiedItemColumns("hydrated_mi"), strings.Join(mediaConditions, " AND "),
qualifiedItemColumns("mi"), episodeCatalogBaseRelation, strings.Join(episodeConditions, " AND "))
rows, err := r.pool.Query(ctx, query, args...)
if err != nil {
return nil, fmt.Errorf("hydrating mixed search items: %w", err)
}
defer rows.Close()
return scanItems(rows)
}
const episodeSearchTitleExpr = `COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text)`
const episodeSearchTitleVector = `setweight(
to_tsvector('simple', public.normalize_search_text(COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text))),
'A'
)`
const episodeSearchOverviewVector = `to_tsvector('english', COALESCE(e.overview, ''))`
const mixedSearchOrder = `exact_title_match DESC, contiguous_title_match DESC, year_match DESC,
phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC,
LOWER(title) ASC, content_id ASC`
// buildMixedSearchSQLWithTotal builds one ranked candidate set from the two
// physical catalog sources. The scored CTE deliberately carries only ranking
// fields; the wide MediaItem projection is hydrated after LIMIT/OFFSET so a
// broad match never sorts posters, arrays, or metadata blobs for every hit.
func (r *ItemRepository) buildMixedSearchSQLWithTotal(
query string,
itemTypes []string,
limit, offset int,
filter AccessFilter,
includeTotal bool,
) (dataSQL, countSQL string, args []any) {
parsed := parseSearchQuery(query)
searchText := parsed.Text
if searchText == "" {
searchText = collapseSearchWhitespace(strings.ReplaceAll(strings.TrimSpace(query), "\"", " "))
}
if searchText == "" {
return "", "", nil
}
mediaTypes, includeEpisodes := splitSearchItemTypes(itemTypes)
includeMediaItems := len(itemTypes) == 0 || len(mediaTypes) > 0
if !includeMediaItems && !includeEpisodes {
return "", "", nil
}
args = []any{searchText, buildTitlePrefixTsQuery(searchText)}
argIdx := 3
var branches []string
mediaConditions := []string{}
if includeMediaItems {
mediaTitleVector := `(
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.title, ''))), 'A') ||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.original_title, ''))), 'A') ||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B')
)`
mediaOverviewVector := `to_tsvector('english', COALESCE(mi.overview, ''))`
mediaConditions = append(mediaConditions, searchMatchCondition(mediaTitleVector, mediaOverviewVector))
if len(itemTypes) > 0 {
mediaConditions = append(mediaConditions, fmt.Sprintf("mi.type = ANY($%d)", argIdx))
args = append(args, mediaTypes)
argIdx++
}
appendLibraryAccessConditions("mi.content_id", filter, &mediaConditions, &args, &argIdx)
applyAccessFilter("mi", AccessFilter{
MaxContentRating: filter.MaxContentRating,
ExcludedMediaTypes: filter.ExcludedMediaTypes,
}, &mediaConditions, &args, &argIdx)
mediaConditions = append(mediaConditions, MangaChapterExclusionWhere("mi"))
}
episodeConditions := []string{}
if includeEpisodes {
episodeConditions = append(episodeConditions,
"si.type = 'series'",
`EXISTS (SELECT 1 FROM episode_libraries available_el WHERE available_el.episode_id = e.content_id)`,
searchMatchCondition(episodeSearchTitleVector, episodeSearchOverviewVector),
)
appendEpisodeLibrarySearchAccess("e.content_id", filter, &episodeConditions, &args, &argIdx)
if filter.MaxContentRating != "" {
allowedRatings := access.AllowedRatingsUpTo(filter.MaxContentRating)
if len(allowedRatings) == 0 {
episodeConditions = append(episodeConditions, "1 = 0")
} else {
episodeConditions = append(episodeConditions, fmt.Sprintf("si.content_rating = ANY($%d)", argIdx))
args = append(args, allowedRatings)
argIdx++
}
}
if len(filter.ExcludedMediaTypes) > 0 {
episodeConditions = append(episodeConditions, fmt.Sprintf("NOT ('episode' = ANY($%d))", argIdx))
args = append(args, filter.ExcludedMediaTypes)
argIdx++
}
}
exactIdx := argIdx
args = append(args, parsed.ExactTitleHint)
argIdx++
var yearArg any
if parsed.Year != nil {
yearArg = *parsed.Year
}
yearIdx := argIdx
args = append(args, yearArg)
argIdx++
phraseIdx := argIdx
args = append(args, parsed.Phrase)
argIdx++
if includeMediaItems {
branches = append(branches, buildMixedSearchCandidateBranch(
"mi.content_id", "mi.type", "mi.title", "mi.year",
`(
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.title, ''))), 'A') ||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.original_title, ''))), 'A') ||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B')
)`,
`to_tsvector('english', COALESCE(mi.overview, ''))`,
[]string{`mi.title_normalized`, `public.normalize_search_text(mi.original_title)`, `public.normalize_search_text(mi.sort_title)`},
"media_items mi", mediaConditions, exactIdx, yearIdx, phraseIdx,
))
}
if includeEpisodes {
episodeNormalizedTitle := `public.normalize_search_text(` + episodeSearchTitleExpr + `)`
branches = append(branches, buildMixedSearchCandidateBranch(
"e.content_id", "'episode'::text", episodeSearchTitleExpr,
"COALESCE(si.year, EXTRACT(YEAR FROM e.air_date)::integer, 0)",
episodeSearchTitleVector, episodeSearchOverviewVector,
[]string{episodeNormalizedTitle},
"episodes e JOIN media_items si ON si.content_id = e.series_id",
episodeConditions, exactIdx, yearIdx, phraseIdx,
))
}
scoredCTE := "WITH scored AS (\n" + strings.Join(branches, "\nUNION ALL\n") + "\n)"
statsCTE := `, stats AS (
SELECT MAX(CASE WHEN title_rank > 0 OR title_prefix_rank > 0 THEN 1 ELSE 0 END) AS has_title_match
FROM scored
)`
postFilter := fmt.Sprintf(`FROM scored
CROSS JOIN stats
WHERE scored.title_rank > 0
OR scored.title_prefix_rank > 0
OR (COALESCE(stats.has_title_match, 0) = 0 AND scored.overview_rank >= %g)`, overviewMatchFloor)
pageTotalColumn := ""
finalTotalColumn := ""
if includeTotal {
pageTotalColumn = ", COUNT(*) OVER () AS total_count"
finalTotalColumn = ", page.total_count"
}
limitIdx, offsetIdx := argIdx, argIdx+1
args = append(args, limit, offset)
pageCTE := fmt.Sprintf(`, page AS (
SELECT scored.*, ROW_NUMBER() OVER (ORDER BY %s) AS ordinal%s
%s
ORDER BY %s
LIMIT $%d OFFSET $%d
)`, mixedSearchOrder, pageTotalColumn, postFilter, mixedSearchOrder, limitIdx, offsetIdx)
hydratedRelation := fmt.Sprintf(`LATERAL (
SELECT %s
FROM media_items hydrated_mi
WHERE page.type <> 'episode'
AND hydrated_mi.content_id = page.content_id
UNION ALL
SELECT %s
FROM %s
WHERE page.type = 'episode'
AND mi.content_id = page.content_id
) hydrated`, qualifiedItemColumns("hydrated_mi"), qualifiedItemColumns("mi"), episodeCatalogBaseRelation)
dataSQL = scoredCTE + statsCTE + pageCTE + fmt.Sprintf(`
SELECT %s%s
FROM page
JOIN %s ON true
ORDER BY page.ordinal`, qualifiedItemColumns("hydrated"), finalTotalColumn, hydratedRelation)
countSQL = scoredCTE + statsCTE + fmt.Sprintf("\nSELECT COUNT(*)\n%s", postFilter)
return dataSQL, countSQL, args
}
func splitSearchItemTypes(itemTypes []string) (mediaTypes []string, includeEpisodes bool) {
if len(itemTypes) == 0 {
return nil, true
}
seen := make(map[string]struct{}, len(itemTypes))
for _, itemType := range itemTypes {
itemType = strings.ToLower(strings.TrimSpace(itemType))
if itemType == "" {
continue
}
if itemType == "episode" {
includeEpisodes = true
continue
}
if _, ok := seen[itemType]; ok {
continue
}
seen[itemType] = struct{}{}
mediaTypes = append(mediaTypes, itemType)
}
return mediaTypes, includeEpisodes
}
func searchMatchCondition(titleVector, overviewVector string) string {
titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))`
prefixQuery := `to_tsquery('simple', $2)`
return fmt.Sprintf(`((%s) @@ %s OR ($2 <> '' AND (%s) @@ %s) OR (%s) @@ websearch_to_tsquery('english', $1))`,
titleVector, titleQuery, titleVector, prefixQuery, overviewVector)
}
func buildMixedSearchCandidateBranch(
contentIDExpr, typeExpr, titleExpr, yearExpr, titleVector, overviewVector string,
exactTitleExprs []string,
fromClause string,
conditions []string,
exactIdx, yearIdx, phraseIdx int,
) string {
exactArms := make([]string, 0, len(exactTitleExprs))
contiguousArms := make([]string, 0, len(exactTitleExprs))
for _, expr := range exactTitleExprs {
exactArms = append(exactArms, fmt.Sprintf("%s = $%d", expr, exactIdx))
contiguousArms = append(contiguousArms, fmt.Sprintf("%s LIKE '%%' || $%d || '%%'", expr, exactIdx))
}
titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))`
prefixQuery := `to_tsquery('simple', $2)`
return fmt.Sprintf(`
SELECT
%s AS content_id,
%s AS type,
%s AS title,
CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS exact_title_match,
CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS contiguous_title_match,
CASE WHEN $%d::int IS NOT NULL AND (%s) = $%d::int THEN 1 ELSE 0 END AS year_match,
ts_rank_cd(%s, %s) AS title_rank,
CASE WHEN $2 <> '' THEN ts_rank_cd(%s, %s) ELSE 0 END AS title_prefix_rank,
ts_rank_cd(%s, websearch_to_tsquery('english', $1)) AS overview_rank,
CASE WHEN $%d <> '' THEN ts_rank_cd(%s, phraseto_tsquery('simple', public.normalize_search_text($%d))) ELSE 0 END AS phrase_rank
FROM %s
WHERE %s`,
contentIDExpr, typeExpr, titleExpr,
exactIdx, strings.Join(exactArms, " OR "),
exactIdx, strings.Join(contiguousArms, " OR "),
yearIdx, yearExpr, yearIdx,
titleVector, titleQuery,
titleVector, prefixQuery,
overviewVector,
phraseIdx, titleVector, phraseIdx,
fromClause, strings.Join(conditions, " AND "))
}
func appendEpisodeLibrarySearchAccess(
episodeIDExpr string,
filter AccessFilter,
conditions *[]string,
args *[]any,
argIdx *int,
) {
if filter.AllowedLibraryIDs != nil {
*conditions = append(*conditions, fmt.Sprintf(
"EXISTS (SELECT 1 FROM episode_libraries allowed_el WHERE allowed_el.episode_id = %s AND allowed_el.media_folder_id = ANY($%d))",
episodeIDExpr, *argIdx))
*args = append(*args, filter.AllowedLibraryIDs)
*argIdx++
}
if len(filter.DisabledLibraryIDs) > 0 {
*conditions = append(*conditions, fmt.Sprintf(
"NOT EXISTS (SELECT 1 FROM episode_libraries disabled_el WHERE disabled_el.episode_id = %s AND disabled_el.media_folder_id = ANY($%d))",
episodeIDExpr, *argIdx))
*args = append(*args, filter.DisabledLibraryIDs)
*argIdx++
}
}
+1 -1
View File
@@ -37,7 +37,7 @@ const (
DefaultMeilisearchIndex = "silo_media_items"
DefaultMeilisearchTimeoutMS = 800
DefaultMeilisearchMatchingStrategy = "last"
SearchMeilisearchSchemaVersion = 2
SearchMeilisearchSchemaVersion = 3
DefaultMeilisearchSyncBatchSize = 500
DefaultMeilisearchRebuildBatchSize = 5000
+135
View File
@@ -150,6 +150,14 @@ func TestCatalogSearchMeilisearchSettingsIncludeEmbeddersWhenSemanticEnabled(t *
}
}
func TestCatalogSearchMeilisearchSettingsFilterLibraryIDs(t *testing.T) {
settings := catalogSearchMeilisearchSettings(DefaultMeilisearchEmbedder, false, false)
filterable, ok := settings["filterableAttributes"].([]string)
if !ok || !reflect.DeepEqual(filterable, []string{"type", "library_ids"}) {
t.Fatalf("filterable attributes = %#v", settings["filterableAttributes"])
}
}
func TestCatalogSearchDocumentPayloadBatchesSplitVectorDocs(t *testing.T) {
vector := make([]float32, 128)
docs := []catalogSearchDocument{
@@ -296,6 +304,133 @@ func TestMeilisearchSearchRequestBuildsHybridWhenCoverageReady(t *testing.T) {
}
}
func TestMeilisearchEpisodeSearchRemainsKeywordOnly(t *testing.T) {
vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}}
provider := &MeilisearchSearchProvider{config: MeilisearchProviderConfig{
SemanticEnabled: true,
SemanticRatio: 0.4,
Embedder: DefaultMeilisearchEmbedder,
Vectorizer: vectorizer,
Coverage: fakeCoverageGate{ready: true},
}}
req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{
Query: "Who Are You",
ItemTypes: []string{"episode"},
Access: AccessFilter{AllowedLibraryIDs: []int{4, 9}, DisabledLibraryIDs: []int{12}},
})
if fallback != "" || req.Hybrid != nil || req.Vector != nil {
t.Fatalf("episode request should be keyword-only, fallback/request = %q/%#v", fallback, req)
}
for _, want := range []string{`type = "episode"`, "library_ids IN [4, 9]", "library_ids NOT IN [12]"} {
if !strings.Contains(req.Filter, want) {
t.Fatalf("episode filter missing %q: %s", want, req.Filter)
}
}
if vectorizer.calls != 0 {
t.Fatalf("episode request generated a vector, calls=%d", vectorizer.calls)
}
}
func TestMeilisearchMixedSemanticSearchBuildsFederation(t *testing.T) {
provider := &MeilisearchSearchProvider{config: MeilisearchProviderConfig{}}
base := meilisearchSearchRequest{
Query: "Who Are You",
AttributesToRetrieve: []string{"content_id"},
MatchingStrategy: "all",
Vector: []float32{0.5, 0.25},
Hybrid: &meilisearchHybridRequest{Embedder: DefaultMeilisearchEmbedder, SemanticRatio: 0.4},
}
federated := provider.buildFederatedSearchRequest("catalog-v3", CatalogSearchRequest{
Access: AccessFilter{AllowedLibraryIDs: []int{4}},
}, base, 100, 25)
if federated.Federation.Offset != 100 || federated.Federation.Limit != 25 {
t.Fatalf("federation pagination = %#v", federated.Federation)
}
if len(federated.Queries) != 2 {
t.Fatalf("federated queries = %#v", federated.Queries)
}
media, episode := federated.Queries[0], federated.Queries[1]
if media.Hybrid == nil || len(media.Vector) == 0 || !strings.Contains(media.Filter, `type != "episode"`) {
t.Fatalf("media federation branch = %#v", media)
}
if episode.Hybrid != nil || episode.Vector != nil || !strings.Contains(episode.Filter, `type = "episode"`) {
t.Fatalf("episode federation branch = %#v", episode)
}
if media.FederationOptions.Weight != 1 || episode.FederationOptions.Weight != 1 {
t.Fatalf("federation weights = %v/%v", media.FederationOptions.Weight, episode.FederationOptions.Weight)
}
}
func TestMeilisearchFederationUnsupportedClassification(t *testing.T) {
tests := []struct {
name string
err error
want bool
}{
{
name: "missing index during rebuild swap is transient",
err: &meilisearchHTTPError{
StatusCode: http.StatusNotFound,
Code: "index_not_found",
Message: "Index catalog_rebuild_old not found.",
},
want: false,
},
{
name: "missing multi-search route is unsupported",
err: &meilisearchHTTPError{
StatusCode: http.StatusNotFound,
Code: "not_found",
Message: "The multi-search route does not exist.",
},
want: true,
},
{
name: "older server rejects federation field",
err: &meilisearchHTTPError{
StatusCode: http.StatusBadRequest,
Code: "bad_request",
Message: "Unknown field federation: expected queries.",
},
want: true,
},
{
name: "invalid federated request is not a capability result",
err: &meilisearchHTTPError{
StatusCode: http.StatusBadRequest,
Code: "invalid_multi_search_query_federated",
Message: "A query has federationOptions but federation is missing.",
},
want: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := isMeilisearchFederationUnsupported(test.err); got != test.want {
t.Fatalf("isMeilisearchFederationUnsupported(%v) = %t, want %t", test.err, got, test.want)
}
})
}
}
func TestCatalogSearchEpisodeDocumentsNeverCarryVectors(t *testing.T) {
docs := []catalogSearchDocument{
{ContentID: "movie-1", Type: "movie"},
{ContentID: "episode-1", Type: "episode"},
}
count := setCatalogSearchDocumentVectors(docs, map[string][]float32{
"movie-1": {0.1, 0.2},
"episode-1": {0.3, 0.4},
}, DefaultMeilisearchEmbedder)
if count != 1 || docs[0].Vectors == nil {
t.Fatalf("media vector assignment = count %d, vectors %#v", count, docs[0].Vectors)
}
if docs[1].Vectors != nil {
t.Fatalf("episode vectors = %#v, want omitted", docs[1].Vectors)
}
}
func TestSemanticModelProviderUsesCommaOkSafety(t *testing.T) {
if got := semanticModelProvider(nil); got != nil {
t.Fatalf("nil vectorizer should yield nil model provider, got %#v", got)
@@ -0,0 +1,53 @@
-- +goose NO TRANSACTION
-- +goose Up
-- Search evaluates these exact expressions in the episode candidate branch.
-- Build concurrently so large episode catalogs remain readable and writable.
-- Remove INVALID remnants first: IF NOT EXISTS otherwise accepts a failed
-- concurrent build and Goose would record a search index that cannot be used.
-- +goose StatementBegin
DO $$
DECLARE
index_name text;
BEGIN
FOREACH index_name IN ARRAY ARRAY[
'idx_episodes_search_title',
'idx_episodes_search_overview'
] LOOP
IF EXISTS (
SELECT 1
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
JOIN pg_index i ON i.indexrelid = c.oid
WHERE n.nspname = 'public'
AND c.relname = index_name
AND NOT i.indisvalid
) THEN
EXECUTE format('DROP INDEX public.%I', index_name);
END IF;
END LOOP;
END;
$$;
-- +goose StatementEnd
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_episodes_search_title
ON public.episodes USING gin (
setweight(
to_tsvector(
'simple',
public.normalize_search_text(
COALESCE(NULLIF(BTRIM(title), ''), 'Episode ' || episode_number::text)
)
),
'A'
)
);
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_episodes_search_overview
ON public.episodes USING gin (
to_tsvector('english', COALESCE(overview, ''))
);
-- +goose Down
DROP INDEX CONCURRENTLY IF EXISTS public.idx_episodes_search_overview;
DROP INDEX CONCURRENTLY IF EXISTS public.idx_episodes_search_title;
@@ -0,0 +1,216 @@
-- +goose Up
-- +goose StatementBegin
CREATE OR REPLACE FUNCTION public.catalog_search_capture_enabled()
RETURNS boolean
LANGUAGE sql
STABLE
AS $$
SELECT lower(COALESCE(
(SELECT value FROM public.server_settings WHERE key = 'catalog.search.provider'),
'postgres'
)) = 'meilisearch'
$$;
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_insert()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'upsert', content_id, '' FROM new_rows GROUP BY content_id;
RETURN NULL;
END;
$$;
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_update()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'delete', old_rows.content_id, ''
FROM old_rows
LEFT JOIN new_rows ON new_rows.content_id = old_rows.content_id
WHERE new_rows.content_id IS NULL
GROUP BY old_rows.content_id;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'upsert', content_id, '' FROM new_rows GROUP BY content_id;
RETURN NULL;
END;
$$;
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_delete()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'delete', content_id, '' FROM old_rows GROUP BY content_id;
RETURN NULL;
END;
$$;
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_library_insert()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'upsert', episode_id, '' FROM new_rows GROUP BY episode_id;
RETURN NULL;
END;
$$;
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_library_update()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'upsert', episode_id, ''
FROM (
SELECT episode_id FROM old_rows
UNION
SELECT episode_id FROM new_rows
) changed;
RETURN NULL;
END;
$$;
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_library_delete()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'upsert', episode_id, '' FROM old_rows GROUP BY episode_id;
RETURN NULL;
END;
$$;
CREATE OR REPLACE FUNCTION public.catalog_search_capture_item_library_insert()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'upsert', content_id, '' FROM new_rows GROUP BY content_id;
RETURN NULL;
END;
$$;
CREATE OR REPLACE FUNCTION public.catalog_search_capture_item_library_update()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'upsert', content_id, ''
FROM (
SELECT content_id FROM old_rows
UNION
SELECT content_id FROM new_rows
) changed;
RETURN NULL;
END;
$$;
CREATE OR REPLACE FUNCTION public.catalog_search_capture_item_library_delete()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'upsert', content_id, '' FROM old_rows GROUP BY content_id;
RETURN NULL;
END;
$$;
CREATE OR REPLACE FUNCTION public.catalog_search_capture_series_type_update()
RETURNS trigger LANGUAGE plpgsql AS $$
BEGIN
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
SELECT 'meilisearch', 'upsert', e.content_id, ''
FROM (
SELECT new_rows.content_id
FROM new_rows
JOIN old_rows USING (content_id)
WHERE new_rows.type IS DISTINCT FROM old_rows.type
AND (new_rows.type = 'series' OR old_rows.type = 'series')
) changed
JOIN public.episodes e ON e.series_id = changed.content_id
GROUP BY e.content_id;
RETURN NULL;
END;
$$;
CREATE TRIGGER trg_catalog_search_episode_insert
AFTER INSERT ON public.episodes
REFERENCING NEW TABLE AS new_rows
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_insert();
CREATE TRIGGER trg_catalog_search_episode_update
AFTER UPDATE ON public.episodes
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_update();
CREATE TRIGGER trg_catalog_search_episode_delete
AFTER DELETE ON public.episodes
REFERENCING OLD TABLE AS old_rows
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_delete();
CREATE TRIGGER trg_catalog_search_episode_library_insert
AFTER INSERT ON public.episode_libraries
REFERENCING NEW TABLE AS new_rows
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_library_insert();
CREATE TRIGGER trg_catalog_search_episode_library_update
AFTER UPDATE ON public.episode_libraries
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_library_update();
CREATE TRIGGER trg_catalog_search_episode_library_delete
AFTER DELETE ON public.episode_libraries
REFERENCING OLD TABLE AS old_rows
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_library_delete();
CREATE TRIGGER trg_catalog_search_item_library_insert
AFTER INSERT ON public.media_item_libraries
REFERENCING NEW TABLE AS new_rows
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_item_library_insert();
CREATE TRIGGER trg_catalog_search_item_library_update
AFTER UPDATE ON public.media_item_libraries
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_item_library_update();
CREATE TRIGGER trg_catalog_search_item_library_delete
AFTER DELETE ON public.media_item_libraries
REFERENCING OLD TABLE AS old_rows
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_item_library_delete();
CREATE TRIGGER trg_catalog_search_series_type_update
AFTER UPDATE ON public.media_items
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_series_type_update();
-- +goose StatementEnd
-- +goose Down
-- +goose StatementBegin
DROP TRIGGER IF EXISTS trg_catalog_search_series_type_update ON public.media_items;
DROP TRIGGER IF EXISTS trg_catalog_search_item_library_delete ON public.media_item_libraries;
DROP TRIGGER IF EXISTS trg_catalog_search_item_library_update ON public.media_item_libraries;
DROP TRIGGER IF EXISTS trg_catalog_search_item_library_insert ON public.media_item_libraries;
DROP TRIGGER IF EXISTS trg_catalog_search_episode_library_delete ON public.episode_libraries;
DROP TRIGGER IF EXISTS trg_catalog_search_episode_library_update ON public.episode_libraries;
DROP TRIGGER IF EXISTS trg_catalog_search_episode_library_insert ON public.episode_libraries;
DROP TRIGGER IF EXISTS trg_catalog_search_episode_delete ON public.episodes;
DROP TRIGGER IF EXISTS trg_catalog_search_episode_update ON public.episodes;
DROP TRIGGER IF EXISTS trg_catalog_search_episode_insert ON public.episodes;
DROP FUNCTION IF EXISTS public.catalog_search_capture_series_type_update();
DROP FUNCTION IF EXISTS public.catalog_search_capture_item_library_delete();
DROP FUNCTION IF EXISTS public.catalog_search_capture_item_library_update();
DROP FUNCTION IF EXISTS public.catalog_search_capture_item_library_insert();
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_library_delete();
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_library_update();
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_library_insert();
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_delete();
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_update();
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_insert();
DROP FUNCTION IF EXISTS public.catalog_search_capture_enabled();
-- +goose StatementEnd
+49
View File
@@ -0,0 +1,49 @@
import { act, fireEvent, render, screen } from "@testing-library/react";
import { MemoryRouter, useLocation } from "react-router";
import { afterEach, describe, expect, it, vi } from "vitest";
import { buildCatalogQueryUpdateHref, parseCatalogSearchParams } from "@/pages/catalogSearchParams";
import SearchBar from "./SearchBar";
function LocationProbe() {
const location = useLocation();
return <output aria-label="location">{`${location.pathname}${location.search}`}</output>;
}
describe("SearchBar", () => {
afterEach(() => {
vi.useRealTimers();
});
it("keeps All Media selected when live search changes the query", () => {
vi.useFakeTimers();
const state = parseCatalogSearchParams(
new URLSearchParams("source=query&q=heat&type=all&genre=Drama"),
);
render(
<MemoryRouter initialEntries={["/catalog?source=query&q=heat&type=all&genre=Drama"]}>
<SearchBar
prominent
initialQuery="heat"
buildSearchHref={(query) => buildCatalogQueryUpdateHref(state, query)}
/>
<LocationProbe />
</MemoryRouter>,
);
fireEvent.change(screen.getByRole("textbox"), { target: { value: "heater" } });
act(() => {
vi.advanceTimersByTime(101);
});
const location = new URL(`http://example.test${screen.getByLabelText("location").textContent}`);
expect(location.searchParams.get("q")).toBe("heater");
expect(location.searchParams.get("type")).toBe("all");
expect(parseCatalogSearchParams(location.searchParams).query_definition.groups).toContainEqual({
match: "all",
rules: [{ field: "genre", op: "contains", value: "Drama" }],
});
});
});
+9 -2
View File
@@ -12,19 +12,26 @@ interface SearchBarProps {
initialQuery?: string;
autoFocus?: boolean;
prominent?: boolean;
buildSearchHref?: (query: string) => string;
}
export default function SearchBar({
initialQuery = "",
autoFocus = false,
prominent = false,
buildSearchHref = buildQueryCatalogHref,
}: SearchBarProps) {
const [query, setQuery] = useState(initialQuery);
const navigate = useViewTransitionNavigate();
const inputRef = useRef<HTMLInputElement>(null);
const isInitialMount = useRef(true);
const buildSearchHrefRef = useRef(buildSearchHref);
const debouncedQuery = useDebounce(query, SEARCH_NAVIGATION_DEBOUNCE_MS);
useEffect(() => {
buildSearchHrefRef.current = buildSearchHref;
}, [buildSearchHref]);
useEffect(() => {
if (autoFocus && inputRef.current) {
inputRef.current.focus();
@@ -39,14 +46,14 @@ export default function SearchBar({
return;
}
if (debouncedQuery.trim()) {
navigate(buildQueryCatalogHref(debouncedQuery.trim()), { replace: true });
navigate(buildSearchHrefRef.current(debouncedQuery.trim()), { replace: true });
}
}, [debouncedQuery, prominent, navigate]);
function handleSubmit(e: FormEvent) {
e.preventDefault();
if (query.trim()) {
navigate(buildQueryCatalogHref(query.trim()));
navigate(buildSearchHref(query.trim()));
}
}
+13 -3
View File
@@ -24,7 +24,8 @@ import {
} from "@/lib/historyRemoval";
import {
buildCatalogApiSearchParams,
buildCatalogFilterSearchParams,
buildCatalogQueryUpdateHref,
catalogSourceAllowsOverlay,
parseCatalogSearchParams,
} from "./catalogSearchParams";
@@ -132,6 +133,10 @@ function CatalogResults({
query_definition: { ...state.query_definition, media_scope: preferredScope },
};
}, [hasExplicitScope, preferredScope, state]);
const buildSearchHref = useCallback(
(query: string) => buildCatalogQueryUpdateHref(effectiveState, query),
[effectiveState],
);
const mediaScope = effectiveState.query_definition.media_scope;
const activeChipScope: SearchMediaScope =
@@ -257,7 +262,12 @@ function CatalogResults({
{state.source === "query" ? (
<div className="flex flex-col items-center gap-3">
<SearchBar prominent initialQuery={state.q ?? ""} autoFocus />
<SearchBar
prominent
initialQuery={state.q ?? ""}
autoFocus
buildSearchHref={buildSearchHref}
/>
<SearchScopeChips activeScope={activeChipScope} onScopeChange={handleChipScopeChange} />
</div>
) : null}
@@ -265,7 +275,7 @@ function CatalogResults({
<CatalogFiltersPanel
state={effectiveState}
onStateChange={(nextState) => {
const nextSearchParams = buildCatalogApiSearchParams(nextState);
const nextSearchParams = buildCatalogFilterSearchParams(nextState);
if (nextSearchParams.toString() !== searchParams.toString()) {
setSearchParams(nextSearchParams);
}
+33
View File
@@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest";
import {
buildCatalogApiSearchParams,
buildCatalogFilterSearchParams,
buildCatalogQueryUpdateHref,
buildCatalogHref,
buildPersonCatalogHref,
buildPersonalCatalogHref,
@@ -127,6 +129,37 @@ describe("catalogSourceAllowsOverlay", () => {
});
describe("buildCatalogHref", () => {
it("preserves type=all when the query filter selects All Media", () => {
const state = parseCatalogSearchParams(params("source=query&q=heat&type=video"));
state.query_definition = {
...state.query_definition,
media_scope: undefined,
};
const built = buildCatalogFilterSearchParams(state);
expect(built.get("type")).toBe("all");
expect(parseCatalogSearchParams(built).query_definition.media_scope).toBeUndefined();
});
it("preserves All Media and other filters when the live query changes", () => {
const state = parseCatalogSearchParams(
params("source=query&q=heat&type=all&genre=Drama&sort=title&order=asc"),
);
const href = buildCatalogQueryUpdateHref(state, "heater");
const built = new URL(`http://example.test${href}`);
expect(built.searchParams.get("q")).toBe("heater");
expect(built.searchParams.get("type")).toBe("all");
expect(built.searchParams.get("sort")).toBe("title");
expect(built.searchParams.get("order")).toBe("asc");
expect(parseCatalogSearchParams(built.searchParams).query_definition.groups).toContainEqual({
match: "all",
rules: [{ field: "genre", op: "contains", value: "Drama" }],
});
});
it("keeps explicit added-at sorting in query-source URLs", () => {
expect(
buildCatalogApiSearchParams({
+22
View File
@@ -181,6 +181,28 @@ export function buildCatalogHref(state: CatalogSearchState): string {
return `/catalog?${params.toString()}`;
}
// Interactive query filters need an explicit `all` sentinel. Internally an
// unscoped query is represented by an undefined media_scope, but omitting the
// URL type makes Catalog reapply the user's saved default (normally `video`).
// Keep this behavior scoped to filter updates so fresh search URLs can still
// inherit that preference.
export function buildCatalogFilterSearchParams(state: CatalogSearchState): URLSearchParams {
const params = buildCatalogApiSearchParams(state);
if (state.source === "query" && !state.type_override && !state.query_definition.media_scope) {
params.set("type", "all");
}
return params;
}
export function buildCatalogQueryUpdateHref(state: CatalogSearchState, q: string): string {
const params = buildCatalogFilterSearchParams({
...state,
source: "query",
q,
});
return `/catalog?${params.toString()}`;
}
export function buildQueryCatalogHref(q?: string): string {
return buildCatalogHref({
source: "query",