feat(search): add scalable TV episode search
Add mixed PostgreSQL and Meilisearch episode search with access-safe hydration, incremental indexing, and regression coverage for explicit all-media searches.\n\nPart of #396
This commit is contained in:
@@ -54,7 +54,7 @@ func buildLegacySearchCatalogValues(values url.Values) (url.Values, bool) {
|
||||
if len(types) > 1 {
|
||||
return nil, false
|
||||
}
|
||||
if len(types) == 1 && types[0] != "movie" && types[0] != "series" {
|
||||
if len(types) == 1 && types[0] != "movie" && types[0] != "series" && types[0] != "episode" {
|
||||
return nil, false
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBuildLegacySearchCatalogValuesAcceptsEpisode(t *testing.T) {
|
||||
values := url.Values{
|
||||
"q": {"Who Are You?"},
|
||||
"type": {"episode"},
|
||||
"limit": {"25"},
|
||||
"offset": {"50"},
|
||||
}
|
||||
|
||||
translated, ok := buildLegacySearchCatalogValues(values)
|
||||
if !ok {
|
||||
t.Fatal("episode search was not routed through the catalog resolver")
|
||||
}
|
||||
for key, want := range map[string]string{
|
||||
"source": "query",
|
||||
"q": "Who Are You?",
|
||||
"type": "episode",
|
||||
"sort": "relevance",
|
||||
"order": "desc",
|
||||
"limit": "25",
|
||||
"offset": "50",
|
||||
} {
|
||||
if got := translated.Get(key); got != want {
|
||||
t.Fatalf("translated %s = %q, want %q", key, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,9 +132,9 @@ func (r *ItemRepository) GetPosterPath(ctx context.Context, contentID string) (s
|
||||
const overviewMatchFloor = 0.15
|
||||
|
||||
// itemColumnNames lists, in scan order, every column selected by media_items
|
||||
// item queries. Shared by itemColumns, qualifiedItemColumns,
|
||||
// qualifiedListItemColumns, and qualifiedItemColumnRefs so the select lists
|
||||
// can never drift from each other or from scanItem.
|
||||
// item queries. Shared by itemColumns, qualifiedItemColumns, and
|
||||
// qualifiedListItemColumns so the select lists can never drift from each
|
||||
// other or from scanItem.
|
||||
var itemColumnNames = []string{
|
||||
"content_id", "type", "title", "sort_title", "default_metadata_language", "original_title", "year", "genres",
|
||||
"content_rating", "runtime", "overview", "tagline",
|
||||
@@ -194,17 +194,6 @@ func qualifiedItemColumns(alias string) string {
|
||||
return joinItemColumns(alias)
|
||||
}
|
||||
|
||||
// qualifiedItemColumnRefs renders plain alias-qualified column references
|
||||
// without COALESCE or AS aliases, for contexts like GROUP BY where output
|
||||
// aliases are invalid.
|
||||
func qualifiedItemColumnRefs(alias string) string {
|
||||
refs := make([]string, len(itemColumnNames))
|
||||
for i, col := range itemColumnNames {
|
||||
refs[i] = alias + "." + col
|
||||
}
|
||||
return strings.Join(refs, ", ")
|
||||
}
|
||||
|
||||
func qualifiedListItemColumns(alias string) string {
|
||||
exprs := make([]string, len(itemColumnNames))
|
||||
for i, col := range itemColumnNames {
|
||||
@@ -965,224 +954,7 @@ func (r *ItemRepository) buildSearchSQL(query string, itemTypes []string, limit,
|
||||
}
|
||||
|
||||
func (r *ItemRepository) buildSearchSQLWithTotal(query string, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool) (dataSQL, countSQL string, args []any) {
|
||||
parsed := parseSearchQuery(query)
|
||||
searchText := parsed.Text
|
||||
if searchText == "" {
|
||||
searchText = collapseSearchWhitespace(strings.ReplaceAll(strings.TrimSpace(query), "\"", " "))
|
||||
}
|
||||
if searchText == "" {
|
||||
return "", "", nil
|
||||
}
|
||||
var conditions []string
|
||||
titlePrefixIdx := 2
|
||||
args = []any{searchText, buildTitlePrefixTsQuery(searchText)}
|
||||
argIdx := 3
|
||||
|
||||
// All title-side text on both sides of @@ flows through
|
||||
// public.normalize_search_text() (migrations 127 / 138), which strips
|
||||
// non-alphanumeric chars, drops standalone "and" tokens, and normalizes
|
||||
// common title numbers. The expression must match
|
||||
// idx_media_items_search_title_fields exactly for the GIN index to be used.
|
||||
//
|
||||
// Overview uses the 'english' config which natively treats "and" as a
|
||||
// stop word, so it does not need explicit normalization.
|
||||
titleVector := `(
|
||||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.title, ''))), 'A') ||
|
||||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.original_title, ''))), 'A') ||
|
||||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B')
|
||||
)`
|
||||
overviewVector := `to_tsvector('english', COALESCE(mi.overview, ''))`
|
||||
normalizedTitleExpr := `public.normalize_search_text(%s)`
|
||||
titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))`
|
||||
titlePrefixQuery := fmt.Sprintf("to_tsquery('simple', $%d)", titlePrefixIdx)
|
||||
titleMatch := fmt.Sprintf("%s @@ %s", titleVector, titleQuery)
|
||||
titlePrefixMatch := fmt.Sprintf("($%d <> '' AND %s @@ %s)", titlePrefixIdx, titleVector, titlePrefixQuery)
|
||||
overviewMatch := fmt.Sprintf("%s @@ websearch_to_tsquery('english', $1)", overviewVector)
|
||||
|
||||
// Keep the base match condition index-friendly; exact-title logic is used as
|
||||
// a ranking boost later, not as an additional scan predicate.
|
||||
conditions = append(conditions, fmt.Sprintf("(%s OR %s OR %s)", titleMatch, titlePrefixMatch, overviewMatch))
|
||||
|
||||
if len(itemTypes) > 0 {
|
||||
placeholders := make([]string, 0, len(itemTypes))
|
||||
for _, itemType := range itemTypes {
|
||||
if strings.TrimSpace(itemType) == "" {
|
||||
continue
|
||||
}
|
||||
placeholders = append(placeholders, fmt.Sprintf("$%d", argIdx))
|
||||
args = append(args, strings.ToLower(strings.TrimSpace(itemType)))
|
||||
argIdx++
|
||||
}
|
||||
if len(placeholders) > 0 {
|
||||
conditions = append(conditions, fmt.Sprintf("mi.type IN (%s)", strings.Join(placeholders, ", ")))
|
||||
}
|
||||
}
|
||||
|
||||
needsLibJoin := filter.AllowedLibraryIDs != nil || len(filter.DisabledLibraryIDs) > 0
|
||||
fromClause := "media_items mi"
|
||||
if filter.AllowedLibraryIDs != nil {
|
||||
// Caller (Search) is expected to short-circuit when len == 0; we still
|
||||
// guard here so the builder is safe to invoke from tests.
|
||||
if len(filter.AllowedLibraryIDs) > 0 {
|
||||
conditions = append(conditions, fmt.Sprintf("mil.media_folder_id = ANY($%d)", argIdx))
|
||||
args = append(args, filter.AllowedLibraryIDs)
|
||||
argIdx++
|
||||
}
|
||||
}
|
||||
if len(filter.DisabledLibraryIDs) > 0 {
|
||||
conditions = append(conditions, fmt.Sprintf("NOT (mil.media_folder_id = ANY($%d))", argIdx))
|
||||
args = append(args, filter.DisabledLibraryIDs)
|
||||
argIdx++
|
||||
}
|
||||
if needsLibJoin {
|
||||
fromClause = "media_items mi JOIN media_item_libraries mil ON mi.content_id = mil.content_id"
|
||||
}
|
||||
applyAccessFilter("mi", AccessFilter{MaxContentRating: filter.MaxContentRating, ExcludedMediaTypes: filter.ExcludedMediaTypes}, &conditions, &args, &argIdx)
|
||||
|
||||
// Manga chapters (type='ebook' rows linked into a manga series) are internal
|
||||
// sub-units and must never surface as standalone search results.
|
||||
conditions = append(conditions, MangaChapterExclusionWhere("mi"))
|
||||
|
||||
whereClause := "WHERE " + strings.Join(conditions, " AND ")
|
||||
|
||||
// Bind ExactTitleHint exactly once. The same arg index is referenced by
|
||||
// both contiguous_title_match and exact_title_match (used as ranking
|
||||
// signals in the ORDER BY). The post-CTE WHERE itself gates on title_rank
|
||||
// > 0 (true title FTS match), not on contiguous_title_match.
|
||||
exactIdx := argIdx
|
||||
args = append(args, parsed.ExactTitleHint)
|
||||
argIdx++
|
||||
|
||||
// mi.title uses the title_normalized stored generated column (migrations
|
||||
// 105 / 127), so the LIKE '%pattern%' arm hits the gin_trgm_ops index
|
||||
// instead of recomputing normalization per row. original_title and
|
||||
// sort_title do not have a trigram index of their own, so their LIKE
|
||||
// arms call public.normalize_search_text() per row. The tsvector @@
|
||||
// path covers all three via idx_media_items_search_title_fields.
|
||||
contiguousTitleMatch := fmt.Sprintf(`(
|
||||
$%d <> '' AND (
|
||||
%s LIKE '%%' || $%d || '%%' OR
|
||||
%s LIKE '%%' || $%d || '%%' OR
|
||||
%s LIKE '%%' || $%d || '%%'
|
||||
)
|
||||
)`,
|
||||
exactIdx,
|
||||
"mi.title_normalized", exactIdx,
|
||||
fmt.Sprintf(normalizedTitleExpr, "mi.original_title"), exactIdx,
|
||||
fmt.Sprintf(normalizedTitleExpr, "mi.sort_title"), exactIdx,
|
||||
)
|
||||
|
||||
var yearArg any
|
||||
if parsed.Year != nil {
|
||||
yearArg = *parsed.Year
|
||||
}
|
||||
yearIdx := argIdx
|
||||
args = append(args, yearArg)
|
||||
argIdx++
|
||||
phraseIdx := argIdx
|
||||
args = append(args, parsed.Phrase)
|
||||
argIdx++
|
||||
|
||||
exactTitleMatch := fmt.Sprintf(`(
|
||||
$%d <> '' AND (
|
||||
%s = $%d OR
|
||||
%s = $%d OR
|
||||
%s = $%d
|
||||
)
|
||||
)`,
|
||||
exactIdx,
|
||||
"mi.title_normalized", exactIdx,
|
||||
fmt.Sprintf(normalizedTitleExpr, "mi.original_title"), exactIdx,
|
||||
fmt.Sprintf(normalizedTitleExpr, "mi.sort_title"), exactIdx,
|
||||
)
|
||||
|
||||
// Use qualified column names inside the CTE to avoid ambiguity when
|
||||
// the FROM clause includes a JOIN to media_item_libraries. The select
|
||||
// list aliases coalesced columns back to their own names (poster_path
|
||||
// etc.) so the outer query can re-reference them; GROUP BY needs the
|
||||
// raw references because output aliases are invalid there.
|
||||
qualifiedCols := qualifiedItemColumns("mi")
|
||||
groupByCols := qualifiedItemColumnRefs("mi")
|
||||
scoredCTE := fmt.Sprintf(`
|
||||
WITH scored AS (
|
||||
SELECT
|
||||
%s,
|
||||
MAX(CASE
|
||||
WHEN %s THEN 1
|
||||
ELSE 0
|
||||
END) AS exact_title_match,
|
||||
MAX(CASE
|
||||
WHEN %s THEN 1
|
||||
ELSE 0
|
||||
END) AS contiguous_title_match,
|
||||
MAX(CASE
|
||||
WHEN $%d::int IS NOT NULL AND mi.year = $%d::int THEN 1
|
||||
ELSE 0
|
||||
END) AS year_match,
|
||||
MAX(ts_rank_cd(%s, %s)) AS title_rank,
|
||||
MAX(CASE
|
||||
WHEN $%d <> '' THEN ts_rank_cd(%s, %s)
|
||||
ELSE 0
|
||||
END) AS title_prefix_rank,
|
||||
MAX(ts_rank_cd(%s, websearch_to_tsquery('english', $1))) AS overview_rank,
|
||||
MAX(CASE
|
||||
WHEN $%d <> '' THEN ts_rank_cd(%s, phraseto_tsquery('simple', public.normalize_search_text($%d)))
|
||||
ELSE 0
|
||||
END) AS phrase_rank
|
||||
FROM %s
|
||||
%s
|
||||
GROUP BY %s
|
||||
)
|
||||
`, qualifiedCols, exactTitleMatch, contiguousTitleMatch, yearIdx, yearIdx, titleVector, titleQuery, titlePrefixIdx, titleVector, titlePrefixQuery, overviewVector, phraseIdx, titleVector, phraseIdx, fromClause, whereClause, groupByCols)
|
||||
|
||||
// COUNT(*) OVER () runs after the GROUP BY in the scored CTE collapses
|
||||
// duplicates from the library JOIN, so the window count preserves the
|
||||
// COUNT(DISTINCT mi.content_id) semantics of the prior 2-query path.
|
||||
// The stats CTE + CROSS JOIN below means the window count reflects the
|
||||
// post-filter row count automatically.
|
||||
//
|
||||
// The stats CTE gates on title_rank > 0 (true title FTS match) rather
|
||||
// than contiguous_title_match (LIKE substring), so reordered-token title
|
||||
// queries like "order law" → "Law and Order" aren't wrongly demoted to
|
||||
// the overview-fallback bucket. The post-CTE WHERE then keeps every title
|
||||
// FTS hit, and admits overview-only rows only when no title hit exists
|
||||
// anywhere in the candidate set AND the overview rank clears
|
||||
// overviewMatchFloor. This suppresses single-occurrence body matches
|
||||
// (which score at PostgreSQL's default D weight of 0.1) for common
|
||||
// single-word queries like "obsession" that would otherwise flood
|
||||
// results with description-only hits.
|
||||
//
|
||||
// countSQL is a count-only sibling that omits LIMIT/OFFSET/ORDER BY. It is
|
||||
// invoked only as a fallback when the data SELECT returns an empty page
|
||||
// past offset 0 — COUNT(*) OVER () emits no rows in that case so total
|
||||
// would otherwise default to 0.
|
||||
statsCTE := `
|
||||
, stats AS (
|
||||
SELECT MAX(CASE WHEN title_rank > 0 OR title_prefix_rank > 0 THEN 1 ELSE 0 END) AS has_title_match
|
||||
FROM scored
|
||||
)`
|
||||
// postFilter is the FROM + CROSS JOIN + WHERE shared by both dataSQL and
|
||||
// countSQL. Keeping it in one string ensures the empty-page fallback count
|
||||
// can never drift from the data query's filter.
|
||||
postFilter := fmt.Sprintf(`FROM scored
|
||||
CROSS JOIN stats
|
||||
WHERE scored.title_rank > 0
|
||||
OR scored.title_prefix_rank > 0
|
||||
OR (COALESCE(stats.has_title_match, 0) = 0 AND scored.overview_rank >= %g)`, overviewMatchFloor)
|
||||
totalColumn := ""
|
||||
if includeTotal {
|
||||
totalColumn = ", COUNT(*) OVER () AS total_count"
|
||||
}
|
||||
dataSQL = scoredCTE + statsCTE + fmt.Sprintf(`
|
||||
SELECT %s%s
|
||||
%s
|
||||
ORDER BY exact_title_match DESC, contiguous_title_match DESC, year_match DESC, phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC, LOWER(title) ASC, content_id ASC
|
||||
LIMIT $%d OFFSET $%d`, itemColumns, totalColumn, postFilter, argIdx, argIdx+1)
|
||||
countSQL = scoredCTE + statsCTE + fmt.Sprintf(`
|
||||
SELECT COUNT(*)
|
||||
%s`, postFilter)
|
||||
args = append(args, limit, offset)
|
||||
return dataSQL, countSQL, args
|
||||
return r.buildMixedSearchSQLWithTotal(query, itemTypes, limit, offset, filter, includeTotal)
|
||||
}
|
||||
|
||||
// ListUnmatchedByFolderAndPathPrefix returns content IDs for unmatched-style
|
||||
|
||||
@@ -490,32 +490,73 @@ func TestItemRepo_Search_ScoredCTEExposesItemColumnNames(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestItemRepo_Search_GroupByHasNoOutputAliases asserts the scored CTE's
|
||||
// GROUP BY uses raw column references: AS aliases are valid in a select list
|
||||
// but are a syntax error inside GROUP BY, so the CTE must not reuse the
|
||||
// aliased projection there.
|
||||
func TestItemRepo_Search_GroupByHasNoOutputAliases(t *testing.T) {
|
||||
// TestItemRepo_Search_ScoredCTEIsLean asserts that relevance ranking carries
|
||||
// only candidate identity/rank fields. Wide MediaItem columns are hydrated
|
||||
// from the page CTE after LIMIT/OFFSET, which keeps broad episode searches
|
||||
// from sorting metadata arrays and image fields for every candidate.
|
||||
func TestItemRepo_Search_ScoredCTEIsLean(t *testing.T) {
|
||||
repo := &ItemRepository{}
|
||||
dataSQL, countSQL, _ := repo.buildSearchSQL("avatar", []string{"movie"}, 20, 0, AccessFilter{})
|
||||
|
||||
for _, sql := range []string{dataSQL, countSQL} {
|
||||
idx := strings.Index(sql, "GROUP BY")
|
||||
if idx < 0 {
|
||||
t.Fatalf("expected GROUP BY in scored CTE; got:\n%s", sql)
|
||||
end := strings.Index(sql, "), stats AS")
|
||||
if end < 0 {
|
||||
t.Fatalf("expected scored CTE boundary; got:\n%s", sql)
|
||||
}
|
||||
clause := sql[idx:]
|
||||
if end := strings.Index(clause, ")"); end >= 0 {
|
||||
clause = clause[:end]
|
||||
scored := sql[:end]
|
||||
for _, wideColumn := range []string{"poster_path", "backdrop_path", "metadata_s3_path", "genres"} {
|
||||
if strings.Contains(scored, wideColumn) {
|
||||
t.Fatalf("scored CTE must not carry %s; got:\n%s", wideColumn, scored)
|
||||
}
|
||||
}
|
||||
if strings.Contains(clause, " AS ") {
|
||||
t.Fatalf("GROUP BY must not contain output aliases; got:\n%s", clause)
|
||||
}
|
||||
if strings.Contains(clause, "COALESCE") {
|
||||
t.Fatalf("GROUP BY should group by raw columns, not COALESCE expressions; got:\n%s", clause)
|
||||
}
|
||||
}
|
||||
|
||||
func TestItemRepo_Search_UnscopedIncludesEpisodeCandidateBranch(t *testing.T) {
|
||||
repo := &ItemRepository{}
|
||||
sql, _, _ := repo.buildSearchSQL("Who Are You?", nil, 20, 0, AccessFilter{})
|
||||
for _, want := range []string{
|
||||
"FROM episodes e JOIN media_items si",
|
||||
"si.type = 'series'",
|
||||
"FROM episode_libraries available_el",
|
||||
"UNION ALL",
|
||||
} {
|
||||
if !strings.Contains(sql, want) {
|
||||
t.Fatalf("mixed search SQL missing %q:\n%s", want, sql)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestItemRepo_Search_EpisodeScopeOmitsMediaItemCandidateBranch(t *testing.T) {
|
||||
repo := &ItemRepository{}
|
||||
sql, _, _ := repo.buildSearchSQL("Who Are You?", []string{"episode"}, 20, 0, AccessFilter{})
|
||||
scoredEnd := strings.Index(sql, "), stats AS")
|
||||
if scoredEnd < 0 {
|
||||
t.Fatalf("expected scored CTE boundary; got:\n%s", sql)
|
||||
}
|
||||
scored := sql[:scoredEnd]
|
||||
if strings.Contains(scored, "FROM media_items mi") {
|
||||
t.Fatalf("episode-only candidate set must not scan media_items directly:\n%s", scored)
|
||||
}
|
||||
if !strings.Contains(scored, "FROM episodes e JOIN media_items si") {
|
||||
t.Fatalf("episode-only candidate set missing episode branch:\n%s", scored)
|
||||
}
|
||||
}
|
||||
|
||||
func TestItemRepo_Search_EpisodeAccessUsesIndependentMembershipPredicates(t *testing.T) {
|
||||
repo := &ItemRepository{}
|
||||
sql, _, _ := repo.buildSearchSQL("Who Are You?", []string{"episode"}, 20, 0, AccessFilter{
|
||||
AllowedLibraryIDs: []int{1},
|
||||
DisabledLibraryIDs: []int{2},
|
||||
})
|
||||
if !strings.Contains(sql, "FROM episode_libraries allowed_el") {
|
||||
t.Fatalf("episode search missing allowed-library EXISTS:\n%s", sql)
|
||||
}
|
||||
if !strings.Contains(sql, "NOT EXISTS (SELECT 1 FROM episode_libraries disabled_el") {
|
||||
t.Fatalf("episode search missing independent disabled-library NOT EXISTS:\n%s", sql)
|
||||
}
|
||||
}
|
||||
|
||||
// TestItemRepo_ListUnmatchedByFolderAndPathPrefix_ExcludesMangaChapters pins
|
||||
// the manga-chapter exclusion in the unmatched-item lister. Manga chapters are
|
||||
// type='ebook' items that stay status='pending' by design (the type='manga'
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
func TestEpisodeSearchPostgresAndDocumentSource(t *testing.T) {
|
||||
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("SILO_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("connect test database: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
|
||||
suffix := time.Now().UnixNano()
|
||||
seriesID := fmt.Sprintf("episode-search-series-%d", suffix)
|
||||
episodeID := fmt.Sprintf("episode-search-episode-%d", suffix)
|
||||
unavailableID := fmt.Sprintf("episode-search-unavailable-%d", suffix)
|
||||
podcastID := fmt.Sprintf("episode-search-podcast-%d", suffix)
|
||||
podcastEpisodeID := fmt.Sprintf("episode-search-podcast-episode-%d", suffix)
|
||||
var folderID int
|
||||
if err := pool.QueryRow(ctx,
|
||||
`INSERT INTO media_folders (type, name, enabled) VALUES ('series', $1, true) RETURNING id`,
|
||||
seriesID,
|
||||
).Scan(&folderID); err != nil {
|
||||
t.Fatalf("seed folder: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, []string{seriesID, podcastID})
|
||||
_, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, folderID)
|
||||
})
|
||||
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO media_items (content_id, type, title, status, genres)
|
||||
VALUES ($1, 'series', 'Silo Search Fixture', 'matched', '{}'::text[]),
|
||||
($2, 'podcast', 'Podcast Search Fixture', 'matched', '{}'::text[])
|
||||
`, seriesID, podcastID); err != nil {
|
||||
t.Fatalf("seed parent items: %v", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO episodes (content_id, series_id, season_number, episode_number, title, overview)
|
||||
VALUES ($1, $2, 3, 1, 'Who Are You?', 'A buried signal returns. A buried signal changes everything.'),
|
||||
($3, $2, 3, 2, 'Who Are You?', 'Unavailable metadata-only episode.'),
|
||||
($4, $5, 1, 1, 'Who Are You?', 'Podcast episode with the same title.')
|
||||
`, episodeID, seriesID, unavailableID, podcastEpisodeID, podcastID); err != nil {
|
||||
t.Fatalf("seed episodes: %v", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `
|
||||
INSERT INTO episode_libraries (episode_id, media_folder_id, first_seen_at)
|
||||
VALUES ($1, $3, NOW()), ($2, $3, NOW())
|
||||
`, episodeID, podcastEpisodeID, folderID); err != nil {
|
||||
t.Fatalf("seed episode memberships: %v", err)
|
||||
}
|
||||
|
||||
repo := NewItemRepository(pool)
|
||||
for _, types := range [][]string{nil, {"episode"}} {
|
||||
items, total, _, exact, err := repo.SearchPage(ctx, "Who Are You?", types, 20, 0, AccessFilter{}, true)
|
||||
if err != nil {
|
||||
t.Fatalf("search types %v: %v", types, err)
|
||||
}
|
||||
found := false
|
||||
for _, item := range items {
|
||||
if item.ContentID == episodeID && item.Type == "episode" {
|
||||
found = true
|
||||
}
|
||||
if item.ContentID == unavailableID || item.ContentID == podcastEpisodeID {
|
||||
t.Fatalf("search types %v leaked ineligible episode %s", types, item.ContentID)
|
||||
}
|
||||
}
|
||||
if !exact || total < 1 || !found {
|
||||
t.Fatalf("search types %v = total %d exact %v items %+v", types, total, exact, items)
|
||||
}
|
||||
}
|
||||
|
||||
overviewItems, _, _, _, err := repo.SearchPage(ctx, "buried signal", []string{"episode"}, 20, 0, AccessFilter{}, true)
|
||||
if err != nil || len(overviewItems) != 1 || overviewItems[0].ContentID != episodeID {
|
||||
t.Fatalf("overview search = items %+v err %v", overviewItems, err)
|
||||
}
|
||||
|
||||
indexer := NewCatalogSearchIndexer(pool, nil)
|
||||
docs, err := indexer.LoadDocumentsByIDs(ctx, []string{episodeID, unavailableID, podcastEpisodeID}, nil, DefaultMeilisearchEmbedder, true, false)
|
||||
if err != nil {
|
||||
t.Fatalf("load episode documents: %v", err)
|
||||
}
|
||||
if len(docs) != 1 || docs[0].ContentID != episodeID || docs[0].Type != "episode" {
|
||||
t.Fatalf("episode documents = %+v", docs)
|
||||
}
|
||||
if len(docs[0].LibraryIDs) != 1 || int(docs[0].LibraryIDs[0]) != folderID {
|
||||
t.Fatalf("episode library ids = %v, want [%d]", docs[0].LibraryIDs, folderID)
|
||||
}
|
||||
if docs[0].Vectors != nil {
|
||||
t.Fatalf("episode document unexpectedly has vectors: %#v", docs[0].Vectors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEpisodeSearchOutboxStatementTriggers(t *testing.T) {
|
||||
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
|
||||
if dsn == "" {
|
||||
t.Skip("SILO_TEST_DATABASE_URL is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
pool, err := pgxpool.New(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatalf("connect test database: %v", err)
|
||||
}
|
||||
t.Cleanup(pool.Close)
|
||||
var captureFunction *string
|
||||
if err := pool.QueryRow(ctx, `SELECT to_regprocedure('public.catalog_search_capture_enabled()')::text`).Scan(&captureFunction); err != nil || captureFunction == nil {
|
||||
t.Skip("episode search outbox trigger migration is not applied")
|
||||
}
|
||||
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
t.Fatalf("begin: %v", err)
|
||||
}
|
||||
defer tx.Rollback(ctx) //nolint:errcheck
|
||||
if _, err := tx.Exec(ctx, `UPDATE server_settings SET value = 'meilisearch' WHERE key = 'catalog.search.provider'`); err != nil {
|
||||
t.Fatalf("enable trigger capture: %v", err)
|
||||
}
|
||||
suffix := time.Now().UnixNano()
|
||||
seriesID := fmt.Sprintf("episode-trigger-series-%d", suffix)
|
||||
episodeID := fmt.Sprintf("episode-trigger-episode-%d", suffix)
|
||||
var folderID int
|
||||
if err := tx.QueryRow(ctx,
|
||||
`INSERT INTO media_folders (type, name, enabled) VALUES ('series', $1, true) RETURNING id`, seriesID,
|
||||
).Scan(&folderID); err != nil {
|
||||
t.Fatalf("seed folder: %v", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO media_items (content_id, type, title, status, genres)
|
||||
VALUES ($1, 'series', 'Trigger Series', 'matched', '{}'::text[])
|
||||
`, seriesID); err != nil {
|
||||
t.Fatalf("seed series: %v", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO media_item_libraries (content_id, media_folder_id) VALUES ($1, $2)`, seriesID, folderID); err != nil {
|
||||
t.Fatalf("seed item membership: %v", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `
|
||||
INSERT INTO episodes (content_id, series_id, season_number, episode_number, title)
|
||||
VALUES ($1, $2, 1, 1, 'Trigger Episode')
|
||||
`, episodeID, seriesID); err != nil {
|
||||
t.Fatalf("seed episode: %v", err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO episode_libraries (episode_id, media_folder_id) VALUES ($1, $2)`, episodeID, folderID); err != nil {
|
||||
t.Fatalf("seed episode membership: %v", err)
|
||||
}
|
||||
|
||||
var upserts int
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM catalog_search_index_events
|
||||
WHERE action = 'upsert' AND content_id = $1
|
||||
`, episodeID).Scan(&upserts); err != nil || upserts < 2 {
|
||||
t.Fatalf("episode insert/membership upserts = %d, err %v", upserts, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, seriesID); err != nil {
|
||||
t.Fatalf("delete series: %v", err)
|
||||
}
|
||||
var deletes int
|
||||
if err := tx.QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM catalog_search_index_events
|
||||
WHERE action = 'delete' AND content_id = $1
|
||||
`, episodeID).Scan(&deletes); err != nil || deletes == 0 {
|
||||
t.Fatalf("episode cascade deletes = %d, err %v", deletes, err)
|
||||
}
|
||||
}
|
||||
@@ -535,7 +535,7 @@ func (i *CatalogSearchIndexer) CheckConnection(ctx context.Context, settings Cat
|
||||
func catalogSearchMeilisearchSettings(embedder string, semanticEnabled, binaryQuantized bool) map[string]any {
|
||||
settings := map[string]any{
|
||||
"displayedAttributes": []string{"content_id", "type"},
|
||||
"filterableAttributes": []string{"type"},
|
||||
"filterableAttributes": []string{"type", "library_ids"},
|
||||
"searchableAttributes": []string{
|
||||
"title",
|
||||
"original_title",
|
||||
@@ -623,6 +623,7 @@ type catalogSearchDocument struct {
|
||||
Countries []string `json:"countries,omitempty"`
|
||||
Keywords []string `json:"keywords,omitempty"`
|
||||
People []string `json:"people,omitempty"`
|
||||
LibraryIDs []int32 `json:"library_ids,omitempty"`
|
||||
SchemaVersion int `json:"schema_version"`
|
||||
Vectors map[string][]float32 `json:"_vectors,omitempty"`
|
||||
}
|
||||
@@ -632,19 +633,14 @@ func (i *CatalogSearchIndexer) LoadDocumentsAfter(ctx context.Context, afterCont
|
||||
return nil, nil
|
||||
}
|
||||
typeFilter := normalizeCatalogSearchItemTypes(itemTypes)
|
||||
whereClause := `
|
||||
WHERE ($1::text = '' OR mi.content_id > $1)
|
||||
AND ` + catalogSearchExcludeMangaChaptersSQL
|
||||
args := []any{afterContentID, limit}
|
||||
typeArg := 0
|
||||
if len(typeFilter) > 0 {
|
||||
whereClause += `
|
||||
AND mi.type = ANY($3)`
|
||||
typeArg = 3
|
||||
args = append(args, typeFilter)
|
||||
}
|
||||
rows, err := i.pool.Query(ctx, catalogSearchDocumentSelectSQL(whereClause, `
|
||||
ORDER BY mi.content_id ASC
|
||||
LIMIT $2
|
||||
`), args...)
|
||||
rows, err := i.pool.Query(ctx, mixedCatalogSearchDocumentSQL(
|
||||
`mi.content_id > $1`, `e.content_id > $1`, typeArg, "LIMIT $2"), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -666,18 +662,14 @@ func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentID
|
||||
return nil, nil
|
||||
}
|
||||
typeFilter := normalizeCatalogSearchItemTypes(itemTypes)
|
||||
whereClause := `
|
||||
WHERE mi.content_id = ANY($1)
|
||||
AND ` + catalogSearchExcludeMangaChaptersSQL
|
||||
args := []any{contentIDs}
|
||||
typeArg := 0
|
||||
if len(typeFilter) > 0 {
|
||||
whereClause += `
|
||||
AND mi.type = ANY($2)`
|
||||
typeArg = 2
|
||||
args = append(args, typeFilter)
|
||||
}
|
||||
rows, err := i.pool.Query(ctx, catalogSearchDocumentSelectSQL(whereClause, `
|
||||
ORDER BY mi.content_id ASC
|
||||
`), args...)
|
||||
rows, err := i.pool.Query(ctx, mixedCatalogSearchDocumentSQL(
|
||||
`mi.content_id = ANY($1)`, `e.content_id = ANY($1)`, typeArg, ""), args...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -693,31 +685,85 @@ func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentID
|
||||
return docs, nil
|
||||
}
|
||||
|
||||
func catalogSearchDocumentSelectSQL(whereClause string, tailClause string) string {
|
||||
return `
|
||||
SELECT
|
||||
mi.content_id,
|
||||
mi.type,
|
||||
COALESCE(mi.title, ''),
|
||||
COALESCE(mi.sort_title, ''),
|
||||
COALESCE(mi.original_title, ''),
|
||||
COALESCE(mi.year, 0),
|
||||
COALESCE(mi.overview, ''),
|
||||
COALESCE(mi.tagline, ''),
|
||||
COALESCE(mi.genres, ARRAY[]::text[]),
|
||||
COALESCE(mi.studios, ARRAY[]::text[]),
|
||||
COALESCE(mi.networks, ARRAY[]::text[]),
|
||||
COALESCE(mi.countries, ARRAY[]::text[]),
|
||||
COALESCE(mi.keywords, ARRAY[]::text[]),
|
||||
COALESCE(array_agg(DISTINCT p.name) FILTER (WHERE p.name IS NOT NULL AND p.name <> ''), ARRAY[]::text[]) AS people
|
||||
FROM media_items mi
|
||||
LEFT JOIN item_people ip ON ip.content_id = mi.content_id
|
||||
LEFT JOIN people p ON p.id = ip.person_id
|
||||
` + whereClause + `
|
||||
GROUP BY
|
||||
mi.content_id, mi.type, mi.title, mi.sort_title, mi.original_title, mi.year,
|
||||
mi.overview, mi.tagline, mi.genres, mi.studios, mi.networks, mi.countries, mi.keywords
|
||||
` + tailClause
|
||||
// mixedCatalogSearchDocumentSQL first pages narrow IDs across both physical
|
||||
// sources, then aggregates people and library memberships only for that batch.
|
||||
// This keeps rebuild work proportional to RebuildBatchSize rather than the
|
||||
// total number of catalog rows.
|
||||
func mixedCatalogSearchDocumentSQL(mediaPredicate, episodePredicate string, typeArg int, limitClause string) string {
|
||||
mediaTypePredicate := ""
|
||||
episodeTypePredicate := ""
|
||||
if typeArg > 0 {
|
||||
mediaTypePredicate = fmt.Sprintf(" AND mi.type = ANY($%d)", typeArg)
|
||||
episodeTypePredicate = fmt.Sprintf(" AND 'episode' = ANY($%d)", typeArg)
|
||||
}
|
||||
return fmt.Sprintf(`
|
||||
WITH candidates AS (
|
||||
SELECT mi.content_id, mi.type
|
||||
FROM media_items mi
|
||||
WHERE %s
|
||||
AND %s%s
|
||||
UNION ALL
|
||||
SELECT e.content_id, 'episode'::text AS type
|
||||
FROM episodes e
|
||||
JOIN media_items si ON si.content_id = e.series_id AND si.type = 'series'
|
||||
WHERE %s%s
|
||||
AND EXISTS (SELECT 1 FROM episode_libraries el WHERE el.episode_id = e.content_id)
|
||||
ORDER BY content_id ASC
|
||||
%s
|
||||
)
|
||||
SELECT * FROM (
|
||||
SELECT
|
||||
mi.content_id,
|
||||
mi.type,
|
||||
COALESCE(mi.title, ''),
|
||||
COALESCE(mi.sort_title, ''),
|
||||
COALESCE(mi.original_title, ''),
|
||||
COALESCE(mi.year, 0),
|
||||
COALESCE(mi.overview, ''),
|
||||
COALESCE(mi.tagline, ''),
|
||||
COALESCE(mi.genres, ARRAY[]::text[]),
|
||||
COALESCE(mi.studios, ARRAY[]::text[]),
|
||||
COALESCE(mi.networks, ARRAY[]::text[]),
|
||||
COALESCE(mi.countries, ARRAY[]::text[]),
|
||||
COALESCE(mi.keywords, ARRAY[]::text[]),
|
||||
COALESCE(people.names, ARRAY[]::text[]),
|
||||
COALESCE(libraries.ids, ARRAY[]::integer[])
|
||||
FROM candidates c
|
||||
JOIN media_items mi ON c.type <> 'episode' AND mi.content_id = c.content_id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT array_agg(DISTINCT p.name) FILTER (WHERE p.name IS NOT NULL AND p.name <> '') AS names
|
||||
FROM item_people ip
|
||||
JOIN people p ON p.id = ip.person_id
|
||||
WHERE ip.content_id = mi.content_id
|
||||
) people ON true
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT array_agg(DISTINCT mil.media_folder_id ORDER BY mil.media_folder_id) AS ids
|
||||
FROM media_item_libraries mil
|
||||
WHERE mil.content_id = mi.content_id
|
||||
) libraries ON true
|
||||
UNION ALL
|
||||
SELECT
|
||||
e.content_id,
|
||||
'episode'::text,
|
||||
COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text),
|
||||
COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text),
|
||||
''::text,
|
||||
COALESCE(EXTRACT(YEAR FROM e.air_date)::integer, 0),
|
||||
COALESCE(e.overview, ''),
|
||||
''::text,
|
||||
ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[], ARRAY[]::text[],
|
||||
COALESCE(libraries.ids, ARRAY[]::integer[])
|
||||
FROM candidates c
|
||||
JOIN episodes e ON c.type = 'episode' AND e.content_id = c.content_id
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT array_agg(DISTINCT el.media_folder_id ORDER BY el.media_folder_id) AS ids
|
||||
FROM episode_libraries el
|
||||
WHERE el.episode_id = e.content_id
|
||||
) libraries ON true
|
||||
) documents
|
||||
ORDER BY content_id ASC`,
|
||||
mediaPredicate, catalogSearchExcludeMangaChaptersSQL, mediaTypePredicate,
|
||||
episodePredicate, episodeTypePredicate, limitClause)
|
||||
}
|
||||
|
||||
func scanCatalogSearchDocuments(rows pgx.Rows) ([]catalogSearchDocument, error) {
|
||||
@@ -739,6 +785,7 @@ func scanCatalogSearchDocuments(rows pgx.Rows) ([]catalogSearchDocument, error)
|
||||
&doc.Countries,
|
||||
&doc.Keywords,
|
||||
&doc.People,
|
||||
&doc.LibraryIDs,
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -759,7 +806,7 @@ func (i *CatalogSearchIndexer) attachDocumentVectors(ctx context.Context, docs [
|
||||
}
|
||||
ids := make([]string, 0, len(docs))
|
||||
for _, doc := range docs {
|
||||
if strings.TrimSpace(doc.ContentID) != "" {
|
||||
if doc.Type != "episode" && strings.TrimSpace(doc.ContentID) != "" {
|
||||
ids = append(ids, doc.ContentID)
|
||||
}
|
||||
}
|
||||
@@ -814,6 +861,10 @@ func setCatalogSearchDocumentVectors(docs []catalogSearchDocument, vectors map[s
|
||||
}
|
||||
count := 0
|
||||
for idx := range docs {
|
||||
if docs[idx].Type == "episode" {
|
||||
docs[idx].Vectors = nil
|
||||
continue
|
||||
}
|
||||
vector := vectors[docs[idx].ContentID]
|
||||
docs[idx].Vectors = map[string][]float32{embedder: nil}
|
||||
if len(vector) > 0 {
|
||||
@@ -891,10 +942,17 @@ func countCatalogSearchEligibleDocuments(ctx context.Context, q coverageQuerier,
|
||||
}
|
||||
var count int
|
||||
if err := q.QueryRow(ctx, `
|
||||
SELECT COUNT(*)
|
||||
FROM media_items mi
|
||||
WHERE `+catalogSearchExcludeMangaChaptersSQL+`
|
||||
AND ($1::text[] IS NULL OR mi.type = ANY($1))
|
||||
SELECT
|
||||
(SELECT COUNT(*)
|
||||
FROM media_items mi
|
||||
WHERE `+catalogSearchExcludeMangaChaptersSQL+`
|
||||
AND ($1::text[] IS NULL OR mi.type = ANY($1)))
|
||||
+
|
||||
(SELECT COUNT(*)
|
||||
FROM episodes e
|
||||
JOIN media_items si ON si.content_id = e.series_id AND si.type = 'series'
|
||||
WHERE ($1::text[] IS NULL OR 'episode' = ANY($1))
|
||||
AND EXISTS (SELECT 1 FROM episode_libraries el WHERE el.episode_id = e.content_id))
|
||||
`, typeArg).Scan(&count); err != nil {
|
||||
return 0, fmt.Errorf("count catalog search eligible documents: %w", err)
|
||||
}
|
||||
|
||||
@@ -106,6 +106,30 @@ type meilisearchSearchResponse struct {
|
||||
Query string `json:"query"`
|
||||
}
|
||||
|
||||
type meilisearchFederatedSearchRequest struct {
|
||||
Federation meilisearchFederationOptions `json:"federation"`
|
||||
Queries []meilisearchFederatedQuery `json:"queries"`
|
||||
}
|
||||
|
||||
type meilisearchFederationOptions struct {
|
||||
Offset int `json:"offset"`
|
||||
Limit int `json:"limit"`
|
||||
}
|
||||
|
||||
type meilisearchFederatedQuery struct {
|
||||
IndexUID string `json:"indexUid"`
|
||||
Query string `json:"q"`
|
||||
Filter string `json:"filter,omitempty"`
|
||||
AttributesToRetrieve []string `json:"attributesToRetrieve"`
|
||||
AttributesToSearchOn []string `json:"attributesToSearchOn,omitempty"`
|
||||
MatchingStrategy string `json:"matchingStrategy,omitempty"`
|
||||
Vector []float32 `json:"vector,omitempty"`
|
||||
Hybrid *meilisearchHybridRequest `json:"hybrid,omitempty"`
|
||||
FederationOptions struct {
|
||||
Weight float64 `json:"weight"`
|
||||
} `json:"federationOptions"`
|
||||
}
|
||||
|
||||
type meilisearchStatsResponse struct {
|
||||
NumberOfDocuments int `json:"numberOfDocuments"`
|
||||
}
|
||||
@@ -217,6 +241,12 @@ func (c *meilisearchClient) Search(ctx context.Context, uid string, req meilisea
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (c *meilisearchClient) FederatedSearch(ctx context.Context, req meilisearchFederatedSearchRequest) (meilisearchSearchResponse, error) {
|
||||
var out meilisearchSearchResponse
|
||||
err := c.do(ctx, http.MethodPost, "/multi-search", req, &out)
|
||||
return out, err
|
||||
}
|
||||
|
||||
func (c *meilisearchClient) AddDocuments(ctx context.Context, uid string, docs []catalogSearchDocument) (meilisearchTaskRef, error) {
|
||||
if len(docs) == 0 {
|
||||
return meilisearchTaskRef{}, nil
|
||||
|
||||
@@ -2,6 +2,7 @@ package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -11,6 +12,45 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMeilisearchFederatedSearchUsesMultiSearchPagination(t *testing.T) {
|
||||
var gotPath string
|
||||
var gotRequest meilisearchFederatedSearchRequest
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotPath = r.URL.Path
|
||||
if err := json.NewDecoder(r.Body).Decode(&gotRequest); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{"hits":[{"content_id":"episode-1"}],"offset":20,"limit":10,"estimatedTotalHits":1}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
client, err := newMeilisearchClient(server.URL, "", time.Second)
|
||||
if err != nil {
|
||||
t.Fatalf("newMeilisearchClient: %v", err)
|
||||
}
|
||||
request := meilisearchFederatedSearchRequest{
|
||||
Federation: meilisearchFederationOptions{Offset: 20, Limit: 10},
|
||||
Queries: []meilisearchFederatedQuery{
|
||||
{IndexUID: "catalog-v3", Query: "Who Are You?"},
|
||||
},
|
||||
}
|
||||
response, err := client.FederatedSearch(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("FederatedSearch: %v", err)
|
||||
}
|
||||
if gotPath != "/multi-search" {
|
||||
t.Fatalf("request path = %q, want /multi-search", gotPath)
|
||||
}
|
||||
if gotRequest.Federation.Offset != 20 || gotRequest.Federation.Limit != 10 || len(gotRequest.Queries) != 1 {
|
||||
t.Fatalf("decoded request = %#v", gotRequest)
|
||||
}
|
||||
if len(response.Hits) != 1 || response.Hits[0].ContentID != "episode-1" {
|
||||
t.Fatalf("response = %#v", response)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeilisearchWaitTaskAcceptsZeroTaskUID(t *testing.T) {
|
||||
var gotPath string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"net"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -43,6 +44,7 @@ var meilisearchTitleSearchAttributes = []string{
|
||||
"original_title",
|
||||
"sort_title",
|
||||
"title_variants",
|
||||
"overview",
|
||||
}
|
||||
|
||||
type meilisearchIndexStateStore interface {
|
||||
@@ -97,6 +99,9 @@ type MeilisearchSearchProvider struct {
|
||||
capMu sync.Mutex
|
||||
capCache CatalogSearchSemanticCapability
|
||||
capCachedAt time.Time
|
||||
|
||||
federationMu sync.Mutex
|
||||
federationUnsupported bool
|
||||
}
|
||||
|
||||
type cachedCatalogSearchQueryVector struct {
|
||||
@@ -151,6 +156,14 @@ func NewMeilisearchSearchProvider(
|
||||
}
|
||||
|
||||
func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearchRequest) (*CatalogSearchResult, error) {
|
||||
if req.Access.AllowedLibraryIDs != nil && len(req.Access.AllowedLibraryIDs) == 0 {
|
||||
return &CatalogSearchResult{
|
||||
Items: []*models.MediaItem{},
|
||||
TotalExact: !req.SkipTotal,
|
||||
Provider: SearchProviderMeilisearch,
|
||||
Mode: "keyword",
|
||||
}, nil
|
||||
}
|
||||
if p == nil || p.client == nil {
|
||||
return p.fallbackSearch(ctx, req, "meilisearch not configured")
|
||||
}
|
||||
@@ -270,6 +283,13 @@ func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req C
|
||||
estimatedTotalHits := 0
|
||||
exhausted := false
|
||||
baseSearchReq, semanticFallback := p.buildMeilisearchSearchRequest(ctx, req)
|
||||
useFederation := baseSearchReq.Hybrid != nil && searchRequestMixesEpisodeAndMedia(req.ItemTypes)
|
||||
if useFederation && p.isFederationUnsupported() {
|
||||
semanticFallback = "meilisearch federation unsupported; using keyword search"
|
||||
baseSearchReq.Vector = nil
|
||||
baseSearchReq.Hybrid = nil
|
||||
useFederation = false
|
||||
}
|
||||
|
||||
for len(accessible) < target && !exhausted {
|
||||
if scanned >= meilisearchCandidateScanCap {
|
||||
@@ -282,8 +302,25 @@ func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req C
|
||||
searchReq := baseSearchReq
|
||||
searchReq.Offset = meiliOffset
|
||||
searchReq.Limit = nextLimit
|
||||
resp, err := p.client.Search(ctx, indexUID, searchReq)
|
||||
if err != nil && baseSearchReq.Hybrid != nil {
|
||||
var resp meilisearchSearchResponse
|
||||
var err error
|
||||
if useFederation {
|
||||
resp, err = p.client.FederatedSearch(ctx, p.buildFederatedSearchRequest(indexUID, req, baseSearchReq, meiliOffset, nextLimit))
|
||||
if err != nil && isMeilisearchFederationUnsupported(err) {
|
||||
p.markFederationUnsupported()
|
||||
semanticFallback = "meilisearch federation unsupported; using keyword search"
|
||||
baseSearchReq.Vector = nil
|
||||
baseSearchReq.Hybrid = nil
|
||||
useFederation = false
|
||||
searchReq = baseSearchReq
|
||||
searchReq.Offset = meiliOffset
|
||||
searchReq.Limit = nextLimit
|
||||
resp, err = p.client.Search(ctx, indexUID, searchReq)
|
||||
}
|
||||
} else {
|
||||
resp, err = p.client.Search(ctx, indexUID, searchReq)
|
||||
}
|
||||
if err != nil && baseSearchReq.Hybrid != nil && !useFederation {
|
||||
semanticFallback = "meilisearch hybrid search failed: " + err.Error()
|
||||
baseSearchReq.Vector = nil
|
||||
baseSearchReq.Hybrid = nil
|
||||
@@ -316,7 +353,7 @@ func (p *MeilisearchSearchProvider) searchMeilisearch(ctx context.Context, req C
|
||||
position[id] = i
|
||||
ids = append(ids, id)
|
||||
}
|
||||
hydrated, err := p.itemRepo.GetByIDsWithAccess(ctx, ids, req.Access)
|
||||
hydrated, err := p.itemRepo.GetSearchItemsByIDsWithAccess(ctx, ids, req.Access)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -371,13 +408,13 @@ func (p *MeilisearchSearchProvider) buildMeilisearchSearchRequest(ctx context.Co
|
||||
if p == nil {
|
||||
return meilisearchSearchRequest{
|
||||
Query: strings.TrimSpace(req.Query),
|
||||
Filter: meilisearchTypeFilter(req.ItemTypes),
|
||||
Filter: meilisearchSearchFilter(req.ItemTypes, req.Access),
|
||||
AttributesToRetrieve: []string{"content_id"},
|
||||
}, ""
|
||||
}
|
||||
searchReq := meilisearchSearchRequest{
|
||||
Query: strings.TrimSpace(req.Query),
|
||||
Filter: meilisearchTypeFilter(req.ItemTypes),
|
||||
Filter: meilisearchSearchFilter(req.ItemTypes, req.Access),
|
||||
AttributesToRetrieve: []string{"content_id"},
|
||||
AttributesToSearchOn: p.attributesToSearchOnForRequest(req),
|
||||
MatchingStrategy: p.matchingStrategyForRequest(req),
|
||||
@@ -412,9 +449,116 @@ func (p *MeilisearchSearchProvider) shouldUseSemanticSearch(req CatalogSearchReq
|
||||
if p == nil || !p.config.SemanticEnabled {
|
||||
return false
|
||||
}
|
||||
mediaTypes, includesEpisodes := splitSearchItemTypes(req.ItemTypes)
|
||||
if includesEpisodes && len(req.ItemTypes) > 0 && len(mediaTypes) == 0 {
|
||||
return false
|
||||
}
|
||||
return len(catalogSearchQueryTerms(req.Query)) >= meilisearchShortHybridMinTerms
|
||||
}
|
||||
|
||||
func searchRequestMixesEpisodeAndMedia(itemTypes []string) bool {
|
||||
mediaTypes, includesEpisodes := splitSearchItemTypes(itemTypes)
|
||||
return includesEpisodes && (len(itemTypes) == 0 || len(mediaTypes) > 0)
|
||||
}
|
||||
|
||||
func (p *MeilisearchSearchProvider) buildFederatedSearchRequest(
|
||||
indexUID string,
|
||||
req CatalogSearchRequest,
|
||||
base meilisearchSearchRequest,
|
||||
offset, limit int,
|
||||
) meilisearchFederatedSearchRequest {
|
||||
mediaTypes, _ := splitSearchItemTypes(req.ItemTypes)
|
||||
mediaFilter := ""
|
||||
if len(req.ItemTypes) == 0 {
|
||||
mediaFilter = joinMeilisearchFilters(`type != "episode"`, meilisearchSearchFilter(nil, req.Access))
|
||||
} else {
|
||||
mediaFilter = meilisearchSearchFilter(mediaTypes, req.Access)
|
||||
}
|
||||
episodeFilter := meilisearchSearchFilter([]string{"episode"}, req.Access)
|
||||
mediaQuery := meilisearchFederatedQuery{
|
||||
IndexUID: indexUID,
|
||||
Query: base.Query,
|
||||
Filter: mediaFilter,
|
||||
AttributesToRetrieve: base.AttributesToRetrieve,
|
||||
AttributesToSearchOn: base.AttributesToSearchOn,
|
||||
MatchingStrategy: base.MatchingStrategy,
|
||||
Vector: base.Vector,
|
||||
Hybrid: base.Hybrid,
|
||||
}
|
||||
mediaQuery.FederationOptions.Weight = 1
|
||||
episodeQuery := meilisearchFederatedQuery{
|
||||
IndexUID: indexUID,
|
||||
Query: base.Query,
|
||||
Filter: episodeFilter,
|
||||
AttributesToRetrieve: base.AttributesToRetrieve,
|
||||
AttributesToSearchOn: base.AttributesToSearchOn,
|
||||
MatchingStrategy: base.MatchingStrategy,
|
||||
}
|
||||
episodeQuery.FederationOptions.Weight = 1
|
||||
return meilisearchFederatedSearchRequest{
|
||||
Federation: meilisearchFederationOptions{Offset: offset, Limit: limit},
|
||||
Queries: []meilisearchFederatedQuery{mediaQuery, episodeQuery},
|
||||
}
|
||||
}
|
||||
|
||||
func joinMeilisearchFilters(parts ...string) string {
|
||||
cleaned := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if part = strings.TrimSpace(part); part != "" {
|
||||
cleaned = append(cleaned, part)
|
||||
}
|
||||
}
|
||||
return strings.Join(cleaned, " AND ")
|
||||
}
|
||||
|
||||
func (p *MeilisearchSearchProvider) isFederationUnsupported() bool {
|
||||
p.federationMu.Lock()
|
||||
defer p.federationMu.Unlock()
|
||||
return p.federationUnsupported
|
||||
}
|
||||
|
||||
func (p *MeilisearchSearchProvider) markFederationUnsupported() {
|
||||
p.federationMu.Lock()
|
||||
p.federationUnsupported = true
|
||||
p.federationMu.Unlock()
|
||||
}
|
||||
|
||||
func isMeilisearchFederationUnsupported(err error) bool {
|
||||
var httpErr *meilisearchHTTPError
|
||||
if !errors.As(err, &httpErr) {
|
||||
return false
|
||||
}
|
||||
code := strings.ToLower(strings.TrimSpace(httpErr.Code))
|
||||
message := strings.ToLower(strings.TrimSpace(httpErr.Message))
|
||||
if httpErr.StatusCode == http.StatusNotFound {
|
||||
// A cached active index can disappear during an atomic rebuild swap.
|
||||
// That is a resource miss, not evidence that this Meilisearch process
|
||||
// lacks federation; caching it would disable mixed hybrid search until
|
||||
// Silo restarts even after the provider refreshes the active index UID.
|
||||
if code == "index_not_found" || code == "document_not_found" {
|
||||
return false
|
||||
}
|
||||
return code == "not_found" ||
|
||||
(code == "" && (strings.Contains(message, "route") || strings.Contains(message, "multi-search")))
|
||||
}
|
||||
if httpErr.StatusCode != http.StatusBadRequest {
|
||||
return false
|
||||
}
|
||||
|
||||
// Older Meilisearch versions expose /multi-search but reject the newer
|
||||
// federation object as an unknown field. Cache only that capability-shaped
|
||||
// failure; malformed or incompatible federated queries should keep falling
|
||||
// through to PostgreSQL so a request bug is not hidden for the process life.
|
||||
details := code + " " + message
|
||||
mentionsFederation := strings.Contains(details, "federation")
|
||||
unsupported := strings.Contains(details, "not supported") ||
|
||||
strings.Contains(details, "unsupported") ||
|
||||
strings.Contains(details, "unknown field") ||
|
||||
strings.Contains(details, "unrecognized field") ||
|
||||
strings.Contains(details, "feature not enabled")
|
||||
return mentionsFederation && unsupported
|
||||
}
|
||||
|
||||
func (p *MeilisearchSearchProvider) matchingStrategyForRequest(req CatalogSearchRequest) string {
|
||||
strategy := strings.TrimSpace(strings.ToLower(p.config.MatchingStrategy))
|
||||
if strategy == "" {
|
||||
@@ -740,6 +884,31 @@ func meilisearchTypeFilter(itemTypes []string) string {
|
||||
return "(" + strings.Join(parts, " OR ") + ")"
|
||||
}
|
||||
|
||||
func meilisearchSearchFilter(itemTypes []string, access AccessFilter) string {
|
||||
parts := make([]string, 0, 4)
|
||||
if typeFilter := meilisearchTypeFilter(itemTypes); typeFilter != "" {
|
||||
parts = append(parts, typeFilter)
|
||||
}
|
||||
if access.AllowedLibraryIDs != nil {
|
||||
parts = append(parts, "library_ids IN ["+joinMeilisearchIntFilterValues(access.AllowedLibraryIDs)+"]")
|
||||
}
|
||||
if len(access.DisabledLibraryIDs) > 0 {
|
||||
if access.AllowedLibraryIDs == nil {
|
||||
parts = append(parts, "library_ids IS NOT EMPTY")
|
||||
}
|
||||
parts = append(parts, "library_ids NOT IN ["+joinMeilisearchIntFilterValues(access.DisabledLibraryIDs)+"]")
|
||||
}
|
||||
return strings.Join(parts, " AND ")
|
||||
}
|
||||
|
||||
func joinMeilisearchIntFilterValues(values []int) string {
|
||||
parts := make([]string, 0, len(values))
|
||||
for _, value := range values {
|
||||
parts = append(parts, strconv.Itoa(value))
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
func normalizeCatalogSearchQueryForVector(query string) string {
|
||||
return strings.Join(strings.Fields(query), " ")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,327 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/access"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
)
|
||||
|
||||
// GetSearchItemsByIDsWithAccess hydrates mixed search hits in one round trip.
|
||||
// Meilisearch IDs are only candidates: both branches reapply the effective
|
||||
// access filter, and callers restore provider order after this method returns.
|
||||
func (r *ItemRepository) GetSearchItemsByIDsWithAccess(
|
||||
ctx context.Context,
|
||||
contentIDs []string,
|
||||
filter AccessFilter,
|
||||
) ([]*models.MediaItem, error) {
|
||||
if len(contentIDs) == 0 || (filter.AllowedLibraryIDs != nil && len(filter.AllowedLibraryIDs) == 0) {
|
||||
return []*models.MediaItem{}, nil
|
||||
}
|
||||
|
||||
args := []any{contentIDs}
|
||||
argIdx := 2
|
||||
mediaConditions := []string{"hydrated_mi.content_id = ANY($1)"}
|
||||
appendLibraryAccessConditions("hydrated_mi.content_id", filter, &mediaConditions, &args, &argIdx)
|
||||
applyAccessFilter("hydrated_mi", AccessFilter{
|
||||
MaxContentRating: filter.MaxContentRating,
|
||||
ExcludedMediaTypes: filter.ExcludedMediaTypes,
|
||||
}, &mediaConditions, &args, &argIdx)
|
||||
|
||||
episodeConditions := []string{"mi.content_id = ANY($1)"}
|
||||
appendEpisodeLibrarySearchAccess("mi.content_id", filter, &episodeConditions, &args, &argIdx)
|
||||
applyAccessFilter("mi", AccessFilter{
|
||||
MaxContentRating: filter.MaxContentRating,
|
||||
ExcludedMediaTypes: filter.ExcludedMediaTypes,
|
||||
}, &episodeConditions, &args, &argIdx)
|
||||
|
||||
query := fmt.Sprintf(`
|
||||
SELECT %s
|
||||
FROM media_items hydrated_mi
|
||||
WHERE %s
|
||||
UNION ALL
|
||||
SELECT %s
|
||||
FROM %s
|
||||
WHERE %s`,
|
||||
qualifiedItemColumns("hydrated_mi"), strings.Join(mediaConditions, " AND "),
|
||||
qualifiedItemColumns("mi"), episodeCatalogBaseRelation, strings.Join(episodeConditions, " AND "))
|
||||
|
||||
rows, err := r.pool.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("hydrating mixed search items: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
return scanItems(rows)
|
||||
}
|
||||
|
||||
const episodeSearchTitleExpr = `COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text)`
|
||||
|
||||
const episodeSearchTitleVector = `setweight(
|
||||
to_tsvector('simple', public.normalize_search_text(COALESCE(NULLIF(BTRIM(e.title), ''), 'Episode ' || e.episode_number::text))),
|
||||
'A'
|
||||
)`
|
||||
|
||||
const episodeSearchOverviewVector = `to_tsvector('english', COALESCE(e.overview, ''))`
|
||||
|
||||
const mixedSearchOrder = `exact_title_match DESC, contiguous_title_match DESC, year_match DESC,
|
||||
phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC,
|
||||
LOWER(title) ASC, content_id ASC`
|
||||
|
||||
// buildMixedSearchSQLWithTotal builds one ranked candidate set from the two
|
||||
// physical catalog sources. The scored CTE deliberately carries only ranking
|
||||
// fields; the wide MediaItem projection is hydrated after LIMIT/OFFSET so a
|
||||
// broad match never sorts posters, arrays, or metadata blobs for every hit.
|
||||
func (r *ItemRepository) buildMixedSearchSQLWithTotal(
|
||||
query string,
|
||||
itemTypes []string,
|
||||
limit, offset int,
|
||||
filter AccessFilter,
|
||||
includeTotal bool,
|
||||
) (dataSQL, countSQL string, args []any) {
|
||||
parsed := parseSearchQuery(query)
|
||||
searchText := parsed.Text
|
||||
if searchText == "" {
|
||||
searchText = collapseSearchWhitespace(strings.ReplaceAll(strings.TrimSpace(query), "\"", " "))
|
||||
}
|
||||
if searchText == "" {
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
mediaTypes, includeEpisodes := splitSearchItemTypes(itemTypes)
|
||||
includeMediaItems := len(itemTypes) == 0 || len(mediaTypes) > 0
|
||||
if !includeMediaItems && !includeEpisodes {
|
||||
return "", "", nil
|
||||
}
|
||||
|
||||
args = []any{searchText, buildTitlePrefixTsQuery(searchText)}
|
||||
argIdx := 3
|
||||
var branches []string
|
||||
|
||||
mediaConditions := []string{}
|
||||
if includeMediaItems {
|
||||
mediaTitleVector := `(
|
||||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.title, ''))), 'A') ||
|
||||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.original_title, ''))), 'A') ||
|
||||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B')
|
||||
)`
|
||||
mediaOverviewVector := `to_tsvector('english', COALESCE(mi.overview, ''))`
|
||||
mediaConditions = append(mediaConditions, searchMatchCondition(mediaTitleVector, mediaOverviewVector))
|
||||
if len(itemTypes) > 0 {
|
||||
mediaConditions = append(mediaConditions, fmt.Sprintf("mi.type = ANY($%d)", argIdx))
|
||||
args = append(args, mediaTypes)
|
||||
argIdx++
|
||||
}
|
||||
appendLibraryAccessConditions("mi.content_id", filter, &mediaConditions, &args, &argIdx)
|
||||
applyAccessFilter("mi", AccessFilter{
|
||||
MaxContentRating: filter.MaxContentRating,
|
||||
ExcludedMediaTypes: filter.ExcludedMediaTypes,
|
||||
}, &mediaConditions, &args, &argIdx)
|
||||
mediaConditions = append(mediaConditions, MangaChapterExclusionWhere("mi"))
|
||||
}
|
||||
|
||||
episodeConditions := []string{}
|
||||
if includeEpisodes {
|
||||
episodeConditions = append(episodeConditions,
|
||||
"si.type = 'series'",
|
||||
`EXISTS (SELECT 1 FROM episode_libraries available_el WHERE available_el.episode_id = e.content_id)`,
|
||||
searchMatchCondition(episodeSearchTitleVector, episodeSearchOverviewVector),
|
||||
)
|
||||
appendEpisodeLibrarySearchAccess("e.content_id", filter, &episodeConditions, &args, &argIdx)
|
||||
if filter.MaxContentRating != "" {
|
||||
allowedRatings := access.AllowedRatingsUpTo(filter.MaxContentRating)
|
||||
if len(allowedRatings) == 0 {
|
||||
episodeConditions = append(episodeConditions, "1 = 0")
|
||||
} else {
|
||||
episodeConditions = append(episodeConditions, fmt.Sprintf("si.content_rating = ANY($%d)", argIdx))
|
||||
args = append(args, allowedRatings)
|
||||
argIdx++
|
||||
}
|
||||
}
|
||||
if len(filter.ExcludedMediaTypes) > 0 {
|
||||
episodeConditions = append(episodeConditions, fmt.Sprintf("NOT ('episode' = ANY($%d))", argIdx))
|
||||
args = append(args, filter.ExcludedMediaTypes)
|
||||
argIdx++
|
||||
}
|
||||
}
|
||||
|
||||
exactIdx := argIdx
|
||||
args = append(args, parsed.ExactTitleHint)
|
||||
argIdx++
|
||||
var yearArg any
|
||||
if parsed.Year != nil {
|
||||
yearArg = *parsed.Year
|
||||
}
|
||||
yearIdx := argIdx
|
||||
args = append(args, yearArg)
|
||||
argIdx++
|
||||
phraseIdx := argIdx
|
||||
args = append(args, parsed.Phrase)
|
||||
argIdx++
|
||||
|
||||
if includeMediaItems {
|
||||
branches = append(branches, buildMixedSearchCandidateBranch(
|
||||
"mi.content_id", "mi.type", "mi.title", "mi.year",
|
||||
`(
|
||||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.title, ''))), 'A') ||
|
||||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.original_title, ''))), 'A') ||
|
||||
setweight(to_tsvector('simple', public.normalize_search_text(COALESCE(mi.sort_title, ''))), 'B')
|
||||
)`,
|
||||
`to_tsvector('english', COALESCE(mi.overview, ''))`,
|
||||
[]string{`mi.title_normalized`, `public.normalize_search_text(mi.original_title)`, `public.normalize_search_text(mi.sort_title)`},
|
||||
"media_items mi", mediaConditions, exactIdx, yearIdx, phraseIdx,
|
||||
))
|
||||
}
|
||||
if includeEpisodes {
|
||||
episodeNormalizedTitle := `public.normalize_search_text(` + episodeSearchTitleExpr + `)`
|
||||
branches = append(branches, buildMixedSearchCandidateBranch(
|
||||
"e.content_id", "'episode'::text", episodeSearchTitleExpr,
|
||||
"COALESCE(si.year, EXTRACT(YEAR FROM e.air_date)::integer, 0)",
|
||||
episodeSearchTitleVector, episodeSearchOverviewVector,
|
||||
[]string{episodeNormalizedTitle},
|
||||
"episodes e JOIN media_items si ON si.content_id = e.series_id",
|
||||
episodeConditions, exactIdx, yearIdx, phraseIdx,
|
||||
))
|
||||
}
|
||||
|
||||
scoredCTE := "WITH scored AS (\n" + strings.Join(branches, "\nUNION ALL\n") + "\n)"
|
||||
statsCTE := `, stats AS (
|
||||
SELECT MAX(CASE WHEN title_rank > 0 OR title_prefix_rank > 0 THEN 1 ELSE 0 END) AS has_title_match
|
||||
FROM scored
|
||||
)`
|
||||
postFilter := fmt.Sprintf(`FROM scored
|
||||
CROSS JOIN stats
|
||||
WHERE scored.title_rank > 0
|
||||
OR scored.title_prefix_rank > 0
|
||||
OR (COALESCE(stats.has_title_match, 0) = 0 AND scored.overview_rank >= %g)`, overviewMatchFloor)
|
||||
|
||||
pageTotalColumn := ""
|
||||
finalTotalColumn := ""
|
||||
if includeTotal {
|
||||
pageTotalColumn = ", COUNT(*) OVER () AS total_count"
|
||||
finalTotalColumn = ", page.total_count"
|
||||
}
|
||||
limitIdx, offsetIdx := argIdx, argIdx+1
|
||||
args = append(args, limit, offset)
|
||||
|
||||
pageCTE := fmt.Sprintf(`, page AS (
|
||||
SELECT scored.*, ROW_NUMBER() OVER (ORDER BY %s) AS ordinal%s
|
||||
%s
|
||||
ORDER BY %s
|
||||
LIMIT $%d OFFSET $%d
|
||||
)`, mixedSearchOrder, pageTotalColumn, postFilter, mixedSearchOrder, limitIdx, offsetIdx)
|
||||
|
||||
hydratedRelation := fmt.Sprintf(`LATERAL (
|
||||
SELECT %s
|
||||
FROM media_items hydrated_mi
|
||||
WHERE page.type <> 'episode'
|
||||
AND hydrated_mi.content_id = page.content_id
|
||||
UNION ALL
|
||||
SELECT %s
|
||||
FROM %s
|
||||
WHERE page.type = 'episode'
|
||||
AND mi.content_id = page.content_id
|
||||
) hydrated`, qualifiedItemColumns("hydrated_mi"), qualifiedItemColumns("mi"), episodeCatalogBaseRelation)
|
||||
|
||||
dataSQL = scoredCTE + statsCTE + pageCTE + fmt.Sprintf(`
|
||||
SELECT %s%s
|
||||
FROM page
|
||||
JOIN %s ON true
|
||||
ORDER BY page.ordinal`, qualifiedItemColumns("hydrated"), finalTotalColumn, hydratedRelation)
|
||||
countSQL = scoredCTE + statsCTE + fmt.Sprintf("\nSELECT COUNT(*)\n%s", postFilter)
|
||||
return dataSQL, countSQL, args
|
||||
}
|
||||
|
||||
func splitSearchItemTypes(itemTypes []string) (mediaTypes []string, includeEpisodes bool) {
|
||||
if len(itemTypes) == 0 {
|
||||
return nil, true
|
||||
}
|
||||
seen := make(map[string]struct{}, len(itemTypes))
|
||||
for _, itemType := range itemTypes {
|
||||
itemType = strings.ToLower(strings.TrimSpace(itemType))
|
||||
if itemType == "" {
|
||||
continue
|
||||
}
|
||||
if itemType == "episode" {
|
||||
includeEpisodes = true
|
||||
continue
|
||||
}
|
||||
if _, ok := seen[itemType]; ok {
|
||||
continue
|
||||
}
|
||||
seen[itemType] = struct{}{}
|
||||
mediaTypes = append(mediaTypes, itemType)
|
||||
}
|
||||
return mediaTypes, includeEpisodes
|
||||
}
|
||||
|
||||
func searchMatchCondition(titleVector, overviewVector string) string {
|
||||
titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))`
|
||||
prefixQuery := `to_tsquery('simple', $2)`
|
||||
return fmt.Sprintf(`((%s) @@ %s OR ($2 <> '' AND (%s) @@ %s) OR (%s) @@ websearch_to_tsquery('english', $1))`,
|
||||
titleVector, titleQuery, titleVector, prefixQuery, overviewVector)
|
||||
}
|
||||
|
||||
func buildMixedSearchCandidateBranch(
|
||||
contentIDExpr, typeExpr, titleExpr, yearExpr, titleVector, overviewVector string,
|
||||
exactTitleExprs []string,
|
||||
fromClause string,
|
||||
conditions []string,
|
||||
exactIdx, yearIdx, phraseIdx int,
|
||||
) string {
|
||||
exactArms := make([]string, 0, len(exactTitleExprs))
|
||||
contiguousArms := make([]string, 0, len(exactTitleExprs))
|
||||
for _, expr := range exactTitleExprs {
|
||||
exactArms = append(exactArms, fmt.Sprintf("%s = $%d", expr, exactIdx))
|
||||
contiguousArms = append(contiguousArms, fmt.Sprintf("%s LIKE '%%' || $%d || '%%'", expr, exactIdx))
|
||||
}
|
||||
titleQuery := `websearch_to_tsquery('simple', public.normalize_search_text($1))`
|
||||
prefixQuery := `to_tsquery('simple', $2)`
|
||||
return fmt.Sprintf(`
|
||||
SELECT
|
||||
%s AS content_id,
|
||||
%s AS type,
|
||||
%s AS title,
|
||||
CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS exact_title_match,
|
||||
CASE WHEN $%d <> '' AND (%s) THEN 1 ELSE 0 END AS contiguous_title_match,
|
||||
CASE WHEN $%d::int IS NOT NULL AND (%s) = $%d::int THEN 1 ELSE 0 END AS year_match,
|
||||
ts_rank_cd(%s, %s) AS title_rank,
|
||||
CASE WHEN $2 <> '' THEN ts_rank_cd(%s, %s) ELSE 0 END AS title_prefix_rank,
|
||||
ts_rank_cd(%s, websearch_to_tsquery('english', $1)) AS overview_rank,
|
||||
CASE WHEN $%d <> '' THEN ts_rank_cd(%s, phraseto_tsquery('simple', public.normalize_search_text($%d))) ELSE 0 END AS phrase_rank
|
||||
FROM %s
|
||||
WHERE %s`,
|
||||
contentIDExpr, typeExpr, titleExpr,
|
||||
exactIdx, strings.Join(exactArms, " OR "),
|
||||
exactIdx, strings.Join(contiguousArms, " OR "),
|
||||
yearIdx, yearExpr, yearIdx,
|
||||
titleVector, titleQuery,
|
||||
titleVector, prefixQuery,
|
||||
overviewVector,
|
||||
phraseIdx, titleVector, phraseIdx,
|
||||
fromClause, strings.Join(conditions, " AND "))
|
||||
}
|
||||
|
||||
func appendEpisodeLibrarySearchAccess(
|
||||
episodeIDExpr string,
|
||||
filter AccessFilter,
|
||||
conditions *[]string,
|
||||
args *[]any,
|
||||
argIdx *int,
|
||||
) {
|
||||
if filter.AllowedLibraryIDs != nil {
|
||||
*conditions = append(*conditions, fmt.Sprintf(
|
||||
"EXISTS (SELECT 1 FROM episode_libraries allowed_el WHERE allowed_el.episode_id = %s AND allowed_el.media_folder_id = ANY($%d))",
|
||||
episodeIDExpr, *argIdx))
|
||||
*args = append(*args, filter.AllowedLibraryIDs)
|
||||
*argIdx++
|
||||
}
|
||||
if len(filter.DisabledLibraryIDs) > 0 {
|
||||
*conditions = append(*conditions, fmt.Sprintf(
|
||||
"NOT EXISTS (SELECT 1 FROM episode_libraries disabled_el WHERE disabled_el.episode_id = %s AND disabled_el.media_folder_id = ANY($%d))",
|
||||
episodeIDExpr, *argIdx))
|
||||
*args = append(*args, filter.DisabledLibraryIDs)
|
||||
*argIdx++
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@ const (
|
||||
DefaultMeilisearchIndex = "silo_media_items"
|
||||
DefaultMeilisearchTimeoutMS = 800
|
||||
DefaultMeilisearchMatchingStrategy = "last"
|
||||
SearchMeilisearchSchemaVersion = 2
|
||||
SearchMeilisearchSchemaVersion = 3
|
||||
|
||||
DefaultMeilisearchSyncBatchSize = 500
|
||||
DefaultMeilisearchRebuildBatchSize = 5000
|
||||
|
||||
@@ -150,6 +150,14 @@ func TestCatalogSearchMeilisearchSettingsIncludeEmbeddersWhenSemanticEnabled(t *
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogSearchMeilisearchSettingsFilterLibraryIDs(t *testing.T) {
|
||||
settings := catalogSearchMeilisearchSettings(DefaultMeilisearchEmbedder, false, false)
|
||||
filterable, ok := settings["filterableAttributes"].([]string)
|
||||
if !ok || !reflect.DeepEqual(filterable, []string{"type", "library_ids"}) {
|
||||
t.Fatalf("filterable attributes = %#v", settings["filterableAttributes"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogSearchDocumentPayloadBatchesSplitVectorDocs(t *testing.T) {
|
||||
vector := make([]float32, 128)
|
||||
docs := []catalogSearchDocument{
|
||||
@@ -296,6 +304,133 @@ func TestMeilisearchSearchRequestBuildsHybridWhenCoverageReady(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeilisearchEpisodeSearchRemainsKeywordOnly(t *testing.T) {
|
||||
vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}}
|
||||
provider := &MeilisearchSearchProvider{config: MeilisearchProviderConfig{
|
||||
SemanticEnabled: true,
|
||||
SemanticRatio: 0.4,
|
||||
Embedder: DefaultMeilisearchEmbedder,
|
||||
Vectorizer: vectorizer,
|
||||
Coverage: fakeCoverageGate{ready: true},
|
||||
}}
|
||||
req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{
|
||||
Query: "Who Are You",
|
||||
ItemTypes: []string{"episode"},
|
||||
Access: AccessFilter{AllowedLibraryIDs: []int{4, 9}, DisabledLibraryIDs: []int{12}},
|
||||
})
|
||||
if fallback != "" || req.Hybrid != nil || req.Vector != nil {
|
||||
t.Fatalf("episode request should be keyword-only, fallback/request = %q/%#v", fallback, req)
|
||||
}
|
||||
for _, want := range []string{`type = "episode"`, "library_ids IN [4, 9]", "library_ids NOT IN [12]"} {
|
||||
if !strings.Contains(req.Filter, want) {
|
||||
t.Fatalf("episode filter missing %q: %s", want, req.Filter)
|
||||
}
|
||||
}
|
||||
if vectorizer.calls != 0 {
|
||||
t.Fatalf("episode request generated a vector, calls=%d", vectorizer.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeilisearchMixedSemanticSearchBuildsFederation(t *testing.T) {
|
||||
provider := &MeilisearchSearchProvider{config: MeilisearchProviderConfig{}}
|
||||
base := meilisearchSearchRequest{
|
||||
Query: "Who Are You",
|
||||
AttributesToRetrieve: []string{"content_id"},
|
||||
MatchingStrategy: "all",
|
||||
Vector: []float32{0.5, 0.25},
|
||||
Hybrid: &meilisearchHybridRequest{Embedder: DefaultMeilisearchEmbedder, SemanticRatio: 0.4},
|
||||
}
|
||||
federated := provider.buildFederatedSearchRequest("catalog-v3", CatalogSearchRequest{
|
||||
Access: AccessFilter{AllowedLibraryIDs: []int{4}},
|
||||
}, base, 100, 25)
|
||||
if federated.Federation.Offset != 100 || federated.Federation.Limit != 25 {
|
||||
t.Fatalf("federation pagination = %#v", federated.Federation)
|
||||
}
|
||||
if len(federated.Queries) != 2 {
|
||||
t.Fatalf("federated queries = %#v", federated.Queries)
|
||||
}
|
||||
media, episode := federated.Queries[0], federated.Queries[1]
|
||||
if media.Hybrid == nil || len(media.Vector) == 0 || !strings.Contains(media.Filter, `type != "episode"`) {
|
||||
t.Fatalf("media federation branch = %#v", media)
|
||||
}
|
||||
if episode.Hybrid != nil || episode.Vector != nil || !strings.Contains(episode.Filter, `type = "episode"`) {
|
||||
t.Fatalf("episode federation branch = %#v", episode)
|
||||
}
|
||||
if media.FederationOptions.Weight != 1 || episode.FederationOptions.Weight != 1 {
|
||||
t.Fatalf("federation weights = %v/%v", media.FederationOptions.Weight, episode.FederationOptions.Weight)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeilisearchFederationUnsupportedClassification(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "missing index during rebuild swap is transient",
|
||||
err: &meilisearchHTTPError{
|
||||
StatusCode: http.StatusNotFound,
|
||||
Code: "index_not_found",
|
||||
Message: "Index catalog_rebuild_old not found.",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "missing multi-search route is unsupported",
|
||||
err: &meilisearchHTTPError{
|
||||
StatusCode: http.StatusNotFound,
|
||||
Code: "not_found",
|
||||
Message: "The multi-search route does not exist.",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "older server rejects federation field",
|
||||
err: &meilisearchHTTPError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Code: "bad_request",
|
||||
Message: "Unknown field federation: expected queries.",
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "invalid federated request is not a capability result",
|
||||
err: &meilisearchHTTPError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
Code: "invalid_multi_search_query_federated",
|
||||
Message: "A query has federationOptions but federation is missing.",
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := isMeilisearchFederationUnsupported(test.err); got != test.want {
|
||||
t.Fatalf("isMeilisearchFederationUnsupported(%v) = %t, want %t", test.err, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogSearchEpisodeDocumentsNeverCarryVectors(t *testing.T) {
|
||||
docs := []catalogSearchDocument{
|
||||
{ContentID: "movie-1", Type: "movie"},
|
||||
{ContentID: "episode-1", Type: "episode"},
|
||||
}
|
||||
count := setCatalogSearchDocumentVectors(docs, map[string][]float32{
|
||||
"movie-1": {0.1, 0.2},
|
||||
"episode-1": {0.3, 0.4},
|
||||
}, DefaultMeilisearchEmbedder)
|
||||
if count != 1 || docs[0].Vectors == nil {
|
||||
t.Fatalf("media vector assignment = count %d, vectors %#v", count, docs[0].Vectors)
|
||||
}
|
||||
if docs[1].Vectors != nil {
|
||||
t.Fatalf("episode vectors = %#v, want omitted", docs[1].Vectors)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSemanticModelProviderUsesCommaOkSafety(t *testing.T) {
|
||||
if got := semanticModelProvider(nil); got != nil {
|
||||
t.Fatalf("nil vectorizer should yield nil model provider, got %#v", got)
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
-- +goose NO TRANSACTION
|
||||
|
||||
-- +goose Up
|
||||
-- Search evaluates these exact expressions in the episode candidate branch.
|
||||
-- Build concurrently so large episode catalogs remain readable and writable.
|
||||
-- Remove INVALID remnants first: IF NOT EXISTS otherwise accepts a failed
|
||||
-- concurrent build and Goose would record a search index that cannot be used.
|
||||
-- +goose StatementBegin
|
||||
DO $$
|
||||
DECLARE
|
||||
index_name text;
|
||||
BEGIN
|
||||
FOREACH index_name IN ARRAY ARRAY[
|
||||
'idx_episodes_search_title',
|
||||
'idx_episodes_search_overview'
|
||||
] LOOP
|
||||
IF EXISTS (
|
||||
SELECT 1
|
||||
FROM pg_class c
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
JOIN pg_index i ON i.indexrelid = c.oid
|
||||
WHERE n.nspname = 'public'
|
||||
AND c.relname = index_name
|
||||
AND NOT i.indisvalid
|
||||
) THEN
|
||||
EXECUTE format('DROP INDEX public.%I', index_name);
|
||||
END IF;
|
||||
END LOOP;
|
||||
END;
|
||||
$$;
|
||||
-- +goose StatementEnd
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_episodes_search_title
|
||||
ON public.episodes USING gin (
|
||||
setweight(
|
||||
to_tsvector(
|
||||
'simple',
|
||||
public.normalize_search_text(
|
||||
COALESCE(NULLIF(BTRIM(title), ''), 'Episode ' || episode_number::text)
|
||||
)
|
||||
),
|
||||
'A'
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_episodes_search_overview
|
||||
ON public.episodes USING gin (
|
||||
to_tsvector('english', COALESCE(overview, ''))
|
||||
);
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX CONCURRENTLY IF EXISTS public.idx_episodes_search_overview;
|
||||
DROP INDEX CONCURRENTLY IF EXISTS public.idx_episodes_search_title;
|
||||
@@ -0,0 +1,216 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_enabled()
|
||||
RETURNS boolean
|
||||
LANGUAGE sql
|
||||
STABLE
|
||||
AS $$
|
||||
SELECT lower(COALESCE(
|
||||
(SELECT value FROM public.server_settings WHERE key = 'catalog.search.provider'),
|
||||
'postgres'
|
||||
)) = 'meilisearch'
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_insert()
|
||||
RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'upsert', content_id, '' FROM new_rows GROUP BY content_id;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_update()
|
||||
RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'delete', old_rows.content_id, ''
|
||||
FROM old_rows
|
||||
LEFT JOIN new_rows ON new_rows.content_id = old_rows.content_id
|
||||
WHERE new_rows.content_id IS NULL
|
||||
GROUP BY old_rows.content_id;
|
||||
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'upsert', content_id, '' FROM new_rows GROUP BY content_id;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_delete()
|
||||
RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'delete', content_id, '' FROM old_rows GROUP BY content_id;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_library_insert()
|
||||
RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'upsert', episode_id, '' FROM new_rows GROUP BY episode_id;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_library_update()
|
||||
RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'upsert', episode_id, ''
|
||||
FROM (
|
||||
SELECT episode_id FROM old_rows
|
||||
UNION
|
||||
SELECT episode_id FROM new_rows
|
||||
) changed;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_episode_library_delete()
|
||||
RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'upsert', episode_id, '' FROM old_rows GROUP BY episode_id;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_item_library_insert()
|
||||
RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'upsert', content_id, '' FROM new_rows GROUP BY content_id;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_item_library_update()
|
||||
RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'upsert', content_id, ''
|
||||
FROM (
|
||||
SELECT content_id FROM old_rows
|
||||
UNION
|
||||
SELECT content_id FROM new_rows
|
||||
) changed;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_item_library_delete()
|
||||
RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'upsert', content_id, '' FROM old_rows GROUP BY content_id;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION public.catalog_search_capture_series_type_update()
|
||||
RETURNS trigger LANGUAGE plpgsql AS $$
|
||||
BEGIN
|
||||
IF NOT public.catalog_search_capture_enabled() THEN RETURN NULL; END IF;
|
||||
INSERT INTO public.catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT 'meilisearch', 'upsert', e.content_id, ''
|
||||
FROM (
|
||||
SELECT new_rows.content_id
|
||||
FROM new_rows
|
||||
JOIN old_rows USING (content_id)
|
||||
WHERE new_rows.type IS DISTINCT FROM old_rows.type
|
||||
AND (new_rows.type = 'series' OR old_rows.type = 'series')
|
||||
) changed
|
||||
JOIN public.episodes e ON e.series_id = changed.content_id
|
||||
GROUP BY e.content_id;
|
||||
RETURN NULL;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE TRIGGER trg_catalog_search_episode_insert
|
||||
AFTER INSERT ON public.episodes
|
||||
REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_insert();
|
||||
|
||||
CREATE TRIGGER trg_catalog_search_episode_update
|
||||
AFTER UPDATE ON public.episodes
|
||||
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_update();
|
||||
|
||||
CREATE TRIGGER trg_catalog_search_episode_delete
|
||||
AFTER DELETE ON public.episodes
|
||||
REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_delete();
|
||||
|
||||
CREATE TRIGGER trg_catalog_search_episode_library_insert
|
||||
AFTER INSERT ON public.episode_libraries
|
||||
REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_library_insert();
|
||||
|
||||
CREATE TRIGGER trg_catalog_search_episode_library_update
|
||||
AFTER UPDATE ON public.episode_libraries
|
||||
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_library_update();
|
||||
|
||||
CREATE TRIGGER trg_catalog_search_episode_library_delete
|
||||
AFTER DELETE ON public.episode_libraries
|
||||
REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_episode_library_delete();
|
||||
|
||||
CREATE TRIGGER trg_catalog_search_item_library_insert
|
||||
AFTER INSERT ON public.media_item_libraries
|
||||
REFERENCING NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_item_library_insert();
|
||||
|
||||
CREATE TRIGGER trg_catalog_search_item_library_update
|
||||
AFTER UPDATE ON public.media_item_libraries
|
||||
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_item_library_update();
|
||||
|
||||
CREATE TRIGGER trg_catalog_search_item_library_delete
|
||||
AFTER DELETE ON public.media_item_libraries
|
||||
REFERENCING OLD TABLE AS old_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_item_library_delete();
|
||||
|
||||
CREATE TRIGGER trg_catalog_search_series_type_update
|
||||
AFTER UPDATE ON public.media_items
|
||||
REFERENCING OLD TABLE AS old_rows NEW TABLE AS new_rows
|
||||
FOR EACH STATEMENT EXECUTE FUNCTION public.catalog_search_capture_series_type_update();
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TRIGGER IF EXISTS trg_catalog_search_series_type_update ON public.media_items;
|
||||
DROP TRIGGER IF EXISTS trg_catalog_search_item_library_delete ON public.media_item_libraries;
|
||||
DROP TRIGGER IF EXISTS trg_catalog_search_item_library_update ON public.media_item_libraries;
|
||||
DROP TRIGGER IF EXISTS trg_catalog_search_item_library_insert ON public.media_item_libraries;
|
||||
DROP TRIGGER IF EXISTS trg_catalog_search_episode_library_delete ON public.episode_libraries;
|
||||
DROP TRIGGER IF EXISTS trg_catalog_search_episode_library_update ON public.episode_libraries;
|
||||
DROP TRIGGER IF EXISTS trg_catalog_search_episode_library_insert ON public.episode_libraries;
|
||||
DROP TRIGGER IF EXISTS trg_catalog_search_episode_delete ON public.episodes;
|
||||
DROP TRIGGER IF EXISTS trg_catalog_search_episode_update ON public.episodes;
|
||||
DROP TRIGGER IF EXISTS trg_catalog_search_episode_insert ON public.episodes;
|
||||
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_series_type_update();
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_item_library_delete();
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_item_library_update();
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_item_library_insert();
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_library_delete();
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_library_update();
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_library_insert();
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_delete();
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_update();
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_episode_insert();
|
||||
DROP FUNCTION IF EXISTS public.catalog_search_capture_enabled();
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,49 @@
|
||||
import { act, fireEvent, render, screen } from "@testing-library/react";
|
||||
import { MemoryRouter, useLocation } from "react-router";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
import { buildCatalogQueryUpdateHref, parseCatalogSearchParams } from "@/pages/catalogSearchParams";
|
||||
|
||||
import SearchBar from "./SearchBar";
|
||||
|
||||
function LocationProbe() {
|
||||
const location = useLocation();
|
||||
return <output aria-label="location">{`${location.pathname}${location.search}`}</output>;
|
||||
}
|
||||
|
||||
describe("SearchBar", () => {
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("keeps All Media selected when live search changes the query", () => {
|
||||
vi.useFakeTimers();
|
||||
const state = parseCatalogSearchParams(
|
||||
new URLSearchParams("source=query&q=heat&type=all&genre=Drama"),
|
||||
);
|
||||
|
||||
render(
|
||||
<MemoryRouter initialEntries={["/catalog?source=query&q=heat&type=all&genre=Drama"]}>
|
||||
<SearchBar
|
||||
prominent
|
||||
initialQuery="heat"
|
||||
buildSearchHref={(query) => buildCatalogQueryUpdateHref(state, query)}
|
||||
/>
|
||||
<LocationProbe />
|
||||
</MemoryRouter>,
|
||||
);
|
||||
|
||||
fireEvent.change(screen.getByRole("textbox"), { target: { value: "heater" } });
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(101);
|
||||
});
|
||||
|
||||
const location = new URL(`http://example.test${screen.getByLabelText("location").textContent}`);
|
||||
expect(location.searchParams.get("q")).toBe("heater");
|
||||
expect(location.searchParams.get("type")).toBe("all");
|
||||
expect(parseCatalogSearchParams(location.searchParams).query_definition.groups).toContainEqual({
|
||||
match: "all",
|
||||
rules: [{ field: "genre", op: "contains", value: "Drama" }],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -12,19 +12,26 @@ interface SearchBarProps {
|
||||
initialQuery?: string;
|
||||
autoFocus?: boolean;
|
||||
prominent?: boolean;
|
||||
buildSearchHref?: (query: string) => string;
|
||||
}
|
||||
|
||||
export default function SearchBar({
|
||||
initialQuery = "",
|
||||
autoFocus = false,
|
||||
prominent = false,
|
||||
buildSearchHref = buildQueryCatalogHref,
|
||||
}: SearchBarProps) {
|
||||
const [query, setQuery] = useState(initialQuery);
|
||||
const navigate = useViewTransitionNavigate();
|
||||
const inputRef = useRef<HTMLInputElement>(null);
|
||||
const isInitialMount = useRef(true);
|
||||
const buildSearchHrefRef = useRef(buildSearchHref);
|
||||
const debouncedQuery = useDebounce(query, SEARCH_NAVIGATION_DEBOUNCE_MS);
|
||||
|
||||
useEffect(() => {
|
||||
buildSearchHrefRef.current = buildSearchHref;
|
||||
}, [buildSearchHref]);
|
||||
|
||||
useEffect(() => {
|
||||
if (autoFocus && inputRef.current) {
|
||||
inputRef.current.focus();
|
||||
@@ -39,14 +46,14 @@ export default function SearchBar({
|
||||
return;
|
||||
}
|
||||
if (debouncedQuery.trim()) {
|
||||
navigate(buildQueryCatalogHref(debouncedQuery.trim()), { replace: true });
|
||||
navigate(buildSearchHrefRef.current(debouncedQuery.trim()), { replace: true });
|
||||
}
|
||||
}, [debouncedQuery, prominent, navigate]);
|
||||
|
||||
function handleSubmit(e: FormEvent) {
|
||||
e.preventDefault();
|
||||
if (query.trim()) {
|
||||
navigate(buildQueryCatalogHref(query.trim()));
|
||||
navigate(buildSearchHref(query.trim()));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,7 +24,8 @@ import {
|
||||
} from "@/lib/historyRemoval";
|
||||
|
||||
import {
|
||||
buildCatalogApiSearchParams,
|
||||
buildCatalogFilterSearchParams,
|
||||
buildCatalogQueryUpdateHref,
|
||||
catalogSourceAllowsOverlay,
|
||||
parseCatalogSearchParams,
|
||||
} from "./catalogSearchParams";
|
||||
@@ -132,6 +133,10 @@ function CatalogResults({
|
||||
query_definition: { ...state.query_definition, media_scope: preferredScope },
|
||||
};
|
||||
}, [hasExplicitScope, preferredScope, state]);
|
||||
const buildSearchHref = useCallback(
|
||||
(query: string) => buildCatalogQueryUpdateHref(effectiveState, query),
|
||||
[effectiveState],
|
||||
);
|
||||
|
||||
const mediaScope = effectiveState.query_definition.media_scope;
|
||||
const activeChipScope: SearchMediaScope =
|
||||
@@ -257,7 +262,12 @@ function CatalogResults({
|
||||
|
||||
{state.source === "query" ? (
|
||||
<div className="flex flex-col items-center gap-3">
|
||||
<SearchBar prominent initialQuery={state.q ?? ""} autoFocus />
|
||||
<SearchBar
|
||||
prominent
|
||||
initialQuery={state.q ?? ""}
|
||||
autoFocus
|
||||
buildSearchHref={buildSearchHref}
|
||||
/>
|
||||
<SearchScopeChips activeScope={activeChipScope} onScopeChange={handleChipScopeChange} />
|
||||
</div>
|
||||
) : null}
|
||||
@@ -265,7 +275,7 @@ function CatalogResults({
|
||||
<CatalogFiltersPanel
|
||||
state={effectiveState}
|
||||
onStateChange={(nextState) => {
|
||||
const nextSearchParams = buildCatalogApiSearchParams(nextState);
|
||||
const nextSearchParams = buildCatalogFilterSearchParams(nextState);
|
||||
if (nextSearchParams.toString() !== searchParams.toString()) {
|
||||
setSearchParams(nextSearchParams);
|
||||
}
|
||||
|
||||
@@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildCatalogApiSearchParams,
|
||||
buildCatalogFilterSearchParams,
|
||||
buildCatalogQueryUpdateHref,
|
||||
buildCatalogHref,
|
||||
buildPersonCatalogHref,
|
||||
buildPersonalCatalogHref,
|
||||
@@ -127,6 +129,37 @@ describe("catalogSourceAllowsOverlay", () => {
|
||||
});
|
||||
|
||||
describe("buildCatalogHref", () => {
|
||||
it("preserves type=all when the query filter selects All Media", () => {
|
||||
const state = parseCatalogSearchParams(params("source=query&q=heat&type=video"));
|
||||
state.query_definition = {
|
||||
...state.query_definition,
|
||||
media_scope: undefined,
|
||||
};
|
||||
|
||||
const built = buildCatalogFilterSearchParams(state);
|
||||
|
||||
expect(built.get("type")).toBe("all");
|
||||
expect(parseCatalogSearchParams(built).query_definition.media_scope).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves All Media and other filters when the live query changes", () => {
|
||||
const state = parseCatalogSearchParams(
|
||||
params("source=query&q=heat&type=all&genre=Drama&sort=title&order=asc"),
|
||||
);
|
||||
|
||||
const href = buildCatalogQueryUpdateHref(state, "heater");
|
||||
const built = new URL(`http://example.test${href}`);
|
||||
|
||||
expect(built.searchParams.get("q")).toBe("heater");
|
||||
expect(built.searchParams.get("type")).toBe("all");
|
||||
expect(built.searchParams.get("sort")).toBe("title");
|
||||
expect(built.searchParams.get("order")).toBe("asc");
|
||||
expect(parseCatalogSearchParams(built.searchParams).query_definition.groups).toContainEqual({
|
||||
match: "all",
|
||||
rules: [{ field: "genre", op: "contains", value: "Drama" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps explicit added-at sorting in query-source URLs", () => {
|
||||
expect(
|
||||
buildCatalogApiSearchParams({
|
||||
|
||||
@@ -181,6 +181,28 @@ export function buildCatalogHref(state: CatalogSearchState): string {
|
||||
return `/catalog?${params.toString()}`;
|
||||
}
|
||||
|
||||
// Interactive query filters need an explicit `all` sentinel. Internally an
|
||||
// unscoped query is represented by an undefined media_scope, but omitting the
|
||||
// URL type makes Catalog reapply the user's saved default (normally `video`).
|
||||
// Keep this behavior scoped to filter updates so fresh search URLs can still
|
||||
// inherit that preference.
|
||||
export function buildCatalogFilterSearchParams(state: CatalogSearchState): URLSearchParams {
|
||||
const params = buildCatalogApiSearchParams(state);
|
||||
if (state.source === "query" && !state.type_override && !state.query_definition.media_scope) {
|
||||
params.set("type", "all");
|
||||
}
|
||||
return params;
|
||||
}
|
||||
|
||||
export function buildCatalogQueryUpdateHref(state: CatalogSearchState, q: string): string {
|
||||
const params = buildCatalogFilterSearchParams({
|
||||
...state,
|
||||
source: "query",
|
||||
q,
|
||||
});
|
||||
return `/catalog?${params.toString()}`;
|
||||
}
|
||||
|
||||
export function buildQueryCatalogHref(q?: string): string {
|
||||
return buildCatalogHref({
|
||||
source: "query",
|
||||
|
||||
Reference in New Issue
Block a user