diff --git a/internal/api/handlers/catalog.go b/internal/api/handlers/catalog.go index 00dd75d1..9ffbc2c6 100644 --- a/internal/api/handlers/catalog.go +++ b/internal/api/handlers/catalog.go @@ -65,12 +65,14 @@ type catalogResponse struct { // /api/v1/catalog only when a relevance-sorted search actually ran through a // CatalogSearchProvider. mode/semantic_used reflect POST-downgrade reality // (a hybrid request that fell back to keyword reports mode="keyword", -// semantic_used=false). fallback_reason is omitted when empty. +// semantic_used=false). fallback_reason and index_pending_updates are omitted +// when empty. type searchDiagnostics struct { - Provider string `json:"provider"` - Mode string `json:"mode"` - SemanticUsed bool `json:"semantic_used"` - FallbackReason string `json:"fallback_reason,omitempty"` + Provider string `json:"provider"` + Mode string `json:"mode"` + SemanticUsed bool `json:"semantic_used"` + FallbackReason string `json:"fallback_reason,omitempty"` + IndexPendingUpdates int `json:"index_pending_updates,omitempty"` } type catalogFiltersResponse struct { @@ -234,10 +236,11 @@ func (h *CatalogHandler) writeCatalogResponse(w http.ResponseWriter, result *cat var diag *searchDiagnostics if result.Provider != "" { diag = &searchDiagnostics{ - Provider: result.Provider, - Mode: result.Mode, - SemanticUsed: result.SemanticUsed, - FallbackReason: result.FallbackReason, + Provider: result.Provider, + Mode: result.Mode, + SemanticUsed: result.SemanticUsed, + FallbackReason: result.FallbackReason, + IndexPendingUpdates: result.IndexPendingEvents, } } diff --git a/internal/api/handlers/catalog_diagnostics_test.go b/internal/api/handlers/catalog_diagnostics_test.go index cc686604..bc659eb6 100644 --- a/internal/api/handlers/catalog_diagnostics_test.go +++ b/internal/api/handlers/catalog_diagnostics_test.go @@ -69,9 +69,10 @@ func TestWriteCatalogResponse_DiagnosticsKeywordFallback(t *testing.T) { func TestWriteCatalogResponse_DiagnosticsHybridOmitsFallbackReason(t *testing.T) { body := decodeCatalogResponse(t, &catalog.CatalogResult{ - Provider: catalog.SearchProviderMeilisearch, - Mode: "hybrid", - SemanticUsed: true, + Provider: catalog.SearchProviderMeilisearch, + Mode: "hybrid", + SemanticUsed: true, + IndexPendingEvents: 7, }, false) diagRaw, ok := body["search_diagnostics"] @@ -88,6 +89,9 @@ func TestWriteCatalogResponse_DiagnosticsHybridOmitsFallbackReason(t *testing.T) if _, ok := diag["fallback_reason"]; ok { t.Fatalf("fallback_reason should be omitted when empty: %v", diag) } + if diag["index_pending_updates"].(float64) != 7 { + t.Fatalf("index_pending_updates = %v, want 7", diag["index_pending_updates"]) + } } func TestWriteCatalogResponse_NoProviderOmitsDiagnostics(t *testing.T) { diff --git a/internal/catalog/catalog_resolver.go b/internal/catalog/catalog_resolver.go index 09cb8910..15b317f2 100644 --- a/internal/catalog/catalog_resolver.go +++ b/internal/catalog/catalog_resolver.go @@ -27,14 +27,16 @@ type CatalogResult struct { HasMore bool TotalExact bool SnapshotAt time.Time // pagination fence timestamp - // Provider, Mode, SemanticUsed and FallbackReason are per-query search - // diagnostics. They are only populated on the direct-search path (where a - // CatalogSearchProvider actually ran); browse / preview / grouped paths - // leave them zero-valued so the handler omits search_diagnostics. - Provider string - Mode string - SemanticUsed bool - FallbackReason string + // Provider, Mode, SemanticUsed, FallbackReason and IndexPendingEvents are + // per-query search diagnostics. They are only populated on the direct-search + // path (where a CatalogSearchProvider actually ran); browse / preview / + // grouped paths leave them zero-valued so the handler omits + // search_diagnostics. + Provider string + Mode string + SemanticUsed bool + FallbackReason string + IndexPendingEvents int } type CatalogFiltersResult struct { @@ -297,14 +299,15 @@ func (r *CatalogResolver) resolveDirectSearchSource(ctx context.Context, req Cat } return &CatalogResult{ - Items: result.Items, - Total: result.Total, - HasMore: result.HasMore, - TotalExact: result.TotalExact, - Provider: result.Provider, - Mode: result.Mode, - SemanticUsed: result.SemanticUsed, - FallbackReason: result.FallbackReason, + Items: result.Items, + Total: result.Total, + HasMore: result.HasMore, + TotalExact: result.TotalExact, + Provider: result.Provider, + Mode: result.Mode, + SemanticUsed: result.SemanticUsed, + FallbackReason: result.FallbackReason, + IndexPendingEvents: result.IndexPendingEvents, }, nil } diff --git a/internal/catalog/catalog_resolver_test.go b/internal/catalog/catalog_resolver_test.go index e2291cff..3e6ef125 100644 --- a/internal/catalog/catalog_resolver_test.go +++ b/internal/catalog/catalog_resolver_test.go @@ -403,10 +403,11 @@ func TestResolveDirectSearchSource_PlumbsDiagnostics(t *testing.T) { { name: "hybrid survived", result: &CatalogSearchResult{ - Items: []*models.MediaItem{}, - Provider: SearchProviderMeilisearch, - Mode: "hybrid", - SemanticUsed: true, + Items: []*models.MediaItem{}, + Provider: SearchProviderMeilisearch, + Mode: "hybrid", + SemanticUsed: true, + IndexPendingEvents: 7, }, }, } @@ -437,6 +438,9 @@ func TestResolveDirectSearchSource_PlumbsDiagnostics(t *testing.T) { if got.FallbackReason != tc.result.FallbackReason { t.Fatalf("FallbackReason = %q, want %q", got.FallbackReason, tc.result.FallbackReason) } + if got.IndexPendingEvents != tc.result.IndexPendingEvents { + t.Fatalf("IndexPendingEvents = %d, want %d", got.IndexPendingEvents, tc.result.IndexPendingEvents) + } }) } } diff --git a/internal/catalog/search_meilisearch_client.go b/internal/catalog/search_meilisearch_client.go index 590a7146..14b91cab 100644 --- a/internal/catalog/search_meilisearch_client.go +++ b/internal/catalog/search_meilisearch_client.go @@ -82,6 +82,7 @@ type meilisearchSearchRequest struct { Limit int `json:"limit"` Filter string `json:"filter,omitempty"` AttributesToRetrieve []string `json:"attributesToRetrieve"` + AttributesToSearchOn []string `json:"attributesToSearchOn,omitempty"` MatchingStrategy string `json:"matchingStrategy,omitempty"` Vector []float32 `json:"vector,omitempty"` Hybrid *meilisearchHybridRequest `json:"hybrid,omitempty"` diff --git a/internal/catalog/search_meilisearch_provider.go b/internal/catalog/search_meilisearch_provider.go index c930d920..046e40d3 100644 --- a/internal/catalog/search_meilisearch_provider.go +++ b/internal/catalog/search_meilisearch_provider.go @@ -19,6 +19,9 @@ const ( meilisearchDefaultBatchSize = 100 meilisearchDefaultCandidateScanCap = 1000 meilisearchDefaultDeepOffsetLimit = 500 + meilisearchShortHybridMinTerms = 2 + meilisearchStrictMatchingTermCount = 2 + meilisearchTitleOnlyTermCount = 2 meilisearchCircuitCooldown = 30 * time.Second meilisearchQueryVectorCacheTTL = 15 * time.Minute meilisearchQueryVectorCacheMax = 1024 @@ -28,6 +31,18 @@ const ( semanticCapabilityProbeTTL = 5 * time.Minute ) +var meilisearchTitleSearchAttributes = []string{ + "title", + "original_title", + "sort_title", + "title_variants", +} + +type meilisearchIndexStateStore interface { + GetState(ctx context.Context, provider string) (SearchIndexState, error) + PendingCount(ctx context.Context, provider string) (int, error) +} + type MeilisearchProviderConfig struct { URL string APIKey string @@ -48,7 +63,7 @@ type MeilisearchProviderConfig struct { type MeilisearchSearchProvider struct { itemRepo *ItemRepository - stateRepo *SearchIndexEventRepository + stateRepo meilisearchIndexStateStore fallback *PostgresSearchProvider client *meilisearchClient config MeilisearchProviderConfig @@ -117,9 +132,13 @@ func NewMeilisearchSearchProvider( if err != nil { return nil, err } + var stateStore meilisearchIndexStateStore + if stateRepo != nil { + stateStore = stateRepo + } return &MeilisearchSearchProvider{ itemRepo: itemRepo, - stateRepo: stateRepo, + stateRepo: stateStore, fallback: fallback, client: client, config: config, @@ -154,13 +173,9 @@ func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearc if state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(p.config.Embedder, p.config.IndexTypes, p.config.SemanticEnabled) { return p.fallbackSearch(ctx, req, "meilisearch index schema mismatch") } - pending, err := p.stateRepo.PendingCount(ctx, SearchProviderMeilisearch) - if err != nil { - p.markFallback("index pending state unavailable") - return p.fallback.Search(ctx, req) - } - if pending > 0 { - return p.fallbackSearch(ctx, req, "meilisearch index has pending updates") + pending := 0 + if count, err := p.stateRepo.PendingCount(ctx, SearchProviderMeilisearch); err == nil && count > 0 { + pending = count } result, err := p.searchMeilisearch(ctx, req, state.ActiveIndexUID) @@ -175,6 +190,7 @@ func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearc } else { p.clearFallback() } + result.IndexPendingEvents = pending return result, nil } @@ -324,7 +340,8 @@ func (p *MeilisearchSearchProvider) buildMeilisearchSearchRequest(ctx context.Co Query: strings.TrimSpace(req.Query), Filter: meilisearchTypeFilter(req.ItemTypes), AttributesToRetrieve: []string{"content_id"}, - MatchingStrategy: p.config.MatchingStrategy, + AttributesToSearchOn: p.attributesToSearchOnForRequest(req), + MatchingStrategy: p.matchingStrategyForRequest(req), } if !p.shouldUseSemanticSearch(req) { return searchReq, "" @@ -356,7 +373,36 @@ func (p *MeilisearchSearchProvider) shouldUseSemanticSearch(req CatalogSearchReq if p == nil || !p.config.SemanticEnabled { return false } - return len(strings.Fields(normalizeCatalogSearchQueryForVector(req.Query))) >= 3 + return len(catalogSearchQueryTerms(req.Query)) >= meilisearchShortHybridMinTerms +} + +func (p *MeilisearchSearchProvider) matchingStrategyForRequest(req CatalogSearchRequest) string { + strategy := strings.TrimSpace(strings.ToLower(p.config.MatchingStrategy)) + if strategy == "" { + strategy = DefaultMeilisearchMatchingStrategy + } + if strategy != "last" { + return strategy + } + // Two-term title searches are easy for Meili's "last" strategy to over-relax + // after typo correction (e.g. "spnge bob" degenerating into "spnge"). + if len(catalogSearchQueryTerms(req.Query)) == meilisearchStrictMatchingTermCount { + return "all" + } + return strategy +} + +func (p *MeilisearchSearchProvider) attributesToSearchOnForRequest(req CatalogSearchRequest) []string { + if len(catalogSearchQueryTerms(req.Query)) != meilisearchTitleOnlyTermCount { + return nil + } + return append([]string(nil), meilisearchTitleSearchAttributes...) +} + +func catalogSearchQueryTerms(query string) []string { + parsed := parseSearchQuery(query) + normalized := normalizeTitleForComparison(firstNonEmptySearchValue(parsed.Text, query)) + return strings.Fields(normalized) } func (p *MeilisearchSearchProvider) cachedQueryVector(ctx context.Context, query string) ([]float32, error) { diff --git a/internal/catalog/search_provider.go b/internal/catalog/search_provider.go index 45227bf7..deab4071 100644 --- a/internal/catalog/search_provider.go +++ b/internal/catalog/search_provider.go @@ -42,7 +42,7 @@ const ( DefaultMeilisearchRebuildBatchSize = 5000 DefaultMeilisearchRebuildQueueDepth = 4 DefaultMeilisearchSemanticEnabled = false - DefaultMeilisearchSemanticRatio = 0.30 + DefaultMeilisearchSemanticRatio = 0.50 DefaultMeilisearchEmbedder = "silo_recommendations" MaxMeilisearchSyncBatchSize = 10000 @@ -62,11 +62,12 @@ type CatalogSearchRequest struct { } type CatalogSearchResult struct { - Items []*models.MediaItem - Total int - HasMore bool - TotalExact bool - Provider string + Items []*models.MediaItem + Total int + HasMore bool + TotalExact bool + Provider string + IndexPendingEvents int // Mode reports which retrieval path actually served this result — // "keyword" or "hybrid". For Meilisearch it reflects POST-downgrade // reality (a hybrid request that fell back to keyword reports "keyword"). diff --git a/internal/catalog/search_provider_test.go b/internal/catalog/search_provider_test.go index 01307f94..cae30730 100644 --- a/internal/catalog/search_provider_test.go +++ b/internal/catalog/search_provider_test.go @@ -4,9 +4,12 @@ import ( "context" "encoding/json" "errors" + "net/http" + "net/http/httptest" "reflect" "strings" "testing" + "time" ) func TestCatalogSearchSettingsFromMapParsesMeilisearchTuning(t *testing.T) { @@ -161,6 +164,9 @@ func TestMeilisearchSearchRequestBuildsKeywordOnlyByDefault(t *testing.T) { if req.Vector != nil || req.Hybrid != nil { t.Fatalf("keyword request should not include vector or hybrid: %#v", req) } + if req.MatchingStrategy != DefaultMeilisearchMatchingStrategy { + t.Fatalf("matching strategy = %q, want %q", req.MatchingStrategy, DefaultMeilisearchMatchingStrategy) + } if req.Filter != `type = "movie"` { t.Fatalf("filter = %q, want movie filter", req.Filter) } @@ -192,6 +198,12 @@ func TestMeilisearchSearchRequestBuildsHybridWhenSemanticEnabled(t *testing.T) { if req.Hybrid.SemanticRatio != 0.4 { t.Fatalf("semantic ratio = %v, want 0.4", req.Hybrid.SemanticRatio) } + if req.MatchingStrategy != DefaultMeilisearchMatchingStrategy { + t.Fatalf("long semantic query matching strategy = %q, want %q", req.MatchingStrategy, DefaultMeilisearchMatchingStrategy) + } + if req.AttributesToSearchOn != nil { + t.Fatalf("long semantic query should search all attributes, got %#v", req.AttributesToSearchOn) + } if len(req.Vector) != 3072 { t.Fatalf("vector len = %d, want 3072", len(req.Vector)) } @@ -302,7 +314,71 @@ func TestMeilisearchSearchRequestBuildsHybridForApproximateInteractiveSearch(t * } } -func TestMeilisearchSearchRequestSkipsHybridForShortTitleSearch(t *testing.T) { +func TestMeilisearchSearchRequestBuildsHybridAndStrictMatchingForTwoTermSearch(t *testing.T) { + vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}} + provider := &MeilisearchSearchProvider{ + config: MeilisearchProviderConfig{ + MatchingStrategy: DefaultMeilisearchMatchingStrategy, + SemanticEnabled: true, + SemanticRatio: 0.3, + Embedder: "silo_recommendations", + Vectorizer: vectorizer, + }, + } + req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{ + Query: "spnge bob", + }) + if fallback != "" { + t.Fatalf("fallback = %q, want empty", fallback) + } + if req.Vector == nil || req.Hybrid == nil { + t.Fatalf("two-term title search should include hybrid search: %#v", req) + } + if req.MatchingStrategy != "all" { + t.Fatalf("two-term title search matching strategy = %q, want all", req.MatchingStrategy) + } + if !reflect.DeepEqual(req.AttributesToSearchOn, meilisearchTitleSearchAttributes) { + t.Fatalf("two-term title search attributes = %#v, want %#v", req.AttributesToSearchOn, meilisearchTitleSearchAttributes) + } + if vectorizer.calls != 1 || vectorizer.lastQuery != "spnge bob" { + t.Fatalf("vectorizer calls/query = %d/%q", vectorizer.calls, vectorizer.lastQuery) + } +} + +func TestMeilisearchSearchRequestUsesStrictMatchingWhenTwoTermHybridNotReady(t *testing.T) { + vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}} + provider := &MeilisearchSearchProvider{ + config: MeilisearchProviderConfig{ + MatchingStrategy: DefaultMeilisearchMatchingStrategy, + SemanticEnabled: true, + SemanticRatio: 0.3, + Embedder: "silo_recommendations", + Vectorizer: vectorizer, + Coverage: fakeCoverageGate{ready: false, reason: `type "movie" coverage 40% below threshold`}, + }, + } + req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{ + Query: "spnge bob", + ItemTypes: []string{"movie"}, + }) + if req.Vector != nil || req.Hybrid != nil { + t.Fatalf("not-ready coverage should stay keyword-only: %#v", req) + } + if req.MatchingStrategy != "all" { + t.Fatalf("not-ready two-term search matching strategy = %q, want all", req.MatchingStrategy) + } + if !reflect.DeepEqual(req.AttributesToSearchOn, meilisearchTitleSearchAttributes) { + t.Fatalf("not-ready two-term search attributes = %#v, want %#v", req.AttributesToSearchOn, meilisearchTitleSearchAttributes) + } + if fallback != `semantic_not_ready: type "movie" coverage 40% below threshold` { + t.Fatalf("fallback = %q, want semantic_not_ready diagnostic", fallback) + } + if vectorizer.calls != 0 { + t.Fatalf("not-ready coverage should not call vectorizer, calls = %d", vectorizer.calls) + } +} + +func TestMeilisearchSearchRequestSkipsHybridForSingleTermTitleSearch(t *testing.T) { vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}} provider := &MeilisearchSearchProvider{ config: MeilisearchProviderConfig{ @@ -313,7 +389,7 @@ func TestMeilisearchSearchRequestSkipsHybridForShortTitleSearch(t *testing.T) { Vectorizer: vectorizer, }, } - for _, query := range []string{"sponge", "spongebob square"} { + for _, query := range []string{"sponge", "spongebob"} { req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{ Query: query, }) @@ -323,6 +399,12 @@ func TestMeilisearchSearchRequestSkipsHybridForShortTitleSearch(t *testing.T) { if req.Vector != nil || req.Hybrid != nil { t.Fatalf("short title search %q should stay keyword-only: %#v", query, req) } + if req.MatchingStrategy != DefaultMeilisearchMatchingStrategy { + t.Fatalf("single-term search %q matching strategy = %q, want %q", query, req.MatchingStrategy, DefaultMeilisearchMatchingStrategy) + } + if req.AttributesToSearchOn != nil { + t.Fatalf("single-term search %q should search all attributes, got %#v", query, req.AttributesToSearchOn) + } } if vectorizer.calls != 0 { t.Fatalf("short title searches should not call vectorizer, calls = %d", vectorizer.calls) @@ -363,6 +445,79 @@ func TestMeilisearchCircuitTripsOnServerAndDecodeErrors(t *testing.T) { } } +type fakeMeilisearchIndexStateStore struct { + state SearchIndexState + pending int +} + +func (f fakeMeilisearchIndexStateStore) GetState(context.Context, string) (SearchIndexState, error) { + return f.state, nil +} + +func (f fakeMeilisearchIndexStateStore) PendingCount(context.Context, string) (int, error) { + return f.pending, nil +} + +func TestMeilisearchProviderUsesActiveIndexWhenPendingUpdatesExist(t *testing.T) { + requests := 0 + var gotMethod, gotPath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + gotMethod = r.Method + gotPath = r.URL.Path + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"hits":[],"estimatedTotalHits":0}`)) + })) + defer server.Close() + + client, err := newMeilisearchClient(server.URL, "", time.Second) + if err != nil { + t.Fatalf("newMeilisearchClient: %v", err) + } + + provider := &MeilisearchSearchProvider{ + stateRepo: fakeMeilisearchIndexStateStore{ + state: SearchIndexState{ + ActiveIndexUID: "search-index", + SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false), + }, + pending: 7, + }, + fallback: &PostgresSearchProvider{}, + client: client, + config: MeilisearchProviderConfig{ + BatchSize: meilisearchDefaultBatchSize, + CandidateScanCap: meilisearchDefaultCandidateScanCap, + DeepOffsetLimit: meilisearchDefaultDeepOffsetLimit, + MatchingStrategy: DefaultMeilisearchMatchingStrategy, + Embedder: DefaultMeilisearchEmbedder, + }, + } + + result, err := provider.Search(context.Background(), CatalogSearchRequest{ + Query: "sponge in the sea", + Limit: 10, + }) + if err != nil { + t.Fatalf("Search returned error: %v", err) + } + if requests != 1 { + t.Fatalf("Meilisearch requests = %d, want 1", requests) + } + if gotMethod != http.MethodPost || gotPath != "/indexes/search-index/search" { + t.Fatalf("unexpected Meilisearch request %s %s", gotMethod, gotPath) + } + if result.Provider != SearchProviderMeilisearch { + t.Fatalf("provider = %q, want %q", result.Provider, SearchProviderMeilisearch) + } + if result.FallbackReason != "" { + t.Fatalf("fallback reason = %q, want empty", result.FallbackReason) + } + if result.IndexPendingEvents != 7 { + t.Fatalf("IndexPendingEvents = %d, want 7", result.IndexPendingEvents) + } +} + func TestMeilisearchProviderFallsBackWhenScopedIndexCannotSatisfyRequest(t *testing.T) { provider := &MeilisearchSearchProvider{ config: MeilisearchProviderConfig{ diff --git a/web/src/pages/admin-settings/SearchSettings.tsx b/web/src/pages/admin-settings/SearchSettings.tsx index dcdb67cf..505c4f2b 100644 --- a/web/src/pages/admin-settings/SearchSettings.tsx +++ b/web/src/pages/admin-settings/SearchSettings.tsx @@ -178,9 +178,9 @@ export default function SearchSettings() { form.setValue("catalog.search.meilisearch.semantic_ratio", value)} - hint="0.30" + hint="0.50" disabled={!meiliEnabled} />