haeaeae/silo-server
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #221 from Silo-Server/codex/meilisearch-short-title-search

Browse Source 
fix(search): tighten short meilisearch title queries
This commit is contained in:
Quick
2026-06-26 10:53:11 -04:00
committed by GitHub
parent 4473f8c60b f57c95e813
commit b21c3eea37
9 changed files with 270 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/handlers/catalog.go
+12 -9
View File
@@ -65,12 +65,14 @@ type catalogResponse struct {
// /api/v1/catalog only when a relevance-sorted search actually ran through a
// CatalogSearchProvider. mode/semantic_used reflect POST-downgrade reality
// (a hybrid request that fell back to keyword reports mode="keyword",
// semantic_used=false). fallback_reason is omitted when empty.
// semantic_used=false). fallback_reason and index_pending_updates are omitted
// when empty.
type searchDiagnostics struct {
Provider string `json:"provider"`
Mode string `json:"mode"`
SemanticUsed bool `json:"semantic_used"`
FallbackReason string `json:"fallback_reason,omitempty"`
Provider string `json:"provider"`
Mode string `json:"mode"`
SemanticUsed bool `json:"semantic_used"`
FallbackReason string `json:"fallback_reason,omitempty"`
IndexPendingUpdates int `json:"index_pending_updates,omitempty"`
}
type catalogFiltersResponse struct {
@@ -234,10 +236,11 @@ func (h *CatalogHandler) writeCatalogResponse(w http.ResponseWriter, result *cat
var diag *searchDiagnostics
if result.Provider != "" {
diag = &searchDiagnostics{
Provider: result.Provider,
Mode: result.Mode,
SemanticUsed: result.SemanticUsed,
FallbackReason: result.FallbackReason,
Provider: result.Provider,
Mode: result.Mode,
SemanticUsed: result.SemanticUsed,
FallbackReason: result.FallbackReason,
IndexPendingUpdates: result.IndexPendingEvents,
}
}
internal/api/handlers/catalog_diagnostics_test.go
+7 -3
View File
@@ -69,9 +69,10 @@ func TestWriteCatalogResponse_DiagnosticsKeywordFallback(t *testing.T) {
func TestWriteCatalogResponse_DiagnosticsHybridOmitsFallbackReason(t *testing.T) {
body := decodeCatalogResponse(t, &catalog.CatalogResult{
Provider: catalog.SearchProviderMeilisearch,
Mode: "hybrid",
SemanticUsed: true,
Provider: catalog.SearchProviderMeilisearch,
Mode: "hybrid",
SemanticUsed: true,
IndexPendingEvents: 7,
}, false)
diagRaw, ok := body["search_diagnostics"]
@@ -88,6 +89,9 @@ func TestWriteCatalogResponse_DiagnosticsHybridOmitsFallbackReason(t *testing.T)
if _, ok := diag["fallback_reason"]; ok {
t.Fatalf("fallback_reason should be omitted when empty: %v", diag)
}
if diag["index_pending_updates"].(float64) != 7 {
t.Fatalf("index_pending_updates = %v, want 7", diag["index_pending_updates"])
}
}
func TestWriteCatalogResponse_NoProviderOmitsDiagnostics(t *testing.T) {
internal/catalog/catalog_resolver.go
+19 -16
View File
@@ -27,14 +27,16 @@ type CatalogResult struct {
HasMore bool
TotalExact bool
SnapshotAt time.Time // pagination fence timestamp
// Provider, Mode, SemanticUsed and FallbackReason are per-query search
// diagnostics. They are only populated on the direct-search path (where a
// CatalogSearchProvider actually ran); browse / preview / grouped paths
// leave them zero-valued so the handler omits search_diagnostics.
Provider string
Mode string
SemanticUsed bool
FallbackReason string
// Provider, Mode, SemanticUsed, FallbackReason and IndexPendingEvents are
// per-query search diagnostics. They are only populated on the direct-search
// path (where a CatalogSearchProvider actually ran); browse / preview /
// grouped paths leave them zero-valued so the handler omits
// search_diagnostics.
Provider string
Mode string
SemanticUsed bool
FallbackReason string
IndexPendingEvents int
}
type CatalogFiltersResult struct {
@@ -297,14 +299,15 @@ func (r *CatalogResolver) resolveDirectSearchSource(ctx context.Context, req Cat
}
return &CatalogResult{
Items: result.Items,
Total: result.Total,
HasMore: result.HasMore,
TotalExact: result.TotalExact,
Provider: result.Provider,
Mode: result.Mode,
SemanticUsed: result.SemanticUsed,
FallbackReason: result.FallbackReason,
Items: result.Items,
Total: result.Total,
HasMore: result.HasMore,
TotalExact: result.TotalExact,
Provider: result.Provider,
Mode: result.Mode,
SemanticUsed: result.SemanticUsed,
FallbackReason: result.FallbackReason,
IndexPendingEvents: result.IndexPendingEvents,
}, nil
}
internal/catalog/catalog_resolver_test.go
+8 -4
View File
@@ -403,10 +403,11 @@ func TestResolveDirectSearchSource_PlumbsDiagnostics(t *testing.T) {
{
name: "hybrid survived",
result: &CatalogSearchResult{
Items: []*models.MediaItem{},
Provider: SearchProviderMeilisearch,
Mode: "hybrid",
SemanticUsed: true,
Items: []*models.MediaItem{},
Provider: SearchProviderMeilisearch,
Mode: "hybrid",
SemanticUsed: true,
IndexPendingEvents: 7,
},
},
}
@@ -437,6 +438,9 @@ func TestResolveDirectSearchSource_PlumbsDiagnostics(t *testing.T) {
if got.FallbackReason != tc.result.FallbackReason {
t.Fatalf("FallbackReason = %q, want %q", got.FallbackReason, tc.result.FallbackReason)
}
if got.IndexPendingEvents != tc.result.IndexPendingEvents {
t.Fatalf("IndexPendingEvents = %d, want %d", got.IndexPendingEvents, tc.result.IndexPendingEvents)
}
})
}
}
internal/catalog/search_meilisearch_client.go
+1
View File
@@ -82,6 +82,7 @@ type meilisearchSearchRequest struct {
Limit int `json:"limit"`
Filter string `json:"filter,omitempty"`
AttributesToRetrieve []string `json:"attributesToRetrieve"`
AttributesToSearchOn []string `json:"attributesToSearchOn,omitempty"`
MatchingStrategy string `json:"matchingStrategy,omitempty"`
Vector []float32 `json:"vector,omitempty"`
Hybrid *meilisearchHybridRequest `json:"hybrid,omitempty"`
internal/catalog/search_meilisearch_provider.go
+57 -11
View File
@@ -19,6 +19,9 @@ const (
meilisearchDefaultBatchSize = 100
meilisearchDefaultCandidateScanCap = 1000
meilisearchDefaultDeepOffsetLimit = 500
meilisearchShortHybridMinTerms = 2
meilisearchStrictMatchingTermCount = 2
meilisearchTitleOnlyTermCount = 2
meilisearchCircuitCooldown = 30 * time.Second
meilisearchQueryVectorCacheTTL = 15 * time.Minute
meilisearchQueryVectorCacheMax = 1024
@@ -28,6 +31,18 @@ const (
semanticCapabilityProbeTTL = 5 * time.Minute
)
var meilisearchTitleSearchAttributes = []string{
"title",
"original_title",
"sort_title",
"title_variants",
}
type meilisearchIndexStateStore interface {
GetState(ctx context.Context, provider string) (SearchIndexState, error)
PendingCount(ctx context.Context, provider string) (int, error)
}
type MeilisearchProviderConfig struct {
URL string
APIKey string
@@ -48,7 +63,7 @@ type MeilisearchProviderConfig struct {
type MeilisearchSearchProvider struct {
itemRepo *ItemRepository
stateRepo *SearchIndexEventRepository
stateRepo meilisearchIndexStateStore
fallback *PostgresSearchProvider
client *meilisearchClient
config MeilisearchProviderConfig
@@ -117,9 +132,13 @@ func NewMeilisearchSearchProvider(
if err != nil {
return nil, err
}
var stateStore meilisearchIndexStateStore
if stateRepo != nil {
stateStore = stateRepo
}
return &MeilisearchSearchProvider{
itemRepo: itemRepo,
stateRepo: stateRepo,
stateRepo: stateStore,
fallback: fallback,
client: client,
config: config,
@@ -154,13 +173,9 @@ func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearc
if state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(p.config.Embedder, p.config.IndexTypes, p.config.SemanticEnabled) {
return p.fallbackSearch(ctx, req, "meilisearch index schema mismatch")
}
pending, err := p.stateRepo.PendingCount(ctx, SearchProviderMeilisearch)
if err != nil {
p.markFallback("index pending state unavailable")
return p.fallback.Search(ctx, req)
}
if pending > 0 {
return p.fallbackSearch(ctx, req, "meilisearch index has pending updates")
pending := 0
if count, err := p.stateRepo.PendingCount(ctx, SearchProviderMeilisearch); err == nil && count > 0 {
pending = count
}
result, err := p.searchMeilisearch(ctx, req, state.ActiveIndexUID)
@@ -175,6 +190,7 @@ func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearc
} else {
p.clearFallback()
}
result.IndexPendingEvents = pending
return result, nil
}
@@ -324,7 +340,8 @@ func (p *MeilisearchSearchProvider) buildMeilisearchSearchRequest(ctx context.Co
Query: strings.TrimSpace(req.Query),
Filter: meilisearchTypeFilter(req.ItemTypes),
AttributesToRetrieve: []string{"content_id"},
MatchingStrategy: p.config.MatchingStrategy,
AttributesToSearchOn: p.attributesToSearchOnForRequest(req),
MatchingStrategy: p.matchingStrategyForRequest(req),
}
if !p.shouldUseSemanticSearch(req) {
return searchReq, ""
@@ -356,7 +373,36 @@ func (p *MeilisearchSearchProvider) shouldUseSemanticSearch(req CatalogSearchReq
if p == nil || !p.config.SemanticEnabled {
return false
}
return len(strings.Fields(normalizeCatalogSearchQueryForVector(req.Query))) >= 3
return len(catalogSearchQueryTerms(req.Query)) >= meilisearchShortHybridMinTerms
}
func (p *MeilisearchSearchProvider) matchingStrategyForRequest(req CatalogSearchRequest) string {
strategy := strings.TrimSpace(strings.ToLower(p.config.MatchingStrategy))
if strategy == "" {
strategy = DefaultMeilisearchMatchingStrategy
}
if strategy != "last" {
return strategy
}
// Two-term title searches are easy for Meili's "last" strategy to over-relax
// after typo correction (e.g. "spnge bob" degenerating into "spnge").
if len(catalogSearchQueryTerms(req.Query)) == meilisearchStrictMatchingTermCount {
return "all"
}
return strategy
}
func (p *MeilisearchSearchProvider) attributesToSearchOnForRequest(req CatalogSearchRequest) []string {
if len(catalogSearchQueryTerms(req.Query)) != meilisearchTitleOnlyTermCount {
return nil
}
return append([]string(nil), meilisearchTitleSearchAttributes...)
}
func catalogSearchQueryTerms(query string) []string {
parsed := parseSearchQuery(query)
normalized := normalizeTitleForComparison(firstNonEmptySearchValue(parsed.Text, query))
return strings.Fields(normalized)
}
func (p *MeilisearchSearchProvider) cachedQueryVector(ctx context.Context, query string) ([]float32, error) {
internal/catalog/search_provider.go
+7 -6
View File
@@ -42,7 +42,7 @@ const (
DefaultMeilisearchRebuildBatchSize = 5000
DefaultMeilisearchRebuildQueueDepth = 4
DefaultMeilisearchSemanticEnabled = false
DefaultMeilisearchSemanticRatio = 0.30
DefaultMeilisearchSemanticRatio = 0.50
DefaultMeilisearchEmbedder = "silo_recommendations"
MaxMeilisearchSyncBatchSize = 10000
@@ -62,11 +62,12 @@ type CatalogSearchRequest struct {
}
type CatalogSearchResult struct {
Items []*models.MediaItem
Total int
HasMore bool
TotalExact bool
Provider string
Items []*models.MediaItem
Total int
HasMore bool
TotalExact bool
Provider string
IndexPendingEvents int
// Mode reports which retrieval path actually served this result —
// "keyword" or "hybrid". For Meilisearch it reflects POST-downgrade
// reality (a hybrid request that fell back to keyword reports "keyword").
internal/catalog/search_provider_test.go
+157 -2
View File
@@ -4,9 +4,12 @@ import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"reflect"
"strings"
"testing"
"time"
)
func TestCatalogSearchSettingsFromMapParsesMeilisearchTuning(t *testing.T) {
@@ -161,6 +164,9 @@ func TestMeilisearchSearchRequestBuildsKeywordOnlyByDefault(t *testing.T) {
if req.Vector != nil || req.Hybrid != nil {
t.Fatalf("keyword request should not include vector or hybrid: %#v", req)
}
if req.MatchingStrategy != DefaultMeilisearchMatchingStrategy {
t.Fatalf("matching strategy = %q, want %q", req.MatchingStrategy, DefaultMeilisearchMatchingStrategy)
}
if req.Filter != `type = "movie"` {
t.Fatalf("filter = %q, want movie filter", req.Filter)
}
@@ -192,6 +198,12 @@ func TestMeilisearchSearchRequestBuildsHybridWhenSemanticEnabled(t *testing.T) {
if req.Hybrid.SemanticRatio != 0.4 {
t.Fatalf("semantic ratio = %v, want 0.4", req.Hybrid.SemanticRatio)
}
if req.MatchingStrategy != DefaultMeilisearchMatchingStrategy {
t.Fatalf("long semantic query matching strategy = %q, want %q", req.MatchingStrategy, DefaultMeilisearchMatchingStrategy)
}
if req.AttributesToSearchOn != nil {
t.Fatalf("long semantic query should search all attributes, got %#v", req.AttributesToSearchOn)
}
if len(req.Vector) != 3072 {
t.Fatalf("vector len = %d, want 3072", len(req.Vector))
}
@@ -302,7 +314,71 @@ func TestMeilisearchSearchRequestBuildsHybridForApproximateInteractiveSearch(t *
}
}
func TestMeilisearchSearchRequestSkipsHybridForShortTitleSearch(t *testing.T) {
func TestMeilisearchSearchRequestBuildsHybridAndStrictMatchingForTwoTermSearch(t *testing.T) {
vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}}
provider := &MeilisearchSearchProvider{
config: MeilisearchProviderConfig{
MatchingStrategy: DefaultMeilisearchMatchingStrategy,
SemanticEnabled: true,
SemanticRatio: 0.3,
Embedder: "silo_recommendations",
Vectorizer: vectorizer,
},
}
req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{
Query: "spnge bob",
})
if fallback != "" {
t.Fatalf("fallback = %q, want empty", fallback)
}
if req.Vector == nil || req.Hybrid == nil {
t.Fatalf("two-term title search should include hybrid search: %#v", req)
}
if req.MatchingStrategy != "all" {
t.Fatalf("two-term title search matching strategy = %q, want all", req.MatchingStrategy)
}
if !reflect.DeepEqual(req.AttributesToSearchOn, meilisearchTitleSearchAttributes) {
t.Fatalf("two-term title search attributes = %#v, want %#v", req.AttributesToSearchOn, meilisearchTitleSearchAttributes)
}
if vectorizer.calls != 1 || vectorizer.lastQuery != "spnge bob" {
t.Fatalf("vectorizer calls/query = %d/%q", vectorizer.calls, vectorizer.lastQuery)
}
}
func TestMeilisearchSearchRequestUsesStrictMatchingWhenTwoTermHybridNotReady(t *testing.T) {
vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}}
provider := &MeilisearchSearchProvider{
config: MeilisearchProviderConfig{
MatchingStrategy: DefaultMeilisearchMatchingStrategy,
SemanticEnabled: true,
SemanticRatio: 0.3,
Embedder: "silo_recommendations",
Vectorizer: vectorizer,
Coverage: fakeCoverageGate{ready: false, reason: `type "movie" coverage 40% below threshold`},
},
}
req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{
Query: "spnge bob",
ItemTypes: []string{"movie"},
})
if req.Vector != nil || req.Hybrid != nil {
t.Fatalf("not-ready coverage should stay keyword-only: %#v", req)
}
if req.MatchingStrategy != "all" {
t.Fatalf("not-ready two-term search matching strategy = %q, want all", req.MatchingStrategy)
}
if !reflect.DeepEqual(req.AttributesToSearchOn, meilisearchTitleSearchAttributes) {
t.Fatalf("not-ready two-term search attributes = %#v, want %#v", req.AttributesToSearchOn, meilisearchTitleSearchAttributes)
}
if fallback != `semantic_not_ready: type "movie" coverage 40% below threshold` {
t.Fatalf("fallback = %q, want semantic_not_ready diagnostic", fallback)
}
if vectorizer.calls != 0 {
t.Fatalf("not-ready coverage should not call vectorizer, calls = %d", vectorizer.calls)
}
}
func TestMeilisearchSearchRequestSkipsHybridForSingleTermTitleSearch(t *testing.T) {
vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}}
provider := &MeilisearchSearchProvider{
config: MeilisearchProviderConfig{
@@ -313,7 +389,7 @@ func TestMeilisearchSearchRequestSkipsHybridForShortTitleSearch(t *testing.T) {
Vectorizer: vectorizer,
},
}
for _, query := range []string{"sponge", "spongebob square"} {
for _, query := range []string{"sponge", "spongebob"} {
req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{
Query: query,
})
@@ -323,6 +399,12 @@ func TestMeilisearchSearchRequestSkipsHybridForShortTitleSearch(t *testing.T) {
if req.Vector != nil || req.Hybrid != nil {
t.Fatalf("short title search %q should stay keyword-only: %#v", query, req)
}
if req.MatchingStrategy != DefaultMeilisearchMatchingStrategy {
t.Fatalf("single-term search %q matching strategy = %q, want %q", query, req.MatchingStrategy, DefaultMeilisearchMatchingStrategy)
}
if req.AttributesToSearchOn != nil {
t.Fatalf("single-term search %q should search all attributes, got %#v", query, req.AttributesToSearchOn)
}
}
if vectorizer.calls != 0 {
t.Fatalf("short title searches should not call vectorizer, calls = %d", vectorizer.calls)
@@ -363,6 +445,79 @@ func TestMeilisearchCircuitTripsOnServerAndDecodeErrors(t *testing.T) {
}
}
type fakeMeilisearchIndexStateStore struct {
state SearchIndexState
pending int
}
func (f fakeMeilisearchIndexStateStore) GetState(context.Context, string) (SearchIndexState, error) {
return f.state, nil
}
func (f fakeMeilisearchIndexStateStore) PendingCount(context.Context, string) (int, error) {
return f.pending, nil
}
func TestMeilisearchProviderUsesActiveIndexWhenPendingUpdatesExist(t *testing.T) {
requests := 0
var gotMethod, gotPath string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
requests++
gotMethod = r.Method
gotPath = r.URL.Path
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"hits":[],"estimatedTotalHits":0}`))
}))
defer server.Close()
client, err := newMeilisearchClient(server.URL, "", time.Second)
if err != nil {
t.Fatalf("newMeilisearchClient: %v", err)
}
provider := &MeilisearchSearchProvider{
stateRepo: fakeMeilisearchIndexStateStore{
state: SearchIndexState{
ActiveIndexUID: "search-index",
SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false),
},
pending: 7,
},
fallback: &PostgresSearchProvider{},
client: client,
config: MeilisearchProviderConfig{
BatchSize: meilisearchDefaultBatchSize,
CandidateScanCap: meilisearchDefaultCandidateScanCap,
DeepOffsetLimit: meilisearchDefaultDeepOffsetLimit,
MatchingStrategy: DefaultMeilisearchMatchingStrategy,
Embedder: DefaultMeilisearchEmbedder,
},
}
result, err := provider.Search(context.Background(), CatalogSearchRequest{
Query: "sponge in the sea",
Limit: 10,
})
if err != nil {
t.Fatalf("Search returned error: %v", err)
}
if requests != 1 {
t.Fatalf("Meilisearch requests = %d, want 1", requests)
}
if gotMethod != http.MethodPost || gotPath != "/indexes/search-index/search" {
t.Fatalf("unexpected Meilisearch request %s %s", gotMethod, gotPath)
}
if result.Provider != SearchProviderMeilisearch {
t.Fatalf("provider = %q, want %q", result.Provider, SearchProviderMeilisearch)
}
if result.FallbackReason != "" {
t.Fatalf("fallback reason = %q, want empty", result.FallbackReason)
}
if result.IndexPendingEvents != 7 {
t.Fatalf("IndexPendingEvents = %d, want 7", result.IndexPendingEvents)
}
}
func TestMeilisearchProviderFallsBackWhenScopedIndexCannotSatisfyRequest(t *testing.T) {
provider := &MeilisearchSearchProvider{
config: MeilisearchProviderConfig{
web/src/pages/admin-settings/SearchSettings.tsx
+2 -2
View File
@@ -178,9 +178,9 @@ export default function SearchSettings() {
<SettingField
label="Semantic Ratio"
type="number"
value={form.getValue("catalog.search.meilisearch.semantic_ratio") || "0.30"}
value={form.getValue("catalog.search.meilisearch.semantic_ratio") || "0.50"}
onChange={(value) => form.setValue("catalog.search.meilisearch.semantic_ratio", value)}
hint="0.30"
hint="0.50"
disabled={!meiliEnabled}
/>
<SettingField