From e96a8a0cf8b0058684dd923bb432947e6e337f78 Mon Sep 17 00:00:00 2001 From: RXWatcher Date: Thu, 9 Jul 2026 15:55:51 +0200 Subject: [PATCH] =?UTF-8?q?feat(search):=20binary-quantized=20embedder=20v?= =?UTF-8?q?ectors=20=E2=80=94=20optional,=20default-on=20for=20fresh=20ins?= =?UTF-8?q?talls=20(#351)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(search): binary quantization setting for the Meilisearch embedder New server setting catalog.search.meilisearch.binary_quantized (default false) threads into the embedder index settings ("binaryQuantized": true) and into the schema-version hash, so flipping it closes the sync gate and mandates a rebuild in both directions — Meilisearch cannot de/re-quantize an index in place. With 3072-dimensional embeddings this cuts vector storage ~32x (≈12KB → 384B per document), keeping the whole vector store in page cache: rebuilds and hybrid queries get sharply cheaper. Hybrid search (keyword + semantic) cushions the small relevance cost of sign-only vectors. The hash token is appended only when the flag is set, so indexes built before this change keep their schema version while it stays off. Co-Authored-By: Claude Fable 5 * feat(search): binary quantization default-on for fresh installs + admin toggle - Migration seeds catalog.search.meilisearch.binary_quantized=true only when no active catalog search index exists. Existing deployments stay unset (= off): flipping quantization changes the index schema-version identity, which closes the incremental-sync gate until a full rebuild runs — that must never happen implicitly on upgrade. Fresh installs have no index yet, so their first rebuild simply starts quantized. - Search settings page gains the toggle with an explicit "requires a full index rebuild" warning, a status row, and settings- search keywords. Prod benchmark (607.9k docs, 3072-dim vectors, N=10 medians, replicated): hybrid 0.5 unchanged (7.5ms float vs 8.0ms quantized, within ±2ms keyword-control jitter); pure semantic 9ms → 4ms; on-disk index 18G → 8.3G; rebuild duration unchanged. Co-Authored-By: Claude Fable 5 * fix(search): address review feedback on binary-quantized embedder - Add catalog.search.meilisearch.binary_quantized to the restart-required registry. The provider freezes BinaryQuantized into MeilisearchProviderConfig at construction, so without this a toggle-then-rebuild in the same process builds a quantized schema while the live provider still compares against the old value and falls back until restart (Codex P2). - Validate binary_quantized in HandleUpdateSetting, mirroring semantic_enabled. A raw API write of a non-bool previously persisted unnormalized, then failed CatalogSearchSettingsFromMap on load and silently reverted the entire search config to Postgres defaults. - Gate the binary_quantized token in the schema-version identity on semanticEnabled: with semantic off the index has no embedders, so the flag has no on-index effect and must not force a pointless rebuild. Stays byte-identical to a pre-flag index. Covered by a new test. - Clarify the seed migration comment (guard is "no active index", which also covers Meilisearch-configured-but-never-indexed deployments) and the UI hint (~30x smaller raw vectors, index roughly halves; only applies with semantic). Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: rxwatcher Co-authored-by: Claude Fable 5 --- internal/api/handlers/admin.go | 7 +++ internal/catalog/search_indexer.go | 24 ++++----- .../catalog/search_meilisearch_provider.go | 4 +- internal/catalog/search_provider.go | 46 ++++++++++++---- internal/catalog/search_provider_test.go | 54 +++++++++++++++---- internal/catalog/search_service.go | 4 +- internal/config/restart_keys.go | 1 + ...ed_binary_quantized_for_fresh_installs.sql | 25 +++++++++ web/src/hooks/queries/admin/settings.ts | 1 + web/src/lib/adminSettingsSearch.ts | 2 + .../pages/admin-settings/SearchSettings.tsx | 15 ++++++ 11 files changed, 148 insertions(+), 35 deletions(-) create mode 100644 migrations/sql/20260709054500_seed_binary_quantized_for_fresh_installs.sql diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go index 59b283a9..0eaa8013 100644 --- a/internal/api/handlers/admin.go +++ b/internal/api/handlers/admin.go @@ -2334,6 +2334,13 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques return } req.Value = embedder + case catalog.SearchSettingMeilisearchBinaryQuantized: + enabled, err := strconv.ParseBool(strings.TrimSpace(req.Value)) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "catalog.search.meilisearch.binary_quantized must be true or false") + return + } + req.Value = strconv.FormatBool(enabled) } if err := h.SettingsRepo.Set(r.Context(), key, req.Value); err != nil { diff --git a/internal/catalog/search_indexer.go b/internal/catalog/search_indexer.go index ced32824..fae05318 100644 --- a/internal/catalog/search_indexer.go +++ b/internal/catalog/search_indexer.go @@ -89,7 +89,7 @@ func (i *CatalogSearchIndexer) ShouldSyncRun(ctx context.Context) (bool, error) if err != nil { return false, err } - if state.ActiveIndexUID == "" || state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled) { + if state.ActiveIndexUID == "" || state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized) { return false, nil } pending, err := i.events.PendingCount(ctx, SearchProviderMeilisearch) @@ -131,7 +131,7 @@ func (i *CatalogSearchIndexer) SyncOutbox(ctx context.Context, progress SearchIn if err != nil { return stats, err } - if state.ActiveIndexUID == "" || state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled) { + if state.ActiveIndexUID == "" || state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized) { stats.Skipped = true stats.Reason = "active search index is missing or stale; run rebuild_catalog_search_index" setSearchIndexTaskResult(progress, stats) @@ -166,7 +166,7 @@ func (i *CatalogSearchIndexer) SyncOutbox(ctx context.Context, progress SearchIn upsertIDs, deleteIDs := coalesceSearchIndexEvents(events) reportSearchIndexProgress(progress, 20, "Building changed catalog search documents") - docs, err := i.LoadDocumentsByIDs(ctx, upsertIDs, settings.IndexTypes, settings.Embedder, settings.SemanticEnabled) + docs, err := i.LoadDocumentsByIDs(ctx, upsertIDs, settings.IndexTypes, settings.Embedder, settings.SemanticEnabled, settings.BinaryQuantized) if err != nil { _ = i.events.MarkFailed(ctx, ids, err) return stats, err @@ -283,7 +283,7 @@ func (i *CatalogSearchIndexer) Rebuild(ctx context.Context, progress SearchIndex if err := client.WaitTask(ctx, taskID); err != nil { return stats, err } - taskID, err = client.UpdateSettings(ctx, buildIndexUID, catalogSearchMeilisearchSettings(settings.Embedder, settings.SemanticEnabled)) + taskID, err = client.UpdateSettings(ctx, buildIndexUID, catalogSearchMeilisearchSettings(settings.Embedder, settings.SemanticEnabled, settings.BinaryQuantized)) if err != nil { return stats, err } @@ -294,7 +294,7 @@ func (i *CatalogSearchIndexer) Rebuild(ctx context.Context, progress SearchIndex lastID := "" queuedTasks := make([]queuedMeilisearchTask, 0, settings.RebuildQueueDepth) for { - docs, err := i.LoadDocumentsAfter(ctx, lastID, settings.RebuildBatchSize, settings.IndexTypes, settings.Embedder, settings.SemanticEnabled) + docs, err := i.LoadDocumentsAfter(ctx, lastID, settings.RebuildBatchSize, settings.IndexTypes, settings.Embedder, settings.SemanticEnabled, settings.BinaryQuantized) if err != nil { return stats, err } @@ -339,7 +339,7 @@ func (i *CatalogSearchIndexer) Rebuild(ctx context.Context, progress SearchIndex // active index as idempotent upserts on the next sync. The reverse order // would mark events processed while the old index is still active, losing // those changes from the served index until the next rebuild. - if err := i.events.UpdateStateAfterRebuild(ctx, SearchProviderMeilisearch, buildIndexUID, catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled), docCount, rebuildEventHighWater); err != nil { + if err := i.events.UpdateStateAfterRebuild(ctx, SearchProviderMeilisearch, buildIndexUID, catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized), docCount, rebuildEventHighWater); err != nil { return stats, err } if err := i.events.MarkProcessedThrough(ctx, SearchProviderMeilisearch, rebuildEventHighWater); err != nil { @@ -532,7 +532,7 @@ func (i *CatalogSearchIndexer) CheckConnection(ctx context.Context, settings Cat return client.Health(ctx) } -func catalogSearchMeilisearchSettings(embedder string, semanticEnabled bool) map[string]any { +func catalogSearchMeilisearchSettings(embedder string, semanticEnabled, binaryQuantized bool) map[string]any { settings := map[string]any{ "displayedAttributes": []string{"content_id", "type"}, "filterableAttributes": []string{"type"}, @@ -558,7 +558,7 @@ func catalogSearchMeilisearchSettings(embedder string, semanticEnabled bool) map if err != nil { embedder = DefaultMeilisearchEmbedder } - settings["embedders"] = catalogSearchMeilisearchEmbedderSettings(embedder) + settings["embedders"] = catalogSearchMeilisearchEmbedderSettings(embedder, binaryQuantized) } return settings } @@ -627,7 +627,7 @@ type catalogSearchDocument struct { Vectors map[string][]float32 `json:"_vectors,omitempty"` } -func (i *CatalogSearchIndexer) LoadDocumentsAfter(ctx context.Context, afterContentID string, limit int, itemTypes []string, embedder string, semanticEnabled bool) ([]catalogSearchDocument, error) { +func (i *CatalogSearchIndexer) LoadDocumentsAfter(ctx context.Context, afterContentID string, limit int, itemTypes []string, embedder string, semanticEnabled, binaryQuantized bool) ([]catalogSearchDocument, error) { if i == nil || i.pool == nil || limit <= 0 { return nil, nil } @@ -653,14 +653,14 @@ func (i *CatalogSearchIndexer) LoadDocumentsAfter(ctx context.Context, afterCont if err != nil { return nil, err } - setCatalogSearchDocumentSchemaVersion(docs, catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, semanticEnabled)) + setCatalogSearchDocumentSchemaVersion(docs, catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, semanticEnabled, binaryQuantized)) if err := i.attachDocumentVectors(ctx, docs, embedder, semanticEnabled); err != nil { return nil, err } return docs, nil } -func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentIDs []string, itemTypes []string, embedder string, semanticEnabled bool) ([]catalogSearchDocument, error) { +func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentIDs []string, itemTypes []string, embedder string, semanticEnabled, binaryQuantized bool) ([]catalogSearchDocument, error) { contentIDs = compactNonEmptyStrings(contentIDs) if i == nil || i.pool == nil || len(contentIDs) == 0 { return nil, nil @@ -686,7 +686,7 @@ func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentID if err != nil { return nil, err } - setCatalogSearchDocumentSchemaVersion(docs, catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, semanticEnabled)) + setCatalogSearchDocumentSchemaVersion(docs, catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, semanticEnabled, binaryQuantized)) if err := i.attachDocumentVectors(ctx, docs, embedder, semanticEnabled); err != nil { return nil, err } diff --git a/internal/catalog/search_meilisearch_provider.go b/internal/catalog/search_meilisearch_provider.go index 06a3598c..3b61c308 100644 --- a/internal/catalog/search_meilisearch_provider.go +++ b/internal/catalog/search_meilisearch_provider.go @@ -58,6 +58,7 @@ type MeilisearchProviderConfig struct { MatchingStrategy string IndexTypes []string SemanticEnabled bool + BinaryQuantized bool SemanticRatio float64 Embedder string Vectorizer CatalogSearchQueryVectorizer @@ -173,7 +174,7 @@ func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearc if strings.TrimSpace(state.ActiveIndexUID) == "" { return p.fallbackSearch(ctx, req, "meilisearch index has not been built") } - if state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(p.config.Embedder, p.config.IndexTypes, p.config.SemanticEnabled) { + if state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(p.config.Embedder, p.config.IndexTypes, p.config.SemanticEnabled, p.config.BinaryQuantized) { return p.fallbackSearch(ctx, req, "meilisearch index schema mismatch") } @@ -591,6 +592,7 @@ func (p *MeilisearchSearchProvider) Status() CatalogSearchMeiliStatus { MatchingStrategy: p.config.MatchingStrategy, IndexTypes: p.config.IndexTypes, SemanticEnabled: p.config.SemanticEnabled, + BinaryQuantized: p.config.BinaryQuantized, SemanticRatio: p.config.SemanticRatio, Embedder: p.config.Embedder, } diff --git a/internal/catalog/search_provider.go b/internal/catalog/search_provider.go index 5a79f7c6..97fbe11d 100644 --- a/internal/catalog/search_provider.go +++ b/internal/catalog/search_provider.go @@ -32,6 +32,7 @@ const ( SearchSettingMeilisearchSemanticEnabled = "catalog.search.meilisearch.semantic_enabled" SearchSettingMeilisearchSemanticRatio = "catalog.search.meilisearch.semantic_ratio" SearchSettingMeilisearchEmbedder = "catalog.search.meilisearch.embedder" + SearchSettingMeilisearchBinaryQuantized = "catalog.search.meilisearch.binary_quantized" DefaultMeilisearchIndex = "silo_media_items" DefaultMeilisearchTimeoutMS = 800 @@ -44,6 +45,7 @@ const ( DefaultMeilisearchSemanticEnabled = false DefaultMeilisearchSemanticRatio = 0.50 DefaultMeilisearchEmbedder = "silo_recommendations" + DefaultMeilisearchBinaryQuantized = false MaxMeilisearchSyncBatchSize = 10000 MaxMeilisearchRebuildBatchSize = 25000 @@ -147,6 +149,7 @@ type CatalogSearchSettings struct { SemanticEnabled bool SemanticRatio float64 Embedder string + BinaryQuantized bool } func DefaultCatalogSearchSettings() CatalogSearchSettings { @@ -161,6 +164,7 @@ func DefaultCatalogSearchSettings() CatalogSearchSettings { SemanticEnabled: DefaultMeilisearchSemanticEnabled, SemanticRatio: DefaultMeilisearchSemanticRatio, Embedder: DefaultMeilisearchEmbedder, + BinaryQuantized: DefaultMeilisearchBinaryQuantized, } } @@ -239,6 +243,13 @@ func CatalogSearchSettingsFromMap(values map[string]string) (CatalogSearchSettin } settings.SemanticEnabled = enabled } + if raw := strings.TrimSpace(values[SearchSettingMeilisearchBinaryQuantized]); raw != "" { + enabled, err := strconv.ParseBool(raw) + if err != nil { + return settings, fmt.Errorf("%s must be true or false", SearchSettingMeilisearchBinaryQuantized) + } + settings.BinaryQuantized = enabled + } if raw := strings.TrimSpace(values[SearchSettingMeilisearchSemanticRatio]); raw != "" { ratio, err := strconv.ParseFloat(raw, 64) if err != nil || math.IsNaN(ratio) || ratio < 0 || ratio > 1 { @@ -399,6 +410,7 @@ type CatalogSearchMeiliStatus struct { MatchingStrategy string `json:"matching_strategy"` IndexTypes []string `json:"index_types,omitempty"` SemanticEnabled bool `json:"semantic_enabled"` + BinaryQuantized bool `json:"binary_quantized"` SemanticRatio float64 `json:"semantic_ratio"` Embedder string `json:"embedder"` } @@ -429,13 +441,18 @@ type CatalogSearchStatusProvider interface { Status(ctx context.Context) CatalogSearchRuntimeStatus } -func catalogSearchMeilisearchEmbedderSettings(embedder string) map[string]any { - return map[string]any{ - embedder: map[string]any{ - "source": "userProvided", - "dimensions": embeddingvectors.CanonicalDimensions, - }, +func catalogSearchMeilisearchEmbedderSettings(embedder string, binaryQuantized bool) map[string]any { + cfg := map[string]any{ + "source": "userProvided", + "dimensions": embeddingvectors.CanonicalDimensions, } + if binaryQuantized { + // One-way per index: Meilisearch cannot de-quantize an index in + // place, so turning this off again requires a rebuild, exactly like + // turning it on. The schema-version hash enforces both transitions. + cfg["binaryQuantized"] = true + } + return map[string]any{embedder: cfg} } // catalogSearchMeilisearchSchemaVersion derives the expected index schema @@ -447,19 +464,28 @@ func catalogSearchMeilisearchEmbedderSettings(embedder string) map[string]any { // rebuilt rather than serving silently degraded hybrid ranking. A mismatch forces // a rebuild (SyncOutbox skips, the provider falls back to keyword) and surfaces as // an ExpectedSchemaVersion divergence in the admin status. -func catalogSearchMeilisearchSchemaVersion(embedder string, itemTypes []string, semanticEnabled bool) int { +func catalogSearchMeilisearchSchemaVersion(embedder string, itemTypes []string, semanticEnabled, binaryQuantized bool) int { embedder, err := NormalizeCatalogSearchEmbedderName(embedder) if err != nil { embedder = DefaultMeilisearchEmbedder } - h := fnv.New32a() - _, _ = fmt.Fprintf( - h, + identity := fmt.Sprintf( "embedder=%s;dimensions=%d;index_types=%s;semantic=%t", embedder, embeddingvectors.CanonicalDimensions, strings.Join(normalizeCatalogSearchItemTypes(itemTypes), ","), semanticEnabled, ) + // Gated on semanticEnabled because binary quantization only affects the + // embedder settings (catalogSearchMeilisearchSettings), which are omitted + // entirely when semantic search is off — so with no vectors on the index + // the flag has no on-index effect and must not force a rebuild. Appended + // only when both are set, so pre-existing indexes built before this flag + // existed (and any index with semantic off) keep their schema version. + if semanticEnabled && binaryQuantized { + identity += ";binary_quantized=true" + } + h := fnv.New32a() + _, _ = h.Write([]byte(identity)) return SearchMeilisearchSchemaVersion*1_000_000 + int(h.Sum32()%1_000_000) } diff --git a/internal/catalog/search_provider_test.go b/internal/catalog/search_provider_test.go index 612892bd..ee74253e 100644 --- a/internal/catalog/search_provider_test.go +++ b/internal/catalog/search_provider_test.go @@ -128,7 +128,7 @@ func TestCatalogSearchDocumentVectorsUseEmbedderAndOptOutMissing(t *testing.T) { } func TestCatalogSearchMeilisearchSettingsOmitEmbeddersWhenSemanticDisabled(t *testing.T) { - settings := catalogSearchMeilisearchSettings("silo_recommendations", false) + settings := catalogSearchMeilisearchSettings("silo_recommendations", false, false) if _, ok := settings["embedders"]; ok { t.Fatalf("semantic-disabled settings should not include embedders: %#v", settings["embedders"]) @@ -139,7 +139,7 @@ func TestCatalogSearchMeilisearchSettingsOmitEmbeddersWhenSemanticDisabled(t *te } func TestCatalogSearchMeilisearchSettingsIncludeEmbeddersWhenSemanticEnabled(t *testing.T) { - settings := catalogSearchMeilisearchSettings("custom_embedder", true) + settings := catalogSearchMeilisearchSettings("custom_embedder", true, false) embedders, ok := settings["embedders"].(map[string]any) if !ok { @@ -512,7 +512,7 @@ func TestMeilisearchProviderCachesIndexStateAcrossRequests(t *testing.T) { store := &countingMeilisearchIndexStateStore{ state: SearchIndexState{ ActiveIndexUID: "search-index", - SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false), + SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false), }, pending: 3, } @@ -557,7 +557,7 @@ func TestMeilisearchProviderInvalidatesStateCacheOnSearchFailure(t *testing.T) { store := &countingMeilisearchIndexStateStore{ state: SearchIndexState{ ActiveIndexUID: "search-index", - SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false), + SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false), }, } provider := &MeilisearchSearchProvider{ @@ -607,7 +607,7 @@ func TestMeilisearchProviderUsesActiveIndexWhenPendingUpdatesExist(t *testing.T) stateRepo: fakeMeilisearchIndexStateStore{ state: SearchIndexState{ ActiveIndexUID: "search-index", - SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false), + SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false), }, pending: 7, }, @@ -683,8 +683,8 @@ func TestNormalizeCatalogSearchIndexTypesValueFormatsCanonicalList(t *testing.T) } func TestMeilisearchSchemaVersionChangesWithEmbedder(t *testing.T) { - defaultVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false) - customVersion := catalogSearchMeilisearchSchemaVersion("custom_embedder", nil, false) + defaultVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false) + customVersion := catalogSearchMeilisearchSchemaVersion("custom_embedder", nil, false, false) if defaultVersion == customVersion { t.Fatal("schema version should change when embedder changes") } @@ -694,8 +694,8 @@ func TestMeilisearchSchemaVersionChangesWithEmbedder(t *testing.T) { } func TestMeilisearchSchemaVersionChangesWithIndexTypes(t *testing.T) { - allTypesVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false) - videoOnlyVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, []string{"movie", "series"}, false) + allTypesVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false) + videoOnlyVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, []string{"movie", "series"}, false, false) if allTypesVersion == videoOnlyVersion { t.Fatal("schema version should change when indexed media scope changes") } @@ -707,8 +707,8 @@ func TestMeilisearchSchemaVersionChangesWithSemanticEnabled(t *testing.T) { // rebuild. Without this, enabling semantic without a rebuild leaves indexed // documents missing _vectors while the Postgres coverage gate reports ready, // silently degrading hybrid ranking. - disabledVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false) - enabledVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, true) + disabledVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false) + enabledVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, true, false) if disabledVersion == enabledVersion { t.Fatal("schema version should change when semantic search is toggled") } @@ -760,3 +760,35 @@ type fakeModelVectorizer struct { func (f *fakeModelVectorizer) ActiveEmbeddingModel(_ context.Context) (string, error) { return f.model, nil } + +func TestCatalogSearchMeilisearchSchemaVersionBinaryQuantized(t *testing.T) { + plain := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, true, false) + quantized := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, true, true) + if plain == quantized { + t.Fatal("binary quantization must change the schema version") + } +} + +func TestCatalogSearchMeilisearchSchemaVersionBinaryQuantizedIgnoredWhenSemanticDisabled(t *testing.T) { + // With semantic search off, the index carries no embedders (see + // catalogSearchMeilisearchSettings), so binary quantization has no on-index + // effect and must not shift the schema version — otherwise toggling it would + // force a pointless full rebuild of a vector-less index. It must also stay + // byte-identical to a pre-flag index so upgrades don't spuriously invalidate. + off := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false) + on := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, true) + if off != on { + t.Fatal("binary quantization must not change the schema version when semantic search is disabled") + } +} + +func TestCatalogSearchMeilisearchEmbedderSettingsBinaryQuantized(t *testing.T) { + off := catalogSearchMeilisearchEmbedderSettings("e", false)["e"].(map[string]any) + if _, ok := off["binaryQuantized"]; ok { + t.Fatal("binaryQuantized key must be absent when off (preserves legacy index settings shape)") + } + on := catalogSearchMeilisearchEmbedderSettings("e", true)["e"].(map[string]any) + if on["binaryQuantized"] != true { + t.Fatal("binaryQuantized must be set when enabled") + } +} diff --git a/internal/catalog/search_service.go b/internal/catalog/search_service.go index 1525c5ad..77ee064d 100644 --- a/internal/catalog/search_service.go +++ b/internal/catalog/search_service.go @@ -75,6 +75,7 @@ func NewCatalogSearchServiceFromSettings( MatchingStrategy: settings.MatchingStrategy, IndexTypes: settings.IndexTypes, SemanticEnabled: settings.SemanticEnabled, + BinaryQuantized: settings.BinaryQuantized, SemanticRatio: settings.SemanticRatio, Embedder: settings.Embedder, Vectorizer: queryVectorizer, @@ -144,11 +145,12 @@ func (s *CatalogSearchService) Status(ctx context.Context) CatalogSearchRuntimeS MatchingStrategy: settings.MatchingStrategy, IndexTypes: settings.IndexTypes, SemanticEnabled: settings.SemanticEnabled, + BinaryQuantized: settings.BinaryQuantized, SemanticRatio: settings.SemanticRatio, Embedder: settings.Embedder, }, Index: CatalogSearchIndexStateStatus{ - ExpectedSchemaVersion: catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled), + ExpectedSchemaVersion: catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized), }, Tasks: []CatalogSearchTaskLink{ {Key: "sync_catalog_search_index", Name: "Sync Catalog Search Index", Href: "/admin/tasks/sync_catalog_search_index"}, diff --git a/internal/config/restart_keys.go b/internal/config/restart_keys.go index 1f388fc1..5aa7a191 100644 --- a/internal/config/restart_keys.go +++ b/internal/config/restart_keys.go @@ -84,6 +84,7 @@ var restartRequiredKeys = map[string]bool{ "catalog.search.meilisearch.semantic_enabled": true, "catalog.search.meilisearch.semantic_ratio": true, "catalog.search.meilisearch.embedder": true, + "catalog.search.meilisearch.binary_quantized": true, } // restartRequiredPrefixes covers whole namespaces of infrastructure settings: diff --git a/migrations/sql/20260709054500_seed_binary_quantized_for_fresh_installs.sql b/migrations/sql/20260709054500_seed_binary_quantized_for_fresh_installs.sql new file mode 100644 index 00000000..2845c9f5 --- /dev/null +++ b/migrations/sql/20260709054500_seed_binary_quantized_for_fresh_installs.sql @@ -0,0 +1,25 @@ +-- +goose Up +-- +goose StatementBegin +-- Binary quantization defaults ON for any install with no active catalog +-- search index — fresh installs, and also deployments that configured +-- Meilisearch but never built an index. Deployments with an active index are +-- left unset (= off): flipping quantization changes the index schema-version +-- identity, which closes the incremental-sync gate until a full rebuild runs +-- — that must never happen implicitly on upgrade. Installs with no index yet +-- have no gate to close, so their first rebuild simply starts quantized. +INSERT INTO server_settings (key, value) +SELECT 'catalog.search.meilisearch.binary_quantized', 'true' +WHERE NOT EXISTS ( + SELECT 1 FROM server_settings + WHERE key = 'catalog.search.meilisearch.binary_quantized') + AND NOT EXISTS ( + SELECT 1 FROM catalog_search_index_state + WHERE active_index_uid IS NOT NULL AND active_index_uid <> ''); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +-- Intentionally a no-op: deleting the setting could silently change the +-- schema-version identity of an index that was built with quantization. +SELECT 1; +-- +goose StatementEnd diff --git a/web/src/hooks/queries/admin/settings.ts b/web/src/hooks/queries/admin/settings.ts index b185a329..72979049 100644 --- a/web/src/hooks/queries/admin/settings.ts +++ b/web/src/hooks/queries/admin/settings.ts @@ -32,6 +32,7 @@ export interface CatalogSearchStatus { matching_strategy: string; index_types?: string[]; semantic_enabled: boolean; + binary_quantized: boolean; semantic_ratio: number; embedder: string; }; diff --git a/web/src/lib/adminSettingsSearch.ts b/web/src/lib/adminSettingsSearch.ts index 2668479a..7c9672ef 100644 --- a/web/src/lib/adminSettingsSearch.ts +++ b/web/src/lib/adminSettingsSearch.ts @@ -143,6 +143,8 @@ export const ADMIN_SETTINGS_GROUPS: AdminSettingsSearchGroup[] = [ "vectors", "embeddings", "embedder", + "binary quantization", + "quantized", ], settings: settingIndex( "Preferred Provider", diff --git a/web/src/pages/admin-settings/SearchSettings.tsx b/web/src/pages/admin-settings/SearchSettings.tsx index d80faf58..0695241b 100644 --- a/web/src/pages/admin-settings/SearchSettings.tsx +++ b/web/src/pages/admin-settings/SearchSettings.tsx @@ -28,6 +28,7 @@ const MEILI_KEYS = [ "catalog.search.meilisearch.semantic_enabled", "catalog.search.meilisearch.semantic_ratio", "catalog.search.meilisearch.embedder", + "catalog.search.meilisearch.binary_quantized", ]; const KEYS = ["catalog.search.provider", ...MEILI_KEYS]; @@ -190,6 +191,16 @@ export default function SearchSettings() { onChange={(value) => form.setValue("catalog.search.meilisearch.embedder", value)} disabled={!meiliEnabled} /> + + form.setValue("catalog.search.meilisearch.binary_quantized", value) + } + hint="~30x smaller raw vectors (overall index size roughly halves) with a small semantic-relevance cost. Only affects the index when Semantic Search is enabled. Changing this requires a full index rebuild (Rebuild Catalog Search Index) before sync resumes." + disabled={!meiliEnabled} + />
+