feat(search): binary-quantized embedder vectors — optional, default-on for fresh installs (#351)
* feat(search): binary quantization setting for the Meilisearch embedder
New server setting catalog.search.meilisearch.binary_quantized
(default false) threads into the embedder index settings
("binaryQuantized": true) and into the schema-version hash, so flipping
it closes the sync gate and mandates a rebuild in both directions —
Meilisearch cannot de/re-quantize an index in place.
With 3072-dimensional embeddings this cuts vector storage ~32x
(≈12KB → 384B per document), keeping the whole vector store in page
cache: rebuilds and hybrid queries get sharply cheaper. Hybrid search
(keyword + semantic) cushions the small relevance cost of sign-only
vectors.
The hash token is appended only when the flag is set, so indexes built
before this change keep their schema version while it stays off.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* feat(search): binary quantization default-on for fresh installs + admin toggle
- Migration seeds catalog.search.meilisearch.binary_quantized=true only
when no active catalog search index exists. Existing deployments stay
unset (= off): flipping quantization changes the index schema-version
identity, which closes the incremental-sync gate until a full rebuild
runs — that must never happen implicitly on upgrade. Fresh installs
have no index yet, so their first rebuild simply starts quantized.
- Search settings page gains the toggle with an explicit
"requires a full index rebuild" warning, a status row, and settings-
search keywords.
Prod benchmark (607.9k docs, 3072-dim vectors, N=10 medians, replicated):
hybrid 0.5 unchanged (7.5ms float vs 8.0ms quantized, within ±2ms
keyword-control jitter); pure semantic 9ms → 4ms; on-disk index 18G →
8.3G; rebuild duration unchanged.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* fix(search): address review feedback on binary-quantized embedder
- Add catalog.search.meilisearch.binary_quantized to the restart-required
registry. The provider freezes BinaryQuantized into MeilisearchProviderConfig
at construction, so without this a toggle-then-rebuild in the same process
builds a quantized schema while the live provider still compares against the
old value and falls back until restart (Codex P2).
- Validate binary_quantized in HandleUpdateSetting, mirroring semantic_enabled.
A raw API write of a non-bool previously persisted unnormalized, then failed
CatalogSearchSettingsFromMap on load and silently reverted the entire search
config to Postgres defaults.
- Gate the binary_quantized token in the schema-version identity on
semanticEnabled: with semantic off the index has no embedders, so the flag
has no on-index effect and must not force a pointless rebuild. Stays
byte-identical to a pre-flag index. Covered by a new test.
- Clarify the seed migration comment (guard is "no active index", which also
covers Meilisearch-configured-but-never-indexed deployments) and the UI hint
(~30x smaller raw vectors, index roughly halves; only applies with semantic).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---------
Co-authored-by: rxwatcher <rxwatcher@users.noreply.github.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
RXWatcher
committed by
GitHub
GitHub
co-authored by
rxwatcher
Claude Fable 5
rxwatcher
Claude Fable 5
parent
3e340552a8
commit
e96a8a0cf8
@@ -2334,6 +2334,13 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
req.Value = embedder
|
||||
case catalog.SearchSettingMeilisearchBinaryQuantized:
|
||||
enabled, err := strconv.ParseBool(strings.TrimSpace(req.Value))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "catalog.search.meilisearch.binary_quantized must be true or false")
|
||||
return
|
||||
}
|
||||
req.Value = strconv.FormatBool(enabled)
|
||||
}
|
||||
|
||||
if err := h.SettingsRepo.Set(r.Context(), key, req.Value); err != nil {
|
||||
|
||||
@@ -89,7 +89,7 @@ func (i *CatalogSearchIndexer) ShouldSyncRun(ctx context.Context) (bool, error)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if state.ActiveIndexUID == "" || state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled) {
|
||||
if state.ActiveIndexUID == "" || state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized) {
|
||||
return false, nil
|
||||
}
|
||||
pending, err := i.events.PendingCount(ctx, SearchProviderMeilisearch)
|
||||
@@ -131,7 +131,7 @@ func (i *CatalogSearchIndexer) SyncOutbox(ctx context.Context, progress SearchIn
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if state.ActiveIndexUID == "" || state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled) {
|
||||
if state.ActiveIndexUID == "" || state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized) {
|
||||
stats.Skipped = true
|
||||
stats.Reason = "active search index is missing or stale; run rebuild_catalog_search_index"
|
||||
setSearchIndexTaskResult(progress, stats)
|
||||
@@ -166,7 +166,7 @@ func (i *CatalogSearchIndexer) SyncOutbox(ctx context.Context, progress SearchIn
|
||||
|
||||
upsertIDs, deleteIDs := coalesceSearchIndexEvents(events)
|
||||
reportSearchIndexProgress(progress, 20, "Building changed catalog search documents")
|
||||
docs, err := i.LoadDocumentsByIDs(ctx, upsertIDs, settings.IndexTypes, settings.Embedder, settings.SemanticEnabled)
|
||||
docs, err := i.LoadDocumentsByIDs(ctx, upsertIDs, settings.IndexTypes, settings.Embedder, settings.SemanticEnabled, settings.BinaryQuantized)
|
||||
if err != nil {
|
||||
_ = i.events.MarkFailed(ctx, ids, err)
|
||||
return stats, err
|
||||
@@ -283,7 +283,7 @@ func (i *CatalogSearchIndexer) Rebuild(ctx context.Context, progress SearchIndex
|
||||
if err := client.WaitTask(ctx, taskID); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
taskID, err = client.UpdateSettings(ctx, buildIndexUID, catalogSearchMeilisearchSettings(settings.Embedder, settings.SemanticEnabled))
|
||||
taskID, err = client.UpdateSettings(ctx, buildIndexUID, catalogSearchMeilisearchSettings(settings.Embedder, settings.SemanticEnabled, settings.BinaryQuantized))
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
@@ -294,7 +294,7 @@ func (i *CatalogSearchIndexer) Rebuild(ctx context.Context, progress SearchIndex
|
||||
lastID := ""
|
||||
queuedTasks := make([]queuedMeilisearchTask, 0, settings.RebuildQueueDepth)
|
||||
for {
|
||||
docs, err := i.LoadDocumentsAfter(ctx, lastID, settings.RebuildBatchSize, settings.IndexTypes, settings.Embedder, settings.SemanticEnabled)
|
||||
docs, err := i.LoadDocumentsAfter(ctx, lastID, settings.RebuildBatchSize, settings.IndexTypes, settings.Embedder, settings.SemanticEnabled, settings.BinaryQuantized)
|
||||
if err != nil {
|
||||
return stats, err
|
||||
}
|
||||
@@ -339,7 +339,7 @@ func (i *CatalogSearchIndexer) Rebuild(ctx context.Context, progress SearchIndex
|
||||
// active index as idempotent upserts on the next sync. The reverse order
|
||||
// would mark events processed while the old index is still active, losing
|
||||
// those changes from the served index until the next rebuild.
|
||||
if err := i.events.UpdateStateAfterRebuild(ctx, SearchProviderMeilisearch, buildIndexUID, catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled), docCount, rebuildEventHighWater); err != nil {
|
||||
if err := i.events.UpdateStateAfterRebuild(ctx, SearchProviderMeilisearch, buildIndexUID, catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized), docCount, rebuildEventHighWater); err != nil {
|
||||
return stats, err
|
||||
}
|
||||
if err := i.events.MarkProcessedThrough(ctx, SearchProviderMeilisearch, rebuildEventHighWater); err != nil {
|
||||
@@ -532,7 +532,7 @@ func (i *CatalogSearchIndexer) CheckConnection(ctx context.Context, settings Cat
|
||||
return client.Health(ctx)
|
||||
}
|
||||
|
||||
func catalogSearchMeilisearchSettings(embedder string, semanticEnabled bool) map[string]any {
|
||||
func catalogSearchMeilisearchSettings(embedder string, semanticEnabled, binaryQuantized bool) map[string]any {
|
||||
settings := map[string]any{
|
||||
"displayedAttributes": []string{"content_id", "type"},
|
||||
"filterableAttributes": []string{"type"},
|
||||
@@ -558,7 +558,7 @@ func catalogSearchMeilisearchSettings(embedder string, semanticEnabled bool) map
|
||||
if err != nil {
|
||||
embedder = DefaultMeilisearchEmbedder
|
||||
}
|
||||
settings["embedders"] = catalogSearchMeilisearchEmbedderSettings(embedder)
|
||||
settings["embedders"] = catalogSearchMeilisearchEmbedderSettings(embedder, binaryQuantized)
|
||||
}
|
||||
return settings
|
||||
}
|
||||
@@ -627,7 +627,7 @@ type catalogSearchDocument struct {
|
||||
Vectors map[string][]float32 `json:"_vectors,omitempty"`
|
||||
}
|
||||
|
||||
func (i *CatalogSearchIndexer) LoadDocumentsAfter(ctx context.Context, afterContentID string, limit int, itemTypes []string, embedder string, semanticEnabled bool) ([]catalogSearchDocument, error) {
|
||||
func (i *CatalogSearchIndexer) LoadDocumentsAfter(ctx context.Context, afterContentID string, limit int, itemTypes []string, embedder string, semanticEnabled, binaryQuantized bool) ([]catalogSearchDocument, error) {
|
||||
if i == nil || i.pool == nil || limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -653,14 +653,14 @@ func (i *CatalogSearchIndexer) LoadDocumentsAfter(ctx context.Context, afterCont
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
setCatalogSearchDocumentSchemaVersion(docs, catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, semanticEnabled))
|
||||
setCatalogSearchDocumentSchemaVersion(docs, catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, semanticEnabled, binaryQuantized))
|
||||
if err := i.attachDocumentVectors(ctx, docs, embedder, semanticEnabled); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return docs, nil
|
||||
}
|
||||
|
||||
func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentIDs []string, itemTypes []string, embedder string, semanticEnabled bool) ([]catalogSearchDocument, error) {
|
||||
func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentIDs []string, itemTypes []string, embedder string, semanticEnabled, binaryQuantized bool) ([]catalogSearchDocument, error) {
|
||||
contentIDs = compactNonEmptyStrings(contentIDs)
|
||||
if i == nil || i.pool == nil || len(contentIDs) == 0 {
|
||||
return nil, nil
|
||||
@@ -686,7 +686,7 @@ func (i *CatalogSearchIndexer) LoadDocumentsByIDs(ctx context.Context, contentID
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
setCatalogSearchDocumentSchemaVersion(docs, catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, semanticEnabled))
|
||||
setCatalogSearchDocumentSchemaVersion(docs, catalogSearchMeilisearchSchemaVersion(embedder, itemTypes, semanticEnabled, binaryQuantized))
|
||||
if err := i.attachDocumentVectors(ctx, docs, embedder, semanticEnabled); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -58,6 +58,7 @@ type MeilisearchProviderConfig struct {
|
||||
MatchingStrategy string
|
||||
IndexTypes []string
|
||||
SemanticEnabled bool
|
||||
BinaryQuantized bool
|
||||
SemanticRatio float64
|
||||
Embedder string
|
||||
Vectorizer CatalogSearchQueryVectorizer
|
||||
@@ -173,7 +174,7 @@ func (p *MeilisearchSearchProvider) Search(ctx context.Context, req CatalogSearc
|
||||
if strings.TrimSpace(state.ActiveIndexUID) == "" {
|
||||
return p.fallbackSearch(ctx, req, "meilisearch index has not been built")
|
||||
}
|
||||
if state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(p.config.Embedder, p.config.IndexTypes, p.config.SemanticEnabled) {
|
||||
if state.SchemaVersion != catalogSearchMeilisearchSchemaVersion(p.config.Embedder, p.config.IndexTypes, p.config.SemanticEnabled, p.config.BinaryQuantized) {
|
||||
return p.fallbackSearch(ctx, req, "meilisearch index schema mismatch")
|
||||
}
|
||||
|
||||
@@ -591,6 +592,7 @@ func (p *MeilisearchSearchProvider) Status() CatalogSearchMeiliStatus {
|
||||
MatchingStrategy: p.config.MatchingStrategy,
|
||||
IndexTypes: p.config.IndexTypes,
|
||||
SemanticEnabled: p.config.SemanticEnabled,
|
||||
BinaryQuantized: p.config.BinaryQuantized,
|
||||
SemanticRatio: p.config.SemanticRatio,
|
||||
Embedder: p.config.Embedder,
|
||||
}
|
||||
|
||||
@@ -32,6 +32,7 @@ const (
|
||||
SearchSettingMeilisearchSemanticEnabled = "catalog.search.meilisearch.semantic_enabled"
|
||||
SearchSettingMeilisearchSemanticRatio = "catalog.search.meilisearch.semantic_ratio"
|
||||
SearchSettingMeilisearchEmbedder = "catalog.search.meilisearch.embedder"
|
||||
SearchSettingMeilisearchBinaryQuantized = "catalog.search.meilisearch.binary_quantized"
|
||||
|
||||
DefaultMeilisearchIndex = "silo_media_items"
|
||||
DefaultMeilisearchTimeoutMS = 800
|
||||
@@ -44,6 +45,7 @@ const (
|
||||
DefaultMeilisearchSemanticEnabled = false
|
||||
DefaultMeilisearchSemanticRatio = 0.50
|
||||
DefaultMeilisearchEmbedder = "silo_recommendations"
|
||||
DefaultMeilisearchBinaryQuantized = false
|
||||
|
||||
MaxMeilisearchSyncBatchSize = 10000
|
||||
MaxMeilisearchRebuildBatchSize = 25000
|
||||
@@ -147,6 +149,7 @@ type CatalogSearchSettings struct {
|
||||
SemanticEnabled bool
|
||||
SemanticRatio float64
|
||||
Embedder string
|
||||
BinaryQuantized bool
|
||||
}
|
||||
|
||||
func DefaultCatalogSearchSettings() CatalogSearchSettings {
|
||||
@@ -161,6 +164,7 @@ func DefaultCatalogSearchSettings() CatalogSearchSettings {
|
||||
SemanticEnabled: DefaultMeilisearchSemanticEnabled,
|
||||
SemanticRatio: DefaultMeilisearchSemanticRatio,
|
||||
Embedder: DefaultMeilisearchEmbedder,
|
||||
BinaryQuantized: DefaultMeilisearchBinaryQuantized,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -239,6 +243,13 @@ func CatalogSearchSettingsFromMap(values map[string]string) (CatalogSearchSettin
|
||||
}
|
||||
settings.SemanticEnabled = enabled
|
||||
}
|
||||
if raw := strings.TrimSpace(values[SearchSettingMeilisearchBinaryQuantized]); raw != "" {
|
||||
enabled, err := strconv.ParseBool(raw)
|
||||
if err != nil {
|
||||
return settings, fmt.Errorf("%s must be true or false", SearchSettingMeilisearchBinaryQuantized)
|
||||
}
|
||||
settings.BinaryQuantized = enabled
|
||||
}
|
||||
if raw := strings.TrimSpace(values[SearchSettingMeilisearchSemanticRatio]); raw != "" {
|
||||
ratio, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil || math.IsNaN(ratio) || ratio < 0 || ratio > 1 {
|
||||
@@ -399,6 +410,7 @@ type CatalogSearchMeiliStatus struct {
|
||||
MatchingStrategy string `json:"matching_strategy"`
|
||||
IndexTypes []string `json:"index_types,omitempty"`
|
||||
SemanticEnabled bool `json:"semantic_enabled"`
|
||||
BinaryQuantized bool `json:"binary_quantized"`
|
||||
SemanticRatio float64 `json:"semantic_ratio"`
|
||||
Embedder string `json:"embedder"`
|
||||
}
|
||||
@@ -429,13 +441,18 @@ type CatalogSearchStatusProvider interface {
|
||||
Status(ctx context.Context) CatalogSearchRuntimeStatus
|
||||
}
|
||||
|
||||
func catalogSearchMeilisearchEmbedderSettings(embedder string) map[string]any {
|
||||
return map[string]any{
|
||||
embedder: map[string]any{
|
||||
"source": "userProvided",
|
||||
"dimensions": embeddingvectors.CanonicalDimensions,
|
||||
},
|
||||
func catalogSearchMeilisearchEmbedderSettings(embedder string, binaryQuantized bool) map[string]any {
|
||||
cfg := map[string]any{
|
||||
"source": "userProvided",
|
||||
"dimensions": embeddingvectors.CanonicalDimensions,
|
||||
}
|
||||
if binaryQuantized {
|
||||
// One-way per index: Meilisearch cannot de-quantize an index in
|
||||
// place, so turning this off again requires a rebuild, exactly like
|
||||
// turning it on. The schema-version hash enforces both transitions.
|
||||
cfg["binaryQuantized"] = true
|
||||
}
|
||||
return map[string]any{embedder: cfg}
|
||||
}
|
||||
|
||||
// catalogSearchMeilisearchSchemaVersion derives the expected index schema
|
||||
@@ -447,19 +464,28 @@ func catalogSearchMeilisearchEmbedderSettings(embedder string) map[string]any {
|
||||
// rebuilt rather than serving silently degraded hybrid ranking. A mismatch forces
|
||||
// a rebuild (SyncOutbox skips, the provider falls back to keyword) and surfaces as
|
||||
// an ExpectedSchemaVersion divergence in the admin status.
|
||||
func catalogSearchMeilisearchSchemaVersion(embedder string, itemTypes []string, semanticEnabled bool) int {
|
||||
func catalogSearchMeilisearchSchemaVersion(embedder string, itemTypes []string, semanticEnabled, binaryQuantized bool) int {
|
||||
embedder, err := NormalizeCatalogSearchEmbedderName(embedder)
|
||||
if err != nil {
|
||||
embedder = DefaultMeilisearchEmbedder
|
||||
}
|
||||
h := fnv.New32a()
|
||||
_, _ = fmt.Fprintf(
|
||||
h,
|
||||
identity := fmt.Sprintf(
|
||||
"embedder=%s;dimensions=%d;index_types=%s;semantic=%t",
|
||||
embedder,
|
||||
embeddingvectors.CanonicalDimensions,
|
||||
strings.Join(normalizeCatalogSearchItemTypes(itemTypes), ","),
|
||||
semanticEnabled,
|
||||
)
|
||||
// Gated on semanticEnabled because binary quantization only affects the
|
||||
// embedder settings (catalogSearchMeilisearchSettings), which are omitted
|
||||
// entirely when semantic search is off — so with no vectors on the index
|
||||
// the flag has no on-index effect and must not force a rebuild. Appended
|
||||
// only when both are set, so pre-existing indexes built before this flag
|
||||
// existed (and any index with semantic off) keep their schema version.
|
||||
if semanticEnabled && binaryQuantized {
|
||||
identity += ";binary_quantized=true"
|
||||
}
|
||||
h := fnv.New32a()
|
||||
_, _ = h.Write([]byte(identity))
|
||||
return SearchMeilisearchSchemaVersion*1_000_000 + int(h.Sum32()%1_000_000)
|
||||
}
|
||||
|
||||
@@ -128,7 +128,7 @@ func TestCatalogSearchDocumentVectorsUseEmbedderAndOptOutMissing(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCatalogSearchMeilisearchSettingsOmitEmbeddersWhenSemanticDisabled(t *testing.T) {
|
||||
settings := catalogSearchMeilisearchSettings("silo_recommendations", false)
|
||||
settings := catalogSearchMeilisearchSettings("silo_recommendations", false, false)
|
||||
|
||||
if _, ok := settings["embedders"]; ok {
|
||||
t.Fatalf("semantic-disabled settings should not include embedders: %#v", settings["embedders"])
|
||||
@@ -139,7 +139,7 @@ func TestCatalogSearchMeilisearchSettingsOmitEmbeddersWhenSemanticDisabled(t *te
|
||||
}
|
||||
|
||||
func TestCatalogSearchMeilisearchSettingsIncludeEmbeddersWhenSemanticEnabled(t *testing.T) {
|
||||
settings := catalogSearchMeilisearchSettings("custom_embedder", true)
|
||||
settings := catalogSearchMeilisearchSettings("custom_embedder", true, false)
|
||||
|
||||
embedders, ok := settings["embedders"].(map[string]any)
|
||||
if !ok {
|
||||
@@ -512,7 +512,7 @@ func TestMeilisearchProviderCachesIndexStateAcrossRequests(t *testing.T) {
|
||||
store := &countingMeilisearchIndexStateStore{
|
||||
state: SearchIndexState{
|
||||
ActiveIndexUID: "search-index",
|
||||
SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false),
|
||||
SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false),
|
||||
},
|
||||
pending: 3,
|
||||
}
|
||||
@@ -557,7 +557,7 @@ func TestMeilisearchProviderInvalidatesStateCacheOnSearchFailure(t *testing.T) {
|
||||
store := &countingMeilisearchIndexStateStore{
|
||||
state: SearchIndexState{
|
||||
ActiveIndexUID: "search-index",
|
||||
SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false),
|
||||
SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false),
|
||||
},
|
||||
}
|
||||
provider := &MeilisearchSearchProvider{
|
||||
@@ -607,7 +607,7 @@ func TestMeilisearchProviderUsesActiveIndexWhenPendingUpdatesExist(t *testing.T)
|
||||
stateRepo: fakeMeilisearchIndexStateStore{
|
||||
state: SearchIndexState{
|
||||
ActiveIndexUID: "search-index",
|
||||
SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false),
|
||||
SchemaVersion: catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false),
|
||||
},
|
||||
pending: 7,
|
||||
},
|
||||
@@ -683,8 +683,8 @@ func TestNormalizeCatalogSearchIndexTypesValueFormatsCanonicalList(t *testing.T)
|
||||
}
|
||||
|
||||
func TestMeilisearchSchemaVersionChangesWithEmbedder(t *testing.T) {
|
||||
defaultVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false)
|
||||
customVersion := catalogSearchMeilisearchSchemaVersion("custom_embedder", nil, false)
|
||||
defaultVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false)
|
||||
customVersion := catalogSearchMeilisearchSchemaVersion("custom_embedder", nil, false, false)
|
||||
if defaultVersion == customVersion {
|
||||
t.Fatal("schema version should change when embedder changes")
|
||||
}
|
||||
@@ -694,8 +694,8 @@ func TestMeilisearchSchemaVersionChangesWithEmbedder(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestMeilisearchSchemaVersionChangesWithIndexTypes(t *testing.T) {
|
||||
allTypesVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false)
|
||||
videoOnlyVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, []string{"movie", "series"}, false)
|
||||
allTypesVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false)
|
||||
videoOnlyVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, []string{"movie", "series"}, false, false)
|
||||
if allTypesVersion == videoOnlyVersion {
|
||||
t.Fatal("schema version should change when indexed media scope changes")
|
||||
}
|
||||
@@ -707,8 +707,8 @@ func TestMeilisearchSchemaVersionChangesWithSemanticEnabled(t *testing.T) {
|
||||
// rebuild. Without this, enabling semantic without a rebuild leaves indexed
|
||||
// documents missing _vectors while the Postgres coverage gate reports ready,
|
||||
// silently degrading hybrid ranking.
|
||||
disabledVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false)
|
||||
enabledVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, true)
|
||||
disabledVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false)
|
||||
enabledVersion := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, true, false)
|
||||
if disabledVersion == enabledVersion {
|
||||
t.Fatal("schema version should change when semantic search is toggled")
|
||||
}
|
||||
@@ -760,3 +760,35 @@ type fakeModelVectorizer struct {
|
||||
func (f *fakeModelVectorizer) ActiveEmbeddingModel(_ context.Context) (string, error) {
|
||||
return f.model, nil
|
||||
}
|
||||
|
||||
func TestCatalogSearchMeilisearchSchemaVersionBinaryQuantized(t *testing.T) {
|
||||
plain := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, true, false)
|
||||
quantized := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, true, true)
|
||||
if plain == quantized {
|
||||
t.Fatal("binary quantization must change the schema version")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogSearchMeilisearchSchemaVersionBinaryQuantizedIgnoredWhenSemanticDisabled(t *testing.T) {
|
||||
// With semantic search off, the index carries no embedders (see
|
||||
// catalogSearchMeilisearchSettings), so binary quantization has no on-index
|
||||
// effect and must not shift the schema version — otherwise toggling it would
|
||||
// force a pointless full rebuild of a vector-less index. It must also stay
|
||||
// byte-identical to a pre-flag index so upgrades don't spuriously invalidate.
|
||||
off := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, false)
|
||||
on := catalogSearchMeilisearchSchemaVersion(DefaultMeilisearchEmbedder, nil, false, true)
|
||||
if off != on {
|
||||
t.Fatal("binary quantization must not change the schema version when semantic search is disabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCatalogSearchMeilisearchEmbedderSettingsBinaryQuantized(t *testing.T) {
|
||||
off := catalogSearchMeilisearchEmbedderSettings("e", false)["e"].(map[string]any)
|
||||
if _, ok := off["binaryQuantized"]; ok {
|
||||
t.Fatal("binaryQuantized key must be absent when off (preserves legacy index settings shape)")
|
||||
}
|
||||
on := catalogSearchMeilisearchEmbedderSettings("e", true)["e"].(map[string]any)
|
||||
if on["binaryQuantized"] != true {
|
||||
t.Fatal("binaryQuantized must be set when enabled")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -75,6 +75,7 @@ func NewCatalogSearchServiceFromSettings(
|
||||
MatchingStrategy: settings.MatchingStrategy,
|
||||
IndexTypes: settings.IndexTypes,
|
||||
SemanticEnabled: settings.SemanticEnabled,
|
||||
BinaryQuantized: settings.BinaryQuantized,
|
||||
SemanticRatio: settings.SemanticRatio,
|
||||
Embedder: settings.Embedder,
|
||||
Vectorizer: queryVectorizer,
|
||||
@@ -144,11 +145,12 @@ func (s *CatalogSearchService) Status(ctx context.Context) CatalogSearchRuntimeS
|
||||
MatchingStrategy: settings.MatchingStrategy,
|
||||
IndexTypes: settings.IndexTypes,
|
||||
SemanticEnabled: settings.SemanticEnabled,
|
||||
BinaryQuantized: settings.BinaryQuantized,
|
||||
SemanticRatio: settings.SemanticRatio,
|
||||
Embedder: settings.Embedder,
|
||||
},
|
||||
Index: CatalogSearchIndexStateStatus{
|
||||
ExpectedSchemaVersion: catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled),
|
||||
ExpectedSchemaVersion: catalogSearchMeilisearchSchemaVersion(settings.Embedder, settings.IndexTypes, settings.SemanticEnabled, settings.BinaryQuantized),
|
||||
},
|
||||
Tasks: []CatalogSearchTaskLink{
|
||||
{Key: "sync_catalog_search_index", Name: "Sync Catalog Search Index", Href: "/admin/tasks/sync_catalog_search_index"},
|
||||
|
||||
@@ -84,6 +84,7 @@ var restartRequiredKeys = map[string]bool{
|
||||
"catalog.search.meilisearch.semantic_enabled": true,
|
||||
"catalog.search.meilisearch.semantic_ratio": true,
|
||||
"catalog.search.meilisearch.embedder": true,
|
||||
"catalog.search.meilisearch.binary_quantized": true,
|
||||
}
|
||||
|
||||
// restartRequiredPrefixes covers whole namespaces of infrastructure settings:
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
-- Binary quantization defaults ON for any install with no active catalog
|
||||
-- search index — fresh installs, and also deployments that configured
|
||||
-- Meilisearch but never built an index. Deployments with an active index are
|
||||
-- left unset (= off): flipping quantization changes the index schema-version
|
||||
-- identity, which closes the incremental-sync gate until a full rebuild runs
|
||||
-- — that must never happen implicitly on upgrade. Installs with no index yet
|
||||
-- have no gate to close, so their first rebuild simply starts quantized.
|
||||
INSERT INTO server_settings (key, value)
|
||||
SELECT 'catalog.search.meilisearch.binary_quantized', 'true'
|
||||
WHERE NOT EXISTS (
|
||||
SELECT 1 FROM server_settings
|
||||
WHERE key = 'catalog.search.meilisearch.binary_quantized')
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM catalog_search_index_state
|
||||
WHERE active_index_uid IS NOT NULL AND active_index_uid <> '');
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
-- Intentionally a no-op: deleting the setting could silently change the
|
||||
-- schema-version identity of an index that was built with quantization.
|
||||
SELECT 1;
|
||||
-- +goose StatementEnd
|
||||
@@ -32,6 +32,7 @@ export interface CatalogSearchStatus {
|
||||
matching_strategy: string;
|
||||
index_types?: string[];
|
||||
semantic_enabled: boolean;
|
||||
binary_quantized: boolean;
|
||||
semantic_ratio: number;
|
||||
embedder: string;
|
||||
};
|
||||
|
||||
@@ -143,6 +143,8 @@ export const ADMIN_SETTINGS_GROUPS: AdminSettingsSearchGroup[] = [
|
||||
"vectors",
|
||||
"embeddings",
|
||||
"embedder",
|
||||
"binary quantization",
|
||||
"quantized",
|
||||
],
|
||||
settings: settingIndex(
|
||||
"Preferred Provider",
|
||||
|
||||
@@ -28,6 +28,7 @@ const MEILI_KEYS = [
|
||||
"catalog.search.meilisearch.semantic_enabled",
|
||||
"catalog.search.meilisearch.semantic_ratio",
|
||||
"catalog.search.meilisearch.embedder",
|
||||
"catalog.search.meilisearch.binary_quantized",
|
||||
];
|
||||
|
||||
const KEYS = ["catalog.search.provider", ...MEILI_KEYS];
|
||||
@@ -190,6 +191,16 @@ export default function SearchSettings() {
|
||||
onChange={(value) => form.setValue("catalog.search.meilisearch.embedder", value)}
|
||||
disabled={!meiliEnabled}
|
||||
/>
|
||||
<SettingField
|
||||
label="Binary Quantized Vectors"
|
||||
type="toggle"
|
||||
value={form.getValue("catalog.search.meilisearch.binary_quantized") || "false"}
|
||||
onChange={(value) =>
|
||||
form.setValue("catalog.search.meilisearch.binary_quantized", value)
|
||||
}
|
||||
hint="~30x smaller raw vectors (overall index size roughly halves) with a small semantic-relevance cost. Only affects the index when Semantic Search is enabled. Changing this requires a full index rebuild (Rebuild Catalog Search Index) before sync resumes."
|
||||
disabled={!meiliEnabled}
|
||||
/>
|
||||
<div className="py-3">
|
||||
<ConnectionCheckAction
|
||||
onClick={handleCheckConnection}
|
||||
@@ -229,6 +240,10 @@ export default function SearchSettings() {
|
||||
label="Indexed Types"
|
||||
value={formatIndexedTypes(status.meilisearch.index_types)}
|
||||
/>
|
||||
<StatusRow
|
||||
label="Binary Quantized"
|
||||
value={status.meilisearch.binary_quantized ? "Enabled" : "Disabled"}
|
||||
/>
|
||||
<StatusRow
|
||||
label="Semantic Search"
|
||||
value={status.meilisearch.semantic_enabled ? "Enabled" : "Disabled"}
|
||||
|
||||
Reference in New Issue
Block a user