From 20f36fdc31703b5b2df608614a7bbcd1187d850c Mon Sep 17 00:00:00 2001 From: Quick <31828688+Quick104@users.noreply.github.com> Date: Thu, 25 Jun 2026 21:31:20 -0400 Subject: [PATCH] fix(search): address provider review comments --- internal/api/handlers/admin.go | 3 +- .../handlers/admin_settings_checks_test.go | 1 + internal/api/router.go | 17 ++- internal/autoscan/service.go | 35 ++++- internal/autoscan/service_test.go | 73 ++++++++++ internal/catalog/folder_repo.go | 11 +- internal/catalog/item_repo.go | 8 +- internal/catalog/search_index_repo.go | 136 ++++++++++++++++-- internal/catalog/search_index_repo_test.go | 24 ++++ .../catalog/search_meilisearch_provider.go | 2 +- internal/catalog/search_provider.go | 3 +- internal/catalog/search_provider_test.go | 13 +- internal/catalogseed/service.go | 28 ++-- internal/catalogseed/service_test.go | 12 +- internal/events/scan_registry.go | 20 ++- internal/events/scan_registry_test.go | 27 ++++ web/src/components/GlobalSearch.test.tsx | 5 +- web/src/components/GlobalSearch.tsx | 8 +- web/src/components/ServerActivity.tsx | 35 ++++- 19 files changed, 401 insertions(+), 60 deletions(-) diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go index 0f51aa30..cf8d7bb3 100644 --- a/internal/api/handlers/admin.go +++ b/internal/api/handlers/admin.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "log/slog" + "math" "net/http" "net/url" "slices" @@ -2216,7 +2217,7 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques req.Value = strconv.FormatBool(enabled) case catalog.SearchSettingMeilisearchSemanticRatio: ratio, err := strconv.ParseFloat(strings.TrimSpace(req.Value), 64) - if err != nil || ratio < 0 || ratio > 1 { + if err != nil || math.IsNaN(ratio) || ratio < 0 || ratio > 1 { writeError(w, http.StatusBadRequest, "bad_request", "catalog.search.meilisearch.semantic_ratio must be a number between 0 and 1") return } diff --git a/internal/api/handlers/admin_settings_checks_test.go b/internal/api/handlers/admin_settings_checks_test.go index a86cdf38..85941bd8 100644 --- a/internal/api/handlers/admin_settings_checks_test.go +++ b/internal/api/handlers/admin_settings_checks_test.go @@ -418,6 +418,7 @@ func TestAdminUpdateCatalogSearchSemanticSettingsValidation(t *testing.T) { {key: catalog.SearchSettingMeilisearchSemanticEnabled, value: "sometimes"}, {key: catalog.SearchSettingMeilisearchSemanticRatio, value: "-0.01"}, {key: catalog.SearchSettingMeilisearchSemanticRatio, value: "1.01"}, + {key: catalog.SearchSettingMeilisearchSemanticRatio, value: "NaN"}, {key: catalog.SearchSettingMeilisearchEmbedder, value: "bad.name"}, } for _, tc := range invalid { diff --git a/internal/api/router.go b/internal/api/router.go index 8b33143c..8e9424d2 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -448,15 +448,24 @@ func NewRouter(deps Dependencies) chi.Router { ebookAnnotationStore = handlers.NewPGEbookReaderAnnotationStore(deps.DB) browseRepo := catalog.NewBrowseRepository(deps.DB) itemRepo = catalog.NewItemRepository(deps.DB) + catalogSearchSettings, err := catalog.LoadCatalogSearchSettings(context.Background(), settingsRepo) + if err != nil { + slog.Warn("catalog search: failed to load settings; using postgres", "err", err) + catalogSearchSettings = catalog.DefaultCatalogSearchSettings() + } searchIndexEvents := catalog.NewSearchIndexEventRepository(deps.DB) - itemRepo.WithSearchIndexEvents(searchIndexEvents) - catalogSearchService = catalog.NewCatalogSearchService( - context.Background(), - settingsRepo, + catalogSearchService = catalog.NewCatalogSearchServiceFromSettings( + catalogSearchSettings, itemRepo, searchIndexEvents, deps.CatalogSearchVectorizer, ) + activeSearchProvider := catalog.SearchProviderPostgres + if _, ok := catalogSearchService.Provider().(*catalog.MeilisearchSearchProvider); ok { + activeSearchProvider = catalog.SearchProviderMeilisearch + } + searchIndexEvents.WithActiveProvider(activeSearchProvider) + itemRepo.WithSearchIndexEvents(searchIndexEvents) episodeRepo = catalog.NewEpisodeRepository(deps.DB) providerIDRepo = catalog.NewProviderIDRepository(deps.DB) calendarRepo = catalog.NewCalendarRepository(deps.DB) diff --git a/internal/autoscan/service.go b/internal/autoscan/service.go index dd396e0b..4209fc65 100644 --- a/internal/autoscan/service.go +++ b/internal/autoscan/service.go @@ -214,12 +214,7 @@ func (s *Service) PollOnce(ctx context.Context) error { var enqueue EnqueueResult if len(targets) > 0 { var eerr error - if eventID != 0 { - enqueue.Created, enqueue.Reused, eerr = s.queue.EnqueueAutoscanScans(ctx, targets, eventID) - } else { - eerr = s.queue.EnqueueScans(ctx, targets) - enqueue.Created = len(targets) - } + enqueue, eerr = s.enqueueScanTargets(ctx, targets, eventID) if eerr != nil { s.releaseClaims(ctx, claimed) slog.WarnContext(ctx, "autoscan: enqueue failed", "source_id", src.ID, "err", eerr) @@ -295,6 +290,34 @@ func (s *Service) PollOnce(ctx context.Context) error { return nil } +func (s *Service) enqueueScanTargets(ctx context.Context, targets []scantrigger.Target, eventID int64) (EnqueueResult, error) { + var result EnqueueResult + if len(targets) == 0 { + return result, nil + } + for start := 0; start < len(targets); start += maxAutoscanTargetsPerPoll { + end := start + maxAutoscanTargetsPerPoll + if end > len(targets) { + end = len(targets) + } + chunk := targets[start:end] + if eventID != 0 { + created, reused, err := s.queue.EnqueueAutoscanScans(ctx, chunk, eventID) + if err != nil { + return result, err + } + result.Created += created + result.Reused += reused + continue + } + if err := s.queue.EnqueueScans(ctx, chunk); err != nil { + return result, err + } + result.Created += len(chunk) + } + return result, nil +} + func (s *Service) createEvent(ctx context.Context, src Source, marker string, startedAt time.Time) (int64, bool) { if s == nil || s.store == nil { return 0, true diff --git a/internal/autoscan/service_test.go b/internal/autoscan/service_test.go index f7c48a71..3048a179 100644 --- a/internal/autoscan/service_test.go +++ b/internal/autoscan/service_test.go @@ -173,6 +173,38 @@ func (fakeResolver) ResolveMissingSubtree(_ context.Context, subtreePath, trigge return nil, &scantrigger.RequestError{Status: 400, Code: "bad_request", Message: "outside media folders"} } +type distinctFolderResolver struct{} + +func (distinctFolderResolver) Resolve(_ context.Context, req scantrigger.Request) (*scantrigger.Target, error) { + if !strings.HasPrefix(req.Path, "/mnt/media/library-") { + return nil, &scantrigger.RequestError{Status: 400, Code: "bad_request", Message: "outside media folders"} + } + rest := strings.TrimPrefix(req.Path, "/mnt/media/library-") + idPart := rest + if slash := strings.IndexByte(rest, '/'); slash >= 0 { + idPart = rest[:slash] + } + id, err := strconv.Atoi(idPart) + if err != nil { + return nil, &scantrigger.RequestError{Status: 400, Code: "bad_request", Message: "outside media folders"} + } + return &scantrigger.Target{ + Folder: &models.MediaFolder{ID: id}, + Mode: scantrigger.ModeFile, + Path: req.Path, + Trigger: req.Trigger, + }, nil +} + +func (distinctFolderResolver) ResolveMissingSubtree(_ context.Context, subtreePath, trigger string) (*scantrigger.Target, error) { + return &scantrigger.Target{ + Folder: &models.MediaFolder{ID: 1}, + Mode: scantrigger.ModeSubtree, + Path: subtreePath, + Trigger: trigger, + }, nil +} + // unresolvableResolver treats every path as outside Silo's media folders, // returning a RequestError — the "none resolved → misconfiguration" signal. type unresolvableResolver struct{} @@ -195,16 +227,19 @@ func (denySuppressor) Release(context.Context, string) error { return nil } type recordingQueuer struct { enqueued []scantrigger.Target + batches [][]scantrigger.Target autoscanEvents []int64 createdCount *int reusedCount int } func (q *recordingQueuer) EnqueueScans(_ context.Context, targets []scantrigger.Target) error { + q.batches = append(q.batches, append([]scantrigger.Target(nil), targets...)) q.enqueued = append(q.enqueued, targets...) return nil } func (q *recordingQueuer) EnqueueAutoscanScans(_ context.Context, targets []scantrigger.Target, eventID int64) (int, int, error) { + q.batches = append(q.batches, append([]scantrigger.Target(nil), targets...)) q.enqueued = append(q.enqueued, targets...) q.autoscanEvents = append(q.autoscanEvents, eventID) if q.createdCount != nil { @@ -496,6 +531,44 @@ func TestPollOnceCollapsesLargeTargetBatchToLibraryScans(t *testing.T) { } } +func TestPollOnceChunksCollapsedLibraryScansAtTargetCap(t *testing.T) { + store := &fakeStore{ + settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60}, + sources: []Source{{ + ID: "s1", PluginID: "silo.autoscan.cephfs", CapabilityID: "cephfs", Enabled: true, + }}, + } + changes := make([]Change, 0, maxAutoscanTargetsPerPoll+1) + for i := 0; i <= maxAutoscanTargetsPerPoll; i++ { + changes = append(changes, Change{ + SourcePath: "/mnt/media/library-" + strconv.Itoa(i+1) + "/Episode.mkv", + Scope: ChangeScopeFile, + }) + } + prov := &fakeProvider{changes: map[string][]Change{"cephfs": changes}, nextMarker: "m1"} + q := &recordingQueuer{} + svc := NewService(store, prov, passthroughConnRes{}, distinctFolderResolver{}, q, allowSuppressor{}, nil) + + if err := svc.PollOnce(context.Background()); err != nil { + t.Fatalf("PollOnce: %v", err) + } + if len(q.enqueued) != maxAutoscanTargetsPerPoll+1 { + t.Fatalf("enqueued scans = %d, want %d", len(q.enqueued), maxAutoscanTargetsPerPoll+1) + } + if len(q.batches) != 2 { + t.Fatalf("queue batches = %d, want 2", len(q.batches)) + } + if len(q.batches[0]) != maxAutoscanTargetsPerPoll { + t.Fatalf("first batch size = %d, want %d", len(q.batches[0]), maxAutoscanTargetsPerPoll) + } + if len(q.batches[1]) != 1 { + t.Fatalf("second batch size = %d, want 1", len(q.batches[1])) + } + if got, ok := store.advanced["s1"]; !ok || got != "m1" { + t.Fatalf("marker must advance after all chunks enqueue, got %q ok=%v", got, ok) + } +} + func TestPollOnceScansDistinctPathsUnderSameFolder(t *testing.T) { // Two imported subtrees resolve to the SAME folder ID (7) but DIFFERENT // target paths. Keying suppression on folder ID alone would drop the second; diff --git a/internal/catalog/folder_repo.go b/internal/catalog/folder_repo.go index 1df19a6e..b3c1ab17 100644 --- a/internal/catalog/folder_repo.go +++ b/internal/catalog/folder_repo.go @@ -762,12 +762,13 @@ func (r *FolderRepository) deleteOrphanedItemsByContentID(ctx context.Context, c return 0, nil, nil } - imageDirs, err := collectImageDirs(ctx, r.pool, orphanIDs) + rawImageDirs, err := collectRawImageDirs(ctx, r.pool, orphanIDs) if err != nil { return 0, nil, err } var deletedIDs []string + var imageDirs []string if err := retryOnDeadlock(ctx, func() error { tx, err := r.pool.Begin(ctx) if err != nil { @@ -783,6 +784,13 @@ func (r *FolderRepository) deleteOrphanedItemsByContentID(ctx context.Context, c if err != nil { return fmt.Errorf("collecting deleted late orphan IDs: %w", err) } + var attemptImageDirs []string + if len(attemptDeletedIDs) > 0 { + attemptImageDirs, err = filterUnreferencedImageDirs(ctx, tx, rawImageDirs, attemptDeletedIDs) + if err != nil { + return err + } + } if err := EnqueueSearchIndexDeletes(ctx, tx, attemptDeletedIDs); err != nil { return fmt.Errorf("enqueueing catalog search late orphan deletes: %w", err) } @@ -790,6 +798,7 @@ func (r *FolderRepository) deleteOrphanedItemsByContentID(ctx context.Context, c return err } deletedIDs = attemptDeletedIDs + imageDirs = attemptImageDirs return nil }); err != nil { return 0, nil, err diff --git a/internal/catalog/item_repo.go b/internal/catalog/item_repo.go index b3fe207e..e460fe45 100644 --- a/internal/catalog/item_repo.go +++ b/internal/catalog/item_repo.go @@ -1206,10 +1206,10 @@ func (r *ItemRepository) buildSearchSQLWithTotal(query string, itemTypes []strin totalColumn = ", COUNT(*) OVER () AS total_count" } dataSQL = scoredCTE + statsCTE + fmt.Sprintf(` - SELECT %s%s - %s - ORDER BY exact_title_match DESC, contiguous_title_match DESC, year_match DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC, LOWER(title) ASC, content_id ASC - LIMIT $%d OFFSET $%d`, itemColumns, totalColumn, postFilter, argIdx, argIdx+1) + SELECT %s%s + %s + ORDER BY exact_title_match DESC, contiguous_title_match DESC, year_match DESC, phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC, LOWER(title) ASC, content_id ASC + LIMIT $%d OFFSET $%d`, itemColumns, totalColumn, postFilter, argIdx, argIdx+1) countSQL = scoredCTE + statsCTE + fmt.Sprintf(` SELECT COUNT(*) %s`, postFilter) diff --git a/internal/catalog/search_index_repo.go b/internal/catalog/search_index_repo.go index c57afb58..c99039d5 100644 --- a/internal/catalog/search_index_repo.go +++ b/internal/catalog/search_index_repo.go @@ -42,13 +42,28 @@ type SearchIndexState struct { } type SearchIndexEventRepository struct { - pool *pgxpool.Pool + pool *pgxpool.Pool + activeProvider string + activeProviderKnown bool } func NewSearchIndexEventRepository(pool *pgxpool.Pool) *SearchIndexEventRepository { return &SearchIndexEventRepository{pool: pool} } +func (r *SearchIndexEventRepository) WithActiveProvider(provider string) *SearchIndexEventRepository { + if r == nil { + return nil + } + provider = normalizeCatalogSearchProvider(provider) + if provider == "" { + provider = SearchProviderPostgres + } + r.activeProvider = provider + r.activeProviderKnown = true + return r +} + func (r *SearchIndexEventRepository) EnqueueUpsert(ctx context.Context, execer itemExecer, contentID string) error { return r.enqueue(ctx, execer, SearchProviderMeilisearch, SearchIndexEventUpsert, contentID, "") } @@ -61,14 +76,14 @@ func (r *SearchIndexEventRepository) EnqueueUpserts(ctx context.Context, execer if len(contentIDs) == 0 { return nil } - _, err := execer.Exec(ctx, ` + if ok, err := r.shouldEnqueue(ctx, execer); err != nil || !ok { + return err + } + err := execSearchIndexEventInsert(ctx, execer, ` INSERT INTO catalog_search_index_events (provider, action, content_id, previous_content_id) SELECT $1, $2, ids.content_id, '' FROM unnest($3::text[]) AS ids(content_id) `, SearchProviderMeilisearch, SearchIndexEventUpsert, contentIDs) - if isSearchIndexSchemaUnavailable(err) { - return nil - } return err } @@ -84,14 +99,14 @@ func (r *SearchIndexEventRepository) EnqueueDeletes(ctx context.Context, execer if len(contentIDs) == 0 { return nil } - _, err := execer.Exec(ctx, ` + if ok, err := r.shouldEnqueue(ctx, execer); err != nil || !ok { + return err + } + err := execSearchIndexEventInsert(ctx, execer, ` INSERT INTO catalog_search_index_events (provider, action, content_id, previous_content_id) SELECT $1, $2, ids.content_id, '' FROM unnest($3::text[]) AS ids(content_id) `, SearchProviderMeilisearch, SearchIndexEventDelete, contentIDs) - if isSearchIndexSchemaUnavailable(err) { - return nil - } return err } @@ -108,16 +123,117 @@ func (r *SearchIndexEventRepository) enqueue(ctx context.Context, execer itemExe if contentID == "" { return nil } - _, err := execer.Exec(ctx, ` + if ok, err := r.shouldEnqueue(ctx, execer); err != nil || !ok { + return err + } + err := execSearchIndexEventInsert(ctx, execer, ` INSERT INTO catalog_search_index_events (provider, action, content_id, previous_content_id) VALUES ($1, $2, $3, $4) `, provider, action, contentID, previousContentID) + return err +} + +func (r *SearchIndexEventRepository) shouldEnqueue(ctx context.Context, execer itemExecer) (bool, error) { + if r == nil || execer == nil { + return false, nil + } + if r.activeProviderKnown { + return r.activeProvider == SearchProviderMeilisearch, nil + } + return searchIndexProviderEnabled(ctx, execer) +} + +type searchIndexProviderQuerier interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +func searchIndexProviderEnabled(ctx context.Context, execer itemExecer) (bool, error) { + querier, ok := execer.(searchIndexProviderQuerier) + if !ok { + return true, nil + } + if tx, ok := execer.(pgx.Tx); ok { + return searchIndexProviderEnabledTx(ctx, tx) + } + enabled, err := querySearchIndexProviderEnabled(ctx, querier) + if isSearchIndexSchemaUnavailable(err) { + return false, nil + } + return enabled, err +} + +func searchIndexProviderEnabledTx(ctx context.Context, tx pgx.Tx) (bool, error) { + if _, err := tx.Exec(ctx, "SAVEPOINT catalog_search_index_provider_check"); err != nil { + return false, err + } + enabled, err := querySearchIndexProviderEnabled(ctx, tx) + if err == nil { + if _, releaseErr := tx.Exec(ctx, "RELEASE SAVEPOINT catalog_search_index_provider_check"); releaseErr != nil { + return false, releaseErr + } + return enabled, nil + } + if rollbackErr := rollbackSearchIndexSavepoint(ctx, tx, "catalog_search_index_provider_check"); rollbackErr != nil { + return false, rollbackErr + } + if isSearchIndexSchemaUnavailable(err) { + return false, nil + } + return false, err +} + +func querySearchIndexProviderEnabled(ctx context.Context, querier searchIndexProviderQuerier) (bool, error) { + var enabled bool + err := querier.QueryRow(ctx, ` + SELECT lower(COALESCE( + (SELECT value FROM server_settings WHERE key = $1), + $2 + )) = $3 + `, SearchSettingProvider, SearchProviderPostgres, SearchProviderMeilisearch).Scan(&enabled) + return enabled, err +} + +func execSearchIndexEventInsert(ctx context.Context, execer itemExecer, sql string, args ...any) error { + if tx, ok := execer.(pgx.Tx); ok { + return execSearchIndexEventInsertTx(ctx, tx, sql, args...) + } + _, err := execer.Exec(ctx, sql, args...) if isSearchIndexSchemaUnavailable(err) { return nil } return err } +func execSearchIndexEventInsertTx(ctx context.Context, tx pgx.Tx, sql string, args ...any) error { + if _, err := tx.Exec(ctx, "SAVEPOINT catalog_search_index_events_insert"); err != nil { + return err + } + _, err := tx.Exec(ctx, sql, args...) + if err == nil { + if _, releaseErr := tx.Exec(ctx, "RELEASE SAVEPOINT catalog_search_index_events_insert"); releaseErr != nil { + return releaseErr + } + return nil + } + if isSearchIndexSchemaUnavailable(err) { + if rollbackErr := rollbackSearchIndexSavepoint(ctx, tx, "catalog_search_index_events_insert"); rollbackErr != nil { + return rollbackErr + } + return nil + } + return err +} + +func rollbackSearchIndexSavepoint(ctx context.Context, tx pgx.Tx, name string) error { + if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT "+name); err != nil { + return err + } + if _, err := tx.Exec(ctx, "RELEASE SAVEPOINT "+name); err != nil { + return err + } + return nil +} + func EnqueueSearchIndexUpsert(ctx context.Context, execer itemExecer, contentID string) error { return NewSearchIndexEventRepository(nil).EnqueueUpsert(ctx, execer, contentID) } diff --git a/internal/catalog/search_index_repo_test.go b/internal/catalog/search_index_repo_test.go index a220972a..71be926a 100644 --- a/internal/catalog/search_index_repo_test.go +++ b/internal/catalog/search_index_repo_test.go @@ -67,3 +67,27 @@ func TestEnqueueSearchIndexUpsertsSkipsEmptyInput(t *testing.T) { t.Fatalf("expected no Exec calls for empty input, got %d", execer.calls) } } + +func TestEnqueueSearchIndexUpsertSkipsWhenProviderIsPostgres(t *testing.T) { + execer := &recordingSearchIndexExecer{} + repo := NewSearchIndexEventRepository(nil).WithActiveProvider(SearchProviderPostgres) + + if err := repo.EnqueueUpsert(context.Background(), execer, "movie-1"); err != nil { + t.Fatalf("EnqueueUpsert returned error: %v", err) + } + if execer.calls != 0 { + t.Fatalf("expected no Exec calls when provider is postgres, got %d", execer.calls) + } +} + +func TestEnqueueSearchIndexUpsertRunsWhenProviderIsMeilisearch(t *testing.T) { + execer := &recordingSearchIndexExecer{} + repo := NewSearchIndexEventRepository(nil).WithActiveProvider(SearchProviderMeilisearch) + + if err := repo.EnqueueUpsert(context.Background(), execer, "movie-1"); err != nil { + t.Fatalf("EnqueueUpsert returned error: %v", err) + } + if execer.calls != 1 { + t.Fatalf("expected one Exec call when provider is meilisearch, got %d", execer.calls) + } +} diff --git a/internal/catalog/search_meilisearch_provider.go b/internal/catalog/search_meilisearch_provider.go index ccd7f141..d4a8f0c2 100644 --- a/internal/catalog/search_meilisearch_provider.go +++ b/internal/catalog/search_meilisearch_provider.go @@ -327,7 +327,7 @@ func (p *MeilisearchSearchProvider) buildMeilisearchSearchRequest(ctx context.Co } func (p *MeilisearchSearchProvider) shouldUseSemanticSearch(req CatalogSearchRequest) bool { - if p == nil || !p.config.SemanticEnabled || req.SkipTotal { + if p == nil || !p.config.SemanticEnabled { return false } return len(strings.Fields(normalizeCatalogSearchQueryForVector(req.Query))) >= 3 diff --git a/internal/catalog/search_provider.go b/internal/catalog/search_provider.go index 2143fbf7..fd8b7d3b 100644 --- a/internal/catalog/search_provider.go +++ b/internal/catalog/search_provider.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "hash/fnv" + "math" "strconv" "strings" "time" @@ -214,7 +215,7 @@ func CatalogSearchSettingsFromMap(values map[string]string) (CatalogSearchSettin } if raw := strings.TrimSpace(values[SearchSettingMeilisearchSemanticRatio]); raw != "" { ratio, err := strconv.ParseFloat(raw, 64) - if err != nil || ratio < 0 || ratio > 1 { + if err != nil || math.IsNaN(ratio) || ratio < 0 || ratio > 1 { return settings, fmt.Errorf("%s must be a number between 0 and 1", SearchSettingMeilisearchSemanticRatio) } settings.SemanticRatio = ratio diff --git a/internal/catalog/search_provider_test.go b/internal/catalog/search_provider_test.go index dba76729..4045b029 100644 --- a/internal/catalog/search_provider_test.go +++ b/internal/catalog/search_provider_test.go @@ -56,6 +56,7 @@ func TestCatalogSearchSettingsFromMapRejectsOutOfRangeTuning(t *testing.T) { {SearchSettingMeilisearchSemanticEnabled: "sometimes"}, {SearchSettingMeilisearchSemanticRatio: "-0.1"}, {SearchSettingMeilisearchSemanticRatio: "1.1"}, + {SearchSettingMeilisearchSemanticRatio: "NaN"}, {SearchSettingMeilisearchEmbedder: "bad.name"}, } @@ -191,7 +192,7 @@ func TestMeilisearchSearchRequestBuildsHybridWhenSemanticEnabled(t *testing.T) { } } -func TestMeilisearchSearchRequestSkipsHybridForApproximateInteractiveSearch(t *testing.T) { +func TestMeilisearchSearchRequestBuildsHybridForApproximateInteractiveSearch(t *testing.T) { vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}} provider := &MeilisearchSearchProvider{ config: MeilisearchProviderConfig{ @@ -203,17 +204,17 @@ func TestMeilisearchSearchRequestSkipsHybridForApproximateInteractiveSearch(t *t }, } req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{ - Query: "sponge", + Query: "found family space opera", SkipTotal: true, }) if fallback != "" { t.Fatalf("fallback = %q, want empty", fallback) } - if req.Vector != nil || req.Hybrid != nil { - t.Fatalf("approximate interactive search should stay keyword-only: %#v", req) + if req.Vector == nil || req.Hybrid == nil { + t.Fatalf("approximate interactive search should include hybrid search: %#v", req) } - if vectorizer.calls != 0 { - t.Fatalf("interactive search should not call vectorizer, calls = %d", vectorizer.calls) + if vectorizer.calls != 1 { + t.Fatalf("interactive search should call vectorizer once, calls = %d", vectorizer.calls) } } diff --git a/internal/catalogseed/service.go b/internal/catalogseed/service.go index 845e6812..86414cdf 100644 --- a/internal/catalogseed/service.go +++ b/internal/catalogseed/service.go @@ -209,7 +209,7 @@ func (s *Service) ImportWithProgress(ctx context.Context, data []byte, opts Impo }); err != nil { return nil, err } - if err := catalog.EnqueueSearchIndexUpserts(ctx, tx, catalogSeedSearchUpsertIDs(itemStates, bundle.Embeddings)); err != nil { + if err := catalog.EnqueueSearchIndexUpserts(ctx, tx, catalogSeedSearchUpsertIDs(itemStates, bundle.Embeddings, nil, nil)); err != nil { return nil, fmt.Errorf("enqueueing catalog search seed import updates: %w", err) } currentWork++ @@ -389,11 +389,6 @@ func (s *Service) ImportWithProgress(ctx context.Context, data []byte, opts Impo }); err != nil { return nil, fmt.Errorf("importing embeddings: %w", err) } - if err := catalog.EnqueueSearchIndexUpserts(ctx, tx, catalogSeedSearchUpsertIDs(itemStates, bundle.Embeddings)); err != nil { - return nil, fmt.Errorf("enqueueing catalog search seed import updates: %w", err) - } - currentWork++ - reportProgress("Importing embeddings", currentWork, totalWork) // Batch import seasons. seasonRows := make([][]any, 0, len(bundle.Seasons)) @@ -588,7 +583,12 @@ func (s *Service) ImportWithProgress(ctx context.Context, data []byte, opts Impo } result.LinksCreated = int(linksAffected) + if err := catalog.EnqueueSearchIndexUpserts(ctx, tx, catalogSeedSearchUpsertIDs(itemStates, bundle.Embeddings, bundle.Files, bundle.LibraryLinks)); err != nil { + return nil, fmt.Errorf("enqueueing catalog search seed import updates: %w", err) + } + currentWork++ reportProgress("Finalizing catalog import", currentWork, totalWork) + if err := tx.Commit(ctx); err != nil { return nil, fmt.Errorf("committing catalog seed import: %w", err) } @@ -2503,8 +2503,8 @@ func dedupeStrings(values []string) []string { return out } -func catalogSeedSearchUpsertIDs(itemStates map[string]bool, embeddings []EmbeddingRecord) []string { - ids := make([]string, 0, len(itemStates)+len(embeddings)) +func catalogSeedSearchUpsertIDs(itemStates map[string]bool, embeddings []EmbeddingRecord, files []FileRecord, links []LibraryLinkRecord) []string { + ids := make([]string, 0, len(itemStates)+len(embeddings)+len(files)+len(links)) for contentID, changed := range itemStates { contentID = strings.TrimSpace(contentID) if changed && contentID != "" { @@ -2517,6 +2517,18 @@ func catalogSeedSearchUpsertIDs(itemStates map[string]bool, embeddings []Embeddi ids = append(ids, contentID) } } + for _, file := range files { + contentID := strings.TrimSpace(file.ContentID) + if contentID != "" { + ids = append(ids, contentID) + } + } + for _, link := range links { + contentID := strings.TrimSpace(link.ContentID) + if contentID != "" { + ids = append(ids, contentID) + } + } ids = dedupeStrings(ids) sort.Strings(ids) return ids diff --git a/internal/catalogseed/service_test.go b/internal/catalogseed/service_test.go index 9cab7c2c..9cd576ae 100644 --- a/internal/catalogseed/service_test.go +++ b/internal/catalogseed/service_test.go @@ -16,9 +16,17 @@ func TestCatalogSeedSearchUpsertIDsIncludesChangedItemsAndEmbeddings(t *testing. {MediaItemID: "movie-1"}, {MediaItemID: ""}, } + files := []FileRecord{ + {ContentID: " movie-4 "}, + {ContentID: ""}, + } + links := []LibraryLinkRecord{ + {ContentID: "movie-5"}, + {ContentID: "movie-2"}, + } - got := catalogSeedSearchUpsertIDs(itemStates, embeddings) - want := []string{"movie-1", "movie-3", "series-1"} + got := catalogSeedSearchUpsertIDs(itemStates, embeddings, files, links) + want := []string{"movie-1", "movie-2", "movie-3", "movie-4", "movie-5", "series-1"} if !reflect.DeepEqual(got, want) { t.Fatalf("catalogSeedSearchUpsertIDs = %#v, want %#v", got, want) diff --git a/internal/events/scan_registry.go b/internal/events/scan_registry.go index 16b1fd72..9eb4da35 100644 --- a/internal/events/scan_registry.go +++ b/internal/events/scan_registry.go @@ -1,6 +1,7 @@ package events import ( + "sort" "sync" "time" ) @@ -73,11 +74,24 @@ func (r *ScanRegistry) ListActiveLimit(limit int) []ScanRun { for _, run := range r.entries { if run.Status == "accepted" || run.Status == "running" { runs = append(runs, run) - if limit > 0 && len(runs) >= limit { - break - } } } + sort.Slice(runs, func(i, j int) bool { + left := runs[i] + right := runs[j] + leftStarted := left.StartedAt != nil + rightStarted := right.StartedAt != nil + if leftStarted != rightStarted { + return leftStarted + } + if leftStarted && !left.StartedAt.Equal(*right.StartedAt) { + return left.StartedAt.Before(*right.StartedAt) + } + return left.ID < right.ID + }) + if limit > 0 && len(runs) > limit { + return runs[:limit] + } return runs } diff --git a/internal/events/scan_registry_test.go b/internal/events/scan_registry_test.go index 3c47f92b..e6664aac 100644 --- a/internal/events/scan_registry_test.go +++ b/internal/events/scan_registry_test.go @@ -3,6 +3,7 @@ package events import ( "strconv" "testing" + "time" ) func TestScanRegistryListActiveLimit(t *testing.T) { @@ -27,3 +28,29 @@ func TestScanRegistryListActiveLimit(t *testing.T) { t.Fatalf("all active runs = %d, want 3", len(all)) } } + +func TestScanRegistryListActiveLimitSortsBeforeLimiting(t *testing.T) { + registry := NewScanRegistry() + newer := time.Date(2026, 6, 26, 12, 1, 0, 0, time.UTC) + older := time.Date(2026, 6, 26, 12, 0, 0, 0, time.UTC) + + registry.Upsert(ScanRun{ID: "queued-b", Status: "accepted"}) + registry.Upsert(ScanRun{ID: "running-newer", Status: "running", StartedAt: &newer}) + registry.Upsert(ScanRun{ID: "queued-a", Status: "accepted"}) + registry.Upsert(ScanRun{ID: "running-older", Status: "running", StartedAt: &older}) + + limited := registry.ListActiveLimit(3) + got := make([]string, 0, len(limited)) + for _, run := range limited { + got = append(got, run.ID) + } + want := []string{"running-older", "running-newer", "queued-a"} + if len(got) != len(want) { + t.Fatalf("limited IDs = %#v, want %#v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("limited IDs = %#v, want %#v", got, want) + } + } +} diff --git a/web/src/components/GlobalSearch.test.tsx b/web/src/components/GlobalSearch.test.tsx index 61328d07..997ac751 100644 --- a/web/src/components/GlobalSearch.test.tsx +++ b/web/src/components/GlobalSearch.test.tsx @@ -123,13 +123,14 @@ describe("GlobalSearch", () => { }); }); - it("renders preview rows and total hint when more results exist than the preview limit", () => { + it("renders preview rows and an approximate more-results hint", () => { const markup = renderSearchMarkup({ defaultOpen: true, initialQuery: "Test" }); expect(markup).toContain('data-testid="dialog"'); expect(markup).toContain('placeholder="Search library..."'); expect(markup).toContain("Test Movie"); - expect(markup).toContain("Showing top 8 of 50"); + expect(markup).toContain("Showing top results"); + expect(markup).not.toContain("of 50"); expect(markup).toContain("Press Enter for all results"); }); diff --git a/web/src/components/GlobalSearch.tsx b/web/src/components/GlobalSearch.tsx index ee8b24c6..94a76ceb 100644 --- a/web/src/components/GlobalSearch.tsx +++ b/web/src/components/GlobalSearch.tsx @@ -213,7 +213,7 @@ export function GlobalSearch({ const showResultsPanel = query.trim().length > 0; const items = previewQuery.data?.items ?? []; - const total = previewQuery.data?.total ?? 0; + const hasMore = previewQuery.data?.has_more ?? false; const showLoading = previewQuery.isFetching && items.length === 0; const showEmpty = !previewQuery.isFetching && @@ -321,10 +321,8 @@ export function GlobalSearch({ : `${items.length} results found`}
- {total > PREVIEW_LIMIT ? ( -

- Showing top {PREVIEW_LIMIT} of {total}. Press Enter for all results. -

+ {hasMore ? ( +

Showing top results. Press Enter for all results.

) : (

Press Enter to open the full search page.

)} diff --git a/web/src/components/ServerActivity.tsx b/web/src/components/ServerActivity.tsx index a82b00e3..ebf8049d 100644 --- a/web/src/components/ServerActivity.tsx +++ b/web/src/components/ServerActivity.tsx @@ -15,6 +15,7 @@ import type { TaskInfo, ScanRun } from "@/api/types"; const CONNECTION_PROBLEM_INDICATOR_DELAY_MS = 4_000; const MAX_ACTIVITY_SCAN_ROWS = 25; const MAX_BADGE_COUNT = 99; +const ACTIVE_SCAN_SNAPSHOT_LIMIT = 500; interface ServerActivityProps { /** Hide the trigger button entirely when there is no activity */ @@ -110,7 +111,8 @@ export default function ServerActivity({ hideWhenEmpty = false }: ServerActivity const showConnectionProblem = useDelayedConnectionProblem(connectionState); const visibleActiveScans = activeScans.slice(0, MAX_ACTIVITY_SCAN_ROWS); const hiddenActiveScanCount = activeScans.length - visibleActiveScans.length; - const totalActiveLabel = formatBadgeCount(totalActive); + const activeScansMayBeTruncated = activeScans.length >= ACTIVE_SCAN_SNAPSHOT_LIMIT; + const totalActiveLabel = formatBadgeCount(totalActive, activeScansMayBeTruncated); // Keep mounted while popover is open so Radix can animate closed if (hideWhenEmpty && totalActive === 0 && !open) return null; @@ -121,7 +123,9 @@ export default function ServerActivity({ hideWhenEmpty = false }: ServerActivity
) : ( @@ -243,6 +252,7 @@ export default function ServerActivity({ hideWhenEmpty = false }: ServerActivity function ActivitySection({ title, count, + countIsLowerBound = false, href, onNavigate, last = false, @@ -250,6 +260,7 @@ function ActivitySection({ }: { title: string; count: number; + countIsLowerBound?: boolean; href: string; onNavigate: () => void; last?: boolean; @@ -270,7 +281,7 @@ function ActivitySection({ {count > 0 && ( - {formatBadgeCount(count)} + {formatBadgeCount(count, countIsLowerBound)} )} @@ -294,7 +305,10 @@ const METHOD_META: Record = { transcode: { label: "Transcode", color: "bg-warning" }, }; -function formatBadgeCount(count: number) { +function formatBadgeCount(count: number, countIsLowerBound = false) { + if (countIsLowerBound) { + return count > MAX_BADGE_COUNT ? `${MAX_BADGE_COUNT}+` : `${count}+`; + } return count > MAX_BADGE_COUNT ? `${MAX_BADGE_COUNT}+` : count.toString(); } @@ -395,9 +409,18 @@ function EmptyRow({ children }: { children: React.ReactNode }) { return
{children}
; } -function MoreRows({ count, label }: { count: number; label: string }) { +function MoreRows({ + count, + label, + countIsLowerBound = false, +}: { + count: number; + label: string; + countIsLowerBound?: boolean; +}) { return (
+ {countIsLowerBound ? "At least " : ""} {count.toLocaleString()} {label}
);