fix(search): address provider review comments
This commit is contained in:
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
@@ -2216,7 +2217,7 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques
|
||||
req.Value = strconv.FormatBool(enabled)
|
||||
case catalog.SearchSettingMeilisearchSemanticRatio:
|
||||
ratio, err := strconv.ParseFloat(strings.TrimSpace(req.Value), 64)
|
||||
if err != nil || ratio < 0 || ratio > 1 {
|
||||
if err != nil || math.IsNaN(ratio) || ratio < 0 || ratio > 1 {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "catalog.search.meilisearch.semantic_ratio must be a number between 0 and 1")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -418,6 +418,7 @@ func TestAdminUpdateCatalogSearchSemanticSettingsValidation(t *testing.T) {
|
||||
{key: catalog.SearchSettingMeilisearchSemanticEnabled, value: "sometimes"},
|
||||
{key: catalog.SearchSettingMeilisearchSemanticRatio, value: "-0.01"},
|
||||
{key: catalog.SearchSettingMeilisearchSemanticRatio, value: "1.01"},
|
||||
{key: catalog.SearchSettingMeilisearchSemanticRatio, value: "NaN"},
|
||||
{key: catalog.SearchSettingMeilisearchEmbedder, value: "bad.name"},
|
||||
}
|
||||
for _, tc := range invalid {
|
||||
|
||||
+13
-4
@@ -448,15 +448,24 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
ebookAnnotationStore = handlers.NewPGEbookReaderAnnotationStore(deps.DB)
|
||||
browseRepo := catalog.NewBrowseRepository(deps.DB)
|
||||
itemRepo = catalog.NewItemRepository(deps.DB)
|
||||
catalogSearchSettings, err := catalog.LoadCatalogSearchSettings(context.Background(), settingsRepo)
|
||||
if err != nil {
|
||||
slog.Warn("catalog search: failed to load settings; using postgres", "err", err)
|
||||
catalogSearchSettings = catalog.DefaultCatalogSearchSettings()
|
||||
}
|
||||
searchIndexEvents := catalog.NewSearchIndexEventRepository(deps.DB)
|
||||
itemRepo.WithSearchIndexEvents(searchIndexEvents)
|
||||
catalogSearchService = catalog.NewCatalogSearchService(
|
||||
context.Background(),
|
||||
settingsRepo,
|
||||
catalogSearchService = catalog.NewCatalogSearchServiceFromSettings(
|
||||
catalogSearchSettings,
|
||||
itemRepo,
|
||||
searchIndexEvents,
|
||||
deps.CatalogSearchVectorizer,
|
||||
)
|
||||
activeSearchProvider := catalog.SearchProviderPostgres
|
||||
if _, ok := catalogSearchService.Provider().(*catalog.MeilisearchSearchProvider); ok {
|
||||
activeSearchProvider = catalog.SearchProviderMeilisearch
|
||||
}
|
||||
searchIndexEvents.WithActiveProvider(activeSearchProvider)
|
||||
itemRepo.WithSearchIndexEvents(searchIndexEvents)
|
||||
episodeRepo = catalog.NewEpisodeRepository(deps.DB)
|
||||
providerIDRepo = catalog.NewProviderIDRepository(deps.DB)
|
||||
calendarRepo = catalog.NewCalendarRepository(deps.DB)
|
||||
|
||||
@@ -214,12 +214,7 @@ func (s *Service) PollOnce(ctx context.Context) error {
|
||||
var enqueue EnqueueResult
|
||||
if len(targets) > 0 {
|
||||
var eerr error
|
||||
if eventID != 0 {
|
||||
enqueue.Created, enqueue.Reused, eerr = s.queue.EnqueueAutoscanScans(ctx, targets, eventID)
|
||||
} else {
|
||||
eerr = s.queue.EnqueueScans(ctx, targets)
|
||||
enqueue.Created = len(targets)
|
||||
}
|
||||
enqueue, eerr = s.enqueueScanTargets(ctx, targets, eventID)
|
||||
if eerr != nil {
|
||||
s.releaseClaims(ctx, claimed)
|
||||
slog.WarnContext(ctx, "autoscan: enqueue failed", "source_id", src.ID, "err", eerr)
|
||||
@@ -295,6 +290,34 @@ func (s *Service) PollOnce(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) enqueueScanTargets(ctx context.Context, targets []scantrigger.Target, eventID int64) (EnqueueResult, error) {
|
||||
var result EnqueueResult
|
||||
if len(targets) == 0 {
|
||||
return result, nil
|
||||
}
|
||||
for start := 0; start < len(targets); start += maxAutoscanTargetsPerPoll {
|
||||
end := start + maxAutoscanTargetsPerPoll
|
||||
if end > len(targets) {
|
||||
end = len(targets)
|
||||
}
|
||||
chunk := targets[start:end]
|
||||
if eventID != 0 {
|
||||
created, reused, err := s.queue.EnqueueAutoscanScans(ctx, chunk, eventID)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Created += created
|
||||
result.Reused += reused
|
||||
continue
|
||||
}
|
||||
if err := s.queue.EnqueueScans(ctx, chunk); err != nil {
|
||||
return result, err
|
||||
}
|
||||
result.Created += len(chunk)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (s *Service) createEvent(ctx context.Context, src Source, marker string, startedAt time.Time) (int64, bool) {
|
||||
if s == nil || s.store == nil {
|
||||
return 0, true
|
||||
|
||||
@@ -173,6 +173,38 @@ func (fakeResolver) ResolveMissingSubtree(_ context.Context, subtreePath, trigge
|
||||
return nil, &scantrigger.RequestError{Status: 400, Code: "bad_request", Message: "outside media folders"}
|
||||
}
|
||||
|
||||
type distinctFolderResolver struct{}
|
||||
|
||||
func (distinctFolderResolver) Resolve(_ context.Context, req scantrigger.Request) (*scantrigger.Target, error) {
|
||||
if !strings.HasPrefix(req.Path, "/mnt/media/library-") {
|
||||
return nil, &scantrigger.RequestError{Status: 400, Code: "bad_request", Message: "outside media folders"}
|
||||
}
|
||||
rest := strings.TrimPrefix(req.Path, "/mnt/media/library-")
|
||||
idPart := rest
|
||||
if slash := strings.IndexByte(rest, '/'); slash >= 0 {
|
||||
idPart = rest[:slash]
|
||||
}
|
||||
id, err := strconv.Atoi(idPart)
|
||||
if err != nil {
|
||||
return nil, &scantrigger.RequestError{Status: 400, Code: "bad_request", Message: "outside media folders"}
|
||||
}
|
||||
return &scantrigger.Target{
|
||||
Folder: &models.MediaFolder{ID: id},
|
||||
Mode: scantrigger.ModeFile,
|
||||
Path: req.Path,
|
||||
Trigger: req.Trigger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (distinctFolderResolver) ResolveMissingSubtree(_ context.Context, subtreePath, trigger string) (*scantrigger.Target, error) {
|
||||
return &scantrigger.Target{
|
||||
Folder: &models.MediaFolder{ID: 1},
|
||||
Mode: scantrigger.ModeSubtree,
|
||||
Path: subtreePath,
|
||||
Trigger: trigger,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// unresolvableResolver treats every path as outside Silo's media folders,
|
||||
// returning a RequestError — the "none resolved → misconfiguration" signal.
|
||||
type unresolvableResolver struct{}
|
||||
@@ -195,16 +227,19 @@ func (denySuppressor) Release(context.Context, string) error { return nil }
|
||||
|
||||
type recordingQueuer struct {
|
||||
enqueued []scantrigger.Target
|
||||
batches [][]scantrigger.Target
|
||||
autoscanEvents []int64
|
||||
createdCount *int
|
||||
reusedCount int
|
||||
}
|
||||
|
||||
func (q *recordingQueuer) EnqueueScans(_ context.Context, targets []scantrigger.Target) error {
|
||||
q.batches = append(q.batches, append([]scantrigger.Target(nil), targets...))
|
||||
q.enqueued = append(q.enqueued, targets...)
|
||||
return nil
|
||||
}
|
||||
func (q *recordingQueuer) EnqueueAutoscanScans(_ context.Context, targets []scantrigger.Target, eventID int64) (int, int, error) {
|
||||
q.batches = append(q.batches, append([]scantrigger.Target(nil), targets...))
|
||||
q.enqueued = append(q.enqueued, targets...)
|
||||
q.autoscanEvents = append(q.autoscanEvents, eventID)
|
||||
if q.createdCount != nil {
|
||||
@@ -496,6 +531,44 @@ func TestPollOnceCollapsesLargeTargetBatchToLibraryScans(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollOnceChunksCollapsedLibraryScansAtTargetCap(t *testing.T) {
|
||||
store := &fakeStore{
|
||||
settings: Settings{Enabled: true, DefaultPollIntervalSeconds: 600, DebounceSeconds: 60},
|
||||
sources: []Source{{
|
||||
ID: "s1", PluginID: "silo.autoscan.cephfs", CapabilityID: "cephfs", Enabled: true,
|
||||
}},
|
||||
}
|
||||
changes := make([]Change, 0, maxAutoscanTargetsPerPoll+1)
|
||||
for i := 0; i <= maxAutoscanTargetsPerPoll; i++ {
|
||||
changes = append(changes, Change{
|
||||
SourcePath: "/mnt/media/library-" + strconv.Itoa(i+1) + "/Episode.mkv",
|
||||
Scope: ChangeScopeFile,
|
||||
})
|
||||
}
|
||||
prov := &fakeProvider{changes: map[string][]Change{"cephfs": changes}, nextMarker: "m1"}
|
||||
q := &recordingQueuer{}
|
||||
svc := NewService(store, prov, passthroughConnRes{}, distinctFolderResolver{}, q, allowSuppressor{}, nil)
|
||||
|
||||
if err := svc.PollOnce(context.Background()); err != nil {
|
||||
t.Fatalf("PollOnce: %v", err)
|
||||
}
|
||||
if len(q.enqueued) != maxAutoscanTargetsPerPoll+1 {
|
||||
t.Fatalf("enqueued scans = %d, want %d", len(q.enqueued), maxAutoscanTargetsPerPoll+1)
|
||||
}
|
||||
if len(q.batches) != 2 {
|
||||
t.Fatalf("queue batches = %d, want 2", len(q.batches))
|
||||
}
|
||||
if len(q.batches[0]) != maxAutoscanTargetsPerPoll {
|
||||
t.Fatalf("first batch size = %d, want %d", len(q.batches[0]), maxAutoscanTargetsPerPoll)
|
||||
}
|
||||
if len(q.batches[1]) != 1 {
|
||||
t.Fatalf("second batch size = %d, want 1", len(q.batches[1]))
|
||||
}
|
||||
if got, ok := store.advanced["s1"]; !ok || got != "m1" {
|
||||
t.Fatalf("marker must advance after all chunks enqueue, got %q ok=%v", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPollOnceScansDistinctPathsUnderSameFolder(t *testing.T) {
|
||||
// Two imported subtrees resolve to the SAME folder ID (7) but DIFFERENT
|
||||
// target paths. Keying suppression on folder ID alone would drop the second;
|
||||
|
||||
@@ -762,12 +762,13 @@ func (r *FolderRepository) deleteOrphanedItemsByContentID(ctx context.Context, c
|
||||
return 0, nil, nil
|
||||
}
|
||||
|
||||
imageDirs, err := collectImageDirs(ctx, r.pool, orphanIDs)
|
||||
rawImageDirs, err := collectRawImageDirs(ctx, r.pool, orphanIDs)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
var deletedIDs []string
|
||||
var imageDirs []string
|
||||
if err := retryOnDeadlock(ctx, func() error {
|
||||
tx, err := r.pool.Begin(ctx)
|
||||
if err != nil {
|
||||
@@ -783,6 +784,13 @@ func (r *FolderRepository) deleteOrphanedItemsByContentID(ctx context.Context, c
|
||||
if err != nil {
|
||||
return fmt.Errorf("collecting deleted late orphan IDs: %w", err)
|
||||
}
|
||||
var attemptImageDirs []string
|
||||
if len(attemptDeletedIDs) > 0 {
|
||||
attemptImageDirs, err = filterUnreferencedImageDirs(ctx, tx, rawImageDirs, attemptDeletedIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := EnqueueSearchIndexDeletes(ctx, tx, attemptDeletedIDs); err != nil {
|
||||
return fmt.Errorf("enqueueing catalog search late orphan deletes: %w", err)
|
||||
}
|
||||
@@ -790,6 +798,7 @@ func (r *FolderRepository) deleteOrphanedItemsByContentID(ctx context.Context, c
|
||||
return err
|
||||
}
|
||||
deletedIDs = attemptDeletedIDs
|
||||
imageDirs = attemptImageDirs
|
||||
return nil
|
||||
}); err != nil {
|
||||
return 0, nil, err
|
||||
|
||||
@@ -1206,10 +1206,10 @@ func (r *ItemRepository) buildSearchSQLWithTotal(query string, itemTypes []strin
|
||||
totalColumn = ", COUNT(*) OVER () AS total_count"
|
||||
}
|
||||
dataSQL = scoredCTE + statsCTE + fmt.Sprintf(`
|
||||
SELECT %s%s
|
||||
%s
|
||||
ORDER BY exact_title_match DESC, contiguous_title_match DESC, year_match DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC, LOWER(title) ASC, content_id ASC
|
||||
LIMIT $%d OFFSET $%d`, itemColumns, totalColumn, postFilter, argIdx, argIdx+1)
|
||||
SELECT %s%s
|
||||
%s
|
||||
ORDER BY exact_title_match DESC, contiguous_title_match DESC, year_match DESC, phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC, LOWER(title) ASC, content_id ASC
|
||||
LIMIT $%d OFFSET $%d`, itemColumns, totalColumn, postFilter, argIdx, argIdx+1)
|
||||
countSQL = scoredCTE + statsCTE + fmt.Sprintf(`
|
||||
SELECT COUNT(*)
|
||||
%s`, postFilter)
|
||||
|
||||
@@ -42,13 +42,28 @@ type SearchIndexState struct {
|
||||
}
|
||||
|
||||
type SearchIndexEventRepository struct {
|
||||
pool *pgxpool.Pool
|
||||
pool *pgxpool.Pool
|
||||
activeProvider string
|
||||
activeProviderKnown bool
|
||||
}
|
||||
|
||||
func NewSearchIndexEventRepository(pool *pgxpool.Pool) *SearchIndexEventRepository {
|
||||
return &SearchIndexEventRepository{pool: pool}
|
||||
}
|
||||
|
||||
func (r *SearchIndexEventRepository) WithActiveProvider(provider string) *SearchIndexEventRepository {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
provider = normalizeCatalogSearchProvider(provider)
|
||||
if provider == "" {
|
||||
provider = SearchProviderPostgres
|
||||
}
|
||||
r.activeProvider = provider
|
||||
r.activeProviderKnown = true
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *SearchIndexEventRepository) EnqueueUpsert(ctx context.Context, execer itemExecer, contentID string) error {
|
||||
return r.enqueue(ctx, execer, SearchProviderMeilisearch, SearchIndexEventUpsert, contentID, "")
|
||||
}
|
||||
@@ -61,14 +76,14 @@ func (r *SearchIndexEventRepository) EnqueueUpserts(ctx context.Context, execer
|
||||
if len(contentIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := execer.Exec(ctx, `
|
||||
if ok, err := r.shouldEnqueue(ctx, execer); err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
err := execSearchIndexEventInsert(ctx, execer, `
|
||||
INSERT INTO catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT $1, $2, ids.content_id, ''
|
||||
FROM unnest($3::text[]) AS ids(content_id)
|
||||
`, SearchProviderMeilisearch, SearchIndexEventUpsert, contentIDs)
|
||||
if isSearchIndexSchemaUnavailable(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -84,14 +99,14 @@ func (r *SearchIndexEventRepository) EnqueueDeletes(ctx context.Context, execer
|
||||
if len(contentIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
_, err := execer.Exec(ctx, `
|
||||
if ok, err := r.shouldEnqueue(ctx, execer); err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
err := execSearchIndexEventInsert(ctx, execer, `
|
||||
INSERT INTO catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
SELECT $1, $2, ids.content_id, ''
|
||||
FROM unnest($3::text[]) AS ids(content_id)
|
||||
`, SearchProviderMeilisearch, SearchIndexEventDelete, contentIDs)
|
||||
if isSearchIndexSchemaUnavailable(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -108,16 +123,117 @@ func (r *SearchIndexEventRepository) enqueue(ctx context.Context, execer itemExe
|
||||
if contentID == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := execer.Exec(ctx, `
|
||||
if ok, err := r.shouldEnqueue(ctx, execer); err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
err := execSearchIndexEventInsert(ctx, execer, `
|
||||
INSERT INTO catalog_search_index_events (provider, action, content_id, previous_content_id)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
`, provider, action, contentID, previousContentID)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *SearchIndexEventRepository) shouldEnqueue(ctx context.Context, execer itemExecer) (bool, error) {
|
||||
if r == nil || execer == nil {
|
||||
return false, nil
|
||||
}
|
||||
if r.activeProviderKnown {
|
||||
return r.activeProvider == SearchProviderMeilisearch, nil
|
||||
}
|
||||
return searchIndexProviderEnabled(ctx, execer)
|
||||
}
|
||||
|
||||
type searchIndexProviderQuerier interface {
|
||||
QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
|
||||
}
|
||||
|
||||
func searchIndexProviderEnabled(ctx context.Context, execer itemExecer) (bool, error) {
|
||||
querier, ok := execer.(searchIndexProviderQuerier)
|
||||
if !ok {
|
||||
return true, nil
|
||||
}
|
||||
if tx, ok := execer.(pgx.Tx); ok {
|
||||
return searchIndexProviderEnabledTx(ctx, tx)
|
||||
}
|
||||
enabled, err := querySearchIndexProviderEnabled(ctx, querier)
|
||||
if isSearchIndexSchemaUnavailable(err) {
|
||||
return false, nil
|
||||
}
|
||||
return enabled, err
|
||||
}
|
||||
|
||||
func searchIndexProviderEnabledTx(ctx context.Context, tx pgx.Tx) (bool, error) {
|
||||
if _, err := tx.Exec(ctx, "SAVEPOINT catalog_search_index_provider_check"); err != nil {
|
||||
return false, err
|
||||
}
|
||||
enabled, err := querySearchIndexProviderEnabled(ctx, tx)
|
||||
if err == nil {
|
||||
if _, releaseErr := tx.Exec(ctx, "RELEASE SAVEPOINT catalog_search_index_provider_check"); releaseErr != nil {
|
||||
return false, releaseErr
|
||||
}
|
||||
return enabled, nil
|
||||
}
|
||||
if rollbackErr := rollbackSearchIndexSavepoint(ctx, tx, "catalog_search_index_provider_check"); rollbackErr != nil {
|
||||
return false, rollbackErr
|
||||
}
|
||||
if isSearchIndexSchemaUnavailable(err) {
|
||||
return false, nil
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func querySearchIndexProviderEnabled(ctx context.Context, querier searchIndexProviderQuerier) (bool, error) {
|
||||
var enabled bool
|
||||
err := querier.QueryRow(ctx, `
|
||||
SELECT lower(COALESCE(
|
||||
(SELECT value FROM server_settings WHERE key = $1),
|
||||
$2
|
||||
)) = $3
|
||||
`, SearchSettingProvider, SearchProviderPostgres, SearchProviderMeilisearch).Scan(&enabled)
|
||||
return enabled, err
|
||||
}
|
||||
|
||||
func execSearchIndexEventInsert(ctx context.Context, execer itemExecer, sql string, args ...any) error {
|
||||
if tx, ok := execer.(pgx.Tx); ok {
|
||||
return execSearchIndexEventInsertTx(ctx, tx, sql, args...)
|
||||
}
|
||||
_, err := execer.Exec(ctx, sql, args...)
|
||||
if isSearchIndexSchemaUnavailable(err) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func execSearchIndexEventInsertTx(ctx context.Context, tx pgx.Tx, sql string, args ...any) error {
|
||||
if _, err := tx.Exec(ctx, "SAVEPOINT catalog_search_index_events_insert"); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := tx.Exec(ctx, sql, args...)
|
||||
if err == nil {
|
||||
if _, releaseErr := tx.Exec(ctx, "RELEASE SAVEPOINT catalog_search_index_events_insert"); releaseErr != nil {
|
||||
return releaseErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if isSearchIndexSchemaUnavailable(err) {
|
||||
if rollbackErr := rollbackSearchIndexSavepoint(ctx, tx, "catalog_search_index_events_insert"); rollbackErr != nil {
|
||||
return rollbackErr
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func rollbackSearchIndexSavepoint(ctx context.Context, tx pgx.Tx, name string) error {
|
||||
if _, err := tx.Exec(ctx, "ROLLBACK TO SAVEPOINT "+name); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, "RELEASE SAVEPOINT "+name); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func EnqueueSearchIndexUpsert(ctx context.Context, execer itemExecer, contentID string) error {
|
||||
return NewSearchIndexEventRepository(nil).EnqueueUpsert(ctx, execer, contentID)
|
||||
}
|
||||
|
||||
@@ -67,3 +67,27 @@ func TestEnqueueSearchIndexUpsertsSkipsEmptyInput(t *testing.T) {
|
||||
t.Fatalf("expected no Exec calls for empty input, got %d", execer.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueueSearchIndexUpsertSkipsWhenProviderIsPostgres(t *testing.T) {
|
||||
execer := &recordingSearchIndexExecer{}
|
||||
repo := NewSearchIndexEventRepository(nil).WithActiveProvider(SearchProviderPostgres)
|
||||
|
||||
if err := repo.EnqueueUpsert(context.Background(), execer, "movie-1"); err != nil {
|
||||
t.Fatalf("EnqueueUpsert returned error: %v", err)
|
||||
}
|
||||
if execer.calls != 0 {
|
||||
t.Fatalf("expected no Exec calls when provider is postgres, got %d", execer.calls)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnqueueSearchIndexUpsertRunsWhenProviderIsMeilisearch(t *testing.T) {
|
||||
execer := &recordingSearchIndexExecer{}
|
||||
repo := NewSearchIndexEventRepository(nil).WithActiveProvider(SearchProviderMeilisearch)
|
||||
|
||||
if err := repo.EnqueueUpsert(context.Background(), execer, "movie-1"); err != nil {
|
||||
t.Fatalf("EnqueueUpsert returned error: %v", err)
|
||||
}
|
||||
if execer.calls != 1 {
|
||||
t.Fatalf("expected one Exec call when provider is meilisearch, got %d", execer.calls)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -327,7 +327,7 @@ func (p *MeilisearchSearchProvider) buildMeilisearchSearchRequest(ctx context.Co
|
||||
}
|
||||
|
||||
func (p *MeilisearchSearchProvider) shouldUseSemanticSearch(req CatalogSearchRequest) bool {
|
||||
if p == nil || !p.config.SemanticEnabled || req.SkipTotal {
|
||||
if p == nil || !p.config.SemanticEnabled {
|
||||
return false
|
||||
}
|
||||
return len(strings.Fields(normalizeCatalogSearchQueryForVector(req.Query))) >= 3
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -214,7 +215,7 @@ func CatalogSearchSettingsFromMap(values map[string]string) (CatalogSearchSettin
|
||||
}
|
||||
if raw := strings.TrimSpace(values[SearchSettingMeilisearchSemanticRatio]); raw != "" {
|
||||
ratio, err := strconv.ParseFloat(raw, 64)
|
||||
if err != nil || ratio < 0 || ratio > 1 {
|
||||
if err != nil || math.IsNaN(ratio) || ratio < 0 || ratio > 1 {
|
||||
return settings, fmt.Errorf("%s must be a number between 0 and 1", SearchSettingMeilisearchSemanticRatio)
|
||||
}
|
||||
settings.SemanticRatio = ratio
|
||||
|
||||
@@ -56,6 +56,7 @@ func TestCatalogSearchSettingsFromMapRejectsOutOfRangeTuning(t *testing.T) {
|
||||
{SearchSettingMeilisearchSemanticEnabled: "sometimes"},
|
||||
{SearchSettingMeilisearchSemanticRatio: "-0.1"},
|
||||
{SearchSettingMeilisearchSemanticRatio: "1.1"},
|
||||
{SearchSettingMeilisearchSemanticRatio: "NaN"},
|
||||
{SearchSettingMeilisearchEmbedder: "bad.name"},
|
||||
}
|
||||
|
||||
@@ -191,7 +192,7 @@ func TestMeilisearchSearchRequestBuildsHybridWhenSemanticEnabled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeilisearchSearchRequestSkipsHybridForApproximateInteractiveSearch(t *testing.T) {
|
||||
func TestMeilisearchSearchRequestBuildsHybridForApproximateInteractiveSearch(t *testing.T) {
|
||||
vectorizer := &fakeCatalogSearchVectorizer{vector: []float32{0.5, 0.25}}
|
||||
provider := &MeilisearchSearchProvider{
|
||||
config: MeilisearchProviderConfig{
|
||||
@@ -203,17 +204,17 @@ func TestMeilisearchSearchRequestSkipsHybridForApproximateInteractiveSearch(t *t
|
||||
},
|
||||
}
|
||||
req, fallback := provider.buildMeilisearchSearchRequest(context.Background(), CatalogSearchRequest{
|
||||
Query: "sponge",
|
||||
Query: "found family space opera",
|
||||
SkipTotal: true,
|
||||
})
|
||||
if fallback != "" {
|
||||
t.Fatalf("fallback = %q, want empty", fallback)
|
||||
}
|
||||
if req.Vector != nil || req.Hybrid != nil {
|
||||
t.Fatalf("approximate interactive search should stay keyword-only: %#v", req)
|
||||
if req.Vector == nil || req.Hybrid == nil {
|
||||
t.Fatalf("approximate interactive search should include hybrid search: %#v", req)
|
||||
}
|
||||
if vectorizer.calls != 0 {
|
||||
t.Fatalf("interactive search should not call vectorizer, calls = %d", vectorizer.calls)
|
||||
if vectorizer.calls != 1 {
|
||||
t.Fatalf("interactive search should call vectorizer once, calls = %d", vectorizer.calls)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -209,7 +209,7 @@ func (s *Service) ImportWithProgress(ctx context.Context, data []byte, opts Impo
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := catalog.EnqueueSearchIndexUpserts(ctx, tx, catalogSeedSearchUpsertIDs(itemStates, bundle.Embeddings)); err != nil {
|
||||
if err := catalog.EnqueueSearchIndexUpserts(ctx, tx, catalogSeedSearchUpsertIDs(itemStates, bundle.Embeddings, nil, nil)); err != nil {
|
||||
return nil, fmt.Errorf("enqueueing catalog search seed import updates: %w", err)
|
||||
}
|
||||
currentWork++
|
||||
@@ -389,11 +389,6 @@ func (s *Service) ImportWithProgress(ctx context.Context, data []byte, opts Impo
|
||||
}); err != nil {
|
||||
return nil, fmt.Errorf("importing embeddings: %w", err)
|
||||
}
|
||||
if err := catalog.EnqueueSearchIndexUpserts(ctx, tx, catalogSeedSearchUpsertIDs(itemStates, bundle.Embeddings)); err != nil {
|
||||
return nil, fmt.Errorf("enqueueing catalog search seed import updates: %w", err)
|
||||
}
|
||||
currentWork++
|
||||
reportProgress("Importing embeddings", currentWork, totalWork)
|
||||
|
||||
// Batch import seasons.
|
||||
seasonRows := make([][]any, 0, len(bundle.Seasons))
|
||||
@@ -588,7 +583,12 @@ func (s *Service) ImportWithProgress(ctx context.Context, data []byte, opts Impo
|
||||
}
|
||||
result.LinksCreated = int(linksAffected)
|
||||
|
||||
if err := catalog.EnqueueSearchIndexUpserts(ctx, tx, catalogSeedSearchUpsertIDs(itemStates, bundle.Embeddings, bundle.Files, bundle.LibraryLinks)); err != nil {
|
||||
return nil, fmt.Errorf("enqueueing catalog search seed import updates: %w", err)
|
||||
}
|
||||
currentWork++
|
||||
reportProgress("Finalizing catalog import", currentWork, totalWork)
|
||||
|
||||
if err := tx.Commit(ctx); err != nil {
|
||||
return nil, fmt.Errorf("committing catalog seed import: %w", err)
|
||||
}
|
||||
@@ -2503,8 +2503,8 @@ func dedupeStrings(values []string) []string {
|
||||
return out
|
||||
}
|
||||
|
||||
func catalogSeedSearchUpsertIDs(itemStates map[string]bool, embeddings []EmbeddingRecord) []string {
|
||||
ids := make([]string, 0, len(itemStates)+len(embeddings))
|
||||
func catalogSeedSearchUpsertIDs(itemStates map[string]bool, embeddings []EmbeddingRecord, files []FileRecord, links []LibraryLinkRecord) []string {
|
||||
ids := make([]string, 0, len(itemStates)+len(embeddings)+len(files)+len(links))
|
||||
for contentID, changed := range itemStates {
|
||||
contentID = strings.TrimSpace(contentID)
|
||||
if changed && contentID != "" {
|
||||
@@ -2517,6 +2517,18 @@ func catalogSeedSearchUpsertIDs(itemStates map[string]bool, embeddings []Embeddi
|
||||
ids = append(ids, contentID)
|
||||
}
|
||||
}
|
||||
for _, file := range files {
|
||||
contentID := strings.TrimSpace(file.ContentID)
|
||||
if contentID != "" {
|
||||
ids = append(ids, contentID)
|
||||
}
|
||||
}
|
||||
for _, link := range links {
|
||||
contentID := strings.TrimSpace(link.ContentID)
|
||||
if contentID != "" {
|
||||
ids = append(ids, contentID)
|
||||
}
|
||||
}
|
||||
ids = dedupeStrings(ids)
|
||||
sort.Strings(ids)
|
||||
return ids
|
||||
|
||||
@@ -16,9 +16,17 @@ func TestCatalogSeedSearchUpsertIDsIncludesChangedItemsAndEmbeddings(t *testing.
|
||||
{MediaItemID: "movie-1"},
|
||||
{MediaItemID: ""},
|
||||
}
|
||||
files := []FileRecord{
|
||||
{ContentID: " movie-4 "},
|
||||
{ContentID: ""},
|
||||
}
|
||||
links := []LibraryLinkRecord{
|
||||
{ContentID: "movie-5"},
|
||||
{ContentID: "movie-2"},
|
||||
}
|
||||
|
||||
got := catalogSeedSearchUpsertIDs(itemStates, embeddings)
|
||||
want := []string{"movie-1", "movie-3", "series-1"}
|
||||
got := catalogSeedSearchUpsertIDs(itemStates, embeddings, files, links)
|
||||
want := []string{"movie-1", "movie-2", "movie-3", "movie-4", "movie-5", "series-1"}
|
||||
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Fatalf("catalogSeedSearchUpsertIDs = %#v, want %#v", got, want)
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package events
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -73,11 +74,24 @@ func (r *ScanRegistry) ListActiveLimit(limit int) []ScanRun {
|
||||
for _, run := range r.entries {
|
||||
if run.Status == "accepted" || run.Status == "running" {
|
||||
runs = append(runs, run)
|
||||
if limit > 0 && len(runs) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.Slice(runs, func(i, j int) bool {
|
||||
left := runs[i]
|
||||
right := runs[j]
|
||||
leftStarted := left.StartedAt != nil
|
||||
rightStarted := right.StartedAt != nil
|
||||
if leftStarted != rightStarted {
|
||||
return leftStarted
|
||||
}
|
||||
if leftStarted && !left.StartedAt.Equal(*right.StartedAt) {
|
||||
return left.StartedAt.Before(*right.StartedAt)
|
||||
}
|
||||
return left.ID < right.ID
|
||||
})
|
||||
if limit > 0 && len(runs) > limit {
|
||||
return runs[:limit]
|
||||
}
|
||||
return runs
|
||||
}
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ package events
|
||||
import (
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestScanRegistryListActiveLimit(t *testing.T) {
|
||||
@@ -27,3 +28,29 @@ func TestScanRegistryListActiveLimit(t *testing.T) {
|
||||
t.Fatalf("all active runs = %d, want 3", len(all))
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanRegistryListActiveLimitSortsBeforeLimiting(t *testing.T) {
|
||||
registry := NewScanRegistry()
|
||||
newer := time.Date(2026, 6, 26, 12, 1, 0, 0, time.UTC)
|
||||
older := time.Date(2026, 6, 26, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
registry.Upsert(ScanRun{ID: "queued-b", Status: "accepted"})
|
||||
registry.Upsert(ScanRun{ID: "running-newer", Status: "running", StartedAt: &newer})
|
||||
registry.Upsert(ScanRun{ID: "queued-a", Status: "accepted"})
|
||||
registry.Upsert(ScanRun{ID: "running-older", Status: "running", StartedAt: &older})
|
||||
|
||||
limited := registry.ListActiveLimit(3)
|
||||
got := make([]string, 0, len(limited))
|
||||
for _, run := range limited {
|
||||
got = append(got, run.ID)
|
||||
}
|
||||
want := []string{"running-older", "running-newer", "queued-a"}
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("limited IDs = %#v, want %#v", got, want)
|
||||
}
|
||||
for i := range want {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("limited IDs = %#v, want %#v", got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -123,13 +123,14 @@ describe("GlobalSearch", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("renders preview rows and total hint when more results exist than the preview limit", () => {
|
||||
it("renders preview rows and an approximate more-results hint", () => {
|
||||
const markup = renderSearchMarkup({ defaultOpen: true, initialQuery: "Test" });
|
||||
|
||||
expect(markup).toContain('data-testid="dialog"');
|
||||
expect(markup).toContain('placeholder="Search library..."');
|
||||
expect(markup).toContain("Test Movie");
|
||||
expect(markup).toContain("Showing top 8 of 50");
|
||||
expect(markup).toContain("Showing top results");
|
||||
expect(markup).not.toContain("of 50");
|
||||
expect(markup).toContain("Press Enter for all results");
|
||||
});
|
||||
|
||||
|
||||
@@ -213,7 +213,7 @@ export function GlobalSearch({
|
||||
|
||||
const showResultsPanel = query.trim().length > 0;
|
||||
const items = previewQuery.data?.items ?? [];
|
||||
const total = previewQuery.data?.total ?? 0;
|
||||
const hasMore = previewQuery.data?.has_more ?? false;
|
||||
const showLoading = previewQuery.isFetching && items.length === 0;
|
||||
const showEmpty =
|
||||
!previewQuery.isFetching &&
|
||||
@@ -321,10 +321,8 @@ export function GlobalSearch({
|
||||
: `${items.length} results found`}
|
||||
</div>
|
||||
<div className="text-muted-foreground border-t px-3 py-2 text-center text-xs">
|
||||
{total > PREVIEW_LIMIT ? (
|
||||
<p>
|
||||
Showing top {PREVIEW_LIMIT} of {total}. Press Enter for all results.
|
||||
</p>
|
||||
{hasMore ? (
|
||||
<p>Showing top results. Press Enter for all results.</p>
|
||||
) : (
|
||||
<p>Press Enter to open the full search page.</p>
|
||||
)}
|
||||
|
||||
@@ -15,6 +15,7 @@ import type { TaskInfo, ScanRun } from "@/api/types";
|
||||
const CONNECTION_PROBLEM_INDICATOR_DELAY_MS = 4_000;
|
||||
const MAX_ACTIVITY_SCAN_ROWS = 25;
|
||||
const MAX_BADGE_COUNT = 99;
|
||||
const ACTIVE_SCAN_SNAPSHOT_LIMIT = 500;
|
||||
|
||||
interface ServerActivityProps {
|
||||
/** Hide the trigger button entirely when there is no activity */
|
||||
@@ -110,7 +111,8 @@ export default function ServerActivity({ hideWhenEmpty = false }: ServerActivity
|
||||
const showConnectionProblem = useDelayedConnectionProblem(connectionState);
|
||||
const visibleActiveScans = activeScans.slice(0, MAX_ACTIVITY_SCAN_ROWS);
|
||||
const hiddenActiveScanCount = activeScans.length - visibleActiveScans.length;
|
||||
const totalActiveLabel = formatBadgeCount(totalActive);
|
||||
const activeScansMayBeTruncated = activeScans.length >= ACTIVE_SCAN_SNAPSHOT_LIMIT;
|
||||
const totalActiveLabel = formatBadgeCount(totalActive, activeScansMayBeTruncated);
|
||||
|
||||
// Keep mounted while popover is open so Radix can animate closed
|
||||
if (hideWhenEmpty && totalActive === 0 && !open) return null;
|
||||
@@ -121,7 +123,9 @@ export default function ServerActivity({ hideWhenEmpty = false }: ServerActivity
|
||||
<button
|
||||
type="button"
|
||||
aria-label={
|
||||
totalActive > 0 ? `Server activity: ${totalActive} active` : "Server activity"
|
||||
totalActive > 0
|
||||
? `Server activity: ${activeScansMayBeTruncated ? "at least " : ""}${totalActive} active`
|
||||
: "Server activity"
|
||||
}
|
||||
className="hover:bg-accent/60 relative flex h-9 w-9 items-center justify-center rounded-xl transition-colors"
|
||||
>
|
||||
@@ -208,6 +212,7 @@ export default function ServerActivity({ hideWhenEmpty = false }: ServerActivity
|
||||
<ActivitySection
|
||||
title="Scans"
|
||||
count={activeScans.length}
|
||||
countIsLowerBound={activeScansMayBeTruncated}
|
||||
href="/admin/libraries"
|
||||
onNavigate={() => setOpen(false)}
|
||||
last
|
||||
@@ -222,7 +227,11 @@ export default function ServerActivity({ hideWhenEmpty = false }: ServerActivity
|
||||
/>
|
||||
))}
|
||||
{hiddenActiveScanCount > 0 && (
|
||||
<MoreRows count={hiddenActiveScanCount} label="more scans queued" />
|
||||
<MoreRows
|
||||
count={hiddenActiveScanCount}
|
||||
label="more scans queued"
|
||||
countIsLowerBound={activeScansMayBeTruncated}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
@@ -243,6 +252,7 @@ export default function ServerActivity({ hideWhenEmpty = false }: ServerActivity
|
||||
function ActivitySection({
|
||||
title,
|
||||
count,
|
||||
countIsLowerBound = false,
|
||||
href,
|
||||
onNavigate,
|
||||
last = false,
|
||||
@@ -250,6 +260,7 @@ function ActivitySection({
|
||||
}: {
|
||||
title: string;
|
||||
count: number;
|
||||
countIsLowerBound?: boolean;
|
||||
href: string;
|
||||
onNavigate: () => void;
|
||||
last?: boolean;
|
||||
@@ -270,7 +281,7 @@ function ActivitySection({
|
||||
</span>
|
||||
{count > 0 && (
|
||||
<span className="bg-primary/10 text-primary rounded-md px-1.5 py-0.5 text-[10px] leading-none font-bold">
|
||||
{formatBadgeCount(count)}
|
||||
{formatBadgeCount(count, countIsLowerBound)}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
@@ -294,7 +305,10 @@ const METHOD_META: Record<string, { label: string; color: string }> = {
|
||||
transcode: { label: "Transcode", color: "bg-warning" },
|
||||
};
|
||||
|
||||
function formatBadgeCount(count: number) {
|
||||
function formatBadgeCount(count: number, countIsLowerBound = false) {
|
||||
if (countIsLowerBound) {
|
||||
return count > MAX_BADGE_COUNT ? `${MAX_BADGE_COUNT}+` : `${count}+`;
|
||||
}
|
||||
return count > MAX_BADGE_COUNT ? `${MAX_BADGE_COUNT}+` : count.toString();
|
||||
}
|
||||
|
||||
@@ -395,9 +409,18 @@ function EmptyRow({ children }: { children: React.ReactNode }) {
|
||||
return <div className="text-muted-foreground py-1 text-[11px]">{children}</div>;
|
||||
}
|
||||
|
||||
function MoreRows({ count, label }: { count: number; label: string }) {
|
||||
function MoreRows({
|
||||
count,
|
||||
label,
|
||||
countIsLowerBound = false,
|
||||
}: {
|
||||
count: number;
|
||||
label: string;
|
||||
countIsLowerBound?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div className="text-muted-foreground border-t border-[color-mix(in_srgb,var(--border)_35%,transparent)] pt-1.5 text-[11px]">
|
||||
{countIsLowerBound ? "At least " : ""}
|
||||
{count.toLocaleString()} {label}
|
||||
</div>
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user