diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 6f5a61d8..8c4da0a6 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -1228,6 +1228,35 @@ func main() { deps.MovieMatchQueueRepo = movieQueueRepo deps.SeriesRootMatchQueueRepo = seriesQueueRepo matchQueueCoordinator = metadata.NewMatchQueueCoordinator(movieQueueRepo, seriesQueueRepo) + backgroundInit = append(backgroundInit, func(ctx context.Context) { + if err := matchQueueCoordinator.WakeForChangedInputs(ctx); err != nil { + slog.WarnContext(ctx, "refresh metadata match queue inputs at startup failed", "component", "app", "error", err) + } + }) + if pluginService != nil { + matchInputChanged := make(chan struct{}, 1) + go func() { + for { + select { + case <-appCtx.Done(): + return + case <-matchInputChanged: + if err := matchQueueCoordinator.WakeForChangedInputs(appCtx); err != nil { + slog.WarnContext(appCtx, "wake metadata matches after plugin lifecycle change failed", "component", "app", "error", err) + } + } + } + }() + pluginService.AddLifecycleHook(func(context.Context) { + // Queue fingerprint reconciliation may touch thousands of parked + // rows. Coalesce lifecycle bursts and keep plugin admin requests + // independent of that background database work. + select { + case matchInputChanged <- struct{}{}: + default: + } + }) + } rootClaimRepo = catalog.NewRootClaimRepository(deps.DB) groupClaimRepo = catalog.NewGroupClaimRepository(deps.DB) pluginResolver := metadata.NewPluginResolverAdapter(pluginService) diff --git a/internal/api/handlers/libraries.go b/internal/api/handlers/libraries.go index 6013831d..195e41af 100644 --- a/internal/api/handlers/libraries.go +++ b/internal/api/handlers/libraries.go @@ -101,19 +101,24 @@ type libraryScanQueuer interface { type libraryMovieMatchQueue interface { SyncForFolder(ctx context.Context, folderID int) error DeleteByFolder(ctx context.Context, folderID int) (int, error) - CountByFolder(ctx context.Context, folderID int) (int, error) + CountStatesByFolder(ctx context.Context, folderID int) (pending int, parked int, err error) + CountStatesByFolders(ctx context.Context, folderIDs []int) (map[int]metadata.MatchQueueStateCounts, error) ListByFolder(ctx context.Context, folderID int, limit int, offset int) ([]models.MovieMatchQueueEntry, int, error) + RetryNowByFolder(ctx context.Context, folderID int) (int, error) } type librarySeriesMatchQueue interface { SyncForFolder(ctx context.Context, folderID int) error DeleteByFolder(ctx context.Context, folderID int) (int, error) - CountByFolder(ctx context.Context, folderID int) (int, error) + CountStatesByFolder(ctx context.Context, folderID int) (pending int, parked int, err error) + CountStatesByFolders(ctx context.Context, folderIDs []int) (map[int]metadata.MatchQueueStateCounts, error) ListByFolder(ctx context.Context, folderID int, limit int, offset int) ([]models.SeriesRootMatchQueueEntry, int, error) + RetryNowByFolder(ctx context.Context, folderID int) (int, error) } type libraryRawMatchBacklog interface { CountUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode) (int, error) + CountUnmatchedMatchBacklogByFolders(ctx context.Context, folderIDs []int, mode scanner.RawMatchBacklogMode) (map[int]int, error) ListUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode, limit int, offset int) ([]*models.MediaFile, int, error) SuppressUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode) (int, error) RetryUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode scanner.RawMatchBacklogMode) (int, error) @@ -713,7 +718,11 @@ func (h *LibraryHandler) HandleUpdateLibrary(w http.ResponseWriter, r *http.Requ // existing items adopt the new language instead of keeping the one // stamped at first match. Quick mode suffices: the refresh item lister // includes complete-but-language-mismatched items. - if h.JobRepo != nil && !strings.EqualFold(strings.TrimSpace(oldFolder.MetadataLanguage), strings.TrimSpace(folder.MetadataLanguage)) { + languageChanged := !strings.EqualFold(strings.TrimSpace(oldFolder.MetadataLanguage), strings.TrimSpace(folder.MetadataLanguage)) + if languageChanged { + h.wakeMetadataMatcher(r.Context(), folder.ID) + } + if h.JobRepo != nil && languageChanged { job, jobErr := h.JobRepo.CreateLibraryRefresh(r.Context(), currentAdminUserID(r), adminjob.LibraryRefreshRequest{ LibraryID: folder.ID, LibraryName: folder.Name, @@ -1370,6 +1379,8 @@ type libraryMetadataMatchQueueStatusResponse struct { SeriesCount int `json:"series_count"` RawFileCount int `json:"raw_file_count"` TotalCount int `json:"total_count"` + PendingCount int `json:"pending_count"` + ParkedCount int `json:"parked_count"` } type libraryMetadataMatchQueueActionResponse struct { @@ -1385,32 +1396,46 @@ type libraryMetadataMatchQueueActionResponse struct { type libraryMetadataMatchQueueDetailResponse struct { libraryMetadataMatchQueueStatusResponse + Limit int `json:"limit"` + Offset int `json:"offset"` Movies []libraryMovieMatchQueueEntryResponse `json:"movies"` Series []librarySeriesMatchQueueEntryResponse `json:"series"` RawFiles []libraryRawMatchBacklogEntryResponse `json:"raw_files"` } type libraryMovieMatchQueueEntryResponse struct { - MediaFileID int `json:"media_file_id"` - MediaFolderID int `json:"media_folder_id"` - FilePath string `json:"file_path"` - FirstQueuedAt time.Time `json:"first_queued_at"` - AvailableAt time.Time `json:"available_at"` - LastAttemptedAt *time.Time `json:"last_attempted_at,omitempty"` - AttemptCount int `json:"attempt_count"` - LastError string `json:"last_error,omitempty"` - UpdatedAt time.Time `json:"updated_at"` + MediaFileID int `json:"media_file_id"` + MediaFolderID int `json:"media_folder_id"` + FilePath string `json:"file_path"` + FirstQueuedAt time.Time `json:"first_queued_at"` + AvailableAt time.Time `json:"available_at"` + LastAttemptedAt *time.Time `json:"last_attempted_at,omitempty"` + AttemptCount int `json:"attempt_count"` + LastError string `json:"last_error,omitempty"` + State string `json:"state"` + FailureKind string `json:"failure_kind,omitempty"` + FailureDetail json.RawMessage `json:"failure_detail,omitempty"` + DeterministicAttemptCount int `json:"deterministic_attempt_count"` + MatcherRevision int `json:"matcher_revision"` + ParkedAt *time.Time `json:"parked_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` } type librarySeriesMatchQueueEntryResponse struct { - MediaFolderID int `json:"media_folder_id"` - ObservedRootPath string `json:"observed_root_path"` - FirstQueuedAt time.Time `json:"first_queued_at"` - AvailableAt time.Time `json:"available_at"` - LastAttemptedAt *time.Time `json:"last_attempted_at,omitempty"` - AttemptCount int `json:"attempt_count"` - LastError string `json:"last_error,omitempty"` - UpdatedAt time.Time `json:"updated_at"` + MediaFolderID int `json:"media_folder_id"` + ObservedRootPath string `json:"observed_root_path"` + FirstQueuedAt time.Time `json:"first_queued_at"` + AvailableAt time.Time `json:"available_at"` + LastAttemptedAt *time.Time `json:"last_attempted_at,omitempty"` + AttemptCount int `json:"attempt_count"` + LastError string `json:"last_error,omitempty"` + State string `json:"state"` + FailureKind string `json:"failure_kind,omitempty"` + FailureDetail json.RawMessage `json:"failure_detail,omitempty"` + DeterministicAttemptCount int `json:"deterministic_attempt_count"` + MatcherRevision int `json:"matcher_revision"` + ParkedAt *time.Time `json:"parked_at,omitempty"` + UpdatedAt time.Time `json:"updated_at"` } type libraryRawMatchBacklogEntryResponse struct { @@ -1438,18 +1463,21 @@ func (h *LibraryHandler) HandleListMetadataMatchQueues(w http.ResponseWriter, r return } - resp := make([]libraryMetadataMatchQueueStatusResponse, 0, len(folders)) + folderIDs := make([]int, 0, len(folders)) for _, folder := range folders { - if folder == nil { - continue + if folder != nil { + folderIDs = append(folderIDs, folder.ID) } - status, err := h.metadataMatchQueueStatus(r.Context(), folder.ID) - if err != nil { - slog.ErrorContext(r.Context(), "metadata queue: failed to load queue status", "component", "api", "library_id", folder.ID, "error", err) - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load metadata matcher queue") - return - } - resp = append(resp, status) + } + statuses, err := h.metadataMatchQueueStatuses(r.Context(), folderIDs) + if err != nil { + slog.ErrorContext(r.Context(), "metadata queue: failed to load queue statuses", "component", "api", "error", err) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load metadata matcher queues") + return + } + resp := make([]libraryMetadataMatchQueueStatusResponse, 0, len(folderIDs)) + for _, folderID := range folderIDs { + resp = append(resp, statuses[folderID]) } writeJSON(w, http.StatusOK, resp) @@ -1481,6 +1509,12 @@ func (h *LibraryHandler) HandleGetMetadataMatchQueue(w http.ResponseWriter, r *h limit = parsed } } + offset := 0 + if value := strings.TrimSpace(r.URL.Query().Get("offset")); value != "" { + if parsed, parseErr := strconv.Atoi(value); parseErr == nil && parsed >= 0 { + offset = parsed + } + } status, err := h.metadataMatchQueueStatus(r.Context(), id) if err != nil { @@ -1491,12 +1525,14 @@ func (h *LibraryHandler) HandleGetMetadataMatchQueue(w http.ResponseWriter, r *h resp := libraryMetadataMatchQueueDetailResponse{ libraryMetadataMatchQueueStatusResponse: status, + Limit: limit, + Offset: offset, Movies: []libraryMovieMatchQueueEntryResponse{}, Series: []librarySeriesMatchQueueEntryResponse{}, RawFiles: []libraryRawMatchBacklogEntryResponse{}, } if h.MovieMatchQueueRepo != nil { - movies, _, err := h.MovieMatchQueueRepo.ListByFolder(r.Context(), id, limit, 0) + movies, _, err := h.MovieMatchQueueRepo.ListByFolder(r.Context(), id, limit, offset) if err != nil { slog.ErrorContext(r.Context(), "metadata queue: failed to list movie queue", "component", "api", "library_id", id, "error", err) writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher queue") @@ -1504,20 +1540,26 @@ func (h *LibraryHandler) HandleGetMetadataMatchQueue(w http.ResponseWriter, r *h } for _, entry := range movies { resp.Movies = append(resp.Movies, libraryMovieMatchQueueEntryResponse{ - MediaFileID: entry.MediaFileID, - MediaFolderID: entry.MediaFolderID, - FilePath: entry.FilePath, - FirstQueuedAt: entry.FirstQueuedAt, - AvailableAt: entry.AvailableAt, - LastAttemptedAt: entry.LastAttemptedAt, - AttemptCount: entry.AttemptCount, - LastError: entry.LastError, - UpdatedAt: entry.UpdatedAt, + MediaFileID: entry.MediaFileID, + MediaFolderID: entry.MediaFolderID, + FilePath: entry.FilePath, + FirstQueuedAt: entry.FirstQueuedAt, + AvailableAt: entry.AvailableAt, + LastAttemptedAt: entry.LastAttemptedAt, + AttemptCount: entry.AttemptCount, + LastError: entry.LastError, + State: entry.State, + FailureKind: entry.FailureKind, + FailureDetail: entry.FailureDetail, + DeterministicAttemptCount: entry.DeterministicAttemptCount, + MatcherRevision: entry.MatcherRevision, + ParkedAt: entry.ParkedAt, + UpdatedAt: entry.UpdatedAt, }) } } if h.SeriesMatchQueueRepo != nil { - series, _, err := h.SeriesMatchQueueRepo.ListByFolder(r.Context(), id, limit, 0) + series, _, err := h.SeriesMatchQueueRepo.ListByFolder(r.Context(), id, limit, offset) if err != nil { slog.ErrorContext(r.Context(), "metadata queue: failed to list series queue", "component", "api", "library_id", id, "error", err) writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher queue") @@ -1525,19 +1567,25 @@ func (h *LibraryHandler) HandleGetMetadataMatchQueue(w http.ResponseWriter, r *h } for _, entry := range series { resp.Series = append(resp.Series, librarySeriesMatchQueueEntryResponse{ - MediaFolderID: entry.MediaFolderID, - ObservedRootPath: entry.ObservedRootPath, - FirstQueuedAt: entry.FirstQueuedAt, - AvailableAt: entry.AvailableAt, - LastAttemptedAt: entry.LastAttemptedAt, - AttemptCount: entry.AttemptCount, - LastError: entry.LastError, - UpdatedAt: entry.UpdatedAt, + MediaFolderID: entry.MediaFolderID, + ObservedRootPath: entry.ObservedRootPath, + FirstQueuedAt: entry.FirstQueuedAt, + AvailableAt: entry.AvailableAt, + LastAttemptedAt: entry.LastAttemptedAt, + AttemptCount: entry.AttemptCount, + LastError: entry.LastError, + State: entry.State, + FailureKind: entry.FailureKind, + FailureDetail: entry.FailureDetail, + DeterministicAttemptCount: entry.DeterministicAttemptCount, + MatcherRevision: entry.MatcherRevision, + ParkedAt: entry.ParkedAt, + UpdatedAt: entry.UpdatedAt, }) } } if h.RawMatchBacklogRepo != nil { - rawFiles, _, err := h.RawMatchBacklogRepo.ListUnmatchedMatchBacklogByFolder(r.Context(), id, h.rawMatchBacklogMode(), limit, 0) + rawFiles, _, err := h.RawMatchBacklogRepo.ListUnmatchedMatchBacklogByFolder(r.Context(), id, h.rawMatchBacklogMode(), limit, offset) if err != nil { slog.ErrorContext(r.Context(), "metadata queue: failed to list raw backlog", "component", "api", "library_id", id, "error", err) writeError(w, http.StatusInternalServerError, "internal_error", "Failed to list metadata matcher backlog") @@ -1590,6 +1638,10 @@ func (h *LibraryHandler) HandleRetryMetadataMatchQueue(w http.ResponseWriter, r writeError(w, http.StatusInternalServerError, "internal_error", "Failed to retry metadata matcher") return } + if _, err := h.SeriesMatchQueueRepo.RetryNowByFolder(r.Context(), id); err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to retry metadata matcher") + return + } } if h.MovieMatchQueueRepo != nil { if err := h.MovieMatchQueueRepo.SyncForFolder(r.Context(), id); err != nil { @@ -1597,6 +1649,10 @@ func (h *LibraryHandler) HandleRetryMetadataMatchQueue(w http.ResponseWriter, r writeError(w, http.StatusInternalServerError, "internal_error", "Failed to retry metadata matcher") return } + if _, err := h.MovieMatchQueueRepo.RetryNowByFolder(r.Context(), id); err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to retry metadata matcher") + return + } } rawFileRetried := 0 if h.RawMatchBacklogRepo != nil { @@ -1693,30 +1749,61 @@ func (h *LibraryHandler) metadataMatchBacklogConfigured() bool { } func (h *LibraryHandler) metadataMatchQueueStatus(ctx context.Context, libraryID int) (libraryMetadataMatchQueueStatusResponse, error) { - resp := libraryMetadataMatchQueueStatusResponse{LibraryID: libraryID} + statuses, err := h.metadataMatchQueueStatuses(ctx, []int{libraryID}) + if err != nil { + return libraryMetadataMatchQueueStatusResponse{LibraryID: libraryID}, err + } + return statuses[libraryID], nil +} + +func (h *LibraryHandler) metadataMatchQueueStatuses(ctx context.Context, libraryIDs []int) (map[int]libraryMetadataMatchQueueStatusResponse, error) { + statuses := make(map[int]libraryMetadataMatchQueueStatusResponse, len(libraryIDs)) + for _, libraryID := range libraryIDs { + statuses[libraryID] = libraryMetadataMatchQueueStatusResponse{LibraryID: libraryID} + } if h.MovieMatchQueueRepo != nil { - count, err := h.MovieMatchQueueRepo.CountByFolder(ctx, libraryID) + counts, err := h.MovieMatchQueueRepo.CountStatesByFolders(ctx, libraryIDs) if err != nil { - return resp, err + return nil, err + } + for libraryID, count := range counts { + status := statuses[libraryID] + status.MovieCount = count.Pending + count.Parked + status.PendingCount += count.Pending + status.ParkedCount += count.Parked + statuses[libraryID] = status } - resp.MovieCount = count } if h.SeriesMatchQueueRepo != nil { - count, err := h.SeriesMatchQueueRepo.CountByFolder(ctx, libraryID) + counts, err := h.SeriesMatchQueueRepo.CountStatesByFolders(ctx, libraryIDs) if err != nil { - return resp, err + return nil, err + } + for libraryID, count := range counts { + status := statuses[libraryID] + status.SeriesCount = count.Pending + count.Parked + status.PendingCount += count.Pending + status.ParkedCount += count.Parked + statuses[libraryID] = status } - resp.SeriesCount = count } if h.RawMatchBacklogRepo != nil { - count, err := h.RawMatchBacklogRepo.CountUnmatchedMatchBacklogByFolder(ctx, libraryID, h.rawMatchBacklogMode()) + counts, err := h.RawMatchBacklogRepo.CountUnmatchedMatchBacklogByFolders(ctx, libraryIDs, h.rawMatchBacklogMode()) if err != nil { - return resp, err + return nil, err + } + for libraryID, count := range counts { + status := statuses[libraryID] + status.RawFileCount = count + status.PendingCount += count + statuses[libraryID] = status } - resp.RawFileCount = count } - resp.TotalCount = resp.MovieCount + resp.SeriesCount + resp.RawFileCount - return resp, nil + for libraryID, status := range statuses { + status.TotalCount = status.MovieCount + status.SeriesCount + status.RawFileCount + statuses[libraryID] = status + } + return statuses, nil } func (h *LibraryHandler) rawMatchBacklogMode() scanner.RawMatchBacklogMode { @@ -2107,10 +2194,24 @@ func (h *LibraryHandler) HandleSetLibraryProviders(w http.ResponseWriter, r *htt if h.chainCacheInvalidator != nil { h.chainCacheInvalidator.InvalidateChainCache() } + h.wakeMetadataMatcher(r.Context(), id) w.WriteHeader(http.StatusNoContent) } +func (h *LibraryHandler) wakeMetadataMatcher(ctx context.Context, libraryID int) { + if h.MovieMatchQueueRepo != nil { + if _, err := h.MovieMatchQueueRepo.RetryNowByFolder(ctx, libraryID); err != nil { + slog.WarnContext(ctx, "wake movie metadata matcher", "component", "api", "library_id", libraryID, "error", err) + } + } + if h.SeriesMatchQueueRepo != nil { + if _, err := h.SeriesMatchQueueRepo.RetryNowByFolder(ctx, libraryID); err != nil { + slog.WarnContext(ctx, "wake series metadata matcher", "component", "api", "library_id", libraryID, "error", err) + } + } +} + // seedDefaultChain builds a default provider chain from plugin manifest defaults // for the given library type. Returns entries for all applicable content levels. func (h *LibraryHandler) seedDefaultChain(ctx context.Context, libraryType string) []metadata.ChainEntry { diff --git a/internal/api/handlers/libraries_match_test.go b/internal/api/handlers/libraries_match_test.go index 6d36a0a9..7806e983 100644 --- a/internal/api/handlers/libraries_match_test.go +++ b/internal/api/handlers/libraries_match_test.go @@ -6,10 +6,13 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/go-chi/chi/v5" + "github.com/Silo-Server/silo-server/internal/metadata" "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/scanner" ) // TestLibraryMatchUnmatchedItems_NilPool verifies that the unmatched-items @@ -234,14 +237,224 @@ func (noopMovieMatchQueue) DeleteByFolder(context.Context, int) (int, error) { return 0, nil } -func (noopMovieMatchQueue) CountByFolder(context.Context, int) (int, error) { - return 0, nil +func (noopMovieMatchQueue) CountStatesByFolder(context.Context, int) (int, int, error) { + return 0, 0, nil +} +func (noopMovieMatchQueue) CountStatesByFolders(context.Context, []int) (map[int]metadata.MatchQueueStateCounts, error) { + return map[int]metadata.MatchQueueStateCounts{}, nil } func (noopMovieMatchQueue) ListByFolder(context.Context, int, int, int) ([]models.MovieMatchQueueEntry, int, error) { return nil, 0, nil } +func (noopMovieMatchQueue) RetryNowByFolder(context.Context, int) (int, error) { + return 0, nil +} + +type fakeMovieMatchQueue struct { + pending int + parked int + entries []models.MovieMatchQueueEntry + retryFolders []int + bulkCalls int +} + +func (f *fakeMovieMatchQueue) SyncForFolder(context.Context, int) error { return nil } +func (f *fakeMovieMatchQueue) DeleteByFolder(context.Context, int) (int, error) { + return 0, nil +} +func (f *fakeMovieMatchQueue) CountStatesByFolder(context.Context, int) (int, int, error) { + return f.pending, f.parked, nil +} +func (f *fakeMovieMatchQueue) CountStatesByFolders(_ context.Context, folderIDs []int) (map[int]metadata.MatchQueueStateCounts, error) { + f.bulkCalls++ + counts := make(map[int]metadata.MatchQueueStateCounts, len(folderIDs)) + for _, folderID := range folderIDs { + counts[folderID] = metadata.MatchQueueStateCounts{Pending: f.pending, Parked: f.parked} + } + return counts, nil +} +func (f *fakeMovieMatchQueue) ListByFolder(context.Context, int, int, int) ([]models.MovieMatchQueueEntry, int, error) { + return f.entries, len(f.entries), nil +} +func (f *fakeMovieMatchQueue) RetryNowByFolder(_ context.Context, folderID int) (int, error) { + f.retryFolders = append(f.retryFolders, folderID) + return len(f.retryFolders), nil +} + +type fakeSeriesMatchQueue struct { + pending int + parked int + entries []models.SeriesRootMatchQueueEntry + retryFolders []int + bulkCalls int +} + +func (f *fakeSeriesMatchQueue) SyncForFolder(context.Context, int) error { return nil } +func (f *fakeSeriesMatchQueue) DeleteByFolder(context.Context, int) (int, error) { + return 0, nil +} +func (f *fakeSeriesMatchQueue) CountStatesByFolder(context.Context, int) (int, int, error) { + return f.pending, f.parked, nil +} +func (f *fakeSeriesMatchQueue) CountStatesByFolders(_ context.Context, folderIDs []int) (map[int]metadata.MatchQueueStateCounts, error) { + f.bulkCalls++ + counts := make(map[int]metadata.MatchQueueStateCounts, len(folderIDs)) + for _, folderID := range folderIDs { + counts[folderID] = metadata.MatchQueueStateCounts{Pending: f.pending, Parked: f.parked} + } + return counts, nil +} +func (f *fakeSeriesMatchQueue) ListByFolder(context.Context, int, int, int) ([]models.SeriesRootMatchQueueEntry, int, error) { + return f.entries, len(f.entries), nil +} +func (f *fakeSeriesMatchQueue) RetryNowByFolder(_ context.Context, folderID int) (int, error) { + f.retryFolders = append(f.retryFolders, folderID) + return len(f.retryFolders), nil +} + +type fakeRawMatchBacklog struct { + count int + bulkCalls int +} + +func (f *fakeRawMatchBacklog) CountUnmatchedMatchBacklogByFolder(context.Context, int, scanner.RawMatchBacklogMode) (int, error) { + return f.count, nil +} +func (f *fakeRawMatchBacklog) CountUnmatchedMatchBacklogByFolders(_ context.Context, folderIDs []int, _ scanner.RawMatchBacklogMode) (map[int]int, error) { + f.bulkCalls++ + counts := make(map[int]int, len(folderIDs)) + for _, folderID := range folderIDs { + counts[folderID] = f.count + } + return counts, nil +} +func (f *fakeRawMatchBacklog) ListUnmatchedMatchBacklogByFolder(context.Context, int, scanner.RawMatchBacklogMode, int, int) ([]*models.MediaFile, int, error) { + return nil, 0, nil +} +func (f *fakeRawMatchBacklog) SuppressUnmatchedMatchBacklogByFolder(context.Context, int, scanner.RawMatchBacklogMode) (int, error) { + return 0, nil +} +func (f *fakeRawMatchBacklog) RetryUnmatchedMatchBacklogByFolder(context.Context, int, scanner.RawMatchBacklogMode) (int, error) { + return 0, nil +} + +// TestMetadataMatchQueueStatus_AggregatesStates verifies the pending/parked +// aggregation rules the admin UI depends on: parked rows stay out of +// pending_count, raw-backlog files count as pending, and the per-queue totals +// still equal pending + parked so movie_count/series_count semantics are +// unchanged from the pre-state API. +func TestMetadataMatchQueueStatus_AggregatesStates(t *testing.T) { + h := &LibraryHandler{ + MovieMatchQueueRepo: &fakeMovieMatchQueue{pending: 2, parked: 1}, + SeriesMatchQueueRepo: &fakeSeriesMatchQueue{pending: 3, parked: 2}, + RawMatchBacklogRepo: &fakeRawMatchBacklog{count: 4}, + } + + status, err := h.metadataMatchQueueStatus(context.Background(), 7) + if err != nil { + t.Fatalf("metadataMatchQueueStatus() error = %v", err) + } + if status.LibraryID != 7 { + t.Errorf("LibraryID = %d, want 7", status.LibraryID) + } + if status.MovieCount != 3 || status.SeriesCount != 5 || status.RawFileCount != 4 { + t.Errorf("per-queue counts = %d/%d/%d, want 3/5/4", status.MovieCount, status.SeriesCount, status.RawFileCount) + } + if status.PendingCount != 9 { + t.Errorf("PendingCount = %d, want 9 (2+3 pending + 4 raw)", status.PendingCount) + } + if status.ParkedCount != 3 { + t.Errorf("ParkedCount = %d, want 3", status.ParkedCount) + } + if status.TotalCount != 12 { + t.Errorf("TotalCount = %d, want 12", status.TotalCount) + } +} + +func TestMetadataMatchQueueStatusesUsesOneAggregateCallPerQueue(t *testing.T) { + movie := &fakeMovieMatchQueue{pending: 1} + series := &fakeSeriesMatchQueue{parked: 1} + raw := &fakeRawMatchBacklog{count: 1} + h := &LibraryHandler{ + MovieMatchQueueRepo: movie, + SeriesMatchQueueRepo: series, + RawMatchBacklogRepo: raw, + } + + statuses, err := h.metadataMatchQueueStatuses(context.Background(), []int{3, 7, 11}) + if err != nil { + t.Fatalf("metadataMatchQueueStatuses() error = %v", err) + } + if len(statuses) != 3 { + t.Fatalf("status count = %d, want 3", len(statuses)) + } + if movie.bulkCalls != 1 || series.bulkCalls != 1 || raw.bulkCalls != 1 { + t.Fatalf("bulk calls = movie:%d series:%d raw:%d, want one each", movie.bulkCalls, series.bulkCalls, raw.bulkCalls) + } +} + +// TestWakeMetadataMatcher_RetriesBothQueues verifies the provider-chain-change +// wake path resets parked work in both queue repos for the changed library. +func TestWakeMetadataMatcher_RetriesBothQueues(t *testing.T) { + movie := &fakeMovieMatchQueue{} + series := &fakeSeriesMatchQueue{} + h := &LibraryHandler{MovieMatchQueueRepo: movie, SeriesMatchQueueRepo: series} + + h.wakeMetadataMatcher(context.Background(), 42) + + if len(movie.retryFolders) != 1 || movie.retryFolders[0] != 42 { + t.Errorf("movie retry folders = %v, want [42]", movie.retryFolders) + } + if len(series.retryFolders) != 1 || series.retryFolders[0] != 42 { + t.Errorf("series retry folders = %v, want [42]", series.retryFolders) + } +} + +// TestMovieMatchQueueEntryResponse_SerializesStateFields pins the JSON contract +// for the queue-state fields the admin UI reads. Additive-only API rule: these +// names must never change once shipped. +func TestMovieMatchQueueEntryResponse_SerializesStateFields(t *testing.T) { + parkedAt := time.Date(2026, 7, 21, 12, 0, 0, 0, time.UTC) + entry := libraryMovieMatchQueueEntryResponse{ + MediaFileID: 11, + MediaFolderID: 7, + FilePath: "/movies/a.mkv", + State: "parked", + FailureKind: "candidate_rejected", + FailureDetail: json.RawMessage(`{"message":"below threshold"}`), + DeterministicAttemptCount: 3, + MatcherRevision: 8, + ParkedAt: &parkedAt, + } + + data, err := json.Marshal(entry) + if err != nil { + t.Fatalf("marshal: %v", err) + } + var m map[string]any + if err := json.Unmarshal(data, &m); err != nil { + t.Fatalf("unmarshal: %v", err) + } + for key, want := range map[string]any{ + "state": "parked", //nolint:goconst // JSON contract key. + "failure_kind": "candidate_rejected", + "deterministic_attempt_count": float64(3), + "matcher_revision": float64(8), + } { + if m[key] != want { + t.Errorf("JSON %q = %v, want %v", key, m[key], want) + } + } + if _, ok := m["parked_at"]; !ok { + t.Error("expected parked_at in JSON output") + } + if _, ok := m["failure_detail"]; !ok { + t.Error("expected failure_detail in JSON output") + } +} + func keys(m map[string]any) []string { out := make([]string, 0, len(m)) for k := range m { diff --git a/internal/metadata/match_queue_coordinator.go b/internal/metadata/match_queue_coordinator.go index b6e5bd18..4ae4abb3 100644 --- a/internal/metadata/match_queue_coordinator.go +++ b/internal/metadata/match_queue_coordinator.go @@ -42,3 +42,23 @@ func (c *MatchQueueCoordinator) EnqueueSeriesRoot(ctx context.Context, folderID } return c.seriesRepo.EnqueueSeriesRoot(ctx, folderID, observedRootPath) } + +// WakeForChangedInputs recalculates queue fingerprints after a provider +// lifecycle event. Only work whose effective provider chain (or another +// fingerprint input) changed is reset, including pending rows in backoff. +func (c *MatchQueueCoordinator) WakeForChangedInputs(ctx context.Context) error { + if c == nil { + return nil + } + if c.movieRepo != nil { + if _, err := c.movieRepo.WakeForChangedInputs(ctx); err != nil { + return err + } + } + if c.seriesRepo != nil { + if _, err := c.seriesRepo.WakeForChangedInputs(ctx); err != nil { + return err + } + } + return nil +} diff --git a/internal/metadata/match_queue_policy.go b/internal/metadata/match_queue_policy.go new file mode 100644 index 00000000..8d903b7a --- /dev/null +++ b/internal/metadata/match_queue_policy.go @@ -0,0 +1,178 @@ +package metadata + +import ( + "fmt" + "regexp" + "sort" + "strings" + "time" +) + +var ( + matchFailureSecretPattern = regexp.MustCompile(`(?i)("?(?:api[_-]?key|access[_-]?token|token|authorization|password)"?)\s*[:=]\s*"?([^"&,;\s}]+)"?`) + matchFailureBearerPattern = regexp.MustCompile(`(?i)\bbearer\s+[^&,;\s]+`) + matchFailureSecretKey = regexp.MustCompile(`(?i)^(?:api[_-]?key|access[_-]?token|token|authorization|password)$`) +) + +const ( + matcherRevision = 9 + movieQueueRetryDelay = 15 * time.Second + seriesRootQueueQuietWindow = 10 * time.Second + seriesRootQueueRetryDelay = 30 * time.Second + + // A claim contains at most one job per available worker. Keep the lease long + // enough that a second server does not reclaim a healthy provider call; a + // crashed process can strand only that worker-sized in-flight window. + // Workers explicitly release unfinished leases on healthy cancellation. + matchQueueClaimLease = 2 * time.Hour + + // Transient provider failures use capped exponential backoff. Deterministic + // outcomes use the fixed one-hour/24-hour/park schedule. + matchQueueRetryMaxDelay = 24 * time.Hour + matchQueueBackoffMaxExponent = 16 +) + +// MatchQueueStateCounts is the pending/parked aggregate for one library. +type MatchQueueStateCounts struct { + Pending int + Parked int +} + +// matchQueueBackoffExpr returns the SQL expression both match queues use to +// schedule the next transient retry. attempt_count is incremented at claim. +func matchQueueBackoffExpr(basePlaceholder, maxPlaceholder string) string { + return fmt.Sprintf( + "NOW() + LEAST(%s::interval * power(2::float8, LEAST(attempt_count, %d)), %s::interval)", + basePlaceholder, matchQueueBackoffMaxExponent, maxPlaceholder, + ) +} + +// matchQueueInputFingerprintSQL returns a deterministic SQL expression for +// inputs that can change a result without changing the queue key. Arguments +// are internal SQL expressions selected by repository code, never user input. +func matchQueueInputFingerprintSQL(pathExpression, typeExpression, folderIDExpression, languageExpression string) string { + return fmt.Sprintf(`md5(%s || '|' || COALESCE(%s, '') || '|' || COALESCE(%s, '') || '|%d|' || COALESCE(( + SELECT string_agg( + chain.priority::text || ':' || installation.id::text || ':' || installation.plugin_id || ':' || + chain.capability_id || ':' || chain.content_level || ':' || + chain.enabled::text || ':' || installation.enabled::text || ':' || installation.version || ':' || + COALESCE(( + SELECT string_agg(config.config_key || ':' || config.updated_at::text, ',' ORDER BY config.config_key) + FROM plugin_runtime_configs config + WHERE config.plugin_installation_id = installation.id + ), ''), + '|' ORDER BY chain.priority, installation.id, chain.capability_id, chain.content_level + ) + FROM library_provider_chains chain + JOIN plugin_installations installation ON installation.id = chain.plugin_installation_id + WHERE chain.media_folder_id = %s + AND chain.capability_type = 'metadata_provider.v1' + ), ''))`, pathExpression, typeExpression, languageExpression, matcherRevision, folderIDExpression) +} + +// seriesMatchQueueInputFingerprintSQL includes the active file-path set because +// episode validation derives both coordinates and episode-title evidence from +// those paths. Adding, removing, or renaming an episode wakes a parked match. +func seriesMatchQueueInputFingerprintSQL(rootExpression, folderIDExpression, languageExpression string) string { + shapeExpression := fmt.Sprintf(`(COALESCE(%s, '') || '|shape:' || COALESCE(( + SELECT md5(string_agg( + md5(shape_file.file_path), '' ORDER BY shape_file.file_path + )) + FROM media_files shape_file + WHERE shape_file.media_folder_id = %s + AND shape_file.observed_root_path = %s + AND shape_file.missing_since IS NULL + AND shape_file.extra_id IS NULL + ), ''))`, rootExpression, folderIDExpression, rootExpression) + return matchQueueInputFingerprintSQL(shapeExpression, "'series'", folderIDExpression, languageExpression) +} + +func boundedMatchFailureMessage(message string) string { + message = matchFailureBearerPattern.ReplaceAllString(strings.TrimSpace(message), "Bearer [redacted]") + message = matchFailureSecretPattern.ReplaceAllString(message, "$1=[redacted]") + runes := []rune(message) + if len(runes) > 1000 { + message = string(runes[:1000]) + } + return message +} + +func truncateMatchFailureField(value string, limit int) string { + runes := []rune(strings.TrimSpace(value)) + if len(runes) > limit { + runes = runes[:limit] + } + return string(runes) +} + +func normalizeMatchFailureKind(kind MatchOutcome) MatchOutcome { + normalized := MatchOutcome(strings.ToLower(strings.TrimSpace(string(kind)))) + switch normalized { + case MatchOutcomeNoCandidates, + MatchOutcomeCandidateRejected, + MatchOutcomeTrustedIDConflict, + MatchOutcomeTrustedIDTypeMismatch, + MatchOutcomeMetadataEmpty, + MatchOutcomeProviderTransient, + MatchOutcomeProviderPermanent: + return normalized + default: + return MatchOutcomeProviderTransient + } +} + +// boundedMatchDecision makes the persistence boundary explicit. Provider IDs, +// titles, and reasons must not be able to grow queue rows without limit. +func boundedMatchDecision(decision *MatchDecision) *MatchDecision { + if decision == nil { + return nil + } + bounded := &MatchDecision{ + Outcome: MatchOutcome(truncateMatchFailureField(string(decision.Outcome), 64)), + CandidateCount: decision.CandidateCount, + Threshold: decision.Threshold, + } + for _, candidate := range decision.TopCandidates { + if len(bounded.TopCandidates) == 3 { + break + } + copyCandidate := MatchDecisionCandidate{ + Title: truncateMatchFailureField(candidate.Title, 256), + MatchedTitle: truncateMatchFailureField(candidate.MatchedTitle, 256), + Year: candidate.Year, + Score: candidate.Score, + } + if len(candidate.ProviderIDs) > 0 { + copyCandidate.ProviderIDs = make(map[string]string, min(len(candidate.ProviderIDs), 8)) + keys := make([]string, 0, len(candidate.ProviderIDs)) + for key := range candidate.ProviderIDs { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if len(copyCandidate.ProviderIDs) == 8 { + break + } + if matchFailureSecretKey.MatchString(strings.TrimSpace(key)) { + continue + } + value := candidate.ProviderIDs[key] + copyCandidate.ProviderIDs[truncateMatchFailureField(key, 64)] = truncateMatchFailureField(value, 256) + } + } + for _, source := range candidate.Sources { + if len(copyCandidate.Sources) == 8 { + break + } + copyCandidate.Sources = append(copyCandidate.Sources, truncateMatchFailureField(source, 64)) + } + for _, reason := range candidate.Reasons { + if len(copyCandidate.Reasons) == 8 { + break + } + copyCandidate.Reasons = append(copyCandidate.Reasons, truncateMatchFailureField(reason, 128)) + } + bounded.TopCandidates = append(bounded.TopCandidates, copyCandidate) + } + return bounded +} diff --git a/internal/metadata/match_queue_state_db_test.go b/internal/metadata/match_queue_state_db_test.go new file mode 100644 index 00000000..32e889b3 --- /dev/null +++ b/internal/metadata/match_queue_state_db_test.go @@ -0,0 +1,567 @@ +package metadata + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" + scannerrepo "github.com/Silo-Server/silo-server/internal/scanner" +) + +const queueStatePending = "pending" + +func TestBoundedMatchFailureMessageRedactsAndLimits(t *testing.T) { + t.Parallel() + message := "provider failed?api_key=secret&token=also-secret " + strings.Repeat("x", 1200) + got := boundedMatchFailureMessage(message) + if strings.Contains(got, "secret") { + t.Fatalf("failure message leaked a secret: %q", got) + } + if len([]rune(got)) != 1000 { + t.Fatalf("failure message length = %d, want 1000", len([]rune(got))) + } +} + +func TestBoundedMatchFailureMessageRedactsHeaderAndBearerForms(t *testing.T) { + t.Parallel() + got := boundedMatchFailureMessage("Authorization: Bearer secret-token; password: hunter2") + for _, secret := range []string{"secret-token", "hunter2"} { + if strings.Contains(got, secret) { + t.Fatalf("failure message leaked %q: %q", secret, got) + } + } +} + +func TestBoundedMatchDecisionLimitsProviderControlledFields(t *testing.T) { + t.Parallel() + decision := &MatchDecision{Outcome: MatchOutcome(strings.Repeat("x", 100)), CandidateCount: 99, Threshold: 55} + for i := 0; i < 5; i++ { + candidate := MatchDecisionCandidate{ + Title: strings.Repeat("t", 400), MatchedTitle: strings.Repeat("m", 400), + ProviderIDs: make(map[string]string), Score: 42, + Sources: make([]string, 12), Reasons: make([]string, 12), + } + for j := 0; j < 12; j++ { + candidate.ProviderIDs[fmt.Sprintf("provider-%02d", j)] = strings.Repeat("i", 400) + candidate.Sources[j] = strings.Repeat("s", 100) + candidate.Reasons[j] = strings.Repeat("r", 200) + } + candidate.ProviderIDs["api_key"] = "must-not-persist" + decision.TopCandidates = append(decision.TopCandidates, candidate) + } + + got := boundedMatchDecision(decision) + if len(got.TopCandidates) != 3 || len(got.TopCandidates[0].ProviderIDs) != 8 || len(got.TopCandidates[0].Sources) != 8 || len(got.TopCandidates[0].Reasons) != 8 { + t.Fatalf("bounded decision sizes = candidates:%d ids:%d sources:%d reasons:%d", len(got.TopCandidates), len(got.TopCandidates[0].ProviderIDs), len(got.TopCandidates[0].Sources), len(got.TopCandidates[0].Reasons)) + } + if len([]rune(got.TopCandidates[0].Title)) != 256 || len([]rune(got.TopCandidates[0].MatchedTitle)) != 256 { + t.Fatalf("bounded title lengths = %d/%d", len([]rune(got.TopCandidates[0].Title)), len([]rune(got.TopCandidates[0].MatchedTitle))) + } + if _, exists := got.TopCandidates[0].ProviderIDs["api_key"]; exists { + t.Fatal("bounded decision retained a credential-shaped provider ID") + } +} + +func TestNormalizeMatchFailureKindTreatsUnknownAsTransient(t *testing.T) { + t.Parallel() + if got := normalizeMatchFailureKind(MatchOutcomeCandidateRejected); got != MatchOutcomeCandidateRejected { + t.Fatalf("known failure kind = %q", got) + } + if got := normalizeMatchFailureKind(MatchOutcome("unexpected-" + strings.Repeat("x", 500))); got != MatchOutcomeProviderTransient { + t.Fatalf("unknown failure kind = %q, want provider_transient", got) + } +} + +func TestMatchQueueFingerprintIncludesMatcherAndProviderConfiguration(t *testing.T) { + t.Parallel() + expression := matchQueueInputFingerprintSQL("mf.file_path", "'movie'", "mf.media_folder_id", "folders.metadata_language") + for _, required := range []string{ + "mf.file_path", "'movie'", "folders.metadata_language", "installation.version", + "chain.priority", "chain.capability_id", "plugin_runtime_configs", "config.updated_at::text", + fmt.Sprintf("|%d|", matcherRevision), + } { + if !strings.Contains(expression, required) { + t.Fatalf("fingerprint expression %q does not contain %q", expression, required) + } + } +} + +func TestSeriesMatchQueueFingerprintIncludesEpisodePathShape(t *testing.T) { + t.Parallel() + expression := seriesMatchQueueInputFingerprintSQL("q.observed_root_path", "q.media_folder_id", "folders.metadata_language") + for _, required := range []string{"shape_file.file_path", "shape_file.observed_root_path", "shape_file.missing_since", "shape_file.extra_id"} { + if !strings.Contains(expression, required) { + t.Fatalf("series fingerprint expression %q does not contain %q", expression, required) + } + } +} + +func TestSeriesMatchQueueDeterministicFailuresParkAndRetryNowResets(t *testing.T) { + pool := chainBuiltinTestPool(t) + ctx := context.Background() + folderID := insertTestFolder(t, pool, "series") + root := fmt.Sprintf("/test/match-queue-%d", time.Now().UnixNano()) + if _, err := pool.Exec(ctx, ` + INSERT INTO series_root_match_queue (media_folder_id, observed_root_path, available_at) + VALUES ($1, $2, NOW())`, folderID, root); err != nil { + t.Fatalf("seed series match queue: %v", err) + } + + repo := NewSeriesRootMatchQueueRepository(pool) + for attempt, wantDelay := range []time.Duration{time.Hour, 24 * time.Hour, 24 * time.Hour} { + leaseToken := fmt.Sprintf("deterministic-lease-%d", attempt) + if _, err := pool.Exec(ctx, `UPDATE series_root_match_queue SET lease_token = $3 WHERE media_folder_id = $1 AND observed_root_path = $2`, folderID, root, leaseToken); err != nil { + t.Fatalf("seed lease token: %v", err) + } + before := time.Now() + if err := repo.UpdateFailure(ctx, folderID, root, leaseToken, MatchFailure{Kind: MatchOutcomeCandidateRejected, Message: "score below threshold"}); err != nil { + t.Fatalf("UpdateFailure(%d): %v", attempt+1, err) + } + var state string + var deterministicCount int + var availableAt time.Time + var parkedAt *time.Time + if err := pool.QueryRow(ctx, ` + SELECT state, deterministic_attempt_count, available_at, parked_at + FROM series_root_match_queue WHERE media_folder_id = $1 AND observed_root_path = $2`, + folderID, root).Scan(&state, &deterministicCount, &availableAt, &parkedAt); err != nil { + t.Fatalf("load queue state: %v", err) + } + if deterministicCount != attempt+1 { + t.Fatalf("deterministic count = %d, want %d", deterministicCount, attempt+1) + } + wantState := queueStatePending + if attempt == 2 { + wantState = "parked" + } + if state != wantState { + t.Fatalf("state = %q, want %q", state, wantState) + } + if attempt == 2 && parkedAt == nil { + t.Fatal("parked_at is nil after third deterministic failure") + } + if availableAt.Before(before.Add(wantDelay - time.Minute)) { + t.Fatalf("available_at = %v, want approximately %v later", availableAt, wantDelay) + } + } + + if _, err := repo.RetryNowByFolder(ctx, folderID); err != nil { + t.Fatalf("RetryNowByFolder(): %v", err) + } + var state, failureKind string + var deterministicCount int + var availableAt time.Time + var parkedAt *time.Time + if err := pool.QueryRow(ctx, ` + SELECT state, failure_kind, deterministic_attempt_count, available_at, parked_at + FROM series_root_match_queue WHERE media_folder_id = $1 AND observed_root_path = $2`, + folderID, root).Scan(&state, &failureKind, &deterministicCount, &availableAt, &parkedAt); err != nil { + t.Fatalf("load retried queue state: %v", err) + } + if state != queueStatePending || failureKind != "" || deterministicCount != 0 || parkedAt != nil { + t.Fatalf("retry state = (%q, %q, %d, %v)", state, failureKind, deterministicCount, parkedAt) + } + if availableAt.After(time.Now().Add(time.Minute)) { + t.Fatalf("RetryNow left available_at in the future: %v", availableAt) + } +} + +func TestSeriesMatchQueueTransientFailureDoesNotConsumeDeterministicBudget(t *testing.T) { + pool := chainBuiltinTestPool(t) + ctx := context.Background() + folderID := insertTestFolder(t, pool, "series") + root := fmt.Sprintf("/test/match-queue-transient-%d", time.Now().UnixNano()) + if _, err := pool.Exec(ctx, ` + INSERT INTO series_root_match_queue (media_folder_id, observed_root_path, available_at) + VALUES ($1, $2, NOW())`, folderID, root); err != nil { + t.Fatalf("seed series match queue: %v", err) + } + + repo := NewSeriesRootMatchQueueRepository(pool) + const leaseToken = "transient-lease" + if _, err := pool.Exec(ctx, `UPDATE series_root_match_queue SET lease_token = $3 WHERE media_folder_id = $1 AND observed_root_path = $2`, folderID, root, leaseToken); err != nil { + t.Fatalf("seed lease token: %v", err) + } + if err := repo.UpdateFailure(ctx, folderID, root, leaseToken, MatchFailure{Kind: MatchOutcomeProviderTransient, Message: "HTTP 429"}); err != nil { + t.Fatalf("UpdateFailure(): %v", err) + } + var state string + var deterministicCount int + if err := pool.QueryRow(ctx, ` + SELECT state, deterministic_attempt_count FROM series_root_match_queue + WHERE media_folder_id = $1 AND observed_root_path = $2`, folderID, root).Scan(&state, &deterministicCount); err != nil { + t.Fatalf("load transient queue state: %v", err) + } + if state != queueStatePending || deterministicCount != 0 { + t.Fatalf("transient state = (%q, %d), want pending with zero deterministic attempts", state, deterministicCount) + } +} + +func TestSeriesMatchQueueWakeForChangedInputsResetsOnlyChangedRows(t *testing.T) { + pool := chainBuiltinTestPool(t) + ctx := context.Background() + folderID := insertTestFolder(t, pool, "series") + root := fmt.Sprintf("/test/match-input-wake-%d", time.Now().UnixNano()) + if _, err := pool.Exec(ctx, ` + INSERT INTO series_root_match_queue ( + media_folder_id, observed_root_path, available_at, state, + failure_kind, failure_detail, deterministic_attempt_count, + input_fingerprint, matcher_revision, parked_at, last_error + ) VALUES ($1, $2, NOW() + interval '24 hours', 'parked', + 'candidate_rejected', '{"message":"old"}'::jsonb, 3, + 'old-fingerprint', 0, NOW(), 'old') + `, folderID, root); err != nil { + t.Fatalf("seed changed-input queue row: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM series_root_match_queue WHERE media_folder_id = $1 AND observed_root_path = $2`, folderID, root) + }) + + repo := NewSeriesRootMatchQueueRepository(pool) + const staleLeaseToken = "series-stale-lease" + if _, err := pool.Exec(ctx, ` + UPDATE series_root_match_queue SET lease_token = $3 + WHERE media_folder_id = $1 AND observed_root_path = $2 + `, folderID, root, staleLeaseToken); err != nil { + t.Fatalf("seed active lease: %v", err) + } + woken, err := repo.WakeForChangedInputs(ctx) + if err != nil { + t.Fatalf("WakeForChangedInputs(): %v", err) + } + if woken < 1 { + t.Fatalf("woken = %d, want at least seeded row", woken) + } + var state, failureKind, lastError, fingerprint, leaseToken string + var deterministicCount, revision int + var availableAt time.Time + var parkedAt *time.Time + var rerunRequested bool + if err := pool.QueryRow(ctx, ` + SELECT state, failure_kind, last_error, deterministic_attempt_count, + input_fingerprint, matcher_revision, available_at, parked_at, + lease_token, rerun_requested + FROM series_root_match_queue + WHERE media_folder_id = $1 AND observed_root_path = $2 + `, folderID, root).Scan(&state, &failureKind, &lastError, &deterministicCount, &fingerprint, &revision, &availableAt, &parkedAt, &leaseToken, &rerunRequested); err != nil { + t.Fatalf("load woken row: %v", err) + } + if state != queueStatePending || failureKind != "" || lastError != "" || deterministicCount != 0 || fingerprint == "" || fingerprint == "old-fingerprint" || revision != matcherRevision || parkedAt != nil { + t.Fatalf("woken row = state:%q failure:%q last:%q deterministic:%d fingerprint:%q revision:%d available:%v parked:%v", state, failureKind, lastError, deterministicCount, fingerprint, revision, availableAt, parkedAt) + } + if leaseToken != staleLeaseToken || !rerunRequested || availableAt.Before(time.Now().Add(time.Hour)) { + t.Fatalf("woken lease ownership was not preserved: rerun:%v available:%v", rerunRequested, availableAt) + } + if woken, err := repo.WakeForChangedInputs(ctx); err != nil || woken != 0 { + t.Fatalf("unchanged WakeForChangedInputs() = (%d, %v), want (0, nil)", woken, err) + } + if err := repo.UpdateFailure(ctx, folderID, root, staleLeaseToken, MatchFailure{ + Kind: MatchOutcomeCandidateRejected, Message: "stale worker result", + }); err != nil { + t.Fatalf("stale UpdateFailure(): %v", err) + } + if err := pool.QueryRow(ctx, ` + SELECT failure_kind FROM series_root_match_queue + WHERE media_folder_id = $1 AND observed_root_path = $2 + `, folderID, root).Scan(&failureKind); err != nil { + t.Fatalf("load row after stale series update: %v", err) + } + if failureKind != "" { + t.Fatalf("stale series worker overwrote awakened row with failure %q", failureKind) + } + if err := pool.QueryRow(ctx, ` + SELECT lease_token, rerun_requested, available_at + FROM series_root_match_queue + WHERE media_folder_id = $1 AND observed_root_path = $2 + `, folderID, root).Scan(&leaseToken, &rerunRequested, &availableAt); err != nil { + t.Fatalf("load released series rerun: %v", err) + } + if leaseToken != "" || !rerunRequested || availableAt.After(time.Now().Add(time.Minute)) { + t.Fatalf("released rerun retained lease ownership: rerun:%v available:%v", rerunRequested, availableAt) + } + + if _, err := pool.Exec(ctx, `UPDATE media_folders SET metadata_language = 'da' WHERE id = $1`, folderID); err != nil { + t.Fatalf("change folder language: %v", err) + } + if _, err := pool.Exec(ctx, ` + UPDATE series_root_match_queue + SET available_at = NOW() + interval '24 hours', deterministic_attempt_count = 2, + failure_kind = 'candidate_rejected', last_error = 'old' + WHERE media_folder_id = $1 AND observed_root_path = $2 + `, folderID, root); err != nil { + t.Fatalf("back off queue row before language change wake: %v", err) + } + if woken, err := repo.WakeForChangedInputs(ctx); err != nil || woken < 1 { + t.Fatalf("language-change WakeForChangedInputs() = (%d, %v), want seeded row", woken, err) + } + + installationID := insertTestInstallation(t, pool, "plugin", true) + insertTestCapability(t, pool, installationID, "config-fingerprint", `{}`) + if _, err := pool.Exec(ctx, ` + INSERT INTO library_provider_chains ( + media_folder_id, plugin_installation_id, capability_id, capability_type, + content_level, priority, enabled + ) VALUES ($1, $2, 'config-fingerprint', 'metadata_provider.v1', 'item', 1, true) + `, folderID, installationID); err != nil { + t.Fatalf("seed relevant provider chain: %v", err) + } + if woken, err := repo.WakeForChangedInputs(ctx); err != nil || woken < 1 { + t.Fatalf("provider-chain WakeForChangedInputs() = (%d, %v), want seeded row", woken, err) + } + if woken, err := repo.WakeForChangedInputs(ctx); err != nil || woken != 0 { + t.Fatalf("stable provider chain wake = (%d, %v), want (0, nil)", woken, err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO media_files ( + media_folder_id, file_path, observed_root_path, base_type, + season_number, episode_number, file_size + ) VALUES ($1, $2, $3, 'series', 1, 8, 0) + `, folderID, root+"/Season 01/Show S01E08.mkv", root); err != nil { + t.Fatalf("add episode path to series shape: %v", err) + } + if woken, err := repo.WakeForChangedInputs(ctx); err != nil || woken < 1 { + t.Fatalf("episode-shape WakeForChangedInputs() = (%d, %v), want seeded row", woken, err) + } + if woken, err := repo.WakeForChangedInputs(ctx); err != nil || woken != 0 { + t.Fatalf("stable episode shape wake = (%d, %v), want (0, nil)", woken, err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO plugin_runtime_configs (plugin_installation_id, config_key, config_value) + VALUES ($1, 'metadata', '{"api_key":"changed-but-never-persisted-in-the-queue"}'::jsonb) + `, installationID); err != nil { + t.Fatalf("change relevant provider config: %v", err) + } + if woken, err := repo.WakeForChangedInputs(ctx); err != nil || woken < 1 { + t.Fatalf("provider-config WakeForChangedInputs() = (%d, %v), want seeded row", woken, err) + } + if err := pool.QueryRow(ctx, ` + SELECT input_fingerprint FROM series_root_match_queue + WHERE media_folder_id = $1 AND observed_root_path = $2 + `, folderID, root).Scan(&fingerprint); err != nil { + t.Fatalf("load provider-config fingerprint: %v", err) + } + if strings.Contains(fingerprint, "api_key") || strings.Contains(fingerprint, "changed-but-never") { + t.Fatalf("queue fingerprint leaked provider configuration: %q", fingerprint) + } +} + +func TestMovieMatchQueueRetryDuringLeaseQueuesFencedRerun(t *testing.T) { + pool := chainBuiltinTestPool(t) + ctx := context.Background() + folderID := insertTestFolder(t, pool, "movie") + root := fmt.Sprintf("/test/claim-lease-%d", time.Now().UnixNano()) + var fileID int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_files (media_folder_id, file_path, base_type, file_size) + VALUES ($1, $2, 'movie', 0) RETURNING id + `, folderID, root+"/Movie.mkv").Scan(&fileID); err != nil { + t.Fatalf("seed movie file: %v", err) + } + + repo := NewMovieMatchQueueRepository(pool, scannerrepo.NewFileRepository(pool)) + if err := repo.EnqueueMovieFile(ctx, fileID); err != nil { + t.Fatalf("EnqueueMovieFile(): %v", err) + } + claimTestMovie := func() ([]models.MovieMatchJob, error) { + return repo.ClaimByFolderAndPathPrefix(ctx, folderID, root, 1, time.Time{}) + } + claimed, err := claimTestMovie() + if err != nil || len(claimed) != 1 || claimed[0].File == nil || claimed[0].File.ID != fileID || claimed[0].LeaseToken == "" { + t.Fatalf("first Claim() = (%#v, %v), want file %d", claimed, err, fileID) + } + claimedAgain, err := claimTestMovie() + if err != nil || len(claimedAgain) != 0 { + t.Fatalf("second Claim() during lease = (%#v, %v), want empty", claimedAgain, err) + } + + var leasedUntil time.Time + if err := pool.QueryRow(ctx, `SELECT available_at FROM movie_match_queue WHERE media_file_id = $1`, fileID).Scan(&leasedUntil); err != nil { + t.Fatalf("load claim lease: %v", err) + } + if leasedUntil.Before(time.Now().Add(time.Hour)) { + t.Fatalf("claim lease = %v, want comfortably beyond one hour", leasedUntil) + } + + if affected, err := repo.RetryNowByFolder(ctx, folderID); err != nil || affected != 1 { + t.Fatalf("RetryNowByFolder() = (%d, %v), want (1, nil)", affected, err) + } + var state, leaseToken string + var availableAt time.Time + var rerunRequested bool + if err := pool.QueryRow(ctx, ` + SELECT state, available_at, lease_token, rerun_requested + FROM movie_match_queue + WHERE media_file_id = $1 + `, fileID).Scan(&state, &availableAt, &leaseToken, &rerunRequested); err != nil { + t.Fatalf("load retried movie row: %v", err) + } + if state != queueStatePending || availableAt.Before(time.Now().Add(time.Hour)) { + t.Fatalf("retried movie row = state %q available %v, want active lease preserved", state, availableAt) + } + if leaseToken != claimed[0].LeaseToken || !rerunRequested { + t.Fatalf("retried movie ownership was not preserved: rerun %v", rerunRequested) + } + if reclaimed, err := claimTestMovie(); err != nil || len(reclaimed) != 0 { + t.Fatalf("Claim() while original worker runs = (%#v, %v), want empty", reclaimed, err) + } + if err := repo.Delete(ctx, fileID, claimed[0].LeaseToken); err != nil { + t.Fatalf("original leased completion: %v", err) + } + var remaining int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM movie_match_queue WHERE media_file_id = $1`, fileID).Scan(&remaining); err != nil { + t.Fatalf("count retried movie row: %v", err) + } + if remaining != 1 { + t.Fatalf("original completion deleted requested rerun; remaining = %d", remaining) + } + if err := repo.UpdateFailure(ctx, fileID, claimed[0].LeaseToken, MatchFailure{ + Kind: MatchOutcomeCandidateRejected, Message: "stale worker result", + }); err != nil { + t.Fatalf("stale leased failure: %v", err) + } + var failureKind string + if err := pool.QueryRow(ctx, `SELECT failure_kind FROM movie_match_queue WHERE media_file_id = $1`, fileID).Scan(&failureKind); err != nil { + t.Fatalf("load retried movie failure kind: %v", err) + } + if failureKind != "" { + t.Fatalf("stale lease overwrote newly awakened row with failure %q", failureKind) + } + + reclaimed, err := claimTestMovie() + if err != nil || len(reclaimed) != 1 { + t.Fatalf("Claim() after original completion = (%#v, %v), want one row", reclaimed, err) + } + if !reclaimed[0].RerunRequested { + t.Fatal("reclaimed job did not carry the forced-rerun marker") + } + if _, err := pool.Exec(ctx, ` + UPDATE movie_match_queue SET available_at = NOW() + WHERE media_file_id = $1 + `, fileID); err != nil { + t.Fatalf("expire forced rerun lease: %v", err) + } + expiredReplacement, err := claimTestMovie() + if err != nil || len(expiredReplacement) != 1 { + t.Fatalf("Claim() after forced lease expiry = (%#v, %v), want one row", expiredReplacement, err) + } + if !expiredReplacement[0].RerunRequested { + t.Fatal("expired forced rerun lost its durable intent") + } + if expiredReplacement[0].LeaseToken == reclaimed[0].LeaseToken { + t.Fatal("expired forced rerun was not assigned fresh ownership") + } + if affected, err := repo.ReleaseLease(ctx, expiredReplacement[0].LeaseToken); err != nil || affected != 1 { + t.Fatalf("ReleaseLease() = (%d, %v), want (1, nil)", affected, err) + } + reclaimedAgain, err := claimTestMovie() + if err != nil || len(reclaimedAgain) != 1 { + t.Fatalf("Claim() after ReleaseLease = (%#v, %v), want one immediately claimable row", reclaimedAgain, err) + } + if reclaimedAgain[0].LeaseToken == expiredReplacement[0].LeaseToken { + t.Fatal("released claim was not assigned a fresh ownership token") + } + if !reclaimedAgain[0].RerunRequested { + t.Fatal("released forced rerun lost its durable intent") + } + if err := repo.UpdateFailure(ctx, fileID, reclaimedAgain[0].LeaseToken, MatchFailure{ + Kind: "provider_transient", Message: "temporary provider outage", + }); err != nil { + t.Fatalf("forced rerun failure: %v", err) + } + var rerunAfterFailure, leaseRerunAfterFailure bool + if err := pool.QueryRow(ctx, ` + SELECT rerun_requested, lease_forced_rerun + FROM movie_match_queue + WHERE media_file_id = $1 + `, fileID).Scan(&rerunAfterFailure, &leaseRerunAfterFailure); err != nil { + t.Fatalf("load failed forced rerun: %v", err) + } + if !rerunAfterFailure || leaseRerunAfterFailure { + t.Fatalf("failed forced rerun state = requested:%v leased:%v", rerunAfterFailure, leaseRerunAfterFailure) + } +} + +func TestMatchQueueSyncDeletesIneligibleReruns(t *testing.T) { + pool := chainBuiltinTestPool(t) + ctx := context.Background() + + t.Run("movie", func(t *testing.T) { + folderID := insertTestFolder(t, pool, "movie") + var fileID int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_files (media_folder_id, file_path, base_type, file_size) + VALUES ($1, $2, 'movie', 0) RETURNING id + `, folderID, fmt.Sprintf("/test/ineligible-rerun-%d/Movie.mkv", time.Now().UnixNano())).Scan(&fileID); err != nil { + t.Fatalf("seed movie file: %v", err) + } + repo := NewMovieMatchQueueRepository(pool, scannerrepo.NewFileRepository(pool)) + if err := repo.EnqueueMovieFile(ctx, fileID); err != nil { + t.Fatalf("EnqueueMovieFile(): %v", err) + } + if _, err := pool.Exec(ctx, ` + UPDATE movie_match_queue + SET rerun_requested = true, lease_token = $2, available_at = NOW() + interval '24 hours' + WHERE media_file_id = $1 + `, fileID, "cleanup-owner"); err != nil { + t.Fatalf("seed movie rerun: %v", err) + } + if _, err := pool.Exec(ctx, `UPDATE media_files SET missing_since = NOW() WHERE id = $1`, fileID); err != nil { + t.Fatalf("mark movie missing: %v", err) + } + if err := repo.SyncForFolder(ctx, folderID); err != nil { + t.Fatalf("SyncForFolder(): %v", err) + } + var remaining int + if err := pool.QueryRow(ctx, `SELECT count(*) FROM movie_match_queue WHERE media_file_id = $1`, fileID).Scan(&remaining); err != nil { + t.Fatalf("count movie rerun: %v", err) + } + if remaining != 0 { + t.Fatalf("ineligible movie reruns remaining = %d, want 0", remaining) + } + }) + + t.Run("series", func(t *testing.T) { + folderID := insertTestFolder(t, pool, "series") + root := fmt.Sprintf("/test/ineligible-series-rerun-%d", time.Now().UnixNano()) + var fileID int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_files ( + media_folder_id, file_path, observed_root_path, base_type, + season_number, episode_number, file_size + ) VALUES ($1, $2, $3, 'series', 1, 1, 0) + RETURNING id + `, folderID, root+"/Season 01/Show S01E01.mkv", root).Scan(&fileID); err != nil { + t.Fatalf("seed series file: %v", err) + } + repo := NewSeriesRootMatchQueueRepository(pool) + if err := repo.EnqueueSeriesRoot(ctx, folderID, root); err != nil { + t.Fatalf("EnqueueSeriesRoot(): %v", err) + } + if _, err := pool.Exec(ctx, ` + UPDATE series_root_match_queue + SET rerun_requested = true, lease_token = $3, available_at = NOW() + interval '24 hours' + WHERE media_folder_id = $1 AND observed_root_path = $2 + `, folderID, root, "cleanup-owner"); err != nil { + t.Fatalf("seed series rerun: %v", err) + } + if _, err := pool.Exec(ctx, `UPDATE media_files SET missing_since = NOW() WHERE id = $1`, fileID); err != nil { + t.Fatalf("mark series file missing: %v", err) + } + if err := repo.SyncForFolder(ctx, folderID); err != nil { + t.Fatalf("SyncForFolder(): %v", err) + } + var remaining int + if err := pool.QueryRow(ctx, ` + SELECT count(*) FROM series_root_match_queue + WHERE media_folder_id = $1 AND observed_root_path = $2 + `, folderID, root).Scan(&remaining); err != nil { + t.Fatalf("count series rerun: %v", err) + } + if remaining != 0 { + t.Fatalf("ineligible series reruns remaining = %d, want 0", remaining) + } + }) +} diff --git a/internal/metadata/movie_match_queue_repo.go b/internal/metadata/movie_match_queue_repo.go index ddaba109..17f49a70 100644 --- a/internal/metadata/movie_match_queue_repo.go +++ b/internal/metadata/movie_match_queue_repo.go @@ -2,6 +2,7 @@ package metadata import ( "context" + "encoding/json" "errors" "fmt" "path/filepath" @@ -10,40 +11,11 @@ import ( "github.com/Silo-Server/silo-server/internal/models" scannerrepo "github.com/Silo-Server/silo-server/internal/scanner" + "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) -const ( - movieQueueRetryDelay = 15 * time.Second - seriesRootQueueQuietWindow = 10 * time.Second - seriesRootQueueRetryDelay = 30 * time.Second - - // matchQueueRetryMaxDelay caps the exponential retry backoff shared by both - // match queues. Terminal outcomes like "no metadata found from any - // provider" stay in the queue but decay to at most one attempt per day - // instead of hot-looping at the base delay forever. A row self-heals by - // being deleted on the first successful attempt. - matchQueueRetryMaxDelay = 24 * time.Hour - // matchQueueBackoffMaxExponent clamps the 2^attempt_count factor so the - // float math stays finite for rows that accumulated very large attempt - // counts before backoff existed. - matchQueueBackoffMaxExponent = 16 -) - -// matchQueueBackoffExpr returns the SQL expression both match queues use to -// schedule the next retry after a failure: base_delay * 2^attempt_count, -// capped at matchQueueRetryMaxDelay. attempt_count is incremented when a row -// is claimed, so the first failure backs off to twice the base delay. -// basePlaceholder and maxPlaceholder name the interval bind parameters (e.g. -// "$3", "$4") in the caller's statement. -func matchQueueBackoffExpr(basePlaceholder, maxPlaceholder string) string { - return fmt.Sprintf( - "NOW() + LEAST(%s::interval * power(2::float8, LEAST(attempt_count, %d)), %s::interval)", - basePlaceholder, matchQueueBackoffMaxExponent, maxPlaceholder, - ) -} - type MovieMatchQueueRepository struct { pool *pgxpool.Pool fileRepo *scannerrepo.FileRepository @@ -74,16 +46,12 @@ func requirePositiveMovieQueueID(label string, value int) error { // Files beneath a root skipped as misplaced series are excluded durably: // their content_id is never set, so without the exclusion every library sync // would re-enqueue them only for the worker to skip them again. -const movieQueueFileEligibleCond = `folders.enabled = true +const movieQueueFileBaseEligibleCond = `folders.enabled = true AND ( lower(trim(folders.type)) IN ('movie', 'movies') OR (lower(trim(folders.type)) = 'mixed' AND lower(trim(mf.base_type)) = 'movie') ) AND mf.missing_since IS NULL AND mf.extra_id IS NULL - AND ( - mf.content_id IS NULL OR mf.content_id = '' OR - lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') - ) AND NOT EXISTS ( SELECT 1 FROM skipped_media_roots sr @@ -92,6 +60,17 @@ const movieQueueFileEligibleCond = `folders.enabled = true AND strpos(mf.file_path, sr.root_path || '/') = 1 )` +const movieQueueFileNeedsMatchCond = `( + mf.content_id IS NULL OR mf.content_id = '' OR + lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') + )` + +const movieQueueFileEligibleCond = movieQueueFileBaseEligibleCond + ` + AND ` + movieQueueFileNeedsMatchCond + +const movieQueueClaimEligibleCond = movieQueueFileBaseEligibleCond + ` + AND (q.rerun_requested OR q.lease_forced_rerun OR ` + movieQueueFileNeedsMatchCond + `)` + func (r *MovieMatchQueueRepository) EnqueueMovieFile(ctx context.Context, fileID int) error { if err := r.requireConfigured(); err != nil { return err @@ -110,12 +89,16 @@ func (r *MovieMatchQueueRepository) EnqueueMovieFile(ctx context.Context, fileID INSERT INTO movie_match_queue ( media_file_id, media_folder_id, + input_fingerprint, + matcher_revision, available_at, updated_at ) SELECT mf.id, mf.media_folder_id, + `+matchQueueInputFingerprintSQL("mf.file_path", "'movie'", "mf.media_folder_id", "folders.metadata_language")+`, + `+fmt.Sprintf("%d", matcherRevision)+`, NOW(), NOW() FROM media_files mf @@ -125,7 +108,20 @@ func (r *MovieMatchQueueRepository) EnqueueMovieFile(ctx context.Context, fileID AND `+movieQueueFileEligibleCond+` ON CONFLICT (media_file_id) DO UPDATE SET media_folder_id = EXCLUDED.media_folder_id, - available_at = GREATEST(movie_match_queue.available_at, EXCLUDED.available_at), + available_at = CASE + WHEN (movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision) AND movie_match_queue.lease_token = '' THEN LEAST(movie_match_queue.available_at, EXCLUDED.available_at) + WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN movie_match_queue.available_at + ELSE GREATEST(movie_match_queue.available_at, EXCLUDED.available_at) + END, + state = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 'pending' ELSE movie_match_queue.state END, + deterministic_attempt_count = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 0 ELSE movie_match_queue.deterministic_attempt_count END, + failure_kind = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE movie_match_queue.failure_kind END, + failure_detail = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '{}'::jsonb ELSE movie_match_queue.failure_detail END, + last_error = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE movie_match_queue.last_error END, + parked_at = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN NULL ELSE movie_match_queue.parked_at END, + rerun_requested = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN true ELSE movie_match_queue.rerun_requested END, + input_fingerprint = EXCLUDED.input_fingerprint, + matcher_revision = EXCLUDED.matcher_revision, updated_at = NOW() `, fileID); err != nil { return fmt.Errorf("upserting movie queue row: %w", err) @@ -140,7 +136,7 @@ func (r *MovieMatchQueueRepository) EnqueueMovieFile(ctx context.Context, fileID JOIN media_folders folders ON folders.id = mf.media_folder_id LEFT JOIN media_items mi ON mi.content_id = mf.content_id WHERE mf.id = q.media_file_id - AND `+movieQueueFileEligibleCond+` + AND `+movieQueueClaimEligibleCond+` ) `, fileID); err != nil { return fmt.Errorf("deleting stale movie queue row: %w", err) @@ -170,12 +166,16 @@ func (r *MovieMatchQueueRepository) SyncForFolder(ctx context.Context, folderID INSERT INTO movie_match_queue ( media_file_id, media_folder_id, + input_fingerprint, + matcher_revision, available_at, updated_at ) SELECT mf.id, mf.media_folder_id, + `+matchQueueInputFingerprintSQL("mf.file_path", "'movie'", "mf.media_folder_id", "folders.metadata_language")+`, + `+fmt.Sprintf("%d", matcherRevision)+`, NOW(), NOW() FROM media_files mf @@ -185,7 +185,20 @@ func (r *MovieMatchQueueRepository) SyncForFolder(ctx context.Context, folderID AND `+movieQueueFileEligibleCond+` ON CONFLICT (media_file_id) DO UPDATE SET media_folder_id = EXCLUDED.media_folder_id, - available_at = GREATEST(movie_match_queue.available_at, EXCLUDED.available_at), + available_at = CASE + WHEN (movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision) AND movie_match_queue.lease_token = '' THEN LEAST(movie_match_queue.available_at, EXCLUDED.available_at) + WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN movie_match_queue.available_at + ELSE GREATEST(movie_match_queue.available_at, EXCLUDED.available_at) + END, + state = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 'pending' ELSE movie_match_queue.state END, + deterministic_attempt_count = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 0 ELSE movie_match_queue.deterministic_attempt_count END, + failure_kind = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE movie_match_queue.failure_kind END, + failure_detail = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '{}'::jsonb ELSE movie_match_queue.failure_detail END, + last_error = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE movie_match_queue.last_error END, + parked_at = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN NULL ELSE movie_match_queue.parked_at END, + rerun_requested = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN true ELSE movie_match_queue.rerun_requested END, + input_fingerprint = EXCLUDED.input_fingerprint, + matcher_revision = EXCLUDED.matcher_revision, updated_at = NOW() `, folderID); err != nil { return fmt.Errorf("upserting movie queue rows for folder: %w", err) @@ -200,7 +213,7 @@ func (r *MovieMatchQueueRepository) SyncForFolder(ctx context.Context, folderID JOIN media_folders folders ON folders.id = mf.media_folder_id LEFT JOIN media_items mi ON mi.content_id = mf.content_id WHERE mf.id = q.media_file_id - AND `+movieQueueFileEligibleCond+` + AND `+movieQueueClaimEligibleCond+` ) `, folderID); err != nil { return fmt.Errorf("deleting stale movie queue rows for folder: %w", err) @@ -235,12 +248,16 @@ func (r *MovieMatchQueueRepository) SyncInScope(ctx context.Context, folderID in INSERT INTO movie_match_queue ( media_file_id, media_folder_id, + input_fingerprint, + matcher_revision, available_at, updated_at ) SELECT mf.id, mf.media_folder_id, + `+matchQueueInputFingerprintSQL("mf.file_path", "'movie'", "mf.media_folder_id", "folders.metadata_language")+`, + `+fmt.Sprintf("%d", matcherRevision)+`, NOW(), NOW() FROM media_files mf @@ -254,7 +271,20 @@ func (r *MovieMatchQueueRepository) SyncInScope(ctx context.Context, folderID in ) ON CONFLICT (media_file_id) DO UPDATE SET media_folder_id = EXCLUDED.media_folder_id, - available_at = GREATEST(movie_match_queue.available_at, EXCLUDED.available_at), + available_at = CASE + WHEN (movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision) AND movie_match_queue.lease_token = '' THEN LEAST(movie_match_queue.available_at, EXCLUDED.available_at) + WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN movie_match_queue.available_at + ELSE GREATEST(movie_match_queue.available_at, EXCLUDED.available_at) + END, + state = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 'pending' ELSE movie_match_queue.state END, + deterministic_attempt_count = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 0 ELSE movie_match_queue.deterministic_attempt_count END, + failure_kind = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE movie_match_queue.failure_kind END, + failure_detail = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '{}'::jsonb ELSE movie_match_queue.failure_detail END, + last_error = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE movie_match_queue.last_error END, + parked_at = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN NULL ELSE movie_match_queue.parked_at END, + rerun_requested = CASE WHEN movie_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR movie_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN true ELSE movie_match_queue.rerun_requested END, + input_fingerprint = EXCLUDED.input_fingerprint, + matcher_revision = EXCLUDED.matcher_revision, updated_at = NOW() `, folderID, scopePath, scopeLike); err != nil { return fmt.Errorf("upserting movie queue rows in scope: %w", err) @@ -285,7 +315,7 @@ func (r *MovieMatchQueueRepository) SyncInScope(ctx context.Context, folderID in JOIN media_folders folders ON folders.id = mf.media_folder_id LEFT JOIN media_items mi ON mi.content_id = mf.content_id WHERE mf.id = q.media_file_id - AND `+movieQueueFileEligibleCond+` + AND `+movieQueueClaimEligibleCond+` ) `, folderID, scopePath, scopeLike); err != nil { return fmt.Errorf("deleting stale movie queue rows in scope: %w", err) @@ -297,23 +327,26 @@ func (r *MovieMatchQueueRepository) SyncInScope(ctx context.Context, folderID in return nil } -func (r *MovieMatchQueueRepository) Claim(ctx context.Context, limit int) ([]*models.MediaFile, error) { +func (r *MovieMatchQueueRepository) Claim(ctx context.Context, limit int) ([]models.MovieMatchJob, error) { if err := r.requireConfigured(); err != nil { return nil, err } if limit <= 0 { limit = 500 } + leaseToken := uuid.NewString() rows, err := r.pool.Query(ctx, ` WITH candidates AS ( - SELECT q.media_file_id, q.available_at, q.last_attempted_at + SELECT q.media_file_id, q.available_at, q.last_attempted_at, + (q.rerun_requested OR q.lease_forced_rerun) AS rerun_requested FROM movie_match_queue q JOIN media_files mf ON mf.id = q.media_file_id JOIN media_folders folders ON folders.id = mf.media_folder_id LEFT JOIN media_items mi ON mi.content_id = mf.content_id - WHERE q.available_at <= NOW() - AND `+movieQueueFileEligibleCond+` + WHERE q.state = 'pending' + AND q.available_at <= NOW() + AND `+movieQueueClaimEligibleCond+` ORDER BY q.available_at ASC, q.last_attempted_at ASC NULLS FIRST, q.media_file_id ASC LIMIT $1 FOR UPDATE OF q SKIP LOCKED @@ -322,26 +355,36 @@ func (r *MovieMatchQueueRepository) Claim(ctx context.Context, limit int) ([]*mo UPDATE movie_match_queue q SET last_attempted_at = NOW(), attempt_count = q.attempt_count + 1, + available_at = NOW() + $2::interval, + lease_token = $3, + lease_forced_rerun = c.rerun_requested, + rerun_requested = false, updated_at = NOW() FROM candidates c WHERE q.media_file_id = c.media_file_id RETURNING q.media_file_id ) - SELECT c.media_file_id + SELECT c.media_file_id, c.rerun_requested FROM candidates c JOIN updated u ON u.media_file_id = c.media_file_id ORDER BY c.available_at ASC, c.last_attempted_at ASC NULLS FIRST, c.media_file_id ASC - `, limit) + `, limit, intervalLiteral(matchQueueClaimLease), leaseToken) if err != nil { return nil, fmt.Errorf("claiming movie queue rows: %w", err) } defer rows.Close() - ids, err := scanClaimedMovieIDs(rows) + claimed, err := scanClaimedMovies(rows) if err != nil { - return nil, err + rows.Close() + return nil, r.releaseClaimAfterError(ctx, leaseToken, err) } - return r.loadFilesByIDs(ctx, ids) + ids := claimedMovieIDs(claimed) + files, err := r.loadFilesByIDs(ctx, ids, leaseToken) + if err != nil { + return nil, r.releaseClaimAfterError(ctx, leaseToken, err) + } + return movieMatchJobs(files, leaseToken, claimed), nil } func (r *MovieMatchQueueRepository) ClaimByFolderAndPathPrefix( @@ -350,7 +393,7 @@ func (r *MovieMatchQueueRepository) ClaimByFolderAndPathPrefix( pathPrefix string, limit int, attemptBefore time.Time, -) ([]*models.MediaFile, error) { +) ([]models.MovieMatchJob, error) { if err := r.requireConfigured(); err != nil { return nil, err } @@ -363,19 +406,22 @@ func (r *MovieMatchQueueRepository) ClaimByFolderAndPathPrefix( if limit <= 0 { limit = 500 } + leaseToken := uuid.NewString() pathPrefix = filepath.Clean(pathPrefix) scopeLike := pathPrefixLike(pathPrefix) rows, err := r.pool.Query(ctx, ` WITH candidates AS ( - SELECT q.media_file_id, q.available_at, q.last_attempted_at + SELECT q.media_file_id, q.available_at, q.last_attempted_at, + (q.rerun_requested OR q.lease_forced_rerun) AS rerun_requested FROM movie_match_queue q JOIN media_files mf ON mf.id = q.media_file_id JOIN media_folders folders ON folders.id = mf.media_folder_id LEFT JOIN media_items mi ON mi.content_id = mf.content_id WHERE q.media_folder_id = $1 + AND q.state = 'pending' AND q.available_at <= NOW() - AND `+movieQueueFileEligibleCond+` + AND `+movieQueueClaimEligibleCond+` AND ( mf.file_path = $2 OR mf.file_path LIKE $3 ESCAPE '\' @@ -389,39 +435,65 @@ func (r *MovieMatchQueueRepository) ClaimByFolderAndPathPrefix( UPDATE movie_match_queue q SET last_attempted_at = NOW(), attempt_count = q.attempt_count + 1, + available_at = NOW() + $6::interval, + lease_token = $7, + lease_forced_rerun = c.rerun_requested, + rerun_requested = false, updated_at = NOW() FROM candidates c WHERE q.media_file_id = c.media_file_id RETURNING q.media_file_id ) - SELECT c.media_file_id + SELECT c.media_file_id, c.rerun_requested FROM candidates c JOIN updated u ON u.media_file_id = c.media_file_id ORDER BY c.available_at ASC, c.last_attempted_at ASC NULLS FIRST, c.media_file_id ASC - `, folderID, pathPrefix, scopeLike, nullTime(attemptBefore), limit) + `, folderID, pathPrefix, scopeLike, nullTime(attemptBefore), limit, intervalLiteral(matchQueueClaimLease), leaseToken) if err != nil { return nil, fmt.Errorf("claiming movie queue rows by scope: %w", err) } defer rows.Close() - ids, err := scanClaimedMovieIDs(rows) + claimed, err := scanClaimedMovies(rows) if err != nil { - return nil, err + rows.Close() + return nil, r.releaseClaimAfterError(ctx, leaseToken, err) } - return r.loadFilesByIDs(ctx, ids) + ids := claimedMovieIDs(claimed) + files, err := r.loadFilesByIDs(ctx, ids, leaseToken) + if err != nil { + return nil, r.releaseClaimAfterError(ctx, leaseToken, err) + } + return movieMatchJobs(files, leaseToken, claimed), nil } -func (r *MovieMatchQueueRepository) Delete(ctx context.Context, mediaFileID int) error { +func (r *MovieMatchQueueRepository) Delete(ctx context.Context, mediaFileID int, leaseToken string) error { if err := r.requireConfigured(); err != nil { return err } if err := requirePositiveMovieQueueID("media file id", mediaFileID); err != nil { return err } + if strings.TrimSpace(leaseToken) == "" { + return errors.New("lease token is required") + } if _, err := r.pool.Exec(ctx, ` + WITH rerun AS ( + UPDATE movie_match_queue + SET available_at = NOW(), + lease_token = '', + lease_forced_rerun = false, + updated_at = NOW() + WHERE media_file_id = $1 + AND lease_token = $2 + AND rerun_requested + RETURNING media_file_id + ) DELETE FROM movie_match_queue WHERE media_file_id = $1 - `, mediaFileID); err != nil { + AND lease_token = $2 + AND NOT rerun_requested + `, mediaFileID, leaseToken); err != nil { return fmt.Errorf("deleting movie queue row: %w", err) } return nil @@ -444,25 +516,195 @@ func (r *MovieMatchQueueRepository) DeleteByFolder(ctx context.Context, folderID return int(tag.RowsAffected()), nil } -func (r *MovieMatchQueueRepository) UpdateError(ctx context.Context, mediaFileID int, errText string) error { +func (r *MovieMatchQueueRepository) UpdateError(ctx context.Context, mediaFileID int, leaseToken, errText string) error { + // Generic processing failures (database errors, interrupted work, provider + // transport errors) are operationally retryable. Deterministic matcher + // outcomes use UpdateFailure with an explicit kind and are the only failures + // allowed to consume the parking budget. + return r.UpdateFailure(ctx, mediaFileID, leaseToken, MatchFailure{Kind: MatchOutcomeProviderTransient, Message: errText}) +} + +func (r *MovieMatchQueueRepository) UpdateFailure(ctx context.Context, mediaFileID int, leaseToken string, failure MatchFailure) error { if err := r.requireConfigured(); err != nil { return err } if err := requirePositiveMovieQueueID("media file id", mediaFileID); err != nil { return err } + if strings.TrimSpace(leaseToken) == "" { + return errors.New("lease token is required") + } + kind := normalizeMatchFailureKind(failure.Kind) + message := boundedMatchFailureMessage(failure.Message) + detail, err := json.Marshal(map[string]any{"message": message, "decision": boundedMatchDecision(failure.Decision)}) + if err != nil { + return fmt.Errorf("encoding movie queue failure: %w", err) + } if _, err := r.pool.Exec(ctx, ` UPDATE movie_match_queue - SET last_error = $2, - available_at = `+matchQueueBackoffExpr("$3", "$4")+`, + SET last_error = CASE WHEN rerun_requested THEN last_error ELSE left($2, 2000) END, + failure_kind = CASE WHEN rerun_requested THEN failure_kind ELSE $3 END, + failure_detail = CASE WHEN rerun_requested THEN failure_detail ELSE $4::jsonb END, + deterministic_attempt_count = CASE + WHEN rerun_requested THEN deterministic_attempt_count + ELSE deterministic_attempt_count + CASE WHEN $3 = 'provider_transient' THEN 0 ELSE 1 END + END, + state = CASE + WHEN rerun_requested THEN 'pending' + WHEN $3 <> 'provider_transient' AND deterministic_attempt_count + 1 >= 3 THEN 'parked' + ELSE 'pending' + END, + parked_at = CASE + WHEN rerun_requested THEN NULL + WHEN $3 <> 'provider_transient' AND deterministic_attempt_count + 1 >= 3 THEN NOW() + ELSE NULL + END, + available_at = CASE + WHEN rerun_requested THEN NOW() + WHEN $3 = 'provider_transient' THEN `+matchQueueBackoffExpr("$5", "$6")+` + WHEN deterministic_attempt_count + 1 = 1 THEN NOW() + interval '1 hour' + ELSE NOW() + interval '24 hours' + END, + lease_token = '', + rerun_requested = rerun_requested OR lease_forced_rerun, + lease_forced_rerun = false, updated_at = NOW() - WHERE media_file_id = $1 - `, mediaFileID, errText, intervalLiteral(movieQueueRetryDelay), intervalLiteral(matchQueueRetryMaxDelay)); err != nil { + WHERE media_file_id = $1 AND lease_token = $7 + `, mediaFileID, message, kind, detail, intervalLiteral(movieQueueRetryDelay), intervalLiteral(matchQueueRetryMaxDelay), leaseToken); err != nil { return fmt.Errorf("updating movie queue error: %w", err) } return nil } +// RetryNowByFolder explicitly wakes every queued row, including parked and +// future-backed-off work. Historical attempt_count remains available for audit. +func (r *MovieMatchQueueRepository) RetryNowByFolder(ctx context.Context, folderID int) (int, error) { + if err := r.requireConfigured(); err != nil { + return 0, err + } + if err := requirePositiveMovieQueueID("folder id", folderID); err != nil { + return 0, err + } + tag, err := r.pool.Exec(ctx, ` + WITH current_inputs AS ( + SELECT q.media_file_id, mf.media_folder_id, + `+matchQueueInputFingerprintSQL("mf.file_path", "'movie'", "mf.media_folder_id", "folders.metadata_language")+` AS input_fingerprint + FROM movie_match_queue q + JOIN media_files mf ON mf.id = q.media_file_id + JOIN media_folders folders ON folders.id = mf.media_folder_id + WHERE q.media_folder_id = $1 + ) + UPDATE movie_match_queue + SET state = 'pending', + available_at = CASE WHEN movie_match_queue.lease_token = '' THEN NOW() ELSE movie_match_queue.available_at END, + deterministic_attempt_count = 0, + failure_kind = '', failure_detail = '{}'::jsonb, last_error = '', parked_at = NULL, + rerun_requested = true, + media_folder_id = current_inputs.media_folder_id, + input_fingerprint = current_inputs.input_fingerprint, matcher_revision = `+fmt.Sprintf("%d", matcherRevision)+`, updated_at = NOW() + FROM current_inputs + WHERE movie_match_queue.media_file_id = current_inputs.media_file_id + `, folderID) + if err != nil { + return 0, fmt.Errorf("retrying movie queue now: %w", err) + } + return int(tag.RowsAffected()), nil +} + +func (r *MovieMatchQueueRepository) WakeForChangedInputs(ctx context.Context) (int, error) { + if err := r.requireConfigured(); err != nil { + return 0, err + } + tag, err := r.pool.Exec(ctx, ` + WITH changed AS ( + SELECT q.media_file_id, mf.media_folder_id, + `+matchQueueInputFingerprintSQL("mf.file_path", "'movie'", "mf.media_folder_id", "folders.metadata_language")+` AS input_fingerprint + FROM movie_match_queue q + JOIN media_files mf ON mf.id = q.media_file_id + JOIN media_folders folders ON folders.id = mf.media_folder_id + ) + UPDATE movie_match_queue q + SET state = 'pending', + available_at = CASE WHEN q.lease_token = '' THEN NOW() ELSE q.available_at END, + deterministic_attempt_count = 0, + failure_kind = '', failure_detail = '{}'::jsonb, last_error = '', parked_at = NULL, + rerun_requested = true, + media_folder_id = changed.media_folder_id, + input_fingerprint = changed.input_fingerprint, matcher_revision = `+fmt.Sprintf("%d", matcherRevision)+`, updated_at = NOW() + FROM changed + WHERE changed.media_file_id = q.media_file_id + AND (q.input_fingerprint <> changed.input_fingerprint OR q.matcher_revision <> `+fmt.Sprintf("%d", matcherRevision)+` OR q.media_folder_id <> changed.media_folder_id) + `) + if err != nil { + return 0, fmt.Errorf("waking movie matches with changed inputs: %w", err) + } + return int(tag.RowsAffected()), nil +} + +// ReleaseLease makes any unfinished rows from one batch immediately +// claimable. Rows completed, failed, expired, or explicitly retried no longer +// carry this token and are therefore untouched. +func (r *MovieMatchQueueRepository) ReleaseLease(ctx context.Context, leaseToken string) (int, error) { + if err := r.requireConfigured(); err != nil { + return 0, err + } + if strings.TrimSpace(leaseToken) == "" { + return 0, errors.New("lease token is required") + } + tag, err := r.pool.Exec(ctx, ` + UPDATE movie_match_queue + SET available_at = NOW(), lease_token = '', + rerun_requested = rerun_requested OR lease_forced_rerun, + lease_forced_rerun = false, + attempt_count = GREATEST(attempt_count - 1, 0), updated_at = NOW() + WHERE lease_token = $1 AND state = 'pending' + `, leaseToken) + if err != nil { + return 0, fmt.Errorf("releasing movie match lease: %w", err) + } + return int(tag.RowsAffected()), nil +} + +func (r *MovieMatchQueueRepository) releaseClaimAfterError(ctx context.Context, leaseToken string, claimErr error) error { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + if _, releaseErr := r.ReleaseLease(releaseCtx, leaseToken); releaseErr != nil { + return errors.Join(claimErr, releaseErr) + } + return claimErr +} + +type claimedMovie struct { + mediaFileID int + rerunRequested bool +} + +func claimedMovieIDs(claimed []claimedMovie) []int { + ids := make([]int, 0, len(claimed)) + for _, row := range claimed { + ids = append(ids, row.mediaFileID) + } + return ids +} + +func movieMatchJobs(files []*models.MediaFile, leaseToken string, claimed []claimedMovie) []models.MovieMatchJob { + rerunByID := make(map[int]bool, len(claimed)) + for _, row := range claimed { + rerunByID[row.mediaFileID] = row.rerunRequested + } + jobs := make([]models.MovieMatchJob, 0, len(files)) + for _, file := range files { + if file != nil { + jobs = append(jobs, models.MovieMatchJob{ + File: file, + LeaseToken: leaseToken, + RerunRequested: rerunByID[file.ID], + }) + } + } + return jobs +} + func (r *MovieMatchQueueRepository) ListByFolder(ctx context.Context, folderID int, limit int, offset int) ([]models.MovieMatchQueueEntry, int, error) { if err := r.requireConfigured(); err != nil { return nil, 0, err @@ -488,11 +730,18 @@ func (r *MovieMatchQueueRepository) ListByFolder(ctx context.Context, folderID i q.last_attempted_at, q.attempt_count, q.last_error, + q.state, + q.failure_kind, + q.failure_detail, + q.deterministic_attempt_count, + q.input_fingerprint, + q.matcher_revision, + q.parked_at, q.updated_at FROM movie_match_queue q LEFT JOIN media_files mf ON mf.id = q.media_file_id WHERE q.media_folder_id = $1 - ORDER BY q.available_at ASC, q.last_attempted_at ASC NULLS FIRST, q.media_file_id ASC + ORDER BY (q.state = 'parked') DESC, q.available_at ASC, q.last_attempted_at ASC NULLS FIRST, q.media_file_id ASC LIMIT $2 OFFSET $3 `, folderID, limit, offset) if err != nil { @@ -512,6 +761,13 @@ func (r *MovieMatchQueueRepository) ListByFolder(ctx context.Context, folderID i &entry.LastAttemptedAt, &entry.AttemptCount, &entry.LastError, + &entry.State, + &entry.FailureKind, + &entry.FailureDetail, + &entry.DeterministicAttemptCount, + &entry.InputFingerprint, + &entry.MatcherRevision, + &entry.ParkedAt, &entry.UpdatedAt, ); err != nil { return nil, 0, fmt.Errorf("scanning movie queue row: %w", err) @@ -540,7 +796,75 @@ func (r *MovieMatchQueueRepository) CountByFolder(ctx context.Context, folderID return total, nil } -func (r *MovieMatchQueueRepository) loadFilesByIDs(ctx context.Context, ids []int) ([]*models.MediaFile, error) { +func (r *MovieMatchQueueRepository) CountStatesByFolder(ctx context.Context, folderID int) (int, int, error) { + if err := r.requireConfigured(); err != nil { + return 0, 0, err + } + if err := requirePositiveMovieQueueID("folder id", folderID); err != nil { + return 0, 0, err + } + var pending, parked int + if err := r.pool.QueryRow(ctx, ` + SELECT + COUNT(*) FILTER (WHERE state = 'pending'), + COUNT(*) FILTER (WHERE state = 'parked') + FROM movie_match_queue + WHERE media_folder_id = $1 + `, folderID).Scan(&pending, &parked); err != nil { + return 0, 0, fmt.Errorf("counting movie queue states: %w", err) + } + return pending, parked, nil +} + +// CountStatesByFolders returns queue aggregates for every requested library in +// one query. Libraries without rows are omitted from the result map. +func (r *MovieMatchQueueRepository) CountStatesByFolders(ctx context.Context, folderIDs []int) (map[int]MatchQueueStateCounts, error) { + if err := r.requireConfigured(); err != nil { + return nil, err + } + counts := make(map[int]MatchQueueStateCounts, len(folderIDs)) + if len(folderIDs) == 0 { + return counts, nil + } + rows, err := r.pool.Query(ctx, ` + SELECT + media_folder_id, + COUNT(*) FILTER (WHERE state = 'pending'), + COUNT(*) FILTER (WHERE state = 'parked') + FROM movie_match_queue + WHERE media_folder_id = ANY($1) + GROUP BY media_folder_id + `, folderIDs) + if err != nil { + return nil, fmt.Errorf("counting movie queue states by folders: %w", err) + } + defer rows.Close() + for rows.Next() { + var folderID int + var count MatchQueueStateCounts + if err := rows.Scan(&folderID, &count.Pending, &count.Parked); err != nil { + return nil, fmt.Errorf("scanning movie queue state counts: %w", err) + } + counts[folderID] = count + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating movie queue state counts: %w", err) + } + return counts, nil +} + +func (r *MovieMatchQueueRepository) CountByFolderAndState(ctx context.Context, folderID int, state string) (int, error) { + if err := r.requireConfigured(); err != nil { + return 0, err + } + var total int + if err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM movie_match_queue WHERE media_folder_id = $1 AND state = $2`, folderID, state).Scan(&total); err != nil { + return 0, fmt.Errorf("counting movie queue state: %w", err) + } + return total, nil +} + +func (r *MovieMatchQueueRepository) loadFilesByIDs(ctx context.Context, ids []int, leaseToken string) ([]*models.MediaFile, error) { if len(ids) == 0 { return nil, nil } @@ -561,7 +885,7 @@ func (r *MovieMatchQueueRepository) loadFilesByIDs(ctx context.Context, ids []in for _, id := range ids { file := filesByID[id] if file == nil { - if err := r.Delete(ctx, id); err != nil { + if err := r.Delete(ctx, id, leaseToken); err != nil { return nil, fmt.Errorf("deleting stale movie queue row %d: %w", id, err) } continue @@ -571,19 +895,19 @@ func (r *MovieMatchQueueRepository) loadFilesByIDs(ctx context.Context, ids []in return files, nil } -func scanClaimedMovieIDs(rows pgx.Rows) ([]int, error) { - ids := make([]int, 0) +func scanClaimedMovies(rows pgx.Rows) ([]claimedMovie, error) { + claimed := make([]claimedMovie, 0) for rows.Next() { - var id int - if err := rows.Scan(&id); err != nil { + var row claimedMovie + if err := rows.Scan(&row.mediaFileID, &row.rerunRequested); err != nil { return nil, fmt.Errorf("scanning claimed movie queue row: %w", err) } - ids = append(ids, id) + claimed = append(claimed, row) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("iterating claimed movie queue rows: %w", err) } - return ids, nil + return claimed, nil } func intervalLiteral(d time.Duration) string { diff --git a/internal/metadata/series_root_match_queue_repo.go b/internal/metadata/series_root_match_queue_repo.go index fcf92731..b9f8f89e 100644 --- a/internal/metadata/series_root_match_queue_repo.go +++ b/internal/metadata/series_root_match_queue_repo.go @@ -2,6 +2,7 @@ package metadata import ( "context" + "encoding/json" "errors" "fmt" "path/filepath" @@ -10,6 +11,7 @@ import ( "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/pathscope" + "github.com/google/uuid" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" ) @@ -71,33 +73,60 @@ func (r *SeriesRootMatchQueueRepository) EnqueueSeriesRoot(ctx context.Context, defer tx.Rollback(ctx) //nolint:errcheck if _, err := tx.Exec(ctx, ` + WITH eligible_roots AS ( + SELECT DISTINCT + mf.media_folder_id, + mf.observed_root_path, + COALESCE(folders.metadata_language, '') AS metadata_language + FROM media_files mf + JOIN media_folders folders ON folders.id = mf.media_folder_id + LEFT JOIN media_items mi ON mi.content_id = mf.content_id + WHERE mf.media_folder_id = $1 + AND mf.observed_root_path = $2 + AND folders.enabled = true + AND ( + lower(trim(folders.type)) IN ('series', 'tv', 'show', 'tvshows') OR + (lower(trim(folders.type)) = 'mixed' AND lower(trim(mf.base_type)) = 'series') + ) + AND mf.missing_since IS NULL AND mf.extra_id IS NULL + AND mf.observed_root_path <> '' + AND ( + mf.content_id IS NULL OR mf.content_id = '' OR + lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') + ) + ) INSERT INTO series_root_match_queue ( media_folder_id, observed_root_path, + input_fingerprint, + matcher_revision, available_at, updated_at ) - SELECT DISTINCT - mf.media_folder_id, - mf.observed_root_path, + SELECT + roots.media_folder_id, + roots.observed_root_path, + `+seriesMatchQueueInputFingerprintSQL("roots.observed_root_path", "roots.media_folder_id", "roots.metadata_language")+`, + `+fmt.Sprintf("%d", matcherRevision)+`, NOW() + $3::interval, NOW() - FROM media_files mf - JOIN media_folders folders ON folders.id = mf.media_folder_id - LEFT JOIN media_items mi ON mi.content_id = mf.content_id - WHERE mf.media_folder_id = $1 - AND mf.observed_root_path = $2 - AND folders.enabled = true - AND lower(trim(folders.type)) IN ('series', 'tv', 'show', 'tvshows') - AND mf.missing_since IS NULL AND mf.extra_id IS NULL - AND mf.observed_root_path <> '' - AND ( - mf.content_id IS NULL OR mf.content_id = '' OR - lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') - ) + FROM eligible_roots roots ON CONFLICT (media_folder_id, observed_root_path) DO UPDATE SET - available_at = GREATEST(series_root_match_queue.available_at, EXCLUDED.available_at), + available_at = CASE + WHEN (series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision) AND series_root_match_queue.lease_token = '' THEN LEAST(series_root_match_queue.available_at, EXCLUDED.available_at) + WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN series_root_match_queue.available_at + ELSE GREATEST(series_root_match_queue.available_at, EXCLUDED.available_at) + END, + state = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 'pending' ELSE series_root_match_queue.state END, + deterministic_attempt_count = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 0 ELSE series_root_match_queue.deterministic_attempt_count END, + failure_kind = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE series_root_match_queue.failure_kind END, + failure_detail = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '{}'::jsonb ELSE series_root_match_queue.failure_detail END, + last_error = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE series_root_match_queue.last_error END, + parked_at = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN NULL ELSE series_root_match_queue.parked_at END, + rerun_requested = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN true ELSE series_root_match_queue.rerun_requested END, + input_fingerprint = EXCLUDED.input_fingerprint, + matcher_revision = EXCLUDED.matcher_revision, updated_at = NOW() `, folderID, observedRootPath, intervalLiteral(seriesRootQueueQuietWindow)); err != nil { return fmt.Errorf("upserting series root queue row: %w", err) @@ -121,10 +150,10 @@ func (r *SeriesRootMatchQueueRepository) EnqueueSeriesRoot(ctx context.Context, ) AND mf.missing_since IS NULL AND mf.extra_id IS NULL AND mf.observed_root_path <> '' - AND ( + AND (q.rerun_requested OR q.lease_forced_rerun OR ( mf.content_id IS NULL OR mf.content_id = '' OR lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') - ) + )) ) `, folderID, observedRootPath); err != nil { return fmt.Errorf("deleting stale series root queue row: %w", err) @@ -151,36 +180,60 @@ func (r *SeriesRootMatchQueueRepository) SyncForFolder(ctx context.Context, fold defer tx.Rollback(ctx) //nolint:errcheck if _, err := tx.Exec(ctx, ` + WITH eligible_roots AS ( + SELECT DISTINCT + mf.media_folder_id, + mf.observed_root_path, + COALESCE(folders.metadata_language, '') AS metadata_language + FROM media_files mf + JOIN media_folders folders ON folders.id = mf.media_folder_id + LEFT JOIN media_items mi ON mi.content_id = mf.content_id + WHERE mf.media_folder_id = $1 + AND folders.enabled = true + AND ( + lower(trim(folders.type)) IN ('series', 'tv', 'show', 'tvshows') OR + (lower(trim(folders.type)) = 'mixed' AND lower(trim(mf.base_type)) = 'series') + ) + AND mf.missing_since IS NULL AND mf.extra_id IS NULL + AND mf.observed_root_path IS NOT NULL + AND mf.observed_root_path <> '' + AND ( + mf.content_id IS NULL OR mf.content_id = '' OR + lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') + ) + ) INSERT INTO series_root_match_queue ( media_folder_id, observed_root_path, + input_fingerprint, + matcher_revision, available_at, updated_at ) - SELECT DISTINCT - mf.media_folder_id, - mf.observed_root_path, + SELECT + roots.media_folder_id, + roots.observed_root_path, + `+seriesMatchQueueInputFingerprintSQL("roots.observed_root_path", "roots.media_folder_id", "roots.metadata_language")+`, + `+fmt.Sprintf("%d", matcherRevision)+`, NOW() + $2::interval, NOW() - FROM media_files mf - JOIN media_folders folders ON folders.id = mf.media_folder_id - LEFT JOIN media_items mi ON mi.content_id = mf.content_id - WHERE mf.media_folder_id = $1 - AND folders.enabled = true - AND ( - lower(trim(folders.type)) IN ('series', 'tv', 'show', 'tvshows') OR - (lower(trim(folders.type)) = 'mixed' AND lower(trim(mf.base_type)) = 'series') - ) - AND mf.missing_since IS NULL AND mf.extra_id IS NULL - AND mf.observed_root_path IS NOT NULL - AND mf.observed_root_path <> '' - AND ( - mf.content_id IS NULL OR mf.content_id = '' OR - lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') - ) + FROM eligible_roots roots ON CONFLICT (media_folder_id, observed_root_path) DO UPDATE SET - available_at = GREATEST(series_root_match_queue.available_at, EXCLUDED.available_at), + available_at = CASE + WHEN (series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision) AND series_root_match_queue.lease_token = '' THEN LEAST(series_root_match_queue.available_at, EXCLUDED.available_at) + WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN series_root_match_queue.available_at + ELSE GREATEST(series_root_match_queue.available_at, EXCLUDED.available_at) + END, + state = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 'pending' ELSE series_root_match_queue.state END, + deterministic_attempt_count = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 0 ELSE series_root_match_queue.deterministic_attempt_count END, + failure_kind = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE series_root_match_queue.failure_kind END, + failure_detail = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '{}'::jsonb ELSE series_root_match_queue.failure_detail END, + last_error = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE series_root_match_queue.last_error END, + parked_at = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN NULL ELSE series_root_match_queue.parked_at END, + rerun_requested = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN true ELSE series_root_match_queue.rerun_requested END, + input_fingerprint = EXCLUDED.input_fingerprint, + matcher_revision = EXCLUDED.matcher_revision, updated_at = NOW() `, folderID, intervalLiteral(seriesRootQueueQuietWindow)); err != nil { return fmt.Errorf("upserting series root queue rows for folder: %w", err) @@ -203,10 +256,10 @@ func (r *SeriesRootMatchQueueRepository) SyncForFolder(ctx context.Context, fold ) AND mf.missing_since IS NULL AND mf.extra_id IS NULL AND mf.observed_root_path <> '' - AND ( + AND (q.rerun_requested OR q.lease_forced_rerun OR ( mf.content_id IS NULL OR mf.content_id = '' OR lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') - ) + )) ) `, folderID); err != nil { return fmt.Errorf("deleting stale series root queue rows for folder: %w", err) @@ -241,7 +294,8 @@ func (r *SeriesRootMatchQueueRepository) SyncInScope(ctx context.Context, folder WITH in_scope_roots AS ( SELECT DISTINCT mf.media_folder_id, - mf.observed_root_path + mf.observed_root_path, + COALESCE(folders.metadata_language, '') AS metadata_language FROM media_files mf JOIN media_folders folders ON folders.id = mf.media_folder_id LEFT JOIN media_items mi ON mi.content_id = mf.content_id @@ -266,14 +320,31 @@ func (r *SeriesRootMatchQueueRepository) SyncInScope(ctx context.Context, folder INSERT INTO series_root_match_queue ( media_folder_id, observed_root_path, + input_fingerprint, + matcher_revision, available_at, updated_at ) - SELECT media_folder_id, observed_root_path, NOW() + $4::interval, NOW() - FROM in_scope_roots + SELECT roots.media_folder_id, roots.observed_root_path, + `+seriesMatchQueueInputFingerprintSQL("roots.observed_root_path", "roots.media_folder_id", "roots.metadata_language")+`, + `+fmt.Sprintf("%d", matcherRevision)+`, NOW() + $4::interval, NOW() + FROM in_scope_roots roots ON CONFLICT (media_folder_id, observed_root_path) DO UPDATE SET - available_at = GREATEST(series_root_match_queue.available_at, EXCLUDED.available_at), + available_at = CASE + WHEN (series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision) AND series_root_match_queue.lease_token = '' THEN LEAST(series_root_match_queue.available_at, EXCLUDED.available_at) + WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN series_root_match_queue.available_at + ELSE GREATEST(series_root_match_queue.available_at, EXCLUDED.available_at) + END, + state = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 'pending' ELSE series_root_match_queue.state END, + deterministic_attempt_count = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN 0 ELSE series_root_match_queue.deterministic_attempt_count END, + failure_kind = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE series_root_match_queue.failure_kind END, + failure_detail = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '{}'::jsonb ELSE series_root_match_queue.failure_detail END, + last_error = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN '' ELSE series_root_match_queue.last_error END, + parked_at = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN NULL ELSE series_root_match_queue.parked_at END, + rerun_requested = CASE WHEN series_root_match_queue.input_fingerprint <> EXCLUDED.input_fingerprint OR series_root_match_queue.matcher_revision <> EXCLUDED.matcher_revision THEN true ELSE series_root_match_queue.rerun_requested END, + input_fingerprint = EXCLUDED.input_fingerprint, + matcher_revision = EXCLUDED.matcher_revision, updated_at = NOW() `, folderID, scopePath, scopeLike, intervalLiteral(seriesRootQueueQuietWindow)); err != nil { return fmt.Errorf("upserting series root queue rows in scope: %w", err) @@ -314,10 +385,10 @@ func (r *SeriesRootMatchQueueRepository) SyncInScope(ctx context.Context, folder ) AND mf.missing_since IS NULL AND mf.extra_id IS NULL AND mf.observed_root_path <> '' - AND ( + AND (q.rerun_requested OR q.lease_forced_rerun OR ( mf.content_id IS NULL OR mf.content_id = '' OR lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') - ) + )) ) `, folderID, scopePath, scopeLike); err != nil { return fmt.Errorf("deleting stale series root queue rows in scope: %w", err) @@ -336,13 +407,16 @@ func (r *SeriesRootMatchQueueRepository) Claim(ctx context.Context, limit int) ( if limit <= 0 { limit = 500 } + leaseToken := uuid.NewString() rows, err := r.pool.Query(ctx, ` WITH candidates AS ( - SELECT q.media_folder_id, q.observed_root_path + SELECT q.media_folder_id, q.observed_root_path, + (q.rerun_requested OR q.lease_forced_rerun) AS rerun_requested FROM series_root_match_queue q JOIN media_folders folders ON folders.id = q.media_folder_id - WHERE q.available_at <= NOW() + WHERE q.state = 'pending' + AND q.available_at <= NOW() AND folders.enabled = true AND lower(trim(folders.type)) IN ('series', 'tv', 'show', 'tvshows', 'mixed') AND EXISTS ( @@ -357,10 +431,10 @@ func (r *SeriesRootMatchQueueRepository) Claim(ctx context.Context, limit int) ( ) AND mf.missing_since IS NULL AND mf.extra_id IS NULL AND mf.observed_root_path <> '' - AND ( + AND (q.rerun_requested OR q.lease_forced_rerun OR ( mf.content_id IS NULL OR mf.content_id = '' OR lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') - ) + )) ) ORDER BY q.available_at ASC, q.last_attempted_at ASC NULLS FIRST, q.media_folder_id ASC, q.observed_root_path ASC LIMIT $1 @@ -370,6 +444,10 @@ func (r *SeriesRootMatchQueueRepository) Claim(ctx context.Context, limit int) ( UPDATE series_root_match_queue q SET last_attempted_at = NOW(), attempt_count = q.attempt_count + 1, + available_at = NOW() + $2::interval, + lease_token = $3, + lease_forced_rerun = c.rerun_requested, + rerun_requested = false, updated_at = NOW() FROM candidates c WHERE q.media_folder_id = c.media_folder_id @@ -386,19 +464,31 @@ func (r *SeriesRootMatchQueueRepository) Claim(ctx context.Context, limit int) ( WHERE mf.media_folder_id = u.media_folder_id AND mf.observed_root_path = u.observed_root_path AND mf.missing_since IS NULL AND mf.extra_id IS NULL - )) AS observed_file_count + )) AS observed_file_count, + c.rerun_requested FROM updated u + JOIN candidates c + ON c.media_folder_id = u.media_folder_id + AND c.observed_root_path = u.observed_root_path LEFT JOIN observed_media_locations loc ON loc.media_folder_id = u.media_folder_id AND loc.observed_root_path = u.observed_root_path ORDER BY u.media_folder_id ASC, u.observed_root_path ASC - `, limit) + `, limit, intervalLiteral(matchQueueClaimLease), leaseToken) if err != nil { return nil, fmt.Errorf("claiming series root queue rows: %w", err) } defer rows.Close() - return scanSeriesRootJobs(rows) + jobs, err := scanSeriesRootJobs(rows) + if err != nil { + rows.Close() + return nil, r.releaseClaimAfterError(ctx, leaseToken, err) + } + for i := range jobs { + jobs[i].LeaseToken = leaseToken + } + return jobs, nil } func (r *SeriesRootMatchQueueRepository) ClaimByFolderAndPathPrefix( @@ -420,15 +510,18 @@ func (r *SeriesRootMatchQueueRepository) ClaimByFolderAndPathPrefix( if limit <= 0 { limit = 500 } + leaseToken := uuid.NewString() pathPrefix = filepath.Clean(pathPrefix) scopeLike := pathPrefixLike(pathPrefix) rows, err := r.pool.Query(ctx, ` WITH candidates AS ( - SELECT q.media_folder_id, q.observed_root_path + SELECT q.media_folder_id, q.observed_root_path, + (q.rerun_requested OR q.lease_forced_rerun) AS rerun_requested FROM series_root_match_queue q JOIN media_folders folders ON folders.id = q.media_folder_id WHERE q.media_folder_id = $1 + AND q.state = 'pending' AND q.available_at <= NOW() AND folders.enabled = true AND lower(trim(folders.type)) IN ('series', 'tv', 'show', 'tvshows', 'mixed') @@ -455,10 +548,10 @@ func (r *SeriesRootMatchQueueRepository) ClaimByFolderAndPathPrefix( ) AND mf.missing_since IS NULL AND mf.extra_id IS NULL AND mf.observed_root_path <> '' - AND ( + AND (q.rerun_requested OR q.lease_forced_rerun OR ( mf.content_id IS NULL OR mf.content_id = '' OR lower(trim(COALESCE(mi.status, ''))) IN ('pending', 'unmatched', 'ambiguous') - ) + )) ) AND ($4::timestamptz IS NULL OR q.last_attempted_at IS NULL OR q.last_attempted_at < $4) ORDER BY q.available_at ASC, q.last_attempted_at ASC NULLS FIRST, q.observed_root_path ASC @@ -469,6 +562,10 @@ func (r *SeriesRootMatchQueueRepository) ClaimByFolderAndPathPrefix( UPDATE series_root_match_queue q SET last_attempted_at = NOW(), attempt_count = q.attempt_count + 1, + available_at = NOW() + $6::interval, + lease_token = $7, + lease_forced_rerun = c.rerun_requested, + rerun_requested = false, updated_at = NOW() FROM candidates c WHERE q.media_folder_id = c.media_folder_id @@ -485,22 +582,34 @@ func (r *SeriesRootMatchQueueRepository) ClaimByFolderAndPathPrefix( WHERE mf.media_folder_id = u.media_folder_id AND mf.observed_root_path = u.observed_root_path AND mf.missing_since IS NULL AND mf.extra_id IS NULL - )) AS observed_file_count + )) AS observed_file_count, + c.rerun_requested FROM updated u + JOIN candidates c + ON c.media_folder_id = u.media_folder_id + AND c.observed_root_path = u.observed_root_path LEFT JOIN observed_media_locations loc ON loc.media_folder_id = u.media_folder_id AND loc.observed_root_path = u.observed_root_path ORDER BY u.observed_root_path ASC - `, folderID, pathPrefix, scopeLike, nullTime(attemptBefore), limit) + `, folderID, pathPrefix, scopeLike, nullTime(attemptBefore), limit, intervalLiteral(matchQueueClaimLease), leaseToken) if err != nil { return nil, fmt.Errorf("claiming series root queue rows by scope: %w", err) } defer rows.Close() - return scanSeriesRootJobs(rows) + jobs, err := scanSeriesRootJobs(rows) + if err != nil { + rows.Close() + return nil, r.releaseClaimAfterError(ctx, leaseToken, err) + } + for i := range jobs { + jobs[i].LeaseToken = leaseToken + } + return jobs, nil } -func (r *SeriesRootMatchQueueRepository) Delete(ctx context.Context, folderID int, observedRootPath string) error { +func (r *SeriesRootMatchQueueRepository) Delete(ctx context.Context, folderID int, observedRootPath, leaseToken string) error { if err := r.requireConfigured(); err != nil { return err } @@ -510,10 +619,28 @@ func (r *SeriesRootMatchQueueRepository) Delete(ctx context.Context, folderID in if strings.TrimSpace(observedRootPath) == "" { return errors.New("observed root path is required") } + if strings.TrimSpace(leaseToken) == "" { + return errors.New("lease token is required") + } _, err := r.pool.Exec(ctx, ` + WITH rerun AS ( + UPDATE series_root_match_queue + SET available_at = NOW(), + lease_token = '', + lease_forced_rerun = false, + updated_at = NOW() + WHERE media_folder_id = $1 + AND observed_root_path = $2 + AND lease_token = $3 + AND rerun_requested + RETURNING media_folder_id + ) DELETE FROM series_root_match_queue - WHERE media_folder_id = $1 AND observed_root_path = $2 - `, folderID, filepath.Clean(observedRootPath)) + WHERE media_folder_id = $1 + AND observed_root_path = $2 + AND lease_token = $3 + AND NOT rerun_requested + `, folderID, filepath.Clean(observedRootPath), leaseToken) if err != nil { return fmt.Errorf("deleting series root queue row: %w", err) } @@ -537,7 +664,11 @@ func (r *SeriesRootMatchQueueRepository) DeleteByFolder(ctx context.Context, fol return int(tag.RowsAffected()), nil } -func (r *SeriesRootMatchQueueRepository) UpdateError(ctx context.Context, folderID int, observedRootPath string, errText string) error { +func (r *SeriesRootMatchQueueRepository) UpdateError(ctx context.Context, folderID int, observedRootPath, leaseToken, errText string) error { + return r.UpdateFailure(ctx, folderID, observedRootPath, leaseToken, MatchFailure{Kind: MatchOutcomeProviderTransient, Message: errText}) +} + +func (r *SeriesRootMatchQueueRepository) UpdateFailure(ctx context.Context, folderID int, observedRootPath, leaseToken string, failure MatchFailure) error { if err := r.requireConfigured(); err != nil { return err } @@ -547,19 +678,145 @@ func (r *SeriesRootMatchQueueRepository) UpdateError(ctx context.Context, folder if strings.TrimSpace(observedRootPath) == "" { return errors.New("observed root path is required") } - _, err := r.pool.Exec(ctx, ` + if strings.TrimSpace(leaseToken) == "" { + return errors.New("lease token is required") + } + kind := normalizeMatchFailureKind(failure.Kind) + message := boundedMatchFailureMessage(failure.Message) + detail, err := json.Marshal(map[string]any{"message": message, "decision": boundedMatchDecision(failure.Decision)}) + if err != nil { + return fmt.Errorf("encoding series root queue failure: %w", err) + } + _, err = r.pool.Exec(ctx, ` UPDATE series_root_match_queue - SET last_error = $3, - available_at = `+matchQueueBackoffExpr("$4", "$5")+`, + SET last_error = CASE WHEN rerun_requested THEN last_error ELSE left($3, 2000) END, + failure_kind = CASE WHEN rerun_requested THEN failure_kind ELSE $4 END, + failure_detail = CASE WHEN rerun_requested THEN failure_detail ELSE $5::jsonb END, + deterministic_attempt_count = CASE + WHEN rerun_requested THEN deterministic_attempt_count + ELSE deterministic_attempt_count + CASE WHEN $4 = 'provider_transient' THEN 0 ELSE 1 END + END, + state = CASE + WHEN rerun_requested THEN 'pending' + WHEN $4 <> 'provider_transient' AND deterministic_attempt_count + 1 >= 3 THEN 'parked' + ELSE 'pending' + END, + parked_at = CASE + WHEN rerun_requested THEN NULL + WHEN $4 <> 'provider_transient' AND deterministic_attempt_count + 1 >= 3 THEN NOW() + ELSE NULL + END, + available_at = CASE + WHEN rerun_requested THEN NOW() + WHEN $4 = 'provider_transient' THEN `+matchQueueBackoffExpr("$6", "$7")+` + WHEN deterministic_attempt_count + 1 = 1 THEN NOW() + interval '1 hour' + ELSE NOW() + interval '24 hours' + END, + lease_token = '', + rerun_requested = rerun_requested OR lease_forced_rerun, + lease_forced_rerun = false, updated_at = NOW() - WHERE media_folder_id = $1 AND observed_root_path = $2 - `, folderID, filepath.Clean(observedRootPath), errText, intervalLiteral(seriesRootQueueRetryDelay), intervalLiteral(matchQueueRetryMaxDelay)) + WHERE media_folder_id = $1 AND observed_root_path = $2 AND lease_token = $8 + `, folderID, filepath.Clean(observedRootPath), message, kind, detail, intervalLiteral(seriesRootQueueRetryDelay), intervalLiteral(matchQueueRetryMaxDelay), leaseToken) if err != nil { return fmt.Errorf("updating series root queue error: %w", err) } return nil } +func (r *SeriesRootMatchQueueRepository) RetryNowByFolder(ctx context.Context, folderID int) (int, error) { + if err := r.requireConfigured(); err != nil { + return 0, err + } + if err := requirePositiveSeriesQueueID("folder id", folderID); err != nil { + return 0, err + } + tag, err := r.pool.Exec(ctx, ` + WITH current_inputs AS ( + SELECT q.media_folder_id, q.observed_root_path, + `+seriesMatchQueueInputFingerprintSQL("q.observed_root_path", "q.media_folder_id", "folders.metadata_language")+` AS input_fingerprint + FROM series_root_match_queue q + JOIN media_folders folders ON folders.id = q.media_folder_id + WHERE q.media_folder_id = $1 + ) + UPDATE series_root_match_queue + SET state = 'pending', + available_at = CASE WHEN series_root_match_queue.lease_token = '' THEN NOW() ELSE series_root_match_queue.available_at END, + deterministic_attempt_count = 0, + failure_kind = '', failure_detail = '{}'::jsonb, last_error = '', parked_at = NULL, + rerun_requested = true, + input_fingerprint = current_inputs.input_fingerprint, matcher_revision = `+fmt.Sprintf("%d", matcherRevision)+`, updated_at = NOW() + FROM current_inputs + WHERE series_root_match_queue.media_folder_id = current_inputs.media_folder_id + AND series_root_match_queue.observed_root_path = current_inputs.observed_root_path + `, folderID) + if err != nil { + return 0, fmt.Errorf("retrying series root queue now: %w", err) + } + return int(tag.RowsAffected()), nil +} + +func (r *SeriesRootMatchQueueRepository) WakeForChangedInputs(ctx context.Context) (int, error) { + if err := r.requireConfigured(); err != nil { + return 0, err + } + tag, err := r.pool.Exec(ctx, ` + WITH changed AS ( + SELECT q.media_folder_id, q.observed_root_path, + `+seriesMatchQueueInputFingerprintSQL("q.observed_root_path", "q.media_folder_id", "folders.metadata_language")+` AS input_fingerprint + FROM series_root_match_queue q + JOIN media_folders folders ON folders.id = q.media_folder_id + ) + UPDATE series_root_match_queue q + SET state = 'pending', + available_at = CASE WHEN q.lease_token = '' THEN NOW() ELSE q.available_at END, + deterministic_attempt_count = 0, + failure_kind = '', failure_detail = '{}'::jsonb, last_error = '', parked_at = NULL, + rerun_requested = true, + input_fingerprint = changed.input_fingerprint, matcher_revision = `+fmt.Sprintf("%d", matcherRevision)+`, updated_at = NOW() + FROM changed + WHERE changed.media_folder_id = q.media_folder_id + AND changed.observed_root_path = q.observed_root_path + AND (q.input_fingerprint <> changed.input_fingerprint OR q.matcher_revision <> `+fmt.Sprintf("%d", matcherRevision)+`) + `) + if err != nil { + return 0, fmt.Errorf("waking series matches with changed inputs: %w", err) + } + return int(tag.RowsAffected()), nil +} + +// ReleaseLease makes unfinished rows from a stopped batch immediately +// claimable without affecting rows whose ownership was revoked or replaced. +func (r *SeriesRootMatchQueueRepository) ReleaseLease(ctx context.Context, leaseToken string) (int, error) { + if err := r.requireConfigured(); err != nil { + return 0, err + } + if strings.TrimSpace(leaseToken) == "" { + return 0, errors.New("lease token is required") + } + tag, err := r.pool.Exec(ctx, ` + UPDATE series_root_match_queue + SET available_at = NOW(), lease_token = '', + rerun_requested = rerun_requested OR lease_forced_rerun, + lease_forced_rerun = false, + attempt_count = GREATEST(attempt_count - 1, 0), updated_at = NOW() + WHERE lease_token = $1 AND state = 'pending' + `, leaseToken) + if err != nil { + return 0, fmt.Errorf("releasing series root match lease: %w", err) + } + return int(tag.RowsAffected()), nil +} + +func (r *SeriesRootMatchQueueRepository) releaseClaimAfterError(ctx context.Context, leaseToken string, claimErr error) error { + releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + if _, releaseErr := r.ReleaseLease(releaseCtx, leaseToken); releaseErr != nil { + return errors.Join(claimErr, releaseErr) + } + return claimErr +} + func (r *SeriesRootMatchQueueRepository) ListByFolder(ctx context.Context, folderID int, limit int, offset int) ([]models.SeriesRootMatchQueueEntry, int, error) { if err := r.requireConfigured(); err != nil { return nil, 0, err @@ -576,10 +833,11 @@ func (r *SeriesRootMatchQueueRepository) ListByFolder(ctx context.Context, folde } rows, err := r.pool.Query(ctx, ` - SELECT media_folder_id, observed_root_path, first_queued_at, available_at, last_attempted_at, attempt_count, last_error, updated_at + SELECT media_folder_id, observed_root_path, first_queued_at, available_at, last_attempted_at, attempt_count, last_error, + state, failure_kind, failure_detail, deterministic_attempt_count, input_fingerprint, matcher_revision, parked_at, updated_at FROM series_root_match_queue WHERE media_folder_id = $1 - ORDER BY available_at ASC, last_attempted_at ASC NULLS FIRST, observed_root_path ASC + ORDER BY (state = 'parked') DESC, available_at ASC, last_attempted_at ASC NULLS FIRST, observed_root_path ASC LIMIT $2 OFFSET $3 `, folderID, limit, offset) if err != nil { @@ -598,6 +856,13 @@ func (r *SeriesRootMatchQueueRepository) ListByFolder(ctx context.Context, folde &entry.LastAttemptedAt, &entry.AttemptCount, &entry.LastError, + &entry.State, + &entry.FailureKind, + &entry.FailureDetail, + &entry.DeterministicAttemptCount, + &entry.InputFingerprint, + &entry.MatcherRevision, + &entry.ParkedAt, &entry.UpdatedAt, ); err != nil { return nil, 0, fmt.Errorf("scanning series root queue row: %w", err) @@ -626,6 +891,74 @@ func (r *SeriesRootMatchQueueRepository) CountByFolder(ctx context.Context, fold return total, nil } +func (r *SeriesRootMatchQueueRepository) CountStatesByFolder(ctx context.Context, folderID int) (int, int, error) { + if err := r.requireConfigured(); err != nil { + return 0, 0, err + } + if err := requirePositiveSeriesQueueID("folder id", folderID); err != nil { + return 0, 0, err + } + var pending, parked int + if err := r.pool.QueryRow(ctx, ` + SELECT + COUNT(*) FILTER (WHERE state = 'pending'), + COUNT(*) FILTER (WHERE state = 'parked') + FROM series_root_match_queue + WHERE media_folder_id = $1 + `, folderID).Scan(&pending, &parked); err != nil { + return 0, 0, fmt.Errorf("counting series queue states: %w", err) + } + return pending, parked, nil +} + +// CountStatesByFolders returns queue aggregates for every requested library in +// one query. Libraries without rows are omitted from the result map. +func (r *SeriesRootMatchQueueRepository) CountStatesByFolders(ctx context.Context, folderIDs []int) (map[int]MatchQueueStateCounts, error) { + if err := r.requireConfigured(); err != nil { + return nil, err + } + counts := make(map[int]MatchQueueStateCounts, len(folderIDs)) + if len(folderIDs) == 0 { + return counts, nil + } + rows, err := r.pool.Query(ctx, ` + SELECT + media_folder_id, + COUNT(*) FILTER (WHERE state = 'pending'), + COUNT(*) FILTER (WHERE state = 'parked') + FROM series_root_match_queue + WHERE media_folder_id = ANY($1) + GROUP BY media_folder_id + `, folderIDs) + if err != nil { + return nil, fmt.Errorf("counting series queue states by folders: %w", err) + } + defer rows.Close() + for rows.Next() { + var folderID int + var count MatchQueueStateCounts + if err := rows.Scan(&folderID, &count.Pending, &count.Parked); err != nil { + return nil, fmt.Errorf("scanning series queue state counts: %w", err) + } + counts[folderID] = count + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating series queue state counts: %w", err) + } + return counts, nil +} + +func (r *SeriesRootMatchQueueRepository) CountByFolderAndState(ctx context.Context, folderID int, state string) (int, error) { + if err := r.requireConfigured(); err != nil { + return 0, err + } + var total int + if err := r.pool.QueryRow(ctx, `SELECT COUNT(*) FROM series_root_match_queue WHERE media_folder_id = $1 AND state = $2`, folderID, state).Scan(&total); err != nil { + return 0, fmt.Errorf("counting series root queue state: %w", err) + } + return total, nil +} + func scanSeriesRootJobs(rows pgx.Rows) ([]models.SeriesRootMatchJob, error) { jobs := make([]models.SeriesRootMatchJob, 0) for rows.Next() { @@ -635,6 +968,7 @@ func scanSeriesRootJobs(rows pgx.Rows) ([]models.SeriesRootMatchJob, error) { &job.ObservedRootPath, &job.SampleFilePath, &job.ObservedFileCount, + &job.RerunRequested, ); err != nil { return nil, fmt.Errorf("scanning series root job: %w", err) } diff --git a/internal/metadata/worker.go b/internal/metadata/worker.go index 3b0dcf0a..49992e46 100644 --- a/internal/metadata/worker.go +++ b/internal/metadata/worker.go @@ -53,6 +53,9 @@ func (w *MatchWorker) SetConcurrency(workers, batchSize int) { func (w *MatchWorker) workerCount() int { return int(w.workers.Load()) } func (w *MatchWorker) claimBatchSize() int { return int(w.batchSize.Load()) } +func (w *MatchWorker) queueClaimSize() int { + return min(w.workerCount(), w.claimBatchSize()) +} type NonSeriesFileClaimer interface { ClaimUnmatchedNonSeries(ctx context.Context, limit int) ([]*models.MediaFile, error) @@ -69,17 +72,21 @@ type MatchSuppressionChecker interface { } type MovieFileClaimer interface { - Claim(ctx context.Context, limit int) ([]*models.MediaFile, error) - ClaimByFolderAndPathPrefix(ctx context.Context, folderID int, pathPrefix string, limit int, attemptBefore time.Time) ([]*models.MediaFile, error) - Delete(ctx context.Context, mediaFileID int) error - UpdateError(ctx context.Context, mediaFileID int, errText string) error + Claim(ctx context.Context, limit int) ([]models.MovieMatchJob, error) + ClaimByFolderAndPathPrefix(ctx context.Context, folderID int, pathPrefix string, limit int, attemptBefore time.Time) ([]models.MovieMatchJob, error) + Delete(ctx context.Context, mediaFileID int, leaseToken string) error + UpdateError(ctx context.Context, mediaFileID int, leaseToken, errText string) error + UpdateFailure(ctx context.Context, mediaFileID int, leaseToken string, failure MatchFailure) error + ReleaseLease(ctx context.Context, leaseToken string) (int, error) } type SeriesRootClaimer interface { Claim(ctx context.Context, limit int) ([]models.SeriesRootMatchJob, error) ClaimByFolderAndPathPrefix(ctx context.Context, folderID int, pathPrefix string, limit int, attemptBefore time.Time) ([]models.SeriesRootMatchJob, error) - Delete(ctx context.Context, folderID int, observedRootPath string) error - UpdateError(ctx context.Context, folderID int, observedRootPath string, errText string) error + Delete(ctx context.Context, folderID int, observedRootPath, leaseToken string) error + UpdateError(ctx context.Context, folderID int, observedRootPath, leaseToken, errText string) error + UpdateFailure(ctx context.Context, folderID int, observedRootPath, leaseToken string, failure MatchFailure) error + ReleaseLease(ctx context.Context, leaseToken string) (int, error) ListByFolder(ctx context.Context, folderID int, limit int, offset int) ([]models.SeriesRootMatchQueueEntry, int, error) CountByFolder(ctx context.Context, folderID int) (int, error) } @@ -152,25 +159,11 @@ func (w *MatchWorker) Run(ctx context.Context) { // processUnmatched fetches a batch of unmatched files and processes them. func (w *MatchWorker) processUnmatched(ctx context.Context) { if w.enableTVSeriesRootQueue && w.seriesClaimer != nil { - jobs, err := w.seriesClaimer.Claim(ctx, w.claimBatchSize()) - if err != nil { - slog.ErrorContext(ctx, "metadata: failed to claim unmatched series roots", "component", "metadata", "error", err) - } else if len(jobs) > 0 { - slog.InfoContext(ctx, "metadata: processing unmatched series roots", "component", "metadata", "count", len(jobs)) - if _, err := w.processSeriesRoots(ctx, jobs); err != nil { - slog.ErrorContext(ctx, "metadata: failed to process unmatched series roots", "component", "metadata", "error", err) - } - } + w.processBackgroundSeriesQueue(ctx) } if w.movieClaimer != nil { - files, err := w.movieClaimer.Claim(ctx, w.claimBatchSize()) - if err != nil { - slog.ErrorContext(ctx, "metadata: failed to claim queued movie files", "component", "metadata", "error", err) - } else if len(files) > 0 { - slog.InfoContext(ctx, "metadata: processing queued movie files", "component", "metadata", "count", len(files)) - w.processQueuedMovieFiles(ctx, files) - } + w.processBackgroundMovieQueue(ctx) } files, err := w.claimBackgroundFiles(ctx) @@ -186,6 +179,51 @@ func (w *MatchWorker) processUnmatched(ctx context.Context) { w.processFiles(ctx, files) } +func (w *MatchWorker) processBackgroundSeriesQueue(ctx context.Context) { + remaining := w.claimBatchSize() + for remaining > 0 && ctx.Err() == nil { + claimLimit := min(w.queueClaimSize(), remaining) + jobs, err := w.seriesClaimer.Claim(ctx, claimLimit) + if err != nil { + slog.ErrorContext(ctx, "metadata: failed to claim unmatched series roots", "component", "metadata", "error", err) + return + } + if len(jobs) == 0 { + return + } + slog.InfoContext(ctx, "metadata: processing unmatched series roots", "component", "metadata", "count", len(jobs)) + if _, err := w.processSeriesRoots(ctx, jobs); err != nil { + slog.ErrorContext(ctx, "metadata: failed to process unmatched series roots", "component", "metadata", "error", err) + return + } + remaining -= len(jobs) + if len(jobs) < claimLimit { + return + } + } +} + +func (w *MatchWorker) processBackgroundMovieQueue(ctx context.Context) { + remaining := w.claimBatchSize() + for remaining > 0 && ctx.Err() == nil { + claimLimit := min(w.queueClaimSize(), remaining) + jobs, err := w.movieClaimer.Claim(ctx, claimLimit) + if err != nil { + slog.ErrorContext(ctx, "metadata: failed to claim queued movie files", "component", "metadata", "error", err) + return + } + if len(jobs) == 0 { + return + } + slog.InfoContext(ctx, "metadata: processing queued movie files", "component", "metadata", "count", len(jobs)) + w.processQueuedMovieFiles(ctx, jobs) + remaining -= len(jobs) + if len(jobs) < claimLimit { + return + } + } +} + func (w *MatchWorker) processFile(ctx context.Context, file *models.MediaFile) { if w.fileLister != nil { defer func() { @@ -429,21 +467,49 @@ func (w *MatchWorker) ProcessFile(ctx context.Context, file *models.MediaFile) { // number of files processed. func (w *MatchWorker) ProcessBatch(ctx context.Context) (processed int, err error) { if w.enableTVSeriesRootQueue && w.seriesClaimer != nil { - jobs, err := w.seriesClaimer.Claim(ctx, w.claimBatchSize()) - if err != nil { - return 0, err + claimed := 0 + for claimed < w.claimBatchSize() { + claimLimit := min(w.queueClaimSize(), w.claimBatchSize()-claimed) + jobs, err := w.seriesClaimer.Claim(ctx, claimLimit) + if err != nil { + return processed, err + } + if len(jobs) == 0 { + break + } + claimed += len(jobs) + batchProcessed, err := w.processSeriesRoots(ctx, jobs) + processed += batchProcessed + if err != nil { + return processed, err + } + if len(jobs) < claimLimit { + break + } } - if len(jobs) > 0 { - return w.processSeriesRoots(ctx, jobs) + if claimed > 0 { + return processed, nil } } if w.movieClaimer != nil { - files, err := w.movieClaimer.Claim(ctx, w.claimBatchSize()) - if err != nil { - return 0, err + claimed := 0 + for claimed < w.claimBatchSize() { + claimLimit := min(w.queueClaimSize(), w.claimBatchSize()-claimed) + jobs, err := w.movieClaimer.Claim(ctx, claimLimit) + if err != nil { + return processed, err + } + if len(jobs) == 0 { + break + } + claimed += len(jobs) + processed += w.processQueuedMovieFiles(ctx, jobs) + if len(jobs) < claimLimit { + break + } } - if len(files) > 0 { - return w.processQueuedMovieFiles(ctx, files), nil + if claimed > 0 { + return processed, nil } } @@ -466,28 +532,51 @@ func (w *MatchWorker) ProcessBatchByFolderAndPathPrefix(ctx context.Context, fol return 0, err } if useSeriesQueue { - jobs, err := w.seriesClaimer.ClaimByFolderAndPathPrefix(ctx, folderID, pathPrefix, w.claimBatchSize(), attemptBefore) - if err != nil { - return 0, err + claimed := 0 + for claimed < w.claimBatchSize() { + claimLimit := min(w.queueClaimSize(), w.claimBatchSize()-claimed) + jobs, err := w.seriesClaimer.ClaimByFolderAndPathPrefix(ctx, folderID, pathPrefix, claimLimit, attemptBefore) + if err != nil { + return processed, err + } + if len(jobs) == 0 { + break + } + claimed += len(jobs) + batchProcessed, err := w.processSeriesRoots(ctx, jobs) + processed += batchProcessed + if err != nil { + return processed, err + } + if len(jobs) < claimLimit { + break + } } - processed, err := w.processSeriesRoots(ctx, jobs) - if err != nil || processed > 0 { - return processed, err + if processed > 0 { + return processed, nil } } if useSeriesQueue && !useMovieQueue { return processed, nil } if useMovieQueue { - files, err := w.movieClaimer.ClaimByFolderAndPathPrefix(ctx, folderID, pathPrefix, w.claimBatchSize(), attemptBefore) - if err != nil { - return 0, err + claimed := 0 + for claimed < w.claimBatchSize() { + claimLimit := min(w.queueClaimSize(), w.claimBatchSize()-claimed) + jobs, err := w.movieClaimer.ClaimByFolderAndPathPrefix(ctx, folderID, pathPrefix, claimLimit, attemptBefore) + if err != nil { + return processed, err + } + if len(jobs) == 0 { + break + } + claimed += len(jobs) + processed += w.processQueuedMovieFiles(ctx, jobs) + if len(jobs) < claimLimit { + break + } } - processed := w.processQueuedMovieFiles(ctx, files) - if processed > 0 { - return processed, nil - } - if !useSeriesQueue { + if processed > 0 || !useSeriesQueue { return processed, nil } } @@ -521,7 +610,7 @@ func (w *MatchWorker) ProcessAllByFolderAndPathPrefix(ctx context.Context, folde } if useSeriesQueue { - jobs, err := w.seriesClaimer.ClaimByFolderAndPathPrefix(ctx, folderID, pathPrefix, w.claimBatchSize(), attemptBefore) + jobs, err := w.seriesClaimer.ClaimByFolderAndPathPrefix(ctx, folderID, pathPrefix, w.queueClaimSize(), attemptBefore) if err != nil { return processed, err } @@ -535,12 +624,12 @@ func (w *MatchWorker) ProcessAllByFolderAndPathPrefix(ctx context.Context, folde } } if useMovieQueue { - files, err := w.movieClaimer.ClaimByFolderAndPathPrefix(ctx, folderID, pathPrefix, w.claimBatchSize(), attemptBefore) + jobs, err := w.movieClaimer.ClaimByFolderAndPathPrefix(ctx, folderID, pathPrefix, w.queueClaimSize(), attemptBefore) if err != nil { return processed, err } - if len(files) > 0 { - batchProcessed := w.processQueuedMovieFiles(ctx, files) + if len(jobs) > 0 { + batchProcessed := w.processQueuedMovieFiles(ctx, jobs) processed += batchProcessed continue } @@ -641,16 +730,17 @@ func (w *MatchWorker) isMatchSuppressed(ctx context.Context, file *models.MediaF return suppressed } -func (w *MatchWorker) processQueuedMovieFiles(ctx context.Context, files []*models.MediaFile) int { - if len(files) == 0 { +func (w *MatchWorker) processQueuedMovieFiles(ctx context.Context, jobs []models.MovieMatchJob) int { + if len(jobs) == 0 { return 0 } + defer w.releaseMovieLeases(jobs) - fileChan := make(chan *models.MediaFile, len(files)) - for _, f := range files { - fileChan <- f + jobChan := make(chan models.MovieMatchJob, len(jobs)) + for _, job := range jobs { + jobChan <- job } - close(fileChan) + close(jobChan) var ( wg sync.WaitGroup @@ -659,11 +749,11 @@ func (w *MatchWorker) processQueuedMovieFiles(ctx context.Context, files []*mode ) for i := 0; i < w.workerCount(); i++ { wg.Go(func() { - for file := range fileChan { + for job := range jobChan { if ctx.Err() != nil { return } - if w.processQueuedMovieFile(ctx, file, &folders) { + if w.processQueuedMovieFile(ctx, job, &folders) { processed.Add(1) } } @@ -674,7 +764,28 @@ func (w *MatchWorker) processQueuedMovieFiles(ctx context.Context, files []*mode return int(processed.Load()) } -func (w *MatchWorker) processQueuedMovieFile(ctx context.Context, file *models.MediaFile, folderEnabledCache *sync.Map) bool { +func (w *MatchWorker) releaseMovieLeases(jobs []models.MovieMatchJob) { + if w == nil || w.movieClaimer == nil { + return + } + tokens := make(map[string]struct{}) + for _, job := range jobs { + if token := strings.TrimSpace(job.LeaseToken); token != "" { + tokens[token] = struct{}{} + } + } + for token := range tokens { + releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _, err := w.movieClaimer.ReleaseLease(releaseCtx, token) + cancel() + if err != nil { + slog.Warn("metadata: failed to release unfinished movie match lease", "component", "metadata", "error", err) + } + } +} + +func (w *MatchWorker) processQueuedMovieFile(ctx context.Context, job models.MovieMatchJob, folderEnabledCache *sync.Map) bool { + file := job.File if file == nil || w == nil || w.service == nil || w.movieClaimer == nil { return false } @@ -682,10 +793,10 @@ func (w *MatchWorker) processQueuedMovieFile(ctx context.Context, file *models.M return false } - skeleton, reusedLinkedItem, err := w.queuedMovieSkeleton(ctx, file) + skeleton, reusedLinkedItem, err := w.queuedMovieSkeleton(ctx, file, job.RerunRequested) if err != nil { queueErr := truncateSeriesQueueError(err.Error()) - if updateErr := w.movieClaimer.UpdateError(ctx, file.ID, queueErr); updateErr != nil { + if updateErr := w.movieClaimer.UpdateError(ctx, file.ID, job.LeaseToken, queueErr); updateErr != nil { slog.WarnContext(ctx, "metadata: failed to update movie queue error", "component", "metadata", "file_id", file.ID, "path", file.FilePath, @@ -705,7 +816,7 @@ func (w *MatchWorker) processQueuedMovieFile(ctx context.Context, file *models.M // Dequeue immediately; the recorded skipped root keeps the file out of // future enqueues (see movieQueueFileEligibleCond), so this drains rows // claimed before the skipped root was recorded. - if err := w.movieClaimer.Delete(ctx, file.ID); err != nil { + if err := w.movieClaimer.Delete(ctx, file.ID, job.LeaseToken); err != nil { slog.WarnContext(ctx, "metadata: failed to delete skipped movie queue row", "component", "metadata", "file_id", file.ID, "path", file.FilePath, @@ -716,7 +827,7 @@ func (w *MatchWorker) processQueuedMovieFile(ctx context.Context, file *models.M return true } if skeleton == nil || strings.TrimSpace(skeleton.ContentID) == "" { - if updateErr := w.movieClaimer.UpdateError(ctx, file.ID, truncateSeriesQueueError("movie queue claimed without a content id")); updateErr != nil { + if updateErr := w.movieClaimer.UpdateError(ctx, file.ID, job.LeaseToken, truncateSeriesQueueError("movie queue claimed without a content id")); updateErr != nil { slog.WarnContext(ctx, "metadata: failed to update movie queue error", "component", "metadata", "file_id", file.ID, "path", file.FilePath, @@ -726,7 +837,7 @@ func (w *MatchWorker) processQueuedMovieFile(ctx context.Context, file *models.M return false } if skeleton.ItemStatus == "ambiguous" { - if err := w.movieClaimer.Delete(ctx, file.ID); err != nil { + if err := w.movieClaimer.Delete(ctx, file.ID, job.LeaseToken); err != nil { slog.WarnContext(ctx, "metadata: failed to delete ambiguous movie queue row", "component", "metadata", "file_id", file.ID, "path", file.FilePath, @@ -742,7 +853,7 @@ func (w *MatchWorker) processQueuedMovieFile(ctx context.Context, file *models.M result, processErr := w.service.Process(ctx, req) if processErr != nil { queueErr := truncateSeriesQueueError(processErr.Error()) - if updateErr := w.movieClaimer.UpdateError(ctx, file.ID, queueErr); updateErr != nil { + if updateErr := w.movieClaimer.UpdateError(ctx, file.ID, job.LeaseToken, queueErr); updateErr != nil { slog.WarnContext(ctx, "metadata: failed to update movie queue error", "component", "metadata", "file_id", file.ID, "path", file.FilePath, @@ -757,7 +868,7 @@ func (w *MatchWorker) processQueuedMovieFile(ctx context.Context, file *models.M w.logStatusUpdateFailure(ctx, skeleton.ContentID, "unmatched", "content_id", skeleton.ContentID, "file_id", file.ID, "path", file.FilePath) return false } else if result != nil && !result.Updated { - if updateErr := w.movieClaimer.UpdateError(ctx, file.ID, truncateSeriesQueueError(ErrMetadataNotFound.Error())); updateErr != nil { + if updateErr := w.updateMovieFailure(ctx, file.ID, job.LeaseToken, result.Decision); updateErr != nil { slog.WarnContext(ctx, "metadata: failed to update movie queue error", "component", "metadata", "file_id", file.ID, "path", file.FilePath, @@ -770,7 +881,7 @@ func (w *MatchWorker) processQueuedMovieFile(ctx context.Context, file *models.M w.publishCatalogItemChanged(ctx, file.MediaFolderID, resultContentID(result, skeleton.ContentID), "metadata_updated") } - if err := w.movieClaimer.Delete(ctx, file.ID); err != nil { + if err := w.movieClaimer.Delete(ctx, file.ID, job.LeaseToken); err != nil { slog.WarnContext(ctx, "metadata: failed to delete movie queue row", "component", "metadata", "file_id", file.ID, "path", file.FilePath, @@ -781,8 +892,8 @@ func (w *MatchWorker) processQueuedMovieFile(ctx context.Context, file *models.M return true } -func (w *MatchWorker) queuedMovieSkeleton(ctx context.Context, file *models.MediaFile) (*skeletonResult, bool, error) { - if skeleton, ok := w.reusableQueuedMovieSkeleton(ctx, file); ok { +func (w *MatchWorker) queuedMovieSkeleton(ctx context.Context, file *models.MediaFile, allowMatched bool) (*skeletonResult, bool, error) { + if skeleton, ok := w.reusableQueuedMovieSkeleton(ctx, file, allowMatched); ok { return skeleton, true, nil } @@ -794,7 +905,7 @@ func (w *MatchWorker) queuedMovieSkeleton(ctx context.Context, file *models.Medi return skeleton, false, nil } -func (w *MatchWorker) reusableQueuedMovieSkeleton(ctx context.Context, file *models.MediaFile) (*skeletonResult, bool) { +func (w *MatchWorker) reusableQueuedMovieSkeleton(ctx context.Context, file *models.MediaFile, allowMatched bool) (*skeletonResult, bool) { if w == nil || w.service == nil || w.service.itemRepo == nil || file == nil { return nil, false } @@ -808,7 +919,10 @@ func (w *MatchWorker) reusableQueuedMovieSkeleton(ctx context.Context, file *mod return nil, false } status := strings.ToLower(strings.TrimSpace(item.Status)) - if !isSkeletonLikeStatus(status) && status != "ambiguous" { + reusableStatus := isSkeletonLikeStatus(status) || + status == "ambiguous" || + (allowMatched && status == string(MatchOutcomeMatched)) + if !reusableStatus { return nil, false } @@ -922,6 +1036,7 @@ func (w *MatchWorker) processSeriesRoots(ctx context.Context, jobs []models.Seri if len(jobs) == 0 { return 0, nil } + defer w.releaseSeriesLeases(jobs) runCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -964,6 +1079,26 @@ func (w *MatchWorker) processSeriesRoots(ctx context.Context, jobs []models.Seri return int(processed.Load()), firstErr } +func (w *MatchWorker) releaseSeriesLeases(jobs []models.SeriesRootMatchJob) { + if w == nil || w.seriesClaimer == nil { + return + } + tokens := make(map[string]struct{}) + for _, job := range jobs { + if token := strings.TrimSpace(job.LeaseToken); token != "" { + tokens[token] = struct{}{} + } + } + for token := range tokens { + releaseCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _, err := w.seriesClaimer.ReleaseLease(releaseCtx, token) + cancel() + if err != nil { + slog.Warn("metadata: failed to release unfinished series match lease", "component", "metadata", "error", err) + } + } +} + func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRootMatchJob, folderEnabledCache *sync.Map) (int, error) { if !w.folderEnabled(ctx, job.MediaFolderID, folderEnabledCache) { return 0, nil @@ -977,7 +1112,7 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo return 0, fmt.Errorf("loading files for series root %d/%s: %w", job.MediaFolderID, job.ObservedRootPath, err) } if len(groupFiles) == 0 { - if err := w.seriesClaimer.Delete(ctx, job.MediaFolderID, job.ObservedRootPath); err != nil { + if err := w.seriesClaimer.Delete(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken); err != nil { return 0, err } slog.InfoContext(ctx, "metadata: series root dropped because files disappeared", "component", "metadata", @@ -989,19 +1124,19 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo representative := selectRepresentativeGroupFile(groupFiles) if representative == nil { - if err := w.seriesClaimer.Delete(ctx, job.MediaFolderID, job.ObservedRootPath); err != nil { + if err := w.seriesClaimer.Delete(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken); err != nil { return 0, err } return 0, nil } if !hasUnlinkedGroupFile(groupFiles) { if strings.TrimSpace(representative.ContentID) != "" { - if skeleton, ok := w.reusableQueuedMovieSkeleton(ctx, representative); ok && skeleton.ItemStatus != "ambiguous" { + if skeleton, ok := w.reusableQueuedMovieSkeleton(ctx, representative, job.RerunRequested); ok && skeleton.ItemStatus != "ambiguous" { req := w.buildProcessRequestForGroup(ctx, representative, skeleton, groupFiles) result, processErr := w.service.Process(ctx, req) if processErr != nil { queueErr := truncateSeriesQueueError(processErr.Error()) - if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, queueErr); updateErr != nil { + if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken, queueErr); updateErr != nil { return 0, updateErr } slog.WarnContext(ctx, "metadata: enrichment failed", "component", "metadata", @@ -1019,7 +1154,7 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo return 0, nil } if result != nil && !result.Updated { - if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, truncateSeriesQueueError(ErrMetadataNotFound.Error())); updateErr != nil { + if updateErr := w.updateSeriesFailure(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken, result.Decision); updateErr != nil { return 0, updateErr } w.logStatusUpdateFailure(ctx, skeleton.ContentID, "unmatched", @@ -1045,14 +1180,14 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo "folder_id", job.MediaFolderID, "observed_root_path", job.ObservedRootPath) } else { - if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, truncateSeriesQueueError(err.Error())); updateErr != nil { + if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken, truncateSeriesQueueError(err.Error())); updateErr != nil { return 0, updateErr } return 0, fmt.Errorf("ensuring series episode links for %s: %w", representative.ContentID, err) } } } - if err := w.seriesClaimer.Delete(ctx, job.MediaFolderID, job.ObservedRootPath); err != nil { + if err := w.seriesClaimer.Delete(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken); err != nil { return 0, err } return len(groupFiles), nil @@ -1062,7 +1197,7 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo skeleton, err := w.service.createOrFindSkeleton(ctx, currentRepresentative, job.MediaFolderID) if err != nil { queueErr := truncateSeriesQueueError(err.Error()) - if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, queueErr); updateErr != nil { + if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken, queueErr); updateErr != nil { return 0, updateErr } slog.WarnContext(ctx, "metadata: series root skeleton creation failed", "component", "metadata", @@ -1078,18 +1213,18 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo } if _, err := w.service.fileRepo.UpdateContentIDByObservedRootPath(ctx, job.MediaFolderID, job.ObservedRootPath, skeleton.ContentID); err != nil { - if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, truncateSeriesQueueError(err.Error())); updateErr != nil { + if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken, truncateSeriesQueueError(err.Error())); updateErr != nil { return 0, updateErr } return 0, fmt.Errorf("relinking series root %d/%s: %w", job.MediaFolderID, job.ObservedRootPath, err) } - needsInitialMatch := skeleton.IsNew + needsInitialMatch := skeleton.IsNew || job.RerunRequested if !needsInitialMatch && strings.TrimSpace(skeleton.ContentID) != "" && w.service.itemRepo != nil { item, loadErr := w.service.itemRepo.GetByID(ctx, skeleton.ContentID) if loadErr != nil { queueErr := truncateSeriesQueueError(loadErr.Error()) - if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, queueErr); updateErr != nil { + if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken, queueErr); updateErr != nil { return 0, updateErr } return 0, fmt.Errorf("loading linked series skeleton %s: %w", skeleton.ContentID, loadErr) @@ -1104,7 +1239,7 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo result, processErr := w.service.Process(ctx, req) if processErr != nil { queueErr := truncateSeriesQueueError(processErr.Error()) - if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, queueErr); updateErr != nil { + if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken, queueErr); updateErr != nil { return 0, updateErr } slog.WarnContext(ctx, "metadata: enrichment failed", "component", "metadata", @@ -1126,7 +1261,7 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo } return 0, nil } else if result != nil && !result.Updated { - if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, truncateSeriesQueueError(ErrMetadataNotFound.Error())); updateErr != nil { + if updateErr := w.updateSeriesFailure(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken, result.Decision); updateErr != nil { return 0, updateErr } w.logStatusUpdateFailure(ctx, skeleton.ContentID, "unmatched", @@ -1149,7 +1284,7 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo finalContentID, err := w.service.fileRepo.FindContentIDByObservedRootPath(ctx, job.MediaFolderID, job.ObservedRootPath, "series") if err != nil { - if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, truncateSeriesQueueError(err.Error())); updateErr != nil { + if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken, truncateSeriesQueueError(err.Error())); updateErr != nil { return 0, updateErr } return 0, fmt.Errorf("resolving final content for series root %d/%s: %w", job.MediaFolderID, job.ObservedRootPath, err) @@ -1165,7 +1300,7 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo "folder_id", job.MediaFolderID, "observed_root_path", job.ObservedRootPath) } else { - if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, truncateSeriesQueueError(err.Error())); updateErr != nil { + if updateErr := w.seriesClaimer.UpdateError(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken, truncateSeriesQueueError(err.Error())); updateErr != nil { return 0, updateErr } return 0, fmt.Errorf("ensuring series episode links for %s: %w", finalContentID, err) @@ -1176,7 +1311,7 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo } } - if err := w.seriesClaimer.Delete(ctx, job.MediaFolderID, job.ObservedRootPath); err != nil { + if err := w.seriesClaimer.Delete(ctx, job.MediaFolderID, job.ObservedRootPath, job.LeaseToken); err != nil { return 0, err } @@ -1189,6 +1324,41 @@ func (w *MatchWorker) processSeriesRoot(ctx context.Context, job models.SeriesRo return len(groupFiles), nil } +func matchFailureFromDecision(decision *MatchDecision) MatchFailure { + kind := MatchOutcomeMetadataEmpty + if decision != nil && strings.TrimSpace(string(decision.Outcome)) != "" { + kind = decision.Outcome + } + message := ErrMetadataNotFound.Error() + if decision != nil { + switch decision.Outcome { + case MatchOutcomeNoCandidates: + message = "no provider candidates returned" + case MatchOutcomeCandidateRejected: + message = "provider candidates did not meet the automatic match threshold" + case MatchOutcomeTrustedIDConflict: + message = "provider candidates conflicted with a trusted external ID" + case MatchOutcomeTrustedIDTypeMismatch: + message = "trusted external ID resolves to the opposite library type" + case MatchOutcomeMetadataEmpty: + message = "selected providers returned no usable metadata" + case MatchOutcomeProviderTransient: + message = "metadata provider is temporarily unavailable" + case MatchOutcomeProviderPermanent: + message = "metadata provider rejected the request permanently" + } + } + return MatchFailure{Kind: kind, Message: message, Decision: decision} +} + +func (w *MatchWorker) updateMovieFailure(ctx context.Context, mediaFileID int, leaseToken string, decision *MatchDecision) error { + return w.movieClaimer.UpdateFailure(ctx, mediaFileID, leaseToken, matchFailureFromDecision(decision)) +} + +func (w *MatchWorker) updateSeriesFailure(ctx context.Context, folderID int, observedRootPath, leaseToken string, decision *MatchDecision) error { + return w.seriesClaimer.UpdateFailure(ctx, folderID, observedRootPath, leaseToken, matchFailureFromDecision(decision)) +} + func (w *MatchWorker) logStatusUpdateFailure(ctx context.Context, contentID, status string, attrs ...any) { if w == nil || w.service == nil { return diff --git a/internal/metadata/worker_test.go b/internal/metadata/worker_test.go index 5b476007..6a9a1642 100644 --- a/internal/metadata/worker_test.go +++ b/internal/metadata/worker_test.go @@ -14,6 +14,18 @@ import ( const queuedShowName = "Show Name" +func TestQueueClaimSizeNeverExceedsAvailableWorkers(t *testing.T) { + worker := NewMatchWorker(nil, nil, 8, 500, time.Second) + if got := worker.queueClaimSize(); got != 8 { + t.Fatalf("queue claim size = %d, want 8 workers", got) + } + + worker.SetConcurrency(32, 4) + if got := worker.queueClaimSize(); got != 4 { + t.Fatalf("queue claim size = %d, want batch cap 4", got) + } +} + type fakeWorkerFolderRepo struct { folders map[int]*models.MediaFolder } @@ -34,6 +46,7 @@ type fakeSeriesQueueRepo struct { claimCalls int scopedClaimCalls int claimErr error + releasedLeases []string } func newFakeSeriesQueueRepo(jobs ...models.SeriesRootMatchJob) *fakeSeriesQueueRepo { @@ -68,6 +81,7 @@ func (r *fakeSeriesQueueRepo) ClaimByFolderAndPathPrefix(_ context.Context, fold if claimedAt := r.lastAttemptedAt[key]; !attemptBefore.IsZero() && !claimedAt.IsZero() && !claimedAt.Before(attemptBefore) { continue } + job.LeaseToken = "fake-series-lease" out = append(out, job) r.lastAttemptedAt[key] = time.Now().UTC() if limit > 0 && len(out) >= limit { @@ -82,14 +96,15 @@ func (r *fakeSeriesQueueRepo) claim(limit int, _ time.Time) ([]models.SeriesRoot limit = len(r.jobs) } out := append([]models.SeriesRootMatchJob(nil), r.jobs[:limit]...) - for _, job := range out { + for i, job := range out { + out[i].LeaseToken = "fake-series-lease" key := fmt.Sprintf("%d:%s", job.MediaFolderID, job.ObservedRootPath) r.lastAttemptedAt[key] = time.Now().UTC() } return out, nil } -func (r *fakeSeriesQueueRepo) Delete(_ context.Context, folderID int, observedRootPath string) error { +func (r *fakeSeriesQueueRepo) Delete(_ context.Context, folderID int, observedRootPath, _ string) error { r.deleted[fmt.Sprintf("%d:%s", folderID, observedRootPath)] = struct{}{} filtered := r.jobs[:0] for _, job := range r.jobs { @@ -102,11 +117,20 @@ func (r *fakeSeriesQueueRepo) Delete(_ context.Context, folderID int, observedRo return nil } -func (r *fakeSeriesQueueRepo) UpdateError(_ context.Context, folderID int, observedRootPath string, errText string) error { +func (r *fakeSeriesQueueRepo) UpdateError(_ context.Context, folderID int, observedRootPath, _ string, errText string) error { r.errors[fmt.Sprintf("%d:%s", folderID, observedRootPath)] = errText return nil } +func (r *fakeSeriesQueueRepo) UpdateFailure(ctx context.Context, folderID int, observedRootPath, leaseToken string, failure MatchFailure) error { + return r.UpdateError(ctx, folderID, observedRootPath, leaseToken, failure.Message) +} + +func (r *fakeSeriesQueueRepo) ReleaseLease(_ context.Context, leaseToken string) (int, error) { + r.releasedLeases = append(r.releasedLeases, leaseToken) + return 1, nil +} + func (r *fakeSeriesQueueRepo) ListByFolder(_ context.Context, folderID int, limit int, offset int) ([]models.SeriesRootMatchQueueEntry, int, error) { out := make([]models.SeriesRootMatchQueueEntry, 0) for _, job := range r.jobs { @@ -147,6 +171,7 @@ type fakeMovieQueueRepo struct { claimCalls int scopedClaimCalls int claimErr error + releasedLeases []string } func newFakeMovieQueueRepo(files ...*models.MediaFile) *fakeMovieQueueRepo { @@ -166,7 +191,7 @@ func newFakeMovieQueueRepo(files ...*models.MediaFile) *fakeMovieQueueRepo { } } -func (r *fakeMovieQueueRepo) Claim(_ context.Context, limit int) ([]*models.MediaFile, error) { +func (r *fakeMovieQueueRepo) Claim(_ context.Context, limit int) ([]models.MovieMatchJob, error) { r.claimCalls++ if r.claimErr != nil { return nil, r.claimErr @@ -174,13 +199,13 @@ func (r *fakeMovieQueueRepo) Claim(_ context.Context, limit int) ([]*models.Medi return r.claim(limit, 0, "", time.Time{}) } -func (r *fakeMovieQueueRepo) ClaimByFolderAndPathPrefix(_ context.Context, folderID int, pathPrefix string, limit int, attemptBefore time.Time) ([]*models.MediaFile, error) { +func (r *fakeMovieQueueRepo) ClaimByFolderAndPathPrefix(_ context.Context, folderID int, pathPrefix string, limit int, attemptBefore time.Time) ([]models.MovieMatchJob, error) { r.scopedClaimCalls++ return r.claim(limit, folderID, pathPrefix, attemptBefore) } -func (r *fakeMovieQueueRepo) claim(limit int, folderID int, pathPrefix string, attemptBefore time.Time) ([]*models.MediaFile, error) { - out := make([]*models.MediaFile, 0, len(r.files)) +func (r *fakeMovieQueueRepo) claim(limit int, folderID int, pathPrefix string, attemptBefore time.Time) ([]models.MovieMatchJob, error) { + out := make([]models.MovieMatchJob, 0, len(r.files)) for _, file := range r.files { if file == nil { continue @@ -195,7 +220,7 @@ func (r *fakeMovieQueueRepo) claim(limit int, folderID int, pathPrefix string, a continue } fileCopy := *file - out = append(out, &fileCopy) + out = append(out, models.MovieMatchJob{File: &fileCopy, LeaseToken: "fake-movie-lease"}) r.lastAttemptedAt[file.ID] = time.Now().UTC() if limit > 0 && len(out) >= limit { break @@ -204,7 +229,7 @@ func (r *fakeMovieQueueRepo) claim(limit int, folderID int, pathPrefix string, a return out, nil } -func (r *fakeMovieQueueRepo) Delete(_ context.Context, mediaFileID int) error { +func (r *fakeMovieQueueRepo) Delete(_ context.Context, mediaFileID int, _ string) error { r.deleted[mediaFileID] = struct{}{} filtered := r.files[:0] for _, file := range r.files { @@ -217,11 +242,66 @@ func (r *fakeMovieQueueRepo) Delete(_ context.Context, mediaFileID int) error { return nil } -func (r *fakeMovieQueueRepo) UpdateError(_ context.Context, mediaFileID int, errText string) error { +func (r *fakeMovieQueueRepo) UpdateError(_ context.Context, mediaFileID int, _ string, errText string) error { r.errors[mediaFileID] = errText return nil } +func (r *fakeMovieQueueRepo) UpdateFailure(ctx context.Context, mediaFileID int, leaseToken string, failure MatchFailure) error { + return r.UpdateError(ctx, mediaFileID, leaseToken, failure.Message) +} + +func (r *fakeMovieQueueRepo) ReleaseLease(_ context.Context, leaseToken string) (int, error) { + r.releasedLeases = append(r.releasedLeases, leaseToken) + return 1, nil +} + +func TestProcessQueuedMovieFiles_ReleasesUnfinishedBatchOnCancellation(t *testing.T) { + const leaseToken = "movie-batch" + repo := newFakeMovieQueueRepo() + worker := NewMatchWorker(nil, nil, 2, 10, 0) + worker.movieClaimer = repo + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + processed := worker.processQueuedMovieFiles(ctx, []models.MovieMatchJob{ + {File: &models.MediaFile{ID: 1}, LeaseToken: leaseToken}, + {File: &models.MediaFile{ID: 2}, LeaseToken: leaseToken}, + }) + + if processed != 0 { + t.Fatalf("processed = %d, want 0", processed) + } + if got := repo.releasedLeases; len(got) != 1 || got[0] != leaseToken { + t.Fatalf("released leases = %v, want [%s]", got, leaseToken) + } +} + +func TestProcessSeriesRoots_ReleasesUnfinishedBatchOnCancellation(t *testing.T) { + const leaseToken = "series-batch" + + repo := newFakeSeriesQueueRepo() + worker := NewMatchWorker(nil, nil, 2, 10, 0) + worker.seriesClaimer = repo + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + processed, err := worker.processSeriesRoots(ctx, []models.SeriesRootMatchJob{ + {MediaFolderID: 1, ObservedRootPath: "/shows/One", LeaseToken: leaseToken}, + {MediaFolderID: 1, ObservedRootPath: "/shows/Two", LeaseToken: leaseToken}, + }) + + if err != nil { + t.Fatalf("process series roots: %v", err) + } + if processed != 0 { + t.Fatalf("processed = %d, want 0", processed) + } + if got := repo.releasedLeases; len(got) != 1 || got[0] != leaseToken { + t.Fatalf("released leases = %v, want [%s]", got, leaseToken) + } +} + // TestWorkerProcessFile_SkeletonCreatedForNoFolderIDs verifies that the worker // processes files under roots without folder IDs and creates skeleton items // (no longer skipping them). @@ -1204,6 +1284,97 @@ func TestWorkerProcessBatchByFolderAndPathPrefix_MovieQueueClaimsOnlyOncePerScan } } +func TestWorkerProcessBatchByFolderAndPathPrefix_ZeroResultSeriesClaimFallsThroughToMovie(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + h.service.folderRepo = &fakeWorkerFolderRepo{ + folders: map[int]*models.MediaFolder{ + 10: {ID: 10, Type: "mixed", Enabled: true}, + }, + } + pathPrefix := "/media/mixed/Example" + seriesQueue := newFakeSeriesQueueRepo(models.SeriesRootMatchJob{ + MediaFolderID: 10, + ObservedRootPath: pathPrefix + "/Missing Show", + SampleFilePath: pathPrefix + "/Missing Show/Show S01E01.mkv", + }) + movieFile := &models.MediaFile{ + ID: 2, + MediaFolderID: 10, + FilePath: pathPrefix + "/Movie (2026)/Movie.mkv", + BaseType: "movie", + } + movieQueue := newFakeMovieQueueRepo(movieFile) + h.service.hooks.process = func(_ context.Context, _ ProcessRequest) (*ProcessResult, error) { + return &ProcessResult{Updated: true}, nil + } + + worker := NewMatchWorker(h.service, h.fileRepo, 1, 1, 0) + worker.SetSeriesRootClaimer(seriesQueue, true) + worker.SetMovieFileClaimer(movieQueue) + _, err := worker.ProcessBatchByFolderAndPathPrefix(ctx, 10, pathPrefix, time.Now().UTC()) + if err != nil { + t.Fatalf("ProcessBatchByFolderAndPathPrefix error = %v", err) + } + if movieQueue.scopedClaimCalls == 0 { + t.Fatal("zero-result series claim suppressed the movie queue") + } + if movieQueue.lastAttemptedAt[movieFile.ID].IsZero() { + t.Fatal("movie fallback job was not claimed") + } +} + +func TestWorkerProcessBatchByFolderAndPathPrefix_ZeroResultMovieClaimFallsThroughToRaw(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + h.service.folderRepo = &fakeWorkerFolderRepo{ + folders: map[int]*models.MediaFolder{ + 10: {ID: 10, Type: "mixed", Enabled: true}, + }, + } + pathPrefix := "/media/mixed/Example" + queuedFile := &models.MediaFile{ + ID: 1, + MediaFolderID: 10, + FilePath: pathPrefix + "/Queued Movie (2026)/Queued.mkv", + BaseType: "movie", + } + rawFile := &models.MediaFile{ + ID: 2, + MediaFolderID: 10, + FilePath: pathPrefix + "/Raw Movie (2025)/Raw.mkv", + BaseType: "movie", + GroupKeyVersion: 1, + ContentGroupKey: "v1|movie|raw_movie|2025", + } + h.fileRepo.setGroupFiles(10, rawFile.GroupKeyVersion, rawFile.ContentGroupKey, rawFile) + processCalls := 0 + h.service.hooks.process = func(_ context.Context, _ ProcessRequest) (*ProcessResult, error) { + processCalls++ + if processCalls == 1 { + return nil, ErrMetadataNotFound + } + return &ProcessResult{Updated: true}, nil + } + + worker := NewMatchWorker(h.service, h.fileRepo, 1, 1, 0) + worker.SetSeriesRootClaimer(newFakeSeriesQueueRepo(), true) + worker.SetMovieFileClaimer(newFakeMovieQueueRepo(queuedFile)) + processed, err := worker.ProcessBatchByFolderAndPathPrefix(ctx, 10, pathPrefix, time.Now().UTC()) + if err != nil { + t.Fatalf("ProcessBatchByFolderAndPathPrefix error = %v", err) + } + if processed != 1 { + t.Fatalf("processed = %d, want raw fallback result", processed) + } + if h.fileRepo.claimMixedCalls == 0 { + t.Fatal("zero-result movie claim suppressed the raw mixed fallback") + } + if processCalls != 2 { + t.Fatalf("process calls = %d, want queued attempt plus raw fallback", processCalls) + } +} + // --------------------------------------------------------------------------- // Tests for concurrent-merge ErrItemNotFound tolerance (hotfix 2026-05-27) // --------------------------------------------------------------------------- @@ -1542,7 +1713,7 @@ func TestReusableQueuedMovieSkeletonPreservesKnownYearWhenCurrentPathHasNone(t * } worker := NewMatchWorker(h.service, h.fileRepo, 1, 1, 0) - skeleton, ok := worker.reusableQueuedMovieSkeleton(ctx, file) + skeleton, ok := worker.reusableQueuedMovieSkeleton(ctx, file, false) if !ok || skeleton == nil { t.Fatal("expected reusable skeleton") } diff --git a/internal/models/media_group.go b/internal/models/media_group.go index 26cdca60..15a4ddea 100644 --- a/internal/models/media_group.go +++ b/internal/models/media_group.go @@ -1,6 +1,9 @@ package models -import "time" +import ( + "encoding/json" + "time" +) // ScannedMediaGroup is the persisted group-inference snapshot for one logical // content group inside a library folder. @@ -100,16 +103,23 @@ type MediaGroupLocation struct { LastSeenAt time.Time } -// SeriesRootMatchQueueEntry represents one pending initial series-root job. +// SeriesRootMatchQueueEntry represents one pending or parked series-root job. type SeriesRootMatchQueueEntry struct { - MediaFolderID int - ObservedRootPath string - FirstQueuedAt time.Time - AvailableAt time.Time - LastAttemptedAt *time.Time - AttemptCount int - LastError string - UpdatedAt time.Time + MediaFolderID int + ObservedRootPath string + FirstQueuedAt time.Time + AvailableAt time.Time + LastAttemptedAt *time.Time + AttemptCount int + LastError string + State string + FailureKind string + FailureDetail json.RawMessage + DeterministicAttemptCount int + InputFingerprint string + MatcherRevision int + ParkedAt *time.Time + UpdatedAt time.Time } // SeriesRootMatchJob is the claimed work payload returned to the matcher. @@ -118,17 +128,34 @@ type SeriesRootMatchJob struct { ObservedRootPath string SampleFilePath string ObservedFileCount int + LeaseToken string + RerunRequested bool } -// MovieMatchQueueEntry represents one pending initial movie-file job. -type MovieMatchQueueEntry struct { - MediaFileID int - MediaFolderID int - FilePath string - FirstQueuedAt time.Time - AvailableAt time.Time - LastAttemptedAt *time.Time - AttemptCount int - LastError string - UpdatedAt time.Time +// MovieMatchJob is a claimed movie file plus the ownership token required to +// complete, fail, or release that exact claim. +type MovieMatchJob struct { + File *MediaFile + LeaseToken string + RerunRequested bool +} + +// MovieMatchQueueEntry represents one pending or parked movie-file job. +type MovieMatchQueueEntry struct { + MediaFileID int + MediaFolderID int + FilePath string + FirstQueuedAt time.Time + AvailableAt time.Time + LastAttemptedAt *time.Time + AttemptCount int + LastError string + State string + FailureKind string + FailureDetail json.RawMessage + DeterministicAttemptCount int + InputFingerprint string + MatcherRevision int + ParkedAt *time.Time + UpdatedAt time.Time } diff --git a/internal/plugins/service.go b/internal/plugins/service.go index 5b4c5faa..5bc79dfd 100644 --- a/internal/plugins/service.go +++ b/internal/plugins/service.go @@ -104,7 +104,7 @@ type Service struct { func (s *Service) SetEventDispatcher(d *EventDispatcher) { s.dispatcher = d } // AddLifecycleHook registers a callback invoked after plugin install, enable, -// disable, uninstall, or preload lifecycle changes. +// disable, uninstall, preload, or runtime-configuration changes. func (s *Service) AddLifecycleHook(hook func(context.Context)) { if s == nil || hook == nil { return diff --git a/internal/plugins/service_admin_config.go b/internal/plugins/service_admin_config.go index d9b2ee4b..4db7f2ee 100644 --- a/internal/plugins/service_admin_config.go +++ b/internal/plugins/service_admin_config.go @@ -109,12 +109,18 @@ func (s *Service) SetGlobalConfigWithClears( return fmt.Errorf("persist plugin config: concurrent updates did not settle") } + var stopErr error if s.host != nil { if err := s.host.Stop(installationID); err != nil && !errors.Is(err, pluginhost.ErrClientNotFound) { - return fmt.Errorf("reload plugin after config update: %w", err) + stopErr = fmt.Errorf("reload plugin after config update: %w", err) } } - return nil + // Configuration is part of metadata match input identity. Notify hooks + // after the old process has been stopped so resolver reloads observe the new + // runtime. The durable config changed even when stopping failed, so hooks + // still run before returning that error and parked rows are not left asleep. + s.OnLifecycleChange(ctx) + return stopErr } func validatedSecretClearSet( diff --git a/internal/scanner/file_repo.go b/internal/scanner/file_repo.go index 82d9c92d..cc46f0ab 100644 --- a/internal/scanner/file_repo.go +++ b/internal/scanner/file_repo.go @@ -2198,6 +2198,50 @@ func (r *FileRepository) CountUnmatchedMatchBacklogByFolder(ctx context.Context, return total, nil } +// CountUnmatchedMatchBacklogByFolders counts raw matcher work for multiple +// libraries in one query. Libraries without eligible files are omitted. +func (r *FileRepository) CountUnmatchedMatchBacklogByFolders(ctx context.Context, folderIDs []int, mode RawMatchBacklogMode) (map[int]int, error) { + counts := make(map[int]int, len(folderIDs)) + if len(folderIDs) == 0 { + return counts, nil + } + rows, err := r.pool.Query(ctx, ` + SELECT mf.media_folder_id, COUNT(*) + FROM media_files mf + JOIN media_folders folders ON folders.id = mf.media_folder_id + WHERE mf.media_folder_id = ANY($1) + AND (mf.content_id IS NULL OR mf.content_id = '') AND mf.extra_id IS NULL + AND mf.missing_since IS NULL + AND mf.match_suppressed_at IS NULL + AND folders.enabled = true + AND ( + $2 = 'generic' + OR ($2 = 'non_series' AND lower(trim(folders.type)) NOT IN ('series', 'tv', 'show', 'tvshows')) + OR ( + $2 = 'mixed' + AND lower(trim(folders.type)) NOT IN ('series', 'tv', 'show', 'tvshows', 'movie', 'movies') + AND lower(trim(COALESCE(mf.base_type, ''))) NOT IN ('series', 'movie') + ) + ) + GROUP BY mf.media_folder_id + `, folderIDs, string(normalizeRawMatchBacklogMode(mode))) + if err != nil { + return nil, fmt.Errorf("counting unmatched match backlog by folders: %w", err) + } + defer rows.Close() + for rows.Next() { + var folderID, count int + if err := rows.Scan(&folderID, &count); err != nil { + return nil, fmt.Errorf("scanning unmatched match backlog counts: %w", err) + } + counts[folderID] = count + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating unmatched match backlog counts: %w", err) + } + return counts, nil +} + // ListUnmatchedMatchBacklogByFolder lists raw unmatched files that are still // eligible for the background matcher. func (r *FileRepository) ListUnmatchedMatchBacklogByFolder(ctx context.Context, folderID int, mode RawMatchBacklogMode, limit int, offset int) ([]*models.MediaFile, int, error) { diff --git a/migrations/sql/20260721211627_metadata_match_queue_state.sql b/migrations/sql/20260721211627_metadata_match_queue_state.sql new file mode 100644 index 00000000..23b1c509 --- /dev/null +++ b/migrations/sql/20260721211627_metadata_match_queue_state.sql @@ -0,0 +1,55 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE movie_match_queue + ADD COLUMN state text NOT NULL DEFAULT 'pending' CHECK (state IN ('pending', 'parked')), + ADD COLUMN failure_kind text NOT NULL DEFAULT '', + ADD COLUMN failure_detail jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN deterministic_attempt_count integer NOT NULL DEFAULT 0, + ADD COLUMN input_fingerprint text NOT NULL DEFAULT '', + ADD COLUMN matcher_revision integer NOT NULL DEFAULT 0, + ADD COLUMN lease_token text NOT NULL DEFAULT '', + ADD COLUMN parked_at timestamptz; + +ALTER TABLE series_root_match_queue + ADD COLUMN state text NOT NULL DEFAULT 'pending' CHECK (state IN ('pending', 'parked')), + ADD COLUMN failure_kind text NOT NULL DEFAULT '', + ADD COLUMN failure_detail jsonb NOT NULL DEFAULT '{}'::jsonb, + ADD COLUMN deterministic_attempt_count integer NOT NULL DEFAULT 0, + ADD COLUMN input_fingerprint text NOT NULL DEFAULT '', + ADD COLUMN matcher_revision integer NOT NULL DEFAULT 0, + ADD COLUMN lease_token text NOT NULL DEFAULT '', + ADD COLUMN parked_at timestamptz; + +CREATE INDEX idx_movie_match_queue_claimable + ON movie_match_queue (available_at, last_attempted_at, media_folder_id, media_file_id) + WHERE state = 'pending'; +CREATE INDEX idx_series_root_match_queue_claimable + ON series_root_match_queue (available_at, last_attempted_at, media_folder_id, observed_root_path) + WHERE state = 'pending'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP INDEX IF EXISTS idx_series_root_match_queue_claimable; +DROP INDEX IF EXISTS idx_movie_match_queue_claimable; + +ALTER TABLE series_root_match_queue + DROP COLUMN IF EXISTS parked_at, + DROP COLUMN IF EXISTS lease_token, + DROP COLUMN IF EXISTS matcher_revision, + DROP COLUMN IF EXISTS input_fingerprint, + DROP COLUMN IF EXISTS deterministic_attempt_count, + DROP COLUMN IF EXISTS failure_detail, + DROP COLUMN IF EXISTS failure_kind, + DROP COLUMN IF EXISTS state; + +ALTER TABLE movie_match_queue + DROP COLUMN IF EXISTS parked_at, + DROP COLUMN IF EXISTS lease_token, + DROP COLUMN IF EXISTS matcher_revision, + DROP COLUMN IF EXISTS input_fingerprint, + DROP COLUMN IF EXISTS deterministic_attempt_count, + DROP COLUMN IF EXISTS failure_detail, + DROP COLUMN IF EXISTS failure_kind, + DROP COLUMN IF EXISTS state; +-- +goose StatementEnd diff --git a/migrations/sql/20260723214954_media_files_series_match_fingerprint_index.sql b/migrations/sql/20260723214954_media_files_series_match_fingerprint_index.sql new file mode 100644 index 00000000..54bf81a7 --- /dev/null +++ b/migrations/sql/20260723214954_media_files_series_match_fingerprint_index.sql @@ -0,0 +1,32 @@ +-- +goose NO TRANSACTION + +-- +goose Up +-- The series queue fingerprint aggregates active paths under each observed +-- root. Build its supporting index without blocking scanner writes to the +-- potentially large media_files table during deployment. +-- A failed concurrent build can leave an INVALID relation that makes +-- IF NOT EXISTS skip every retry. Remove only that unusable artifact. +-- +goose StatementBegin +DO $$ +BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + JOIN pg_index i ON i.indexrelid = c.oid + WHERE n.nspname = 'public' + AND c.relname = 'idx_media_files_folder_root_active' + AND NOT i.indisvalid + ) THEN + DROP INDEX public.idx_media_files_folder_root_active; + END IF; +END; +$$; +-- +goose StatementEnd + +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_media_files_folder_root_active + ON public.media_files (media_folder_id, observed_root_path) + WHERE missing_since IS NULL AND extra_id IS NULL; + +-- +goose Down +DROP INDEX CONCURRENTLY IF EXISTS idx_media_files_folder_root_active; diff --git a/migrations/sql/20260724135931_add_metadata_match_queue_rerun_requests.sql b/migrations/sql/20260724135931_add_metadata_match_queue_rerun_requests.sql new file mode 100644 index 00000000..a1a24325 --- /dev/null +++ b/migrations/sql/20260724135931_add_metadata_match_queue_rerun_requests.sql @@ -0,0 +1,17 @@ +-- +goose Up +ALTER TABLE movie_match_queue + ADD COLUMN rerun_requested boolean NOT NULL DEFAULT false, + ADD COLUMN lease_forced_rerun boolean NOT NULL DEFAULT false; + +ALTER TABLE series_root_match_queue + ADD COLUMN rerun_requested boolean NOT NULL DEFAULT false, + ADD COLUMN lease_forced_rerun boolean NOT NULL DEFAULT false; + +-- +goose Down +ALTER TABLE series_root_match_queue + DROP COLUMN lease_forced_rerun, + DROP COLUMN rerun_requested; + +ALTER TABLE movie_match_queue + DROP COLUMN lease_forced_rerun, + DROP COLUMN rerun_requested; diff --git a/web/src/api/types.ts b/web/src/api/types.ts index 0b7f9d17..4beb0114 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -3038,6 +3038,8 @@ export interface LibraryMetadataMatchQueueStatus { series_count: number; raw_file_count: number; total_count: number; + pending_count: number; + parked_count: number; } export interface LibraryMovieMatchQueueEntry { @@ -3049,6 +3051,12 @@ export interface LibraryMovieMatchQueueEntry { last_attempted_at?: string; attempt_count: number; last_error?: string; + state: "pending" | "parked"; + failure_kind?: string; + failure_detail?: LibraryMetadataMatchFailureDetail; + deterministic_attempt_count: number; + matcher_revision: number; + parked_at?: string; updated_at: string; } @@ -3060,9 +3068,34 @@ export interface LibrarySeriesMatchQueueEntry { last_attempted_at?: string; attempt_count: number; last_error?: string; + state: "pending" | "parked"; + failure_kind?: string; + failure_detail?: LibraryMetadataMatchFailureDetail; + deterministic_attempt_count: number; + matcher_revision: number; + parked_at?: string; updated_at: string; } +export interface LibraryMetadataMatchFailureDetail { + message?: string; + decision?: { + outcome: string; + candidate_count: number; + threshold: number; + top_candidates?: Array<{ + title: string; + matched_title?: string; + year?: number; + score: number; + provider_ids?: Record; + sources?: string[]; + reasons?: string[]; + }>; + }; + [key: string]: unknown; +} + export interface LibraryRawMatchBacklogEntry { media_file_id: number; media_folder_id: number; @@ -3076,6 +3109,8 @@ export interface LibraryRawMatchBacklogEntry { } export interface LibraryMetadataMatchQueueDetail extends LibraryMetadataMatchQueueStatus { + limit: number; + offset: number; movies: LibraryMovieMatchQueueEntry[]; series: LibrarySeriesMatchQueueEntry[]; raw_files: LibraryRawMatchBacklogEntry[]; diff --git a/web/src/components/admin/CollapsibleDiagnosticsSection.tsx b/web/src/components/admin/CollapsibleDiagnosticsSection.tsx new file mode 100644 index 00000000..77ad4b63 --- /dev/null +++ b/web/src/components/admin/CollapsibleDiagnosticsSection.tsx @@ -0,0 +1,63 @@ +import type { ReactNode } from "react"; +import { ChevronDown } from "lucide-react"; + +import { Badge } from "@/components/ui/badge"; +import { cn } from "@/lib/utils"; + +export function CollapsibleDiagnosticsSection({ + title, + description, + count, + icon, + iconClassName, + open, + onOpenChange, + children, +}: { + title: string; + description: string; + count: number; + icon: ReactNode; + iconClassName?: string; + open: boolean; + onOpenChange: (open: boolean) => void; + children: ReactNode; +}) { + return ( +
+ + {open ?
{children}
: null} +
+ ); +} diff --git a/web/src/components/admin/libraries/MetadataMatcherQueuesSection.test.tsx b/web/src/components/admin/libraries/MetadataMatcherQueuesSection.test.tsx new file mode 100644 index 00000000..1b31b325 --- /dev/null +++ b/web/src/components/admin/libraries/MetadataMatcherQueuesSection.test.tsx @@ -0,0 +1,105 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; + +import type { Library } from "@/api/types"; +import { MetadataMatcherQueuesSection } from "./MetadataMatcherQueuesSection"; + +const mocks = vi.hoisted(() => ({ + useQueues: vi.fn(), + useDetail: vi.fn(), + useRetry: vi.fn(), +})); + +vi.mock("@/hooks/queries/admin/libraries", () => ({ + useLibraryMetadataMatchQueues: (...args: unknown[]) => mocks.useQueues(...args), + useLibraryMetadataMatchQueueDetail: (...args: unknown[]) => mocks.useDetail(...args), + useRetryLibraryMetadataMatchQueue: (...args: unknown[]) => mocks.useRetry(...args), +})); + +describe("MetadataMatcherQueuesSection", () => { + it("renders the structured failure kind with the parked item detail", async () => { + mocks.useQueues.mockReturnValue({ + data: [ + { + library_id: 1, + movie_count: 1, + series_count: 0, + raw_file_count: 0, + total_count: 1, + pending_count: 0, + parked_count: 1, + }, + ], + }); + mocks.useDetail.mockReturnValue({ + data: { + movies: [ + { + media_file_id: 42, + file_path: "/media/movies/Unknown.mkv", + state: "parked", + failure_kind: "candidate_rejected", + failure_detail: { message: "Score below threshold" }, + }, + ], + series: [], + raw_files: [], + }, + }); + mocks.useRetry.mockReturnValue({ mutate: vi.fn(), isPending: false, variables: undefined }); + const libraries = [{ id: 1, name: "Movies" }] as Library[]; + + render(); + await userEvent.click(screen.getByRole("button", { name: /metadata matcher/i })); + await userEvent.click(screen.getByText("Movies")); + + expect(screen.getByText("candidate rejected")).toBeInTheDocument(); + expect(screen.getByText("Score below threshold")).toBeInTheDocument(); + expect(screen.getByText("parked")).toBeInTheDocument(); + }); + + it("pages through every queue entry instead of stopping at the first ten", async () => { + mocks.useQueues.mockReturnValue({ + data: [ + { + library_id: 1, + movie_count: 15, + series_count: 0, + raw_file_count: 0, + total_count: 15, + pending_count: 15, + parked_count: 0, + }, + ], + }); + mocks.useDetail.mockImplementation((_libraryID: number | null, offset: number) => ({ + data: { + limit: 10, + offset, + movie_count: 15, + series_count: 0, + raw_file_count: 0, + movies: Array.from({ length: offset === 0 ? 10 : 5 }, (_, index) => ({ + media_file_id: offset + index + 1, + file_path: `/media/movies/Movie ${offset + index + 1}.mkv`, + state: "pending", + })), + series: [], + raw_files: [], + }, + isFetching: false, + })); + mocks.useRetry.mockReturnValue({ mutate: vi.fn(), isPending: false, variables: undefined }); + const libraries = [{ id: 1, name: "Movies" }] as Library[]; + + render(); + await userEvent.click(screen.getByRole("button", { name: /metadata matcher/i })); + await userEvent.click(screen.getByText("Movies")); + await userEvent.click(screen.getByRole("button", { name: "Next" })); + + expect(mocks.useDetail).toHaveBeenLastCalledWith(1, 10); + expect(screen.getByText("Page 2")).toBeInTheDocument(); + expect(screen.getByText("/media/movies/Movie 15.mkv")).toBeInTheDocument(); + }); +}); diff --git a/web/src/components/admin/libraries/MetadataMatcherQueuesSection.tsx b/web/src/components/admin/libraries/MetadataMatcherQueuesSection.tsx new file mode 100644 index 00000000..ec64d17b --- /dev/null +++ b/web/src/components/admin/libraries/MetadataMatcherQueuesSection.tsx @@ -0,0 +1,260 @@ +import { Fragment, useState } from "react"; +import { Wrench } from "lucide-react"; + +import type { Library } from "@/api/types"; +import { CollapsibleDiagnosticsSection } from "@/components/admin/CollapsibleDiagnosticsSection"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from "@/components/ui/table"; +import { + useLibraryMetadataMatchQueueDetail, + useLibraryMetadataMatchQueues, + useRetryLibraryMetadataMatchQueue, +} from "@/hooks/queries/admin/libraries"; + +function formatFailureKind(kind: string): string { + return kind.replace(/_/g, " "); +} + +export function MetadataMatcherQueuesSection({ libraries }: { libraries: Library[] }) { + const [open, setOpen] = useState(false); + const [selectedLibraryID, setSelectedLibraryID] = useState(null); + const [detailOffset, setDetailOffset] = useState(0); + const { data: queues = [] } = useLibraryMetadataMatchQueues(); + // Passing null while collapsed disables the detail query entirely so a + // hidden expansion does not keep polling the per-library endpoint. + const { data: detail, isFetching: detailFetching } = useLibraryMetadataMatchQueueDetail( + open ? selectedLibraryID : null, + detailOffset, + ); + const retry = useRetryLibraryMetadataMatchQueue(); + const total = queues.reduce((sum, queue) => sum + queue.total_count, 0); + const detailLimit = detail?.limit ?? 10; + const hasPreviousDetailPage = detailOffset > 0; + const hasNextDetailPage = detail + ? detailOffset + detail.movies.length < detail.movie_count || + detailOffset + detail.series.length < detail.series_count || + detailOffset + detail.raw_files.length < detail.raw_file_count + : false; + + const detailEntries = detail + ? [ + ...detail.movies.map((entry) => ({ + key: `movie-${entry.media_file_id}`, + path: entry.file_path, + state: entry.state, + failureKind: entry.failure_kind, + message: entry.failure_detail?.message ?? entry.last_error, + decision: entry.failure_detail?.decision, + })), + ...detail.series.map((entry) => ({ + key: `series-${entry.media_folder_id}-${entry.observed_root_path}`, + path: entry.observed_root_path, + state: entry.state, + failureKind: entry.failure_kind, + message: entry.failure_detail?.message ?? entry.last_error, + decision: entry.failure_detail?.decision, + })), + ...detail.raw_files.map((entry) => { + const identity = [ + entry.base_title, + entry.base_year ? `(${entry.base_year})` : "", + entry.base_type ? `[${entry.base_type}]` : "", + ] + .filter(Boolean) + .join(" "); + return { + key: `raw-${entry.media_file_id}`, + path: entry.file_path, + state: "pending" as const, + failureKind: null, + message: identity ? `Awaiting initial match: ${identity}` : "Awaiting initial match", + decision: undefined, + }; + }), + ] + : []; + + if (total === 0) return null; + + return ( + } + open={open} + onOpenChange={(next) => { + setOpen(next); + if (!next) { + setSelectedLibraryID(null); + setDetailOffset(0); + } + }} + > +
+ + + + Library + Pending + Parked + + + + + {queues.map((queue) => { + const selected = selectedLibraryID === queue.library_id; + const library = libraries.find((entry) => entry.id === queue.library_id); + return ( + + { + setSelectedLibraryID(selected ? null : queue.library_id); + setDetailOffset(0); + }} + > + + {library?.name ?? `Library ${queue.library_id}`} + + {queue.pending_count} + + {queue.parked_count > 0 ? ( + + {queue.parked_count} + + ) : ( + 0 + )} + + + + + + {selected ? ( + + +
+ {detailEntries.map((entry) => { + const { path, failureKind, message, decision } = entry; + return ( +
+
+ {path} +
+ {failureKind ? ( + + {formatFailureKind(failureKind)} + + ) : null} + + {entry.state} + +
+
+ {message ? ( +

{message}

+ ) : null} + {decision?.top_candidates?.length ? ( +
+ {decision.top_candidates.map((candidate, candidateIndex) => ( +
+ + {candidate.title} + {candidate.year ? ` (${candidate.year})` : ""} + + + score {candidate.score.toFixed(1)} / {decision.threshold} + + {candidate.matched_title && + candidate.matched_title !== candidate.title ? ( + matched “{candidate.matched_title}” + ) : null} + {candidate.reasons?.length ? ( + {candidate.reasons.join(", ")} + ) : null} + {candidate.sources?.length ? ( + via {candidate.sources.join(", ")} + ) : null} +
+ ))} +
+ ) : null} +
+ ); + })} + {detail && detailEntries.length === 0 ? ( +

+ No queued item details. +

+ ) : null} + {hasPreviousDetailPage || hasNextDetailPage ? ( +
+ + Page {Math.floor(detailOffset / detailLimit) + 1} + +
+ + +
+
+ ) : null} +
+ + + ) : null} + + ); + })} + +
+
+ + ); +} diff --git a/web/src/hooks/queries/admin/libraries.ts b/web/src/hooks/queries/admin/libraries.ts index 406d9b88..c2089a41 100644 --- a/web/src/hooks/queries/admin/libraries.ts +++ b/web/src/hooks/queries/admin/libraries.ts @@ -423,15 +423,20 @@ export function useLibraryMetadataMatchQueues() { }); } -export function useLibraryMetadataMatchQueueDetail(libraryId: number | null) { +const METADATA_MATCH_QUEUE_PAGE_SIZE = 10; + +export function useLibraryMetadataMatchQueueDetail(libraryId: number | null, offset = 0) { + const pageActivity = usePageActivity(); + return useQuery({ - queryKey: adminKeys.libraryMatchQueueDetail(libraryId ?? 0), + queryKey: [...adminKeys.libraryMatchQueueDetail(libraryId ?? 0), offset], queryFn: () => api( - `/libraries/${encodeURIComponent(String(libraryId))}/metadata-match-queue?limit=10`, + `/libraries/${encodeURIComponent(String(libraryId))}/metadata-match-queue?limit=${METADATA_MATCH_QUEUE_PAGE_SIZE}&offset=${offset}`, ), enabled: libraryId !== null, staleTime: 0, + refetchInterval: pageActivity.canApplyRealtimeUpdates ? 10_000 : false, }); } diff --git a/web/src/pages/AdminLibraries.test.tsx b/web/src/pages/AdminLibraries.test.tsx index 5874c235..b49d2206 100644 --- a/web/src/pages/AdminLibraries.test.tsx +++ b/web/src/pages/AdminLibraries.test.tsx @@ -243,7 +243,7 @@ describe("AdminLibraries", () => { expect(markup).toContain("Scanner roots that stay visible"); }); - it("does not show metadata matcher queue counts in the library status", () => { + it("shows metadata matcher pending and parked counts", () => { mocks.useLibraryMetadataMatchQueues.mockReturnValue({ data: [ { @@ -252,6 +252,8 @@ describe("AdminLibraries", () => { series_count: 2, raw_file_count: 0, total_count: 3, + pending_count: 2, + parked_count: 1, }, ], isLoading: false, @@ -259,9 +261,11 @@ describe("AdminLibraries", () => { const markup = renderPage(); - expect(markup).toContain("Enabled"); - expect(markup).not.toContain("3 matching"); - expect(markup).not.toContain("View backlog"); + expect(markup).toContain("Metadata Matcher"); + expect(markup).toContain("Pending and parked items that still need a provider match."); + // The total renders as element text (">3<"); a bare "3" would also match + // Tailwind class names like p-3 and prove nothing. + expect(markup).toMatch(/>\s*3\s* { diff --git a/web/src/pages/AdminLibraries.tsx b/web/src/pages/AdminLibraries.tsx index 36cd53ba..8b8c5d2b 100644 --- a/web/src/pages/AdminLibraries.tsx +++ b/web/src/pages/AdminLibraries.tsx @@ -1,5 +1,4 @@ import { Fragment, useState, useEffect, useCallback, useMemo, useRef } from "react"; -import type { ReactNode } from "react"; import { useDebounce } from "@/hooks/useDebounce"; import { useEventChannel } from "@/components/realtimeEventsContext"; import type { @@ -36,6 +35,8 @@ import { useActiveScans } from "@/hooks/queries/admin/scans"; import { buildLibraryReorderEntries } from "./adminLibraryOrder"; import MatchItemDialog from "@/components/MatchItemDialog"; import { LibraryEditorDialog } from "@/components/admin/libraries/LibraryEditorDialog"; +import { MetadataMatcherQueuesSection } from "@/components/admin/libraries/MetadataMatcherQueuesSection"; +import { CollapsibleDiagnosticsSection } from "@/components/admin/CollapsibleDiagnosticsSection"; import { Button } from "@/components/ui/button"; import { Input } from "@/components/ui/input"; import { Label } from "@/components/ui/label"; @@ -717,6 +718,7 @@ export default function AdminLibraries() { + {skippedRoots.length > 0 ? : null} {staleIDs.length > 0 && } @@ -1215,64 +1217,6 @@ function usePagination(items: T[], pageSize = PAGE_SIZE) { }; } -function CollapsibleDiagnosticsSection({ - title, - description, - count, - icon, - iconClassName, - open, - onOpenChange, - children, -}: { - title: string; - description: string; - count: number; - icon: ReactNode; - iconClassName?: string; - open: boolean; - onOpenChange: (open: boolean) => void; - children: ReactNode; -}) { - return ( -
- - {open ?
{children}
: null} -
- ); -} - function PaginationBar({ total, rangeStart,