From 9cd00a877bda129b4ded01e863a89495fcc680ef Mon Sep 17 00:00:00 2001 From: CoffeeKnyte <67730400+CoffeeKnyte@users.noreply.github.com> Date: Wed, 17 Jun 2026 06:16:33 +0800 Subject: [PATCH] fix(metadata): cap un-completable episode refresh debt as terminal (#168) Episodes a provider genuinely has no data for stay actionable forever and, via syncRefreshDebtForItem, pin the whole series item at RefreshDebtReasonEpisodeIncomplete -> priority 300 (the top band). Those debt rows never drain and always cut the queue ahead of legitimately-due content, every backoff interval, indefinitely. Classify episode-incomplete debt as terminal once its persistent attempt_count reaches a small cap (3, ~5 days under the existing stepped backoff): demote it off the priority-300 band (to a still-fixable reason's band if one is present, else a terminal floor) and re-check it rarely (90d) instead of every 30d. The signal is purely attempt_count on the existing row -- no schema change. Demote rather than delete: deleting would let the next series pass recreate the row at attempt_count 0 and restart the cycle. Giving up fast is safe: the library "refresh incomplete" and per-item refresh endpoints re-fetch on demand and bypass the backoff, and media_items.episode_metadata_incomplete is left untouched so they still find these items. Logs the terminal transition (no silent demotion). AI-use disclosure: implemented with AI assistance (Claude Code). --- internal/metadata/refresh_debt.go | 65 ++++++++++++++++++++++ internal/metadata/refresh_debt_test.go | 77 +++++++++++++++++++++++++- internal/metadata/service.go | 18 ++++-- 3 files changed, 151 insertions(+), 9 deletions(-) diff --git a/internal/metadata/refresh_debt.go b/internal/metadata/refresh_debt.go index eaca1fb3..4819da0c 100644 --- a/internal/metadata/refresh_debt.go +++ b/internal/metadata/refresh_debt.go @@ -1,6 +1,7 @@ package metadata import ( + "log/slog" "strings" "time" @@ -38,6 +39,48 @@ func hasRefreshDebtReason(mask, reason int64) bool { return mask&reason != 0 } +const ( + // refreshDebtEpisodeTerminalAttempts is the number of fruitless attempts after + // which episode-incomplete debt is treated as "the provider has no data" and given + // up on: demoted off the priority-300 band and re-checked only rarely. We give up + // fast — with the stepped backoff below (1d, 1d, 3d, ...) attempt 3 lands after + // ~5 days of trying. This is safe because operators can force an immediate re-check + // anytime via the library "refresh incomplete" / per-item refresh endpoints, and + // because media_items.episode_metadata_incomplete is left untouched so those paths + // still find the item. + refreshDebtEpisodeTerminalAttempts = 3 + // refreshDebtTerminalPriority sorts terminal debt below the default band (100) so it + // never front-runs legitimately-due content. + refreshDebtTerminalPriority = 50 + // refreshDebtTerminalDelay is the rare automatic safety re-check for terminal debt, + // so late-arriving provider data is eventually picked up without operator action. + refreshDebtTerminalDelay = 90 * 24 * time.Hour +) + +// isTerminalEpisodeDebt reports that an episode-incomplete debt row has exhausted its +// attempts and almost certainly cannot be improved by another provider fetch. The signal +// is purely the persistent attempt_count on the row: resolved rows are deleted, so a row +// that still exists with a high attempt count has been re-processed many times without +// ever completing. +func isTerminalEpisodeDebt(reasonMask int64, attemptCount int) bool { + return attemptCount >= refreshDebtEpisodeTerminalAttempts && + hasRefreshDebtReason(reasonMask, RefreshDebtReasonEpisodeIncomplete) +} + +// effectiveRefreshDebtPriority demotes terminal episode-incomplete debt off the priority-300 +// band. If the row also carries a still-fixable reason it falls to that reason's band (not +// the floor), so a series with real core/provider-id debt keeps refreshing at the right +// cadence; pure episode-incomplete debt falls to the terminal floor. +func effectiveRefreshDebtPriority(reasonMask int64, attemptCount int) int { + if isTerminalEpisodeDebt(reasonMask, attemptCount) { + if demoted := reasonMask &^ RefreshDebtReasonEpisodeIncomplete; demoted != 0 { + return refreshDebtPriority(demoted) + } + return refreshDebtTerminalPriority + } + return refreshDebtPriority(reasonMask) +} + func refreshDebtPriority(reasonMask int64) int { switch { case hasRefreshDebtReason(reasonMask, RefreshDebtReasonEpisodeIncomplete): @@ -56,6 +99,13 @@ func refreshDebtPriority(reasonMask int64) int { } func nextRefreshDelay(reasonMask int64, attemptCount int) time.Duration { + // Only pure episode-incomplete debt is parked on the rare terminal cadence. If the row + // also carries a still-fixable reason, that reason keeps driving the normal backoff (the + // priority demotion in effectiveRefreshDebtPriority handles the queue ordering). + if isTerminalEpisodeDebt(reasonMask, attemptCount) && + reasonMask&^RefreshDebtReasonEpisodeIncomplete == 0 { + return refreshDebtTerminalDelay + } if hasRefreshDebtReason(reasonMask, RefreshDebtReasonEpisodeIncomplete) { switch { case attemptCount <= 1: @@ -89,6 +139,21 @@ func nextRefreshAtForDebt(reasonMask int64, attemptCount int, now time.Time) tim return now.Add(nextRefreshDelay(reasonMask, attemptCount)) } +// logRefreshDebtTerminal emits a one-time notice when an episode-incomplete debt row first +// crosses into the terminal give-up state, so the demotion is observable in logs rather +// than silent. Logging on the exact transition attempt keeps it to a single line per row. +func logRefreshDebtTerminal(targetType, contentID string, reasonMask int64, attemptCount int) { + if attemptCount == refreshDebtEpisodeTerminalAttempts && + isTerminalEpisodeDebt(reasonMask, attemptCount) { + slog.Warn("metadata: episode-incomplete refresh debt reached terminal attempts; demoting off top priority", + "target_type", NormalizeRefreshTargetType(targetType), + "content_id", contentID, + "attempt_count", attemptCount, + "reason_mask", reasonMask, + ) + } +} + func refreshDebtReasonsForItem(item *models.MediaItem) int64 { if item == nil { return 0 diff --git a/internal/metadata/refresh_debt_test.go b/internal/metadata/refresh_debt_test.go index b69d0f81..92ac3564 100644 --- a/internal/metadata/refresh_debt_test.go +++ b/internal/metadata/refresh_debt_test.go @@ -105,12 +105,14 @@ func TestNextRefreshDelayEpisodeSchedule(t *testing.T) { attempts int want time.Duration }{ + // Stepped backoff up to the terminal cap... {attempts: 0, want: 24 * time.Hour}, {attempts: 1, want: 24 * time.Hour}, {attempts: 2, want: 3 * 24 * time.Hour}, - {attempts: 3, want: 7 * 24 * time.Hour}, - {attempts: 4, want: 14 * 24 * time.Hour}, - {attempts: 5, want: 30 * 24 * time.Hour}, + // ...then we give up: episode-incomplete debt goes terminal at attempt 3+. + {attempts: 3, want: refreshDebtTerminalDelay}, + {attempts: 4, want: refreshDebtTerminalDelay}, + {attempts: 5, want: refreshDebtTerminalDelay}, } for _, tc := range cases { @@ -119,3 +121,72 @@ func TestNextRefreshDelayEpisodeSchedule(t *testing.T) { } } } + +func TestNextRefreshDelayNonEpisodeScheduleUnaffected(t *testing.T) { + // Non-episode reasons keep the full stepped backoff and never go terminal. + reasonMask := RefreshDebtReasonCoreMetadataIncomplete + cases := []struct { + attempts int + want time.Duration + }{ + {attempts: 2, want: 3 * 24 * time.Hour}, + {attempts: 3, want: 7 * 24 * time.Hour}, + {attempts: 4, want: 14 * 24 * time.Hour}, + {attempts: 5, want: 30 * 24 * time.Hour}, + } + for _, tc := range cases { + if got := nextRefreshDelay(reasonMask, tc.attempts); got != tc.want { + t.Fatalf("attempts=%d delay=%s want %s", tc.attempts, got, tc.want) + } + } +} + +func TestNextRefreshDelayTerminalOnlyWhenPureEpisodeDebt(t *testing.T) { + // Pure episode-incomplete debt at/over the cap is parked on the rare terminal cadence. + if got := nextRefreshDelay(RefreshDebtReasonEpisodeIncomplete, refreshDebtEpisodeTerminalAttempts); got != refreshDebtTerminalDelay { + t.Fatalf("pure terminal delay = %s, want %s", got, refreshDebtTerminalDelay) + } + // Episode-incomplete + a still-fixable reason keeps the normal backoff, not 90d, so the + // fixable reason is not parked for a quarter of a year. + combined := RefreshDebtReasonEpisodeIncomplete | RefreshDebtReasonRefreshFailure + if got := nextRefreshDelay(combined, refreshDebtEpisodeTerminalAttempts); got == refreshDebtTerminalDelay { + t.Fatalf("mixed-reason delay must not use the terminal cadence, got %s", got) + } +} + +func TestIsTerminalEpisodeDebt(t *testing.T) { + if isTerminalEpisodeDebt(RefreshDebtReasonEpisodeIncomplete, refreshDebtEpisodeTerminalAttempts-1) { + t.Fatalf("debt below the attempt cap must not be terminal") + } + if !isTerminalEpisodeDebt(RefreshDebtReasonEpisodeIncomplete, refreshDebtEpisodeTerminalAttempts) { + t.Fatalf("episode-incomplete debt at the attempt cap must be terminal") + } + if isTerminalEpisodeDebt(RefreshDebtReasonCoreMetadataIncomplete, refreshDebtEpisodeTerminalAttempts+5) { + t.Fatalf("non-episode debt must never be terminal regardless of attempts") + } +} + +func TestEffectiveRefreshDebtPriorityDemotesTerminalEpisodeDebt(t *testing.T) { + threshold := refreshDebtEpisodeTerminalAttempts + + // Below the cap: pure episode-incomplete debt keeps the top priority band. + if got := effectiveRefreshDebtPriority(RefreshDebtReasonEpisodeIncomplete, threshold-1); got != 300 { + t.Fatalf("pre-terminal episode priority = %d, want 300", got) + } + + // At/over the cap: pure episode-incomplete debt falls to the terminal floor. + if got := effectiveRefreshDebtPriority(RefreshDebtReasonEpisodeIncomplete, threshold); got != refreshDebtTerminalPriority { + t.Fatalf("terminal episode priority = %d, want %d", got, refreshDebtTerminalPriority) + } + + // At/over the cap with a still-fixable reason: falls to that reason's band, not the floor. + combined := RefreshDebtReasonEpisodeIncomplete | RefreshDebtReasonProviderIDIncomplete + if got := effectiveRefreshDebtPriority(combined, threshold); got != 240 { + t.Fatalf("terminal episode+provider priority = %d, want provider band 240", got) + } + + // Non-episode debt is never demoted. + if got := effectiveRefreshDebtPriority(RefreshDebtReasonCoreMetadataIncomplete, threshold+5); got != 150 { + t.Fatalf("core metadata priority = %d, want 150", got) + } +} diff --git a/internal/metadata/service.go b/internal/metadata/service.go index 0e553950..27b65862 100644 --- a/internal/metadata/service.go +++ b/internal/metadata/service.go @@ -2015,10 +2015,11 @@ func (s *MetadataService) syncRefreshDebtForItem(ctx context.Context, contentID return err } now := time.Now().UTC() + logRefreshDebtTerminal(RefreshTargetItem, contentID, reasonMask, attemptCount) return s.refreshDebtRepo.MarkSuccess( ctx, contentID, - refreshDebtPriority(reasonMask), + effectiveRefreshDebtPriority(reasonMask, attemptCount), reasonMask, nextRefreshAtForDebt(reasonMask, attemptCount, now), ) @@ -2066,11 +2067,12 @@ func (s *MetadataService) syncRefreshDebtForSeason(ctx context.Context, seasonID if err != nil { return err } + logRefreshDebtTerminal(RefreshTargetSeason, seasonID, reasonMask, attemptCount) return s.refreshDebtRepo.MarkTargetSuccess( ctx, RefreshTargetSeason, seasonID, - refreshDebtPriority(reasonMask), + effectiveRefreshDebtPriority(reasonMask, attemptCount), reasonMask, nextRefreshAtForDebt(reasonMask, attemptCount, now), ) @@ -2099,11 +2101,12 @@ func (s *MetadataService) syncRefreshDebtForEpisode(ctx context.Context, episode if err != nil { return err } + logRefreshDebtTerminal(RefreshTargetEpisode, episodeID, reasonMask, attemptCount) return s.refreshDebtRepo.MarkTargetSuccess( ctx, RefreshTargetEpisode, episodeID, - refreshDebtPriority(reasonMask), + effectiveRefreshDebtPriority(reasonMask, attemptCount), reasonMask, nextRefreshAtForDebt(reasonMask, attemptCount, now), ) @@ -2154,10 +2157,11 @@ func (s *MetadataService) syncRefreshDebtFailure(ctx context.Context, contentID attemptCount++ } now := time.Now().UTC() + logRefreshDebtTerminal(RefreshTargetItem, contentID, reasonMask, attemptCount) return s.refreshDebtRepo.MarkFailure( ctx, contentID, - refreshDebtPriority(reasonMask), + effectiveRefreshDebtPriority(reasonMask, attemptCount), reasonMask, nextRefreshAtForDebt(reasonMask, attemptCount, now), attemptCount, @@ -2183,11 +2187,12 @@ func (s *MetadataService) syncRefreshDebtTargetFailure(ctx context.Context, targ attemptCount++ } now := time.Now().UTC() + logRefreshDebtTerminal(targetType, contentID, reasonMask, attemptCount) return s.refreshDebtRepo.MarkTargetFailure( ctx, targetType, contentID, - refreshDebtPriority(reasonMask), + effectiveRefreshDebtPriority(reasonMask, attemptCount), reasonMask, nextRefreshAtForDebt(reasonMask, attemptCount, now), attemptCount, @@ -3632,11 +3637,12 @@ func (s *MetadataService) syncVisibleEpisodeRefreshDebt(ctx context.Context, epi if err != nil { return err } + logRefreshDebtTerminal(RefreshTargetEpisode, episode.ContentID, reasonMask, attemptCount) return s.refreshDebtRepo.MarkTargetSuccess( ctx, RefreshTargetEpisode, episode.ContentID, - refreshDebtPriority(reasonMask), + effectiveRefreshDebtPriority(reasonMask, attemptCount), reasonMask, nextRefreshAtForDebt(reasonMask, attemptCount, now.UTC()), )