diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d94c3fdd..8ee62e85 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -19,7 +19,15 @@ env: jobs: build: - runs-on: [self-hosted, linux] + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + runner: ubuntu-latest + - platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} permissions: contents: read packages: write @@ -28,6 +36,13 @@ jobs: - name: Checkout uses: actions/checkout@v5 + - name: Prepare platform slug and lowercase image name + run: | + echo "PLATFORM_PAIR=${PLATFORM//\//-}" >> "$GITHUB_ENV" + echo "IMAGE_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" + env: + PLATFORM: ${{ matrix.platform }} + - name: Set up pnpm uses: pnpm/action-setup@v4 with: @@ -76,9 +91,6 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - with: - name: silo-builder - keep-state: true - name: Log in to GitHub Container Registry uses: docker/login-action@v3 @@ -92,19 +104,81 @@ jobs: uses: docker/metadata-action@v5 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=sha,prefix=,format=short - - name: Build and push + - name: Build and push by digest + id: build uses: docker/build-push-action@v6 with: context: . - push: true - tags: ${{ steps.meta.outputs.tags }} + platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} build-contexts: | frontend_dist=./web/dist build-args: | BUILD_REVISION=${{ github.sha }} BUILD_DIRTY=false + cache-from: type=gha,scope=docker-${{ env.PLATFORM_PAIR }} + cache-to: type=gha,scope=docker-${{ env.PLATFORM_PAIR }},mode=max + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_LC }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p "${RUNNER_TEMP}/digests" + digest="${{ steps.build.outputs.digest }}" + touch "${RUNNER_TEMP}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + runs-on: ubuntu-latest + needs: build + permissions: + packages: write + + steps: + - name: Prepare lowercase image name + run: echo "IMAGE_LC=${IMAGE_NAME,,}" >> "$GITHUB_ENV" + + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: ${{ runner.temp }}/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_LC }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=sha,prefix=,format=short + + - name: Create and push manifest list + working-directory: ${{ runner.temp }}/digests + run: | + # shellcheck disable=SC2046 + docker buildx imagetools create \ + $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf "${REGISTRY}/${IMAGE_LC}@sha256:%s " *) + + - name: Inspect manifest list + run: | + docker buildx imagetools inspect "${REGISTRY}/${IMAGE_LC}:${GITHUB_SHA::7}" diff --git a/cmd/silo/main.go b/cmd/silo/main.go index b0939170..6f4db971 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -50,6 +50,7 @@ import ( "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/config" "github.com/Silo-Server/silo-server/internal/database" + "github.com/Silo-Server/silo-server/internal/diagnostics" "github.com/Silo-Server/silo-server/internal/downloads" "github.com/Silo-Server/silo-server/internal/ebooks" evt "github.com/Silo-Server/silo-server/internal/events" @@ -67,6 +68,11 @@ import ( "github.com/Silo-Server/silo-server/internal/markers" "github.com/Silo-Server/silo-server/internal/mdblist" "github.com/Silo-Server/silo-server/internal/metadata" + + // Built-in metadata providers self-register into the metadata package's + // builtin registry on import; buildProviders resolves their seeded chain + // entries in-process (no gRPC). + _ "github.com/Silo-Server/silo-server/internal/metadata/nfo" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/nodeconfig" "github.com/Silo-Server/silo-server/internal/nodepool" @@ -189,6 +195,9 @@ func configureOperationalLogging( if err := opslog.SeedDefaults(ctx, settingsRepo); err != nil { log.Fatalf("seed opslog defaults: %v", err) } + if err := diagnostics.SeedDefaults(ctx, settingsRepo); err != nil { + log.Fatalf("seed diagnostics defaults: %v", err) + } opsPM := partman.NewManager(pool, "operational_logs", partman.Daily, 3) if err := opsPM.EnsureFuturePartitions(ctx); err != nil { // Non-fatal: a partition hiccup must not crash-loop the server (see the @@ -665,6 +674,9 @@ func main() { // start, so this node can rebuild a Jellyfin transcode after its own // restart (the node hop token is recipe-less). Shares the offload Redis. srv.SetRecipeStore(noderecipe.NewStore(redisClient, 0)) + // Reclaim orphaned transcode dirs at boot and hourly thereafter, bound + // to appCtx so it stops on shutdown. + srv.StartOrphanSweeper(appCtx) handler = srv.Handler() } @@ -996,6 +1008,11 @@ func main() { } out := make([]pluginhost.InstalledPluginRecord, 0, len(installations)) for _, installation := range installations { + // The reserved builtin row is not a plugin; keep it out + // of the host's installed-plugin listing. + if installation.IsBuiltin() { + continue + } capabilities, err := installationStore.ListCapabilities(ctx, installation.ID) if err != nil { return nil, err @@ -1145,6 +1162,17 @@ func main() { var mangaEnricher *manga.Enricher if needsWorkers && deps.DB != nil && deps.FileRepo != nil { chainRepo := metadata.NewChainRepository(deps.DB) + // Make every existing library chain aware of the built-in providers + // before serving: materialize legacy content_level='' chains per level, + // then append registered builtins disabled (idempotent; also the repair + // path after a stale chain-editor save drops a builtin row). Runs before + // the metadata service exists, so no chain cache to invalidate here. + syncCtx, syncCancel := context.WithTimeout(appCtx, 30*time.Second) + syncErr := metadata.SyncBuiltinProviderChains(syncCtx, chainRepo) + syncCancel() + if syncErr != nil { + log.Fatalf("sync builtin provider chains: %v", syncErr) + } skippedRootRepo = metadata.NewSkippedRootRepository(deps.DB) itemRepo = catalog.NewItemRepository(deps.DB).WithActiveSearchProvider(activeCatalogSearchProvider) episodeRepo = catalog.NewEpisodeRepository(deps.DB) @@ -1252,6 +1280,7 @@ func main() { // admin image applies can succeed even if automatic metadata caching is off. if deps.S3Public != nil { imageCacher := imagecache.New(deps.S3Public) + imageCacher.SetArtworkRevisionTracker(catalog.NewArtworkRevisionTracker(deps.DB)) metadataService.SetImageCacher(imageCacher) imageCacheJobs := metadata.NewImageCacheJobRepository(deps.DB) metadataService.SetImageCacheJobEnqueuer(imageCacheJobs) @@ -1268,6 +1297,11 @@ func main() { People: personRepo, }, ) + // Local file:// artwork (NFO sidecars): confine reads to the owning + // library's roots and sweep stale hashed local/ prefixes on re-cache. + // The processor host must mount the libraries, like the metadata worker. + metadataImageCacheProcessor.SetLibraryRootResolver(deps.FolderRepo) + metadataImageCacheProcessor.SetImagePrefixDeleter(deps.S3Public) metadataService.SetAutoCacheImages(cfg.Metadata.CacheImages) metadataImageCacheProcessor.SetEnabled(cfg.Metadata.CacheImages) configWatcher.OnChange(func(_, updated *config.Config) { @@ -1550,6 +1584,7 @@ func main() { // Step 6: Create playback session manager and wire into dependencies. sessionMgr := playback.NewSessionManager(6, 2) // defaults from plan: max_streams=6, max_transcodes=2 + var compatTerminalRecoveryReady <-chan struct{} if userStoreProvider != nil { deps.UserStoreProvider = userStoreProvider } @@ -1561,6 +1596,13 @@ func main() { WithWatchState(watchstate.NewService(userStoreProvider).WithStableIdentityResolver(historyIdentity)). WithUserStoreProvider(userStoreProvider) backgroundInit = append(backgroundInit, func(ctx context.Context) { + if compatTerminalRecoveryReady != nil { + select { + case <-compatTerminalRecoveryReady: + case <-ctx.Done(): + return + } + } if err := watchProviderService.SweepOpenScrobbles(ctx); err != nil { slog.WarnContext(ctx, "failed to sweep open watch provider scrobbles", "component", "app", "error", err) } @@ -1879,6 +1921,11 @@ func main() { taskMgr.Register(tasks.NewScanLibrariesTask(deps.FolderRepo, deps.LibraryScanQueue, deps.EventBus)) } taskMgr.Register(tasks.NewCleanupOrphanedMediaItemsTask(catalog.NewOrphanedProvisionalCleaner(deps.DB))) + if deps.S3Public != nil { + taskMgr.Register(tasks.NewCleanupArtworkRevisionsTask( + metadata.NewArtworkRevisionGarbageCollector(deps.DB, deps.S3Public), + )) + } catalogSearchIndexer := catalog.NewCatalogSearchIndexer(deps.DB, settingsRepo) taskMgr.Register(tasks.NewSyncCatalogSearchIndexTask(catalogSearchIndexer)) taskMgr.Register(tasks.NewRebuildCatalogSearchIndexTask(catalogSearchIndexer)) @@ -1895,6 +1942,15 @@ func main() { } taskMgr.Register(tasks.NewActivityLogCleanupTask(deps.DB, settingsRepo, activityPM)) taskMgr.Register(tasks.NewOperationalLogCleanupTask(deps.DB, settingsRepo, opsPM)) + var diagnosticsStore diagnostics.ObjectStore + if deps.S3Private != nil { + diagnosticsStore = diagnostics.NewS3ObjectStore(deps.S3Private) + } + taskMgr.Register(tasks.NewClientDiagnosticsCleanupTask( + diagnostics.NewPostgresRepository(deps.DB), + settingsRepo, + diagnosticsStore, + )) taskMgr.Register(tasks.NewPolicyDecisionLogCleanupTask(deps.DB, settingsRepo, policyPM)) if deps.FileRepo != nil { // Download prepare-to-file pipeline (Phase 3): a durable, leased encode @@ -2359,6 +2415,7 @@ func main() { if (mode == "integrated" || mode == "api") && cfg.JellyfinCompat.Enabled && cfg.JellyfinCompat.Listen != "" { compatDeps := jellycompat.Dependencies{ Config: cfg, + AppContext: appCtx, LiveConfig: configWatcher.Config, DB: deps.DB, SecretCipher: dataCipher, @@ -2402,6 +2459,7 @@ func main() { compatDeps.SeasonRepo = seasonRepo compatDeps.EpisodeRepo = episodeRepo compatDeps.ProviderIDRepo = providerIDRepo + compatDeps.StableIdentityResolver = watchstate.NewStableIdentityResolver(itemRepo, episodeRepo, providerIDRepo) compatDeps.DetailSvc = detailSvc compatDeps.FolderRepo = folderRepo compatDeps.SessionMgr = sessionMgr @@ -2409,6 +2467,9 @@ func main() { compatDeps.WatchCompletionObserver = deps.WatchCompletionObserver compatDeps.SettingsRepo = settingsRepo compatDeps.PersonRepo = personRepo + if watchProviderService != nil { + compatDeps.WatchScrobbler = watchProviderService + } compatSearchService := catalog.NewCatalogSearchService( appCtx, settingsRepo, @@ -2488,7 +2549,7 @@ func main() { compat := jellycompat.NewServerWithDependencies(compatDeps) compatServer = compat - compat.StartBackgroundTasks(context.Background()) + compatTerminalRecoveryReady = compat.StartBackgroundTasks(context.Background()) compatSrv = compat.HTTPServer() compatSrv.ReadTimeout = 30 * time.Second compatSrv.WriteTimeout = 0 @@ -2784,6 +2845,12 @@ func reloadPluginImageResolvers( if installation == nil { continue } + // Builtin installations resolve in-process metadata providers only; + // registering them here would claim their capability id as a gRPC + // image-resolver scheme with no binary behind it. + if installation.IsBuiltin() { + continue + } capabilities, err := store.ListCapabilities(ctx, installation.ID) if err != nil { return fmt.Errorf("list image resolver capabilities for installation %d: %w", installation.ID, err) @@ -2970,6 +3037,11 @@ func reloadMarkerPluginProviders( if installation == nil { continue } + // Builtin installations expose no marker providers; defense in depth + // alongside the capability-type filter below. + if installation.IsBuiltin() { + continue + } capabilities, err := store.ListCapabilities(ctx, installation.ID) if err != nil { return fmt.Errorf("list marker provider capabilities for installation %d: %w", installation.ID, err) diff --git a/docs/architecture/local-nfo-metadata.md b/docs/architecture/local-nfo-metadata.md new file mode 100644 index 00000000..d30d43ea --- /dev/null +++ b/docs/architecture/local-nfo-metadata.md @@ -0,0 +1,126 @@ +# Local NFO metadata and sidecar artwork (as built) + +Implements #216: Kodi/Jellyfin-style `.nfo` files and sidecar images +(`poster.jpg`, `fanart.jpg`, …) next to movies and shows are read by a builtin +metadata provider, so curated local libraries work with partial or zero +TMDB/TVDB coverage. Admin-facing usage lives in +`docs/wiki/admin/nfo-local-metadata.md`; this page records the architecture for +maintainers. + +## Registration: a builtin provider in the plugin chain + +Provider chains are keyed on `plugin_installations` rows, so builtins are +registered **as data**: a reserved installation (`plugin_id='silo.builtin'`, +`kind='builtin'`, sentinel `install_path`, `update_policy='manual'`) plus a +`metadata_provider.v1` capability `nfo` with `default_priority` 1 for +movie/series/season/episode and `default_enabled=false` +(`migrations/sql/20260712100125_builtin_host_providers.sql`, +`...124741_nfo_capability_season_episode_levels.sql`). + +- `internal/metadata/builtin.go` is the in-process registry; providers + self-register (`internal/metadata/nfo/register.go`) and `buildProviders` + (`internal/metadata/chain.go`) returns the in-process provider for + `kind='builtin'` rows instead of constructing a gRPC plugin. +- Guard rails keep the reserved row out of every plugin surface (user plugin + settings, installations list, image resolvers, preload, auto-update, store + delete, mutation handlers); `silo.builtin` is a reserved manifest id. Generic + installation reads must **not** filter builtins — the chain enabled-check + depends on them. +- On startup, `SyncBuiltinProviderChains` (`internal/metadata/builtin_sync.go`) + first materializes legacy `content_level=''` chains per level, then appends + builtin capabilities to all existing chains, disabled. New libraries pick the + capability up through normal chain seeding. The chain-less fallback respects + `default_enabled=false`. +- **Everything is inert until an admin enables "NFO Files" on a library.** + +## Identity: hint-first, never candidate-first + +NFO `` values (tmdb/imdb/tvdb) are **trusted identity hints** seeded +before remote search (`IdentityHintProvider`, applied in +`internal/metadata/service.go`), unlocking full remote enrichment even when +search-by-title fails. Conflict policy by mode: + +- initial match / scheduled refresh: NFO hints beat folder-name hints; stored + durable IDs beat NFO hints (no background identity flips); +- manual refresh: NFO hints beat stored IDs — the recovery path for a corrected + NFO. The item's `content_id` re-anchors too (not just its stored provider + IDs): a provider-anchored item whose fixed `` now derives a + different anchor is renamed to it, or merged onto the existing item already + holding that id; +- manual Identify: the user's choice wins; the NFO provider is skipped for the + whole operation. + +The NFO search candidate exists only for the title-only case and is defanged: +ID-less candidates lose provider-priority tie-breaks to ID-bearing group +members, `nfo` never counts as corroboration, and Phase-2 NFO results cannot +inject provider-id keys. A title-only NFO yields a `matched` item under a +path-deterministic `local:` content id; adding a `` later promotes it +to that provider anchor on the next refresh (this promotion from an unanchored +id runs on scheduled refresh too — only an *already-anchored* identity is held +stable in the background). NFO field edits propagate on **manual refresh only** +(scheduled refresh is fill-empty by design). + +## Sidecar artwork: copy into the S3 image cache + +Sidecar images (`poster`/`folder`/`cover`, `fanart`/`backdrop`/`background`, +`logo`/`clearlogo`, and `-poster/-fanart/-logo` variants; extensions +`.jpg/.jpeg/.png/.webp`, 8 MiB cap, symlinked leaves rejected) are discovered +by the provider's `GetImages` (`internal/metadata/nfo/images.go`). Clients — +including jellycompat — always receive the normal presigned +`poster_url`/`backdrop_url`/`logo_url`; library files are never served +directly, and API nodes never need filesystem access to libraries. + +- Sources are recorded as `file://` in `*_source_path` + columns and cached by the metadata image-cache processor under + `local/{contentType}/{contentID}/{hash8}/{imageType}/...` (`hash8` = content + hash; the discriminator sits before the imageType segment because variant + clamping reads the parent directory). Editing the file and refreshing rotates + the key; the stale hashed prefix is deleted after a successful re-cache, and + item deletion trims local paths to the content prefix. +- The processor confines each read to the owning library's `media_folders` + roots: a lexical check on the logical path, then a symlink-resolving re-check + (both path and roots are resolved, so a legitimately symlinked root stays + valid while an intermediate directory symlink escaping a root is rejected). + Missing/unreadable/out-of-root and structurally-unusable (non-regular, + over-cap) paths are stable failures with a long retry deferral; recovery is + refresh-driven — the provider-artwork backfill sweep deliberately skips + `file://` sources. +- Generic filenames attach only when the directory holds a single content + group; a shared `folder.jpg` in a flat multi-movie folder applies to none. + `-poster.jpg` style names always attach to their file. +- `applyBestImages`' final write has a local exemption so rating-0 local art + can fill already-matched items and cannot be stickily displaced by remote + art; the localization pass skips language-neutral local candidates. + +**Deployment constraint:** the host running the metadata image-cache processor +must mount the media libraries at the same paths as the scanner/metadata +worker, otherwise local artwork jobs fail until the mount is present. + +## Series depth and mixed libraries + +`SeasonsRequest`/`EpisodesRequest` carry local path context; `season.nfo` +supplies season name/plot, `.nfo` + `-thumb.ext` supply +episode metadata/thumbs (`internal/metadata/nfo/series_depth.go`). **Naming +supplies structure; NFOs supply metadata**: NFO season/episode numbers are +advisory — directory/filename-derived numbers win with a warning. Episode NFOs +work without a `season.nfo`; `SynthesizeFallbackEpisodes` still covers +NFO-less episodes. + +In `mixed` libraries, movie-vs-series is decided per file by naming before any +provider runs (`internal/naming/filename.go`); the NFO provider is seeded into +all levels and behaves as in dedicated libraries. The provider's root-element +type guard means a `tvshow.nfo` next to a movie-classified file is ignored, +never applied; the per-root Type override is the correction path. This is the +contract for sports/mixed libraries (events as movies, weekly shows as series). + +## Known limitations + +- The admin image picker does not surface local art (automatic chain path + only). +- Multi-part movies: a basename-mismatched NFO in a folder holding multiple + content groups is not found (directory candidates are suppressed there). +- No NFO writing, no music/audiobook/ebook NFO (those ecosystems use + OPF/metadata.json — future builtin providers via the same registry). +- Feature detection: the `nfo` capability's presence in + `GET /api/v1/libraries/provider-defaults` (see + `docs/architecture/v1-scope.md`). diff --git a/docs/architecture/v1-scope.md b/docs/architecture/v1-scope.md index 610510b9..46e3d762 100644 --- a/docs/architecture/v1-scope.md +++ b/docs/architecture/v1-scope.md @@ -17,3 +17,8 @@ When the scope locks, this file becomes the source of truth and will contain: Until lock: treat any capability not tracked as `Proposed`/`Locked` on the project as out of scope for feature PRs (see the scope gate in `CLAUDE.md`). + +Feature-detection precedent: clients discover which metadata providers (including the +built-in NFO provider, #216) apply to a library type via +`GET /api/v1/libraries/provider-defaults` rather than version sniffing. New capabilities +should follow the same capability-endpoint pattern. diff --git a/docs/design/2026-07-19-client-diagnostics.md b/docs/design/2026-07-19-client-diagnostics.md new file mode 100644 index 00000000..cec95ddd --- /dev/null +++ b/docs/design/2026-07-19-client-diagnostics.md @@ -0,0 +1,771 @@ +# Client Diagnostics: crash reports and debug log upload + +Status: Draft (spec + cross-repo plan) +Date: 2026-07-19 +Repos affected: `silo-server`, `silo-android`, `silo-apple` + +## Summary + +Add an opt-in, self-hosted diagnostics pipeline: Silo clients detect crashes +and abnormal exits, collect structured debug logs (playback, TV focus, +network, lifecycle) plus a hardware snapshot (device model, display/HDMI, +audio path, codec capabilities), and — only with explicit user consent — +upload a compressed diagnostics bundle to the user's own Silo server. Admins +review bundles through new acting-admin API endpoints alongside the existing +server logs tooling. + +Everything ships **disabled by default**: + +- Server: a `diagnostics.uploads_enabled` server setting gates the whole + feature (default off), and it cannot be enabled unless the server's private + object storage is configured and validated. +- Client: verbose "debug logging" is a per-device setting (default off). + Crash reporting defaults to **Ask** — after an abnormal exit the app shows + an incident-specific prompt (e.g. *"Silo closed unexpectedly during + playback. Send a diagnostic report to your server?"*) with + **Send / Don't send / Always send**. Nothing uploads without one of: an + explicit prompt confirmation, a prior "Always send" election bound to this + server and account, or the user explicitly sending a manual capture. + +## Motivation + +Today we are blind to client-side failures: + +- **No crash reporting anywhere.** Neither the Android repo nor the Apple + repo installs an uncaught-exception handler, uses MetricKit, or bundles a + crash SDK (verified: no handler/SDK references in either repo; the only + Firebase dependency is messaging). A user saying "the app crashed" is a + dead end. +- **Diagnostics don't survive to the report.** Android logs go to logcat (raw + `android.util.Log` in ~40 files, no buffer or file sink); the Apple + player's rich `[CMP]`/`[CMP-DIAG]` diagnostics go through `print()` + (`PlayerLog.swift` / `PlayerCore.swift`) with no app-owned persistent sink. + By the time a user reports a problem, the evidence is gone. +- **The hardest bugs are device-specific.** Shield HLG quirks, Fire TV + passthrough suppression, HDMI mode switches, TV focus traps — reproducing + them requires knowing the exact device, panel, audio chain, and codec path. + Users can't reliably report any of that. +- **The server already has half the story.** `operational_logs` records + `playback_session_id` (migration `028_log_partitioning.sql`), the admin + logs API filters by it, and the web admin has a logs viewer. What's missing + is the client half, and a way to join the two. + +## Goals + +- Capture crashes and abnormal exits (JVM/native/ANR on Android; crash and + hang diagnostics on iOS; abnormal-exit detection on tvOS) with enough + context to act on: stack or exit trace where the platform provides one, + recent structured logs, and a device snapshot. +- Capture deep opt-in debug logs for playback, TV focus behavior, networking, + and navigation/lifecycle — everything needed to troubleshoot user issues. +- Always include a hardware snapshot: device (Shield / Fire TV / onn / + phone), OS, app build, display and HDMI mode with HDR capabilities, audio + output path and passthrough capabilities, decoder inventory — with honest + `unknown` / `not_collected` states where a platform can't provide a field. +- One payload contract across Android and Apple; one server endpoint; one + admin surface. The contract is a checked-in versioned JSON Schema with + golden fixtures consumed by all three repos' CI. +- Explicit, server-bound consent for every byte uploaded; feature fully off + by default on both server and client. +- Bounded cost: size caps, per-account quotas, concurrency limits, and + server-side retention. + +## Non-goals (v1) + +- Live/continuous log streaming from clients to the server. +- Screenshots or screen recordings in bundles. +- Native/tvOS crash *stack* capture via an in-process crash reporter + (PLCrashReporter/KSCrash-style). tvOS v1 detects abnormal exits and + attaches breadcrumbs, not stacks (see Apple design). Adopting an in-process + crash reporter is a separate decision with its own signal-safety and review + implications. +- Symbolication. Bundles store what the OS gives us; a server-side + symbolication job can come later. +- Native-app admin viewer screens (per current product policy, admin surfaces + beyond STATS on the clients are a separate product decision; v1 admin + consumption is via API + web admin). +- macOS. The Apple repo has a macOS target, but v1 scopes to + `android`, `android-tv`, `ios`, `tvos`; the platform enum reserves `macos` + without accepting it. +- Third-party telemetry, metrics aggregation, or fleet analytics. This is a + support/debugging tool, not analytics. + +## Why this route + +**Upload to the user's own Silo server, not a third-party crash service.** +Crashlytics/Sentry SaaS would send user data to a third party, require vendor +accounts/keys we'd have to ship, and put the reports in front of *us* rather +than the person who can actually act — the self-hosting admin. Requiring +every admin to also run self-hosted Sentry is far too heavy. The Silo server +is the natural, privacy-aligned destination, and it already has auth, admin +tooling, object storage, and retention patterns to build on. + +**Bundle-on-event, not streaming.** A bundle captured at the moment of +failure (crash, or a user-triggered capture) contains exactly the context +needed: recent structured logs, the playback stats timeline, and the hardware +snapshot. Continuous streaming would cost bandwidth and privacy while mostly +capturing noise, and would still miss pre-crash buffers. + +**Hybrid storage: Postgres row + object-store blob.** Report *metadata* (a +small manifest: device summary, truncated crash summary, type, timestamps) +goes in a queryable table; the payload itself (potentially megabytes of logs +and crash artifacts) goes to the S3-compatible private bucket as a single +archive. Ingesting client log lines into the partitioned `operational_logs` +table was considered and rejected: bundles are consumed as whole documents by +a human, not queried row-wise; client lines are untrusted input with +different retention needs; and multi-MB uploads would bloat a table tuned for +the server's own high-churn logs. The server *does* emit normalized +operational events about diagnostics activity (report accepted / rejected / +downloaded / deleted, with report id, account, sizes, result) so ingest and +admin actions are auditable and correlate with existing logs — raw client +lines never enter `operational_logs`. + +**A versioned archive, not a bare gzipped log stream.** Crash evidence is +heterogeneous: JSONL logs, ANR trace text, Android native tombstones +(protobuf on API 31+), MetricKit diagnostic JSON. A tar.gz archive with named +entries carries each artifact losslessly with a declared media type, keeps +large traces out of Postgres, and can evolve by adding entries. + +**Build small in-house rather than adopt ACRA/other SDKs.** The surface we +need on Android (one exception handler writing a bounded marker, a ring +buffer, one upload call) is a few hundred lines; ACRA would add a dependency +and its own report format without covering Apple. On Apple, the OS provides +MetricKit natively (iOS; not tvOS — see Apple design). + +## Architecture overview + +```text + Android / Apple client silo-server +┌────────────────────────────────┐ ┌──────────────────────────────────────┐ +│ Safe-logging facade → ring │ │ GET /api/v1/diagnostics/status │ +│ └ debug mode: rotating │ │ └ available|disabled| │ +│ segment files │ │ storage_unavailable + limits │ +│ Crash/abnormal-exit capture │ │ POST /api/v1/diagnostics/reports │ +│ ├ UEH marker (Android) │──▶│ ├ gate + storage check │ +│ ├ ApplicationExitInfo (30+) │ │ ├ streaming, bounded multipart │ +│ ├ MetricKit (iOS) │ │ ├ account quotas + concurrency │ +│ └ exit sentinel (tvOS) │ │ ├ manifest → client_diagnostic_ │ +│ Pending reports (server-bound) │ │ │ reports (state machine) │ +│ DeviceSnapshot (probes+new │ │ └ archive → S3 private bucket │ +│ accessors) │ │ Admin (acting admin): list/detail/ │ +│ Consent UI (ask/always/never, │ │ download/delete + audit events │ +│ per server+account) │ │ Retention + orphan reconciler task │ +└────────────────────────────────┘ └──────────────────────────────────────┘ +``` + +## Consent model + +Consent is **scoped, versioned, and destination-bound** — not a global +boolean. Both clients support multiple servers and accounts, so a device-wide +"Always send" would let a bundle full of server A's titles and URLs upload to +server B after a switch. Rules: + +- Consent (including "Always send") is **account-scoped**: every consent + record and every pending report is bound to + `(server_instance_id, account_user_id, consent_notice_version)`, where + `server_instance_id` is a stable opaque ID returned by the status + endpoint. The capturing `profile_id` is recorded on reports for + *attribution*, not as a consent scope — quotas and elections follow the + account. +- A pending report is only ever uploaded to the server+account it was + captured under. Reports are never retargeted. Switching server or account + neither uploads nor migrates pending reports; signing out of an account or + removing a server deletes its pending reports. +- "Always send" is an election for one server+account. Bumping + `consent_notice_version` (when we materially change what bundles contain) + invalidates prior elections and reverts to Ask. +- Setting crash reports to **Never** deletes pending reports for that + server+account and stops persistent capture (breadcrumbs/debug files); the + in-memory ring keeps running (memory-only, never uploaded without a later + explicit action). +- **Child/managed profiles**: the Diagnostics section is not shown to + profiles with `is_child`, and every diagnostics action — changing + settings, electing Always, the crash prompt, and manual capture/send — + requires a non-child profile. A crash captured while a child profile was + active is held pending and prompted for the next time a non-child profile + is active. Reports are attributed to account + capturing profile. +- The crash prompt and the manual flow both offer **View report** — a + human-readable summary (incident, categories and line counts, device + identity, destination server) before anything is sent. +- Uploads carry `consent: {mode, notice_version}` in the manifest so the + server can reject stale-consent uploads outright. + +## Bundle format (shared contract) + +An upload is `multipart/form-data` with exactly two parts, in order: + +1. `manifest` — `application/json`, ≤ 64 KiB. Queryable summary; stored in + Postgres. +2. `bundle` — `application/gzip`: a tar.gz archive, `manifest.json` first. + Entry names come from a **fixed allowlist**, and each name implies its + media type — there is no per-entry type metadata to drift: + +```jsonc +manifest.json (manifest minus the `archive` object; see hashing note) +device.json (full device snapshot) +logs.jsonl (structured log lines, newest-last) +crash/summary.json (normalized crash info incl. provenance) +crash/stack.txt (JVM stack / ANR trace, when present) +crash/tombstone.pb (Android native tombstone, opaque bytes, API 31+) +crash/metrickit.json (raw MetricKit diagnostic JSON, iOS) +breadcrumbs.jsonl (persisted pre-crash context journal, when present) +``` + +Hashing note: `archive.sha256` and the size fields in part 1 describe the +finalized `bundle` part bytes as transmitted. Since the archive is hashed +after it is built, the embedded `manifest.json` is the manifest **without** +the `archive` object; the server verifies part 1's `archive` fields against +the received blob. `archive.bytes` is the transmitted (gzip) size; +`archive.uncompressed_bytes` is the total decompressed tar stream size, +including tar headers, the end-of-archive marker, and record padding — +i.e. the byte count between the client's tar writer and gzip writer, and +what `gzip -l` reports. Standard end-of-archive zero padding (up to 64 KiB) +is accepted; any other trailing data is rejected. + +The contract lives in this repo as a versioned JSON Schema +(`docs/design/schemas/client-diagnostics/v1/`) with canonical valid and +invalid fixtures; Go, Kotlin, and Swift CI all validate against the same +fixtures. Servers accept all schema versions they support and ignore unknown +additive fields. + +### Manifest (part 1) + +```json +{ + "schema_version": 1, + "report": { + "type": "crash", // crash | anr | native_crash | hang | abnormal_exit | manual + "captured_at": "2026-07-19T18:22:31Z", + "capture_session_id": "run_c7d…", // UUID per app run, also stamped on log lines + "app_version": "1.4.2", + "app_build": "20841", + "platform": "android-tv", // android | android-tv | ios | tvos (macos reserved) + "os_version": "11 (API 30)", + "profile_id": "prof_…" // capturing profile, attribution only + }, + "destination": { "server_instance_id": "srv_…" }, // consent binding; server rejects mismatches + "consent": { "mode": "prompt", "notice_version": 1 }, // prompt | always | manual + "crash": { // absent for manual + "summary": "NullPointerException in PlaybackSessionManager.start", + "stack_excerpt": "…truncated ≤ 8 KiB; full artifact in archive…", + "thread": "main", + "foreground": true, + "source": "ueh", // ueh | exit_info | metrickit | exit_sentinel + "provenance": "pre_failure", // pre_failure | post_restart | metric_reporting_period + "occurred_at": "2026-07-19T18:22:31Z" // best known; MetricKit reports carry period bounds + }, + "device_summary": { "manufacturer":"NVIDIA", "model":"SHIELD Android TV", "os":"11", "form_factor":"tv" }, + "playback_session_ids": ["ps_9f2…"], + "log_summary": { "lines": 3841, "bytes_gz": 412034, "dropped_lines": 12, "categories": ["playback","focus","network"], "debug_logging": true }, + "archive": { "entries": ["manifest.json","device.json","logs.jsonl","crash/summary.json","crash/stack.txt"], "bytes": 498112, "uncompressed_bytes": 3110400, "sha256": "…" } +} +``` + +### Log line schema (`logs.jsonl`) + +```json +{"ts":"2026-07-19T18:22:29.412Z","run":"run_c7d…","lvl":"W","cat":"playback","tag":"AudioCapabilityMgr","msg":"passthrough suppressed: HDMI sink lost TrueHD","attrs":{"sink":"HDMI","fmt":"truehd"}} +``` + +- `lvl`: `V|D|I|W|E`. `cat`: `playback`, `focus`, `network`, `lifecycle`, + `browse`, `cast`, `download`, `crash`, `other`. +- `run` (capture_session_id) distinguishes lines from prior app runs when + breadcrumbs or debug files span sessions. +- Playback additionally emits periodic (~5s) player-stats snapshot lines + (`"cat":"playback","tag":"StatsSnapshot"`) built from the existing stats + models — a timeline of decoder, resolution, HDR mode, bitrate, dropped + frames, and audio underruns around the failure. + +### Safe logging contract (redaction happens at collection time) + +Best-effort scrubbing at package time is not enough: today's client logs +already emit full URLs with query strings, token suffixes, and raw exception +strings (e.g. the Apple `HTTPClient` response logging and player URL prints; +Android exception messages containing backend URLs). Therefore: + +- The facade accepts **typed attributes**, not preformatted strings, for + anything risky: URL values are normalized (userinfo, query, and fragment + always stripped; path preserved), exceptions pass through a sanitizing + formatter, and free-text `msg` is authored, not interpolated from network + data. +- Attribute keys are **normatively registered**: a per-category key registry + (checked in next to the JSON Schema) declares each key and its value type + (scalar types plus a few enumerated units). Unregistered keys fail debug + builds/CI and are dropped in production builds. +- Existing call sites are migrated **curated, not mechanically** — each + migrated line is reviewed against the contract. Unmigrated `Log.*`/`print` + call sites simply never enter bundles. +- Defense in depth at bundle time: values matching the account's current + access/profile tokens are replaced before packaging. +- Golden leak-fixture tests (JWTs, query tokens, signed URLs, cookies, token + fragments) run in all three repos' CI against the facade and the bundle + builder. +- The network category logs `method, templated path, status, duration`; + never headers, bodies, query strings, or full URLs. + +### Device snapshot (`device.json`) + +Fields are tri-state where a platform can't provide them: a value, +`"unknown"` (probe exists, couldn't determine), or `"not_collected"` +(no probe on this platform). Every snapshot carries `captured_at` and +`provenance` (`pre_failure` for snapshots persisted with the incident, +`post_restart` for snapshots taken at bundle-build time). + +```json +{ + "captured_at": "2026-07-19T18:22:31Z", + "provenance": "pre_failure", + "identity": { "manufacturer":"NVIDIA", "model":"SHIELD Android TV", "device":"mdarcy", "form_factor":"tv", "device_id":"…" }, + "display": { "mode":"3840x2160@59.94", "modes_supported":[…], "hdr_types":["HDR10","HLG","DV"], "wide_gamut":true }, + "audio": { "outputs":[{"type":"HDMI","encodings":["AC3","EAC3","TRUEHD"],"channels":8}], "passthrough":["AC3","EAC3","TRUEHD"], "suppressions":[…] }, + "video_codecs": [ { "mime":"video/hevc", "hw":true, "profiles":["Main10","DV P8"], "max":"4096x2160@60" } ], + "network": { "transport":"ethernet" } +} +``` + +Android sources: `AndroidDeviceMetadataProvider` (id/model/platform/app +version), `MediaCodecCapabilitiesProbe` (codec + HDR decode caps), +`AudioCapabilityManager` (aggregate passthrough). **New accessors required** +(the probes exist but don't currently export everything): current + supported +`Display.Mode` list (DisplayHdrProbe exposes HDR support only), exact +identity fields (`Build.DEVICE`, form factor), `AudioManager.getDevices()` +output enumeration, and a snapshot/export API on +`PassthroughSuppressionRegistry` (its set is currently private). + +Apple sources: `AppleDeviceIdentity.current` (persisted id, device name, +platform) and `ApplePlaybackV3Capabilities.snapshot()` (codec/HDR/device +context). **Honesty constraints:** the current `outputSnapshot()` helper is +private, reports route UIDs/types rather than full output capabilities, and +its passthrough payload is a placeholder — so Apple v1 reports audio as +route metadata with `passthrough: "unknown"` until a real probe exists, and +the private helper is refactored into an internal shared accessor so +diagnostics can reuse it without duplicating route logic. Audio route UIDs +are persistent peripheral identifiers and are one-way hashed before +inclusion. + +### Playback session correlation + +Clients include recent server playback session IDs so admins can pivot into +the existing server-side logs (`operational_logs.playback_session_id` + +the admin logs filter). Both clients currently retain only the *active* +session ID (Android `PlaybackSessionManager`, Apple `PlaybackSessionBridge`), +so each client adds a small **bounded, persisted recent-session tracker** +(last ~10 session IDs with timestamps) that survives restarts — required for +next-launch crash reports. + +## Server design (`silo-server`) + +### Feature gate + +Borrows the *behavior* of the requests gate (status endpoint + per-call +guard that rereads settings) but stores keys in the general `server_settings` +store — requests uses its own dedicated table, which diagnostics doesn't +need. Key naming follows existing namespaces (`opslog.*`, +`audiobooks.enabled`, `playback.protocol_v3_enabled`): + +- `diagnostics.uploads_enabled` (bool, default `false`) +- `diagnostics.max_bundle_bytes` (default 10 MiB compressed) +- `diagnostics.max_uncompressed_bytes` (default 64 MiB) +- `diagnostics.max_reports_per_user_per_day` (default 20) +- `diagnostics.retention_days` (default 30) +- `diagnostics.max_bytes_per_user` (default 200 MiB) + +Enabling `diagnostics.uploads_enabled` **requires a storage validation** +(private object store configured and writable); the admin settings API +refuses to enable it otherwise. `S3Private` is a nullable dependency in the +router and is only populated when configured, so the feature must degrade +predictably, not 500. + +### Status endpoint + +`GET /api/v1/diagnostics/status` — authenticated (`RequireAuth`), **no +profile required**: crashes during login/profile selection must still be +reportable, so capability discovery and upload are account-scoped with +optional profile attribution. + +```json +{ + "status": "available", // available | disabled | storage_unavailable + "server_instance_id": "srv_…", // stable; binds consent + pending reports + "accepted_schema_versions": [1], + "max_bundle_bytes": 10485760, + "max_manifest_bytes": 65536, + "retention_days": 30, + "consent_notice_version": 1 +} +``` + +Clients cache this alongside existing server-driven config refresh. + +### Ingest endpoint + +`POST /api/v1/diagnostics/reports` — `RequireAuth`; access-token +`Authorization` header only (no query credentials; `sa_` API keys rejected +for diagnostics). Optional validated profile attribution. Hardened parsing — +the avatar handler is the *precedent* (multipart + `PutObject`), but its +buffer-everything approach (`ParseMultipartForm` + `io.ReadAll`) is not +reused: + +1. Guard: `diagnostics.uploads_enabled` + storage present, else stable error + codes (`disabled`, `storage_unavailable`). +2. `http.MaxBytesReader` with `max_bundle_bytes` + manifest overhead before + any parsing; request deadline; per-account concurrency cap (1 in-flight) + and a server-global concurrency ceiling, on top of the existing rate-limit + middleware. +3. Streaming `multipart.Reader`: exactly the two named parts in order; + manifest capped at 64 KiB and schema-validated; unknown parts rejected. +4. Quota reservation: in one transaction holding a per-account advisory + lock (`pg_advisory_xact_lock(user_id)` — serializing concurrent uploads + from multiple devices on one account), check the per-day count and + per-user byte quota and insert the report row in state `receiving` + (server-generated UUID + `short_id`). Failures return `quota_exceeded` + (429, with `retry_after`) or `too_large` (413) — machine-readable so + clients back off and keep the report pending locally. +5. Stream the `bundle` part directly to its final key + (`diagnostics/{user_id}/{report_id}.tar.gz`) while hashing and counting + bytes; enforce the compressed cap during streaming. Validate the + gzip/tar structure and per-entry limits (entry count, name allowlist, + uncompressed total ≤ `max_uncompressed_bytes`, compression ratio bound) + in the same bounded streaming pass. Reject on violation + (`invalid_bundle`, `unsupported_schema`). There is no staging/rename + step — S3-style stores have no cheap rename, and the row's `state` is + the source of truth: a blob is not a report until its row is `ready`. +6. On success, update the row to `ready` with sizes + sha256. On any + failure: delete the object, mark the row `failed` (compensation); the + reconciler (below) cleans up stragglers from process death mid-request. +7. Respond `201 {"report_id":"…","short_id":"SILO-…"}` and emit a normalized + operational event (accepted/rejected, account, sizes, result). + +`short_id` is 12 Crockford-Base32 characters, server-generated with +collision retry, case-insensitively unique, shown to the user for support +conversations, filterable in the admin API — and explicitly not an +authorization secret. + +New code: `internal/diagnostics/` (service + store interface), handler in +`internal/api/handlers/diagnostics.go`, wired through `Dependencies` in +`internal/api/router.go`. The store interface is diagnostics-owned (the +avatar handler's is package-private and presign-only) and covers everything +the design uses: a **streaming** put (`io.Reader` — the existing s3client +`PutObject` takes `[]byte`, so this is a small s3client addition), +`GetObject` (streamed admin download fallback), `DeleteObject`, +`ListObjects(prefix)` (reconciler), and `PresignGetURL`. + +### Storage + +```sql +CREATE TABLE client_diagnostic_reports ( + id UUID PRIMARY KEY, + short_id TEXT NOT NULL, -- unique, case-insensitive (expression index) + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + profile_id TEXT, -- capturing profile, nullable + state TEXT NOT NULL, -- receiving | ready | failed + captured_at TIMESTAMPTZ NOT NULL, -- client claim + received_at TIMESTAMPTZ NOT NULL DEFAULT now(), -- server truth; retention keys off this + report_type TEXT NOT NULL, -- crash|anr|native_crash|hang|abnormal_exit|manual + platform TEXT NOT NULL, + app_version TEXT NOT NULL, + crash_summary TEXT, -- truncated ≤ 8 KiB + manifest JSONB NOT NULL, -- part 1 verbatim (≤ 64 KiB by construction) + playback_session_ids TEXT[] NOT NULL DEFAULT '{}', + blob_bucket TEXT, + blob_key TEXT, + blob_bytes BIGINT, + uncompressed_bytes BIGINT, + blob_sha256 TEXT +); +CREATE UNIQUE INDEX ON client_diagnostic_reports (lower(short_id)); +CREATE INDEX ON client_diagnostic_reports (user_id, received_at DESC); +CREATE INDEX ON client_diagnostic_reports (received_at); +``` + +Plain table, not partitioned — volume is orders of magnitude below +`operational_logs`; the byte-heavy payload lives in object storage. Full +stacks/traces live only in the archive; the row holds a truncated summary. + +### Admin API + +Under the acting-admin subtree (same placement as the `admin_logs.go` +routes): + +- `GET /api/v1/admin/diagnostics/reports` — list; filters: user, platform, + type, date range, `short_id` (exact, case-insensitive); paginated. +- `GET /api/v1/admin/diagnostics/reports/{id}` — row incl. manifest. +- `GET /api/v1/admin/diagnostics/reports/{id}/download` — presigned GET URL + (primary, matching avatar behavior) with a streamed `GetObject` proxy + fallback for deployments where presigned URLs aren't reachable — the + fallback is new behavior, not something the avatar path provides. +- `DELETE /api/v1/admin/diagnostics/reports/{id}` — row + blob. + +Admin download and delete emit audit/operational events. Web admin gets a +list/detail/download page next to the logs viewer, with a +playback-session-id link into the existing filtered logs view. Native apps +add no admin screens in v1 (product policy). + +### Retention + reconciliation + +A taskmanager task `internal/taskmanager/tasks/cleanup_client_diagnostics.go` +mirroring `cleanup_operational_log.go`, registered in `cmd/silo/main.go` +with the other cleanup tasks: + +- Delete reports older than `diagnostics.retention_days` (by `received_at`), + then enforce `diagnostics.max_bytes_per_user` oldest-first. Blob deleted + first (tolerating already-missing objects), then the row; deletes are + idempotent. +- Reconcile: remove `receiving`/`failed` rows older than a grace window, + and (via `ListObjects`) any `diagnostics/*` objects whose row is absent, + `failed`, or stale-`receiving` — so partial failures and process deaths + mid-request never leak storage. +- Account deletion cascades rows (FK) ; the reconciler then removes orphaned + blobs. + +## Android design (`silo-android`) + +New package +`android-shared/src/androidMain/kotlin/org/siloserver/silo/common/diagnostics/`. + +### Logging + +- `SiloLog.kt` — safe-logging facade per the contract above: + `SiloLog.d(cat, tag, msg, attrs)` (+`i/w/e`, sanitized `Throwable`). + Always forwards to `android.util.Log` (today's behavior unchanged) and + offers the entry to the ring buffer. +- `LogRing.kt` — always-on in-memory ring: fixed-size array of pre-rendered + compact entries, **non-blocking** writes (atomic index; a torn final entry + during crash snapshot is acceptable), ~4000 entries / ~1.5 MB cap, lazy + attribute formatting off the hot path, dropped-entry counter. Must add no + measurable contention to playback; a playback benchmark gate accompanies + the PR that wires player events in. +- `DiagnosticsFileLogger.kt` — only while debug logging is on: a + single-writer coroutine draining a bounded channel (drop + count under + backpressure) into **append-only framed segment files** (5 × 2 MB, + app-private `files/diagnostics/logs/`, `noBackup`), compressed only at + bundle-build time — a process death never corrupts more than the tail of + the active segment (unlike a single long-lived gzip stream). +- Playback wiring: each `PlaybackAnalyticsListener` event plus a ~5s periodic + `PlayerStatsSnapshot` line while debug logging is on (reuses the existing + reducer). TV focus logging hooks the TV app's focus observers under + `cat=focus`. Network: a Ktor hook logging `method, templated path, status, + duration_ms` only. + +### Crash and abnormal-exit capture + +- `CrashCapture.kt` — installed as early as possible in both `Application` + classes; idempotent; captures the previous + `Thread.getDefaultUncaughtExceptionHandler` once. On crash it does the + minimum on the dying thread: render the stack (bounded), copy what the + non-blocking ring exposes within a hard time budget, write **one bounded + marker file** (≤ ~512 KiB) via temp-write + atomic rename, and invoke the + prior handler in a `finally`. No locks shared with the ring's write path, + no coroutines, no allocation-heavy JSON building (preformatted line + buffer). Bundle assembly happens on next launch, never in the handler. + At most 3 pending reports are kept (oldest dropped) as a crash-loop guard. +- `ExitInfoCollector.kt` — API 30+ enhancement (minSdk is 24; many Fire TV + devices ship below API 30, where the handler is the only source): reads + `ActivityManager.getHistoricalProcessExitReasons()` on launch. + `REASON_CRASH`, `REASON_CRASH_NATIVE`, `REASON_ANR` become pending + reports. Dedupe uses a persisted bounded **fingerprint set** (process + name, pid, timestamp, reason, status, trace hash) — not a raw timestamp + watermark, which loses same-stamp events and breaks on clock changes. + `getTraceInputStream()` is treated as optional opaque bytes with a bounded + read: ANR text → `crash/stack.txt`; API 31+ native tombstone protobuf → + `crash/tombstone.pb` (never decoded into the manifest). UEH-captured + events are correlated by fingerprint so the same death isn't reported + twice. +- Next foreground launch, per the consent model: Ask → aggregated, + incident-specific prompt (one prompt covering N pending reports; default + focus **Don't send** on TV; "Always send" requires a second confirmation; + full-screen review flow on TV rather than a small dialog); Always → + silent upload, at most one bundle per crash fingerprint per day; + Never → purge. Per-fingerprint prompt suppression with backoff prevents + crash-loop prompt fatigue. Pending reports expire after 7 days — + visibly (the Diagnostics screen shows pending reports with expiry), not + silently. + +### Device snapshot + +`DeviceSnapshotCollector.kt` assembles `device.json` from the existing +probes plus the new accessors listed in the contract section (display mode +list, `Build.DEVICE`/form factor, `AudioManager.getDevices()` enumeration, +suppression-registry export). A compact snapshot is persisted with each +crash marker (`provenance: pre_failure`) since post-restart audio/display +state can differ from crash-time state. + +### Settings + consent UI + +- `DiagnosticsSettingsStore.kt` — DataStore. Deliberately **not** the + per-profile pattern of `AndroidPlayerSettingsStore`: consent and pending + reports are keyed by server+account (per the consent model), and debug + logging is per-device. Holds consent records, crash-report mode, + debug-logging flag, fingerprint set, session tracker, and status cache. +- Phone: "Diagnostics" section in `SettingsScreen.kt`; TV: rail category in + `TvSettingsScreen.kt`. For non-child profiles the section is **always + visible** and state-aware rather than hidden when the server gate is off — it distinguishes + `available` / `disabled by server` / `storage unavailable` / `offline`, + and always shows local pending reports (count, size, expiry) with + review/delete controls. Uploads are simply unavailable in non-available + states; a report captured while the gate was off is not auto-uploaded + later unless consent for that server+account exists. +- Manual capture is the primary support flow and is stronger than a bare + "send now": **Start diagnostic capture** (turns on verbose capture, + shows an indicator) → user reproduces the issue → **Stop & review** → + summary (categories, time range, sizes, destination) → send. A one-shot + "Send diagnostics now" remains for quick cases but warns when only the + in-memory ring is available ("this bundle contains only the last few + minutes of basic logs"). +- After upload, the `short_id` is displayed and kept in a small sent + history so a user can read it to their admin later. + +### Upload + +- `shared/commonMain/…/network/api/DiagnosticsApi.kt` — `getStatus()` and + `uploadReport(manifestJson, bundleBytes)` via Ktor + `MultiPartFormDataContent` (first multipart use in the client — verified + none exists — written as a small reusable helper), using the existing + `safeApiCall`/`ApiResult` pattern, wired through Koin like other `*Api` + classes. +- `DiagnosticsUploader.kt` — builds the tar.gz (marker + ring/segment files + overlapping the incident window + breadcrumbs + snapshot), streams it to + the endpoint, deletes the pending report only on 201, records `short_id`. + On `quota_exceeded`/`too_large`/`disabled`/`storage_unavailable` it keeps + the pending report and backs off per `retry_after`; on + `unsupported_schema` it surfaces "server needs an update" rather than + silently retrying until expiry. + +## Apple design (`silo-apple`) + +Mirrors Android with platform-native mechanisms, and with tvOS scoped +honestly — **MetricKit diagnostic payloads (crash/hang) are iOS-only; the +SDK marks them unavailable on tvOS**, and delivery is delayed (typically +daily, while the app runs), not "next launch". + +- **Ring + facade**: `DiagLog` implementing the same safe-logging contract. + The player's existing `cmpLog` helper (`PlayerLog.swift`) is **extended** + into a tee — it keeps `print()`ing (preserving `devicectl --console` + capture on tvOS) and also feeds the ring; remaining direct `print()` call + sites in `Screens/Player/` migrate to it curated, per the redaction + contract. OSLog-based subsystems are harvested at bundle-build time via + `OSLogStore(scope: .currentProcessIdentifier)` (deployment targets are + iOS/tvOS 26, far above the OSLogStore 15+ floor), merged into + `logs.jsonl` with provenance. +- **Breadcrumb journal**: because MetricKit reports arrive in a *later* + process, the in-memory ring is gone by delivery time. While crash + reporting is enabled, a small append-only breadcrumb journal (bounded, + rotating, crash-safe appends: lifecycle transitions, playback start/stop, + route changes, screen changes) persists across runs and becomes + `breadcrumbs.jsonl`, matched to a diagnostic by `capture_session_id` and + time window. Snapshots taken at bundle-build time are labeled + `provenance: post_restart`; the design never presents post-restart state + as crash-time state. +- **iOS crash/hang detection**: an `MXMetricManager` subscriber. Each + received `MXCrashDiagnostic`/`MXHangDiagnostic` becomes its **own** + pending report: raw diagnostic JSON preserved as `crash/metrickit.json`, + deduped by a canonical hash of the diagnostic (not timestamps), carrying + its own app version and reporting-period bounds. Prompt copy is + type-specific ("crashed" vs "was not responding") and honest about timing + ("yesterday during playback"). Delivery is verified on physical devices; + parsing is tested via injected fixtures. +- **tvOS abnormal-exit detection**: no crash stacks in v1. A dirty-exit + sentinel (marker written at launch, cleared on clean + background/termination) plus the breadcrumb journal yields + `type: abnormal_exit` reports — "Silo did not shut down cleanly last + time" — with breadcrumbs and device snapshot but no stack. An in-process + crash reporter for tvOS is explicitly deferred (see Non-goals). +- **Device snapshot**: `AppleDeviceIdentity.current` + + `ApplePlaybackV3Capabilities.snapshot()` for identity/codec/HDR context; + audio reported as route metadata with hashed route UIDs and + `passthrough: "unknown"` until a real output-capability probe exists (the + current one is a private placeholder). +- **Settings/consent**: Diagnostics section in `SettingsView` (iOS) and + `TVSettingsView` (tvOS) with the same state-aware presentation, manual + capture flow, and consent rules. +- **Upload**: a small multipart body writer added to `HTTPClient` (verified: + JSON-only today, no multipart/uploadTask), same status/error handling as + Android. +- Housekeeping note: product bundle IDs are already `org.siloserver.silo`; + only legacy keychain/logging identifiers (`com.continuum.*`) remain, so + nothing here blocks on rebranding. + +## Privacy summary + +- **Consent boundaries**: no network transmission of diagnostic data without + prompt confirmation, a server+account-bound Always election, or an + explicit manual send. The always-on ring is memory-only; persistent + capture (segments, breadcrumbs, markers) is app-private, size-capped, + backup-excluded, and purged by Never/sign-out/server removal. +- **Never collected**: credentials, `Authorization`/profile-token values, + request bodies, query strings, full URLs (normalized at collection), + unsanitized exception dumps. Bearer-token matching at bundle time as + defense in depth. Golden leak fixtures in CI. +- **Deliberately collected** (disclosed in View report): device identity + incl. device name, media titles appearing in playback logs, templated + server paths. The recipient is the user's own server admin, who already + has server-side access to this information; that is the trust model this + feature is scoped to — and why third-party destinations are out. +- **Persistent identifiers**: the existing device IDs (already sent to the + server today) plus hashed audio-route UIDs; each is enumerated in the + schema docs with its purpose and retention. +- **User control**: state-aware Diagnostics screen with pending list, + review, delete, sent history; admin-side deletion; account deletion + cascades reports; admin access is acting-admin-only and audited. +- **Bounded retention**: defaults 30 days / 200 MiB per user, admin-tunable, + enforced server-side. + +## Rollout plan (vertical slices, each independently shippable) + +1. **Server foundation** — settings + storage-validated gate, status + endpoint, hardened ingest with state machine, schema + fixtures, admin + API, retention/reconciler task, operational audit events. Feature stays + off; nothing user-visible. +2. **Contract fixtures in clients** — JSON Schema + golden fixtures wired + into Android/Apple CI; safe-logging facades + rings land **dark** (no + UI, no capture persistence, no prompts). +3. **Android end-to-end slice** — consent model + Diagnostics settings UI + + crash marker/ExitInfo capture + device snapshot + uploader + manual + capture flow, with the initial curated logging migration (player tags + + TV focus + network hook + stats snapshots). First release where a user + can be asked for, and send, a useful report. +4. **Android coverage expansion** — remaining curated call-site migration, + breadcrumbs-on-Android if wanted, prompt-fatigue tuning from the field. +5. **Apple slice** — cmpLog tee + OSLogStore harvest + breadcrumbs + + MetricKit (iOS) + exit sentinel (tvOS) + settings/consent + multipart + upload. +6. **Web admin viewer + correlation UX** — list/detail/download page, + playback-session pivot into the logs viewer. +7. **Later / separate decisions** — symbolication job, tvOS in-process + crash reporter, native-app admin viewer, macOS. + +Slices 3 and 5 ship complete consent→capture→upload flows; intermediate +releases never prompt for reports they can't send. + +## Open questions + +1. Should users be able to delete their **uploaded** reports from the client + (self-service deletion), or is pending-report control + admin deletion + enough for v1? +2. tvOS: is abnormal-exit detection (no stacks) acceptable for v1, or does + the Shield/Fire TV debugging need justify evaluating an in-process crash + reporter (PLCrashReporter/KSCrash) with its signal-safety and review + trade-offs? +3. Do we want breadcrumbs on Android in v1 (cheap once the facade exists), + or is the UEH marker + ExitInfo enough there? +4. Localization of prompt/consent strings at launch vs English-first. +5. Exact quota defaults (20/day, 200 MiB/user, 10 MiB/bundle) — tune before + enabling anywhere real. +6. Whether the status payload should also fold into an existing server-driven + config refresh response later (clients poll both today); the dedicated + endpoint ships first either way. + +## Review notes + +This spec was adversarially reviewed before the draft PR: two independent +review passes (a fact-check of every codebase claim against the three repos, +and a design attack across consent, crash-capture correctness, ingest +robustness, storage consistency, UX, and phasing). Material corrections +adopted include: tvOS MetricKit unavailability and honest MetricKit timing; +server+account-bound consent; storage-validated gating (private object store +is optional in real deployments); streaming bounded ingest with a report +state machine and orphan reconciliation; the archive container replacing a +bare log gzip; collection-time typed redaction replacing scrub-at-package; +fingerprint-based ExitInfo dedupe; non-blocking ring + bounded crash marker; +recent-session trackers (only the active playback session ID exists today); +and honest Apple snapshot capabilities (passthrough currently unknown). diff --git a/docs/design/schemas/client-diagnostics/README.md b/docs/design/schemas/client-diagnostics/README.md new file mode 100644 index 00000000..a58cf126 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/README.md @@ -0,0 +1,5 @@ +The canonical client-diagnostics v1 contract lives in `v1/`. +Clients vendor the schema files, attribute registry, and fixtures from this directory. +Servers accept supported schema versions and ignore unknown additive fields. +Within v1, changes are additive only: new optional fields, enum values after coordinated client support, and new allowlisted archive entries. +Breaking changes require a new versioned contract directory and explicit server support. diff --git a/docs/design/schemas/client-diagnostics/v1/attr-registry.json b/docs/design/schemas/client-diagnostics/v1/attr-registry.json new file mode 100644 index 00000000..25b710b7 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/attr-registry.json @@ -0,0 +1,89 @@ +{ + "$schema": "https://silo-server.dev/schemas/client-diagnostics/v1/attr-registry.schema.json", + "schema_version": 1, + "policy": "Attribute keys are registered per category. Debug builds and CI should fail unregistered keys; production clients drop unregistered keys before upload.", + "categories": { + "playback": { + "sink": { + "type": "string", + "description": "Current audio output sink, such as HDMI or speaker." + }, + "fmt": { + "type": "string", + "description": "Media or passthrough format identifier, such as truehd." + }, + "decoder": { + "type": "string", + "description": "Selected decoder name." + }, + "width": { + "type": "integer", + "description": "Video width in pixels." + }, + "height": { + "type": "integer", + "description": "Video height in pixels." + }, + "hdr_mode": { + "type": "string", + "description": "Active HDR mode or selected tone-mapping mode." + }, + "bitrate_kbps": { + "type": "integer", + "description": "Observed or selected bitrate in kilobits per second." + }, + "dropped_frames": { + "type": "integer", + "description": "Dropped video frame count." + }, + "audio_underruns": { + "type": "integer", + "description": "Audio underrun count." + } + }, + "focus": { + "target": { + "type": "string", + "description": "Focusable element or logical destination." + }, + "action": { + "type": "string", + "description": "Focus action or navigation direction." + } + }, + "network": { + "method": { + "type": "string", + "description": "HTTP method." + }, + "path": { + "type": "string", + "description": "Templated path without query string." + }, + "status": { + "type": "integer", + "description": "HTTP status code." + }, + "duration_ms": { + "type": "integer", + "description": "Request duration in milliseconds." + } + }, + "lifecycle": { + "state": { + "type": "string", + "description": "App lifecycle state." + } + }, + "crash": { + "fingerprint": { + "type": "string", + "description": "Client-side dedupe fingerprint." + }, + "source": { + "type": "string", + "description": "Crash source matching manifest crash.source." + } + } + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/device.schema.json b/docs/design/schemas/client-diagnostics/v1/device.schema.json new file mode 100644 index 00000000..b9a988cb --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/device.schema.json @@ -0,0 +1,113 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://silo-server.dev/schemas/client-diagnostics/v1/device.schema.json", + "title": "Silo Client Diagnostics Device Snapshot v1", + "type": "object", + "required": [ + "captured_at", + "provenance", + "identity", + "display", + "audio", + "video_codecs", + "network" + ], + "properties": { + "captured_at": { + "type": "string", + "format": "date-time", + "maxLength": 64 + }, + "provenance": { + "type": "string", + "enum": [ + "pre_failure", + "post_restart" + ], + "maxLength": 32 + }, + "identity": { + "$ref": "#/$defs/tri_state_object" + }, + "display": { + "$ref": "#/$defs/tri_state_object" + }, + "audio": { + "$ref": "#/$defs/tri_state_object" + }, + "video_codecs": { + "anyOf": [ + { + "type": "array", + "maxItems": 128, + "items": { + "$ref": "#/$defs/open_object" + } + }, + { + "$ref": "#/$defs/sentinel" + } + ] + }, + "network": { + "$ref": "#/$defs/tri_state_object" + } + }, + "$defs": { + "sentinel": { + "type": "string", + "enum": [ + "unknown", + "not_collected" + ] + }, + "tri_state_object": { + "anyOf": [ + { + "$ref": "#/$defs/open_object" + }, + { + "$ref": "#/$defs/sentinel" + } + ] + }, + "open_object": { + "type": "object", + "maxProperties": 64, + "additionalProperties": { + "$ref": "#/$defs/tri_state_value" + } + }, + "tri_state_value": { + "anyOf": [ + { + "type": "string", + "maxLength": 512 + }, + { + "type": "number" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array", + "maxItems": 128, + "items": { + "$ref": "#/$defs/tri_state_value" + } + }, + { + "$ref": "#/$defs/open_object" + } + ] + } + }, + "additionalProperties": true +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/archive-entry-outside-allowlist.json b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/archive-entry-outside-allowlist.json new file mode 100644 index 00000000..4c58edaa --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/archive-entry-outside-allowlist.json @@ -0,0 +1,45 @@ +{ + "schema_version": 1, + "report": { + "type": "manual", + "captured_at": "2026-07-19T18:22:31Z", + "capture_session_id": "run_bad_archive_entry", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "android", + "os_version": "15 (API 35)", + "profile_id": "prof_tablet" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "consent": { + "mode": "manual", + "notice_version": 1 + }, + "device_summary": { + "manufacturer": "Google", + "model": "Pixel Tablet", + "os": "15", + "form_factor": "tablet" + }, + "playback_session_ids": [], + "log_summary": { + "lines": 1, + "bytes_gz": 100, + "dropped_lines": 0, + "categories": [ + "other" + ], + "debug_logging": false + }, + "archive": { + "entries": [ + "manifest.json", + "../secrets.txt" + ], + "bytes": 100, + "uncompressed_bytes": 200, + "sha256": "3333333333333333333333333333333333333333333333333333333333333333" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/bad-schema-version.json b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/bad-schema-version.json new file mode 100644 index 00000000..d5015b74 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/bad-schema-version.json @@ -0,0 +1,44 @@ +{ + "schema_version": 2, + "report": { + "type": "manual", + "captured_at": "2026-07-19T18:22:31Z", + "capture_session_id": "run_bad_schema", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "android", + "os_version": "15 (API 35)", + "profile_id": "prof_tablet" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "consent": { + "mode": "manual", + "notice_version": 1 + }, + "device_summary": { + "manufacturer": "Google", + "model": "Pixel Tablet", + "os": "15", + "form_factor": "tablet" + }, + "playback_session_ids": [], + "log_summary": { + "lines": 1, + "bytes_gz": 100, + "dropped_lines": 0, + "categories": [ + "other" + ], + "debug_logging": false + }, + "archive": { + "entries": [ + "manifest.json" + ], + "bytes": 100, + "uncompressed_bytes": 200, + "sha256": "2222222222222222222222222222222222222222222222222222222222222222" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/missing-consent.json b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/missing-consent.json new file mode 100644 index 00000000..ea4d35d9 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/missing-consent.json @@ -0,0 +1,40 @@ +{ + "schema_version": 1, + "report": { + "type": "manual", + "captured_at": "2026-07-19T18:22:31Z", + "capture_session_id": "run_missing_consent", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "android", + "os_version": "15 (API 35)", + "profile_id": "prof_tablet" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "device_summary": { + "manufacturer": "Google", + "model": "Pixel Tablet", + "os": "15", + "form_factor": "tablet" + }, + "playback_session_ids": [], + "log_summary": { + "lines": 1, + "bytes_gz": 100, + "dropped_lines": 0, + "categories": [ + "other" + ], + "debug_logging": false + }, + "archive": { + "entries": [ + "manifest.json" + ], + "bytes": 100, + "uncompressed_bytes": 200, + "sha256": "1111111111111111111111111111111111111111111111111111111111111111" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/platform-macos-reserved.json b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/platform-macos-reserved.json new file mode 100644 index 00000000..27d205b4 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/platform-macos-reserved.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "report": { + "type": "manual", + "captured_at": "2026-07-19T18:22:31Z", + "capture_session_id": "run_macos_reserved", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "macos", + "os_version": "26.0", + "profile_id": "prof_mac" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "consent": { + "mode": "manual", + "notice_version": 1 + }, + "device_summary": { + "manufacturer": "Apple", + "model": "Mac", + "os": "26.0", + "form_factor": "desktop" + }, + "playback_session_ids": [], + "log_summary": { + "lines": 1, + "bytes_gz": 100, + "dropped_lines": 0, + "categories": [ + "other" + ], + "debug_logging": false + }, + "archive": { + "entries": [ + "manifest.json" + ], + "bytes": 100, + "uncompressed_bytes": 200, + "sha256": "ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/stack-excerpt-over-8kib.json b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/stack-excerpt-over-8kib.json new file mode 100644 index 00000000..b2f03b63 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/stack-excerpt-over-8kib.json @@ -0,0 +1,54 @@ +{ + "schema_version": 1, + "report": { + "type": "crash", + "captured_at": "2026-07-19T18:22:31Z", + "capture_session_id": "run_stack_too_large", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "android-tv", + "os_version": "11 (API 30)", + "profile_id": "prof_living_room" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "consent": { + "mode": "prompt", + "notice_version": 1 + }, + "crash": { + "summary": "Oversized stack excerpt", + "stack_excerpt": "000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "thread": "main", + "foreground": true, + "source": "ueh", + "provenance": "pre_failure", + "occurred_at": "2026-07-19T18:22:31Z" + }, + "device_summary": { + "manufacturer": "NVIDIA", + "model": "SHIELD Android TV", + "os": "11", + "form_factor": "tv" + }, + "playback_session_ids": [], + "log_summary": { + "lines": 1, + "bytes_gz": 100, + "dropped_lines": 0, + "categories": [ + "crash" + ], + "debug_logging": false + }, + "archive": { + "entries": [ + "manifest.json", + "crash/stack.txt" + ], + "bytes": 100, + "uncompressed_bytes": 200, + "sha256": "4444444444444444444444444444444444444444444444444444444444444444" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/unknown-report-type.json b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/unknown-report-type.json new file mode 100644 index 00000000..7bcfa37b --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/invalid/unknown-report-type.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "report": { + "type": "kernel_panic", + "captured_at": "2026-07-19T18:22:31Z", + "capture_session_id": "run_bad_type", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "android-tv", + "os_version": "11 (API 30)", + "profile_id": "prof_living_room" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "consent": { + "mode": "prompt", + "notice_version": 1 + }, + "device_summary": { + "manufacturer": "NVIDIA", + "model": "SHIELD Android TV", + "os": "11", + "form_factor": "tv" + }, + "playback_session_ids": [], + "log_summary": { + "lines": 1, + "bytes_gz": 100, + "dropped_lines": 0, + "categories": [ + "crash" + ], + "debug_logging": false + }, + "archive": { + "entries": [ + "manifest.json" + ], + "bytes": 100, + "uncompressed_bytes": 200, + "sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/valid/android-manual-no-profile.json b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/android-manual-no-profile.json new file mode 100644 index 00000000..b2722d6a --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/android-manual-no-profile.json @@ -0,0 +1,46 @@ +{ + "schema_version": 1, + "report": { + "type": "manual", + "captured_at": "2026-07-19T23:45:00Z", + "capture_session_id": "run_android_manual_preprofile_006", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "android", + "os_version": "15 (API 35)" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "consent": { + "mode": "manual", + "notice_version": 1 + }, + "device_summary": { + "manufacturer": "Google", + "model": "Pixel Tablet", + "os": "15", + "form_factor": "tablet" + }, + "playback_session_ids": [], + "log_summary": { + "lines": 12, + "bytes_gz": 2048, + "dropped_lines": 0, + "categories": [ + "lifecycle", + "network" + ], + "debug_logging": false + }, + "archive": { + "entries": [ + "manifest.json", + "device.json", + "logs.jsonl" + ], + "bytes": 4096, + "uncompressed_bytes": 16384, + "sha256": "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/valid/android-manual.json b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/android-manual.json new file mode 100644 index 00000000..6ae848f0 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/android-manual.json @@ -0,0 +1,47 @@ +{ + "schema_version": 1, + "report": { + "type": "manual", + "captured_at": "2026-07-19T23:40:00Z", + "capture_session_id": "run_android_manual_005", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "android", + "os_version": "15 (API 35)", + "profile_id": "prof_tablet" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "consent": { + "mode": "manual", + "notice_version": 1 + }, + "device_summary": { + "manufacturer": "Google", + "model": "Pixel Tablet", + "os": "15", + "form_factor": "tablet" + }, + "playback_session_ids": [], + "log_summary": { + "lines": 60, + "bytes_gz": 5400, + "dropped_lines": 0, + "categories": [ + "browse", + "network" + ], + "debug_logging": false + }, + "archive": { + "entries": [ + "manifest.json", + "device.json", + "logs.jsonl" + ], + "bytes": 8000, + "uncompressed_bytes": 43000, + "sha256": "dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/valid/android-tv-crash-ueh.json b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/android-tv-crash-ueh.json new file mode 100644 index 00000000..7ff5b314 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/android-tv-crash-ueh.json @@ -0,0 +1,61 @@ +{ + "schema_version": 1, + "report": { + "type": "crash", + "captured_at": "2026-07-19T18:22:31Z", + "capture_session_id": "run_android_tv_ueh_001", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "android-tv", + "os_version": "11 (API 30)", + "profile_id": "prof_living_room" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "consent": { + "mode": "prompt", + "notice_version": 1 + }, + "crash": { + "summary": "NullPointerException in PlaybackSessionManager.start", + "stack_excerpt": "java.lang.NullPointerException: player was null\n\tat org.siloserver.silo.PlaybackSessionManager.start(PlaybackSessionManager.kt:42)", + "thread": "main", + "foreground": true, + "source": "ueh", + "provenance": "pre_failure", + "occurred_at": "2026-07-19T18:22:31Z" + }, + "device_summary": { + "manufacturer": "NVIDIA", + "model": "SHIELD Android TV", + "os": "11", + "form_factor": "tv" + }, + "playback_session_ids": [ + "ps_9f2a" + ], + "log_summary": { + "lines": 3841, + "bytes_gz": 412034, + "dropped_lines": 12, + "categories": [ + "playback", + "focus", + "network" + ], + "debug_logging": true + }, + "archive": { + "entries": [ + "manifest.json", + "device.json", + "logs.jsonl", + "crash/summary.json", + "crash/stack.txt" + ], + "bytes": 498112, + "uncompressed_bytes": 3110400, + "sha256": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/valid/device.json b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/device.json new file mode 100644 index 00000000..0c2dcd70 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/device.json @@ -0,0 +1,57 @@ +{ + "captured_at": "2026-07-19T18:22:31Z", + "provenance": "pre_failure", + "identity": { + "manufacturer": "NVIDIA", + "model": "SHIELD Android TV", + "device": "mdarcy", + "form_factor": "tv", + "device_id": "dev_8f4a" + }, + "display": { + "mode": "3840x2160@59.94", + "modes_supported": [ + "3840x2160@59.94", + "1920x1080@60" + ], + "hdr_types": [ + "HDR10", + "HLG", + "DV" + ], + "wide_gamut": true + }, + "audio": { + "outputs": [ + { + "type": "HDMI", + "encodings": [ + "AC3", + "EAC3", + "TRUEHD" + ], + "channels": 8 + } + ], + "passthrough": [ + "AC3", + "EAC3", + "TRUEHD" + ], + "suppressions": [] + }, + "video_codecs": [ + { + "mime": "video/hevc", + "hw": true, + "profiles": [ + "Main10", + "DV P8" + ], + "max": "4096x2160@60" + } + ], + "network": { + "transport": "ethernet" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/valid/ios-hang-metrickit.json b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/ios-hang-metrickit.json new file mode 100644 index 00000000..fcaf7446 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/ios-hang-metrickit.json @@ -0,0 +1,62 @@ +{ + "schema_version": 1, + "report": { + "type": "hang", + "captured_at": "2026-07-19T21:15:00Z", + "capture_session_id": "run_ios_metric_042", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "ios", + "os_version": "26.0", + "profile_id": "prof_phone" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "consent": { + "mode": "always", + "notice_version": 1 + }, + "crash": { + "summary": "Main thread hang reported by MetricKit", + "stack_excerpt": "Thread 0 name: main\n0 Silo PlayerCore.waitForRouteChange", + "thread": "main", + "foreground": true, + "source": "metrickit", + "provenance": "metric_reporting_period", + "occurred_at": "2026-07-19T21:15:00Z" + }, + "device_summary": { + "manufacturer": "Apple", + "model": "iPhone17,2", + "os": "26.0", + "form_factor": "phone" + }, + "playback_session_ids": [ + "ps_ios_103" + ], + "log_summary": { + "lines": 902, + "bytes_gz": 84511, + "dropped_lines": 0, + "categories": [ + "playback", + "lifecycle", + "network" + ], + "debug_logging": true + }, + "archive": { + "entries": [ + "manifest.json", + "device.json", + "logs.jsonl", + "crash/summary.json", + "crash/metrickit.json", + "breadcrumbs.jsonl" + ], + "bytes": 141120, + "uncompressed_bytes": 812400, + "sha256": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/valid/loglines.jsonl b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/loglines.jsonl new file mode 100644 index 00000000..201e2bf9 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/loglines.jsonl @@ -0,0 +1,3 @@ +{"ts":"2026-07-19T18:22:29.412Z","run":"run_android_tv_ueh_001","lvl":"W","cat":"playback","tag":"AudioCapabilityMgr","msg":"passthrough suppressed: HDMI sink lost TrueHD","attrs":{"sink":"HDMI","fmt":"truehd"}} +{"ts":"2026-07-19T18:22:30Z","run":"run_android_tv_ueh_001","lvl":"I","cat":"network","tag":"HTTPClient","msg":"request completed","attrs":{"method":"GET","path":"/api/v1/playback/{id}","status":200,"duration_ms":84}} +{"ts":"2026-07-19T18:22:31Z","run":"run_android_tv_ueh_001","lvl":"D","cat":"focus","tag":"TvFocus","msg":"focus moved","attrs":{"target":"play-button","action":"right"}} diff --git a/docs/design/schemas/client-diagnostics/v1/fixtures/valid/tvos-abnormal-exit.json b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/tvos-abnormal-exit.json new file mode 100644 index 00000000..833e386c --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/fixtures/valid/tvos-abnormal-exit.json @@ -0,0 +1,60 @@ +{ + "schema_version": 1, + "report": { + "type": "abnormal_exit", + "captured_at": "2026-07-19T22:05:11Z", + "capture_session_id": "run_tvos_exit_007", + "app_version": "1.4.2", + "app_build": "20841", + "platform": "tvos", + "os_version": "26.0", + "profile_id": "prof_den" + }, + "destination": { + "server_instance_id": "srv_home_01" + }, + "consent": { + "mode": "prompt", + "notice_version": 1 + }, + "crash": { + "summary": "Silo did not shut down cleanly last time", + "thread": "unknown", + "foreground": true, + "source": "exit_sentinel", + "provenance": "post_restart", + "occurred_at": "2026-07-19T22:05:11Z" + }, + "device_summary": { + "manufacturer": "Apple", + "model": "Apple TV", + "os": "26.0", + "form_factor": "tv" + }, + "playback_session_ids": [ + "ps_tvos_11", + "ps_tvos_10" + ], + "log_summary": { + "lines": 120, + "bytes_gz": 12044, + "dropped_lines": 3, + "categories": [ + "lifecycle", + "playback" + ], + "debug_logging": false + }, + "archive": { + "entries": [ + "manifest.json", + "device.json", + "logs.jsonl", + "crash/summary.json", + "breadcrumbs.jsonl" + ], + "bytes": 30001, + "uncompressed_bytes": 104000, + "sha256": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc" + } +} diff --git a/docs/design/schemas/client-diagnostics/v1/logline.schema.json b/docs/design/schemas/client-diagnostics/v1/logline.schema.json new file mode 100644 index 00000000..6efd898b --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/logline.schema.json @@ -0,0 +1,95 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://silo-server.dev/schemas/client-diagnostics/v1/logline.schema.json", + "title": "Silo Client Diagnostics Log Line v1", + "type": "object", + "required": [ + "ts", + "run", + "lvl", + "cat", + "tag", + "msg" + ], + "properties": { + "ts": { + "type": "string", + "format": "date-time", + "maxLength": 64 + }, + "run": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "lvl": { + "type": "string", + "enum": [ + "V", + "D", + "I", + "W", + "E" + ], + "maxLength": 1 + }, + "cat": { + "type": "string", + "enum": [ + "playback", + "focus", + "network", + "lifecycle", + "browse", + "cast", + "download", + "crash", + "other" + ], + "maxLength": 32 + }, + "tag": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "msg": { + "type": "string", + "minLength": 1, + "maxLength": 2048 + }, + "attrs": { + "type": "object", + "description": "Attribute keys are registered in attr-registry.json. Production clients drop unregistered keys before upload.", + "additionalProperties": { + "anyOf": [ + { + "type": "string", + "maxLength": 512 + }, + { + "type": "number" + }, + { + "type": "integer" + }, + { + "type": "boolean" + }, + { + "type": "null" + }, + { + "type": "array", + "maxItems": 32, + "items": { + "type": "string", + "maxLength": 256 + } + } + ] + } + } + }, + "additionalProperties": true +} diff --git a/docs/design/schemas/client-diagnostics/v1/manifest.schema.json b/docs/design/schemas/client-diagnostics/v1/manifest.schema.json new file mode 100644 index 00000000..b0da6ff0 --- /dev/null +++ b/docs/design/schemas/client-diagnostics/v1/manifest.schema.json @@ -0,0 +1,392 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://silo-server.dev/schemas/client-diagnostics/v1/manifest.schema.json", + "title": "Silo Client Diagnostics Manifest v1", + "description": "Canonical manifest contract for a client diagnostics bundle. STRING LENGTHS: every `maxLength` in this schema is a character-count sanity bound. The server enforces the matching limit as a UTF-8 *byte* length (Go `len(string)`), so a value near a cap that contains multi-byte characters (CJK, emoji, accented text) can satisfy `maxLength` here yet be rejected at ingest. Treat each numeric cap as a byte budget for non-ASCII text. JSON Schema cannot express byte length, so this divergence is intentional and documented rather than encoded.", + "type": "object", + "required": [ + "schema_version", + "report", + "destination", + "consent", + "device_summary", + "playback_session_ids", + "log_summary", + "archive" + ], + "properties": { + "schema_version": { + "type": "integer", + "const": 1 + }, + "report": { + "type": "object", + "required": [ + "type", + "captured_at", + "capture_session_id", + "app_version", + "app_build", + "platform", + "os_version" + ], + "properties": { + "type": { + "type": "string", + "enum": [ + "crash", + "anr", + "native_crash", + "hang", + "abnormal_exit", + "manual" + ], + "maxLength": 32 + }, + "captured_at": { + "type": "string", + "format": "date-time", + "maxLength": 64 + }, + "capture_session_id": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "app_version": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "app_build": { + "type": "string", + "minLength": 1, + "maxLength": 64 + }, + "platform": { + "type": "string", + "enum": [ + "android", + "android-tv", + "ios", + "tvos" + ], + "maxLength": 32 + }, + "os_version": { + "type": "string", + "minLength": 1, + "maxLength": 128 + }, + "profile_id": { + "type": "string", + "maxLength": 128 + } + }, + "additionalProperties": true + }, + "destination": { + "type": "object", + "required": [ + "server_instance_id" + ], + "properties": { + "server_instance_id": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "additionalProperties": true + }, + "consent": { + "type": "object", + "required": [ + "mode", + "notice_version" + ], + "properties": { + "mode": { + "type": "string", + "enum": [ + "prompt", + "always", + "manual" + ], + "maxLength": 32 + }, + "notice_version": { + "type": "integer", + "minimum": 1 + } + }, + "additionalProperties": true + }, + "crash": { + "type": "object", + "required": [ + "summary", + "source", + "provenance", + "occurred_at" + ], + "properties": { + "summary": { + "type": "string", + "minLength": 1, + "description": "Free-text crash summary; may contain non-ASCII. maxLength counts characters — the server enforces 8192 UTF-8 bytes (see the schema-level string-length note).", + "maxLength": 8192 + }, + "stack_excerpt": { + "type": "string", + "description": "Free-text stack excerpt; may contain non-ASCII. maxLength counts characters — the server enforces 8192 UTF-8 bytes (see the schema-level string-length note).", + "maxLength": 8192 + }, + "thread": { + "type": "string", + "description": "Thread name; may contain non-ASCII. maxLength counts characters — the server enforces 128 UTF-8 bytes (see the schema-level string-length note).", + "maxLength": 128 + }, + "foreground": { + "type": "boolean" + }, + "source": { + "type": "string", + "enum": [ + "ueh", + "exit_info", + "metrickit", + "exit_sentinel" + ], + "maxLength": 32 + }, + "provenance": { + "type": "string", + "enum": [ + "pre_failure", + "post_restart", + "metric_reporting_period" + ], + "maxLength": 32 + }, + "occurred_at": { + "type": "string", + "format": "date-time", + "maxLength": 64 + } + }, + "additionalProperties": true + }, + "device_summary": { + "type": "object", + "required": [ + "manufacturer", + "model", + "os", + "form_factor" + ], + "properties": { + "manufacturer": { + "type": "string", + "minLength": 1, + "description": "Device manufacturer; may contain non-ASCII. maxLength counts characters — the server enforces 128 UTF-8 bytes (see the schema-level string-length note).", + "maxLength": 128 + }, + "model": { + "type": "string", + "minLength": 1, + "description": "Device model; may contain non-ASCII (e.g. CJK). maxLength counts characters — the server enforces 128 UTF-8 bytes, so a 128-character multi-byte value can pass this schema yet be rejected at ingest (see the schema-level string-length note).", + "maxLength": 128 + }, + "os": { + "type": "string", + "minLength": 1, + "description": "OS name/version; may contain non-ASCII. maxLength counts characters — the server enforces 128 UTF-8 bytes (see the schema-level string-length note).", + "maxLength": 128 + }, + "form_factor": { + "type": "string", + "minLength": 1, + "description": "Device form factor; may contain non-ASCII. maxLength counts characters — the server enforces 64 UTF-8 bytes (see the schema-level string-length note).", + "maxLength": 64 + } + }, + "additionalProperties": { + "type": "string", + "maxLength": 256 + } + }, + "playback_session_ids": { + "type": "array", + "maxItems": 20, + "items": { + "type": "string", + "minLength": 1, + "maxLength": 128 + } + }, + "log_summary": { + "type": "object", + "required": [ + "lines", + "bytes_gz", + "dropped_lines", + "categories", + "debug_logging" + ], + "properties": { + "lines": { + "type": "integer", + "minimum": 0 + }, + "bytes_gz": { + "type": "integer", + "minimum": 0 + }, + "dropped_lines": { + "type": "integer", + "minimum": 0 + }, + "categories": { + "type": "array", + "uniqueItems": true, + "maxItems": 9, + "items": { + "type": "string", + "enum": [ + "playback", + "focus", + "network", + "lifecycle", + "browse", + "cast", + "download", + "crash", + "other" + ], + "maxLength": 32 + } + }, + "debug_logging": { + "type": "boolean" + } + }, + "additionalProperties": true + }, + "archive": { + "type": "object", + "required": [ + "entries", + "bytes", + "uncompressed_bytes", + "sha256" + ], + "properties": { + "entries": { + "type": "array", + "description": "Tar member names in bundle order. The first entry MUST be `manifest.json`: the server reads and validates the leading member before any other and compares this order against the actual bundle. `prefixItems` pins index 0, while `contains` keeps `manifest.json` required for validators that do not implement `prefixItems`.", + "uniqueItems": true, + "minItems": 1, + "maxItems": 8, + "prefixItems": [ + { + "const": "manifest.json" + } + ], + "contains": { + "const": "manifest.json" + }, + "items": { + "type": "string", + "enum": [ + "manifest.json", + "device.json", + "logs.jsonl", + "crash/summary.json", + "crash/stack.txt", + "crash/tombstone.pb", + "crash/metrickit.json", + "breadcrumbs.jsonl" + ], + "maxLength": 64 + } + }, + "bytes": { + "type": "integer", + "minimum": 0 + }, + "uncompressed_bytes": { + "type": "integer", + "minimum": 0 + }, + "sha256": { + "type": "string", + "pattern": "^[A-Fa-f0-9]{64}$", + "maxLength": 64 + } + }, + "additionalProperties": true + } + }, + "additionalProperties": true, + "allOf": [ + { + "$comment": "Event reports (crash/anr/native_crash/hang/abnormal_exit) must carry the crash object; mirrors ValidateManifest in internal/diagnostics/contract/contract.go.", + "if": { + "properties": { + "report": { + "properties": { + "type": { + "enum": [ + "crash", + "anr", + "native_crash", + "hang", + "abnormal_exit" + ] + } + }, + "required": [ + "type" + ] + } + }, + "required": [ + "report" + ] + }, + "then": { + "required": [ + "crash" + ] + } + }, + { + "$comment": "Manual captures must not carry a crash object.", + "if": { + "properties": { + "report": { + "properties": { + "type": { + "const": "manual" + } + }, + "required": [ + "type" + ] + } + }, + "required": [ + "report" + ] + }, + "then": { + "not": { + "required": [ + "crash" + ] + } + } + } + ] +} diff --git a/docs/superpowers/plans/2026-07-12-playback-protocol-v3.md b/docs/superpowers/plans/2026-07-12-playback-protocol-v3.md new file mode 100644 index 00000000..2ddef63d --- /dev/null +++ b/docs/superpowers/plans/2026-07-12-playback-protocol-v3.md @@ -0,0 +1,822 @@ +# Playback Protocol v3 Server Implementation Plan + +**Status:** Proposed +**Scope:** `silo-server`, with coordinated contract fixtures in `silo-android` +**Protocol owner:** Silo server +**Client target:** Android Media3-only playback; legacy web and Apple playback remain supported + +## 1. Goal + +Implement the server half of Silo playback protocol v3 so a client can report +its current device and output capabilities and receive one complete executable +plan for: + +- authenticated original HTTP playback; +- progressive or HLS remux; +- video-copy with audio adaptation; +- HLS video transcode; +- subtitle render, conversion, or burn-in; or +- a terminal `adaptation_unavailable` outcome. + +The client must never choose FFmpeg recipes. The server must never claim that a +route preserves Dolby Vision, HDR, lossless audio, or subtitle fidelity unless +the source metadata, client/output capability, selected recipe, and installed +tooling support that claim. + +The normative behavior remains in the sibling Android repository: + +- `silo-android/docs/playback/01-media3-only-player-architecture.md` +- `silo-android/docs/playback/02-migration-compatibility-validation.md` +- `silo-android/shared/src/commonMain/kotlin/org/siloserver/silo/model/playback/PlaybackProtocolV3.kt` + +This document maps that contract onto the current server. It does not redefine +the protocol. + +## 2. Verified server baseline + +The implementation should reuse the current playback stack rather than replace +it: + +| Existing behavior | Current owner | v3 implication | +| --- | --- | --- | +| Legacy start, authorization, profile checks, track preferences, session creation | `internal/api/handlers/playback.go` | Preserve the legacy branch byte-for-byte where possible; move shared operations behind helpers. | +| Direct/remux/transcode choice | `internal/playback/resolver.go` | Replace its broad codec/container booleans with a v3 planner for v3 requests only. Do not change legacy decisions during the first rollout. | +| Byte-range original delivery | `internal/playback/directplay.go`, `internal/api/handlers/stream.go` | Reuse as `original_http`. | +| Progressive MP4 remux and audio-to-AAC | `internal/playback/remux.go`, `internal/api/handlers/stream.go` | Reuse as `server_remux_progressive`; make every transformation explicit in the plan. | +| HLS copy/transcode, timelines, restart/reconstruct | `internal/playback/transcode.go`, `internal/playback/transcode_manager.go` | Reuse as `server_remux_hls` or `server_transcode_hls`. | +| Local and remote transcode start | `HandleStartTranscode`, `internal/transcodenode/server.go`, `internal/nodepool` | Extract a server-callable starter so v3 can return the final manifest without a second client recipe request. | +| Playback sessions, policy and limits | `internal/playback/session.go` | Extend session state with v3 attempt/plan identity; keep current admission controls authoritative. | +| Source video/audio/subtitle metadata | `internal/models/media.go`, `internal/scanner/probe.go` | Most required fields already exist, including DV profile, BL compatibility ID, EL presence, color/range, bit depth, layout, and channels. MEL/FEL classification and transformation provenance are not yet represented. | +| Subtitle extract/conversion/burn-in | `internal/playback/subtitles.go`, `subtitle_stream.go`, transcode arguments | Reuse the executors; add an explicit policy decision and artifact contract. | +| Restart-safe signed stream recipes | `internal/playback/recipecard.go`, `internal/streamtoken` | Continue using signed recipe tokens; v3 plans describe the resulting URL and timeline. | + +The largest correctness gaps in the current resolver are expected and must not +be carried into v3: + +- transcode-disabled incompatibility currently falls back to attempting direct + playback instead of returning a terminal result; +- codec support is modeled as a string union, without profile, level, bit depth, + frame rate, range, sink layout, or subtitle-fidelity validation; +- clients currently supply transcode codec, bitrate, segment, and burn-in + recipe details through `/playback/transcode/start`; +- the start response does not contain an authoritative effective recipe, + validation claims, transformations, or timeline for every delivery; +- no idempotent replan or attempt-scoped route-event API exists. + +## 3. Architecture decision + +### 3.1 Additive dispatch on the existing start endpoint + +Keep `POST /api/v1/playback/start` for compatibility. Refactor its handler into +a small protocol dispatcher: + +1. Decode only an envelope containing `protocol_version`. +2. When it equals `3`, decode strictly as `PlaybackStartRequestV3` and call the + v3 orchestration service. +3. When it is absent or not `3`, pass the original body to the existing legacy + decoder and behavior. + +Use buffered request bytes or `json.RawMessage`; never decode the body twice +from the socket. Unknown fields remain tolerated for additive compatibility, +but required v3 fields and closed enum values are validated explicitly. + +Do not overwrite the existing request/response structs with a union of legacy +and v3 fields. That would make missing nested capabilities silently look like a +valid legacy request and recreate the current false-direct failure mode. + +### 3.2 Introduce one v3 orchestration service + +Add a service in the `internal/playback` domain that owns: + +- input normalization and validation; +- stable track resolution; +- source and capability evaluation; +- ordered candidate construction; +- policy and capacity checks; +- deterministic plan identity and loop prevention; +- transport startup; +- terminal outcome mapping; +- replan idempotency; and +- plan/session state transitions. + +HTTP handlers authenticate, authorize, decode, call the service, and encode the +result. They do not contain route-selection rules. + +Recommended files: + +```text +internal/playback/protocol_v3.go wire/domain types and enums +internal/playback/plan_v3.go candidate planner +internal/playback/capabilities_v3.go source/capability validators +internal/playback/recipe_v3.go normalized executable recipes +internal/playback/plan_key_v3.go canonical IDs and attempt keys +internal/playback/tracks_v3.go stable track identities/remapping +internal/playback/subtitle_policy_v3.go subtitle decision policy +internal/playback/transformations_v3.go installed-tool/recipe registry +internal/playback/protocol_v3_test.go +internal/playback/testdata/protocol_v3/ golden contract/planner fixtures + +internal/playback/planstore/store.go persistence interfaces +internal/playback/planstore/postgres.go attempts, replans and route events + +internal/api/handlers/playback_v3.go start/replan/event handlers +internal/api/handlers/playback_transport.go reusable transport starter +internal/api/handlers/playback_v3_test.go +``` + +If implementation reveals that the wire and domain structs need different +types, keep JSON structs in `handlers` and convert at the boundary. Do not add +JSON tags throughout unrelated session and FFmpeg types merely to save mapping +code. + +### 3.3 Reconcile additive client-contract gaps before freezing v3 + +The current Kotlin v3 types are sufficient to negotiate and execute a plan, +but they do not yet carry every fact the normative direct-play rules require. +The server cannot manufacture those facts. Phase 0 therefore includes a +coordinated additive Android contract change: + +| Current gap | Required additive contract | +| --- | --- | +| Video capability is a codec-name list plus a global maximum resolution. | Add per-codec decode entries with supported profiles/levels, bit depths, maximum width/height/frame rate/bitrate, and hardware/software status. | +| Audio passthrough is a codec-name list plus one global maximum channel count. | Add per-codec supported channel counts/layouts; retain the old fields for tolerant readers. | +| A plan does not expose requested versus effective media file when alternate-version selection occurs. | Add requested/effective media file IDs and a normalized source descriptor to the plan. | +| Subtitle decision does not expose the effective fidelity policy. | Add `subtitle_fidelity_policy` beside mode/artifact. | +| Route events cannot explicitly report replan identity, local PCM recovery, retry outcome, decoder timing, or requested/effective quality. | Add optional structured fields or a versioned, bounded diagnostic schema; the server enriches delivery/recipe/source facts from stored plan state rather than trusting duplicates from the client. | +| Android's profile-quality canonicalizer passes unknown stored labels through and does not produce `original`. | Canonicalize the closed v3 values and aliases client-side; keep server normalization tolerant for rolling upgrades. | + +Gate the detailed behavior with additive client feature tokens such as +`detailed_decode_capabilities`, `layout_aware_passthrough`, and +`playback_route_diagnostics`. A v3 client missing them remains parseable, but +unknown profile/layout facts cannot satisfy a strict direct claim. The planner +must choose a conservative adapted route or terminal result. + +Freeze the exact JSON field names only after Go/Kotlin golden fixtures exist. +Do not implement a server-only guess and call the contract complete. + +The response engine mapping is fixed: + +| Delivery | `engine` | +| --- | --- | +| `original_http` | `media3_direct` | +| `server_remux_progressive` | `media3_progressive_remux` | +| `server_remux_hls` | `media3_hls` | +| `server_transcode_hls` | `media3_hls` | + +Android currently accepts header refresh modes `none` and `session`; do not +emit `refresh_endpoint` until the client implements it. + +Never emit legacy/client-owned runtime values in a v3 plan: +`mpv_direct`, `client_local_loopback`, `external_player`, or +`client_local_normalization`. They remain decode-only compatibility values on +Android. + +### 3.4 Keep v3 dark until the complete route is executable + +Add a dynamic server setting `playback.protocol_v3_enabled`, default `false`. +Read it through `PlaybackSettingsReader.Get` on each capability/start request, +as the existing `allow_4k_transcode` path does. Do not load it only through +`internal/config/db_loader.go`; that would turn rollback into a restart-required +change. +Also add `GET /api/v1/playback/capability` because `/api/v1` features use +capability detection rather than server-version sniffing. + +When a v3 start arrives while disabled, return a successful negotiation +response with no allocated session and without `playback_plan_v3` in +`server_features`. This intentionally triggers Android's +`server_upgrade_required` path without leaking a legacy session. + +The capability response should contain: + +```json +{ + "enabled": false, + "protocol_versions": [3], + "features": [], + "deliveries": [], + "transformations": [], + "reason": "disabled" +} +``` + +Only advertise `playback_plan_v3` after every enabled delivery and +transformation passes Phase 0 fixtures on the deployed build. + +## 4. HTTP contract and ownership + +### 4.1 Endpoints + +| Endpoint | Success | Responsibility | +| --- | --- | --- | +| `GET /api/v1/playback/capability` | `200` | Feature/protocol and installed transformation discovery. | +| `POST /api/v1/playback/start` | `201` | Authorize source, create a session, select/start a route, return playable or terminal v3 response. | +| `POST /api/v1/playback/{session_id}/replan` | `200` | Idempotently replace the active plan at the supplied media position. | +| `POST /api/v1/playback/route-events` | `202` | Validate and enqueue attempt-scoped diagnostics. | + +All four routes use normal account authentication. Start, replan, and route +events require profile auth. `X-Profile-Id` remains authoritative; a body +`profile_id` must match it. Replan additionally verifies session ownership and +that `playback_attempt_id` belongs to the session. + +Playable and `adaptation_unavailable` are protocol outcomes, not HTTP errors. +Use HTTP errors only for transport/API failures such as malformed input, +unauthorized access, missing files, or an idempotency conflict. + +### 4.2 Strict request validation + +Reject before allocating a session when any of these fail: + +- protocol is not exactly `3`; +- `playback_attempt_id` is absent or not a bounded UUID/ULID-like identifier; +- file/profile authorization fails; +- track ID and fallback index disagree; +- output-route generation is negative or disagrees with the nested output + context; +- codec/container/feature lists exceed bounded counts or string lengths; +- bandwidth, bitrate, resolution, attempt count, or position is outside sane + bounds; or +- a replan's failed plan is not the session's active plan. + +Normalize codecs, containers, layouts, and dynamic-range labels once at the +boundary. Accept bounded known quality aliases such as `4k`; fold an unknown +rolling-upgrade quality value to `auto` with a decision warning instead of +hard-failing playback. Preserve the original values only in bounded +diagnostics. + +### 4.3 Complete response invariant + +A `playable` response is valid only after its transport can be fetched or has +successfully entered its startup state. It includes: + +- protocol and feature negotiation; +- stable `session_id` and deterministic `plan_id`; +- delivery and Media3 engine; +- final URL, stream protocol, container/MIME, scoped headers, and supported + header-refresh mode; +- exact player/source timeline mapping and seek window; +- selected stable audio/subtitle identities; +- effective codec, range, dimensions, frame rate, bitrate, channels and layout; +- route validation claims; +- subtitle mode/artifact; +- named transformations and degradation warnings; and +- a stable decision reason code. + +Never return a placeholder transcode URL and expect v3 Android to call the +legacy transcode-start endpoint. + +## 5. Domain invariants + +### 5.1 Stable tracks + +Use the current Android-compatible identity initially: + +```text +file:{media_file_id}:audio:{ffmpeg_audio_ordinal} +file:{media_file_id}:subtitle:{combined_subtitle_ordinal} +``` + +Freeze the combined subtitle ordering used by `buildSubtitleURLs`: + +1. external subtitles in stored order, starting at zero; +2. embedded subtitles at `len(external) + embedded_ordinal`; skipped legacy + bitmap entries retain their ordinal hole; and +3. downloaded subtitles in repository `created_at` order at + `len(external) + len(all_embedded) + downloaded_ordinal`. + +The identity is paired with the effective media file ID. Centralize generation, +parsing, URL mapping, and alternate-version remapping in `tracks_v3.go`, and +freeze the exact inventory in Phase 0 fixtures. A plan that switches file +version must return the effective file's identity after matching the requested +track by signature. Do not echo a requested-file track ID with an +effective-file index. + +Longer term, scanner-persisted immutable stream IDs can replace ordinal IDs in a +future protocol version; do not change v3 identity semantics after release. + +### 5.2 Deterministic plan IDs and loop prevention + +The Android attempt key currently includes `plan_id`. Therefore a random plan +ID would let the same failed recipe reappear under a new key and defeat loop +prevention. + +Derive `plan_id` deterministically within a playback attempt from: + +- `playback_attempt_id`; +- effective file ID; +- delivery and stream protocol/container; +- normalized video/audio recipe; +- effective track IDs; +- subtitle mode/artifact type; +- sorted transformation names; and +- policy/recipe version. + +Do not include expiring URLs, JWTs, timestamps, node hostnames, or the output +route generation in `plan_id`. The same effective recipe has the same plan ID; +an output-route change remains distinguishable because Android includes +`output_route_generation` in its attempt key. + +Implement the client's FNV-1a 64-bit `v3:<16-lowercase-hex>` attempt-key +algorithm in Go. The UTF-8 canonical string is exactly: + +```text +plan_id +|KOTLIN_DELIVERY_ENUM_NAME +|KOTLIN_STREAM_PROTOCOL_ENUM_NAME +|lowercase_container_or_empty +|lowercase_video_codec_or_empty +|lowercase_audio_codec_or_empty +|(width_or_0)x(height_or_0) +|bitrate_kbps_or_0 +|lowercase_dynamic_range_or_empty +|KOTLIN_SUBTITLE_MODE_ENUM_NAME +|comma_joined_transformation_names_sorted_by_name +|output_route_generation +|comma_joined_local_mutations_sorted_lexically +``` + +The newlines above are explanatory only; the hashed value is one string joined +by literal `|` characters. Enum components are Kotlin constant names such as +`ORIGINAL_HTTP`, `HTTP_PROGRESSIVE`, `HLS`, and `BURN_IN`, not lowercase wire +tokens. Hash with offset basis `0xcbf29ce484222325` and prime +`0x100000001b3`, then zero-pad to 16 lowercase hex digits. + +Before committing a candidate, compute its base key for the request's output +route and reject it when present in `attempted_plan_keys`. Canonical fixtures +must be generated by the checked-in Kotlin implementation and consumed as +opaque expected values by Go tests; a Go-generated fixture would make the +parity test circular. Include transformation-order, empty/default-field, +output-route, and local PCM-mutation cases. + +Server-side ordered candidate IDs are not security tokens. + +### 5.3 Session and replacement semantics + +Treat replan as one replacement transaction, serialized with the per-session +lifecycle lock: + +1. validate policy, source, candidate and tooling without touching the active + route; +2. reserve the active session's existing stream/transcode capacity slot so the + replacement is neither double-counted nor denied by its own predecessor; +3. start the successor in a plan-scoped staging directory/transport generation, + or under an explicitly replacement-linked new session ID; +4. wait until the successor URL is fetchable or its startup state is validated; +5. atomically commit effective file, tracks, recipe, plan ID, timeline and + signed route; +6. close and reap the predecessor; and +7. release or transfer the capacity reservation exactly once. + +If any pre-commit step fails, leave the old route and session usable and return +a typed terminal/retryable result. Never run two FFmpeg writers against the +same output directory. Progressive request-scoped transports may drain until +the old client request disconnects; they do not require a destructive +close-first swap. + +Prefer retaining the same logical playback session so progress, scrobble and +admin history remain continuous. If the transport design requires a new public +session ID, return it explicitly and keep the old ID alive until the successor +is ready. Android will stop the old ID afterward and ignore stop failures. Keep +the legacy stop endpoint's behavior unchanged; internal v3 replacement and +cleanup paths must treat an already-stopped session as success even though the +current `SessionManager.StopSession` returns `ErrSessionNotFound`. + +### 5.4 Replan idempotency + +Create a timestamped Goose migration, generated with: + +```text +make migrate-create NAME=add_playback_protocol_v3 +``` + +Add short-lived persistence for: + +- playback attempt/session/current-plan state; +- `(session_id, replan_request_id)` request digest, state, lease and serialized + response; and +- route events. + +Current native restart reconstruction is carried by signed stream tokens, not +a durable plan database. This plan store is the first persisted v3 control- +plane state; it complements rather than replaces recipe tokens and therefore +needs explicit expiry/cleanup and token-to-plan reconciliation tests. + +For a duplicate replan ID: + +- same digest and completed state returns the cached response; +- same digest and active lease waits on/single-flights the owner, then returns + the response; +- different digest returns `409 idempotency_key_reused`; and +- an expired lease may be reclaimed after reconciling the session's current + recipe/plan state. + +Store only the minimum response needed for replay and expire it with the +playback attempt. Never put source paths or authorization headers in route +events. Signed URLs in cached responses inherit the attempt TTL and are removed +by cleanup. + +## 6. Planner rules + +### 6.1 Candidate order + +Build candidates in least-destructive order, but validate every candidate +independently: + +1. `original_http`; +2. `server_remux_progressive` when the container alone is incompatible and + progressive seek semantics are acceptable; +3. `server_remux_hls` when HLS packaging is required; +4. `server_remux_hls` with copied video plus audio adaptation, with the audio + change named in the effective recipe, transformations, claims and warnings; +5. `server_transcode_hls`; and +6. terminal `adaptation_unavailable`. + +Remux is not mandatory before transcode. Skip directly to the first valid +candidate. A selected subtitle may independently add an artifact or force +burn-in. + +### 6.2 Direct eligibility + +Original delivery requires all of: + +- validated source file and byte-range path; +- client container extractor support; +- selected video codec/profile/level/bit depth/resolution/frame rate/bitrate; +- compatible source dynamic range and current display output; +- selected audio local decode or exact sink passthrough by codec and channel + layout; +- renderable selected subtitle at the required fidelity, or an independent + converted artifact; and +- no requested quality reduction. + +Missing metadata is `unknown`, not compatible. A probe repair may run before +planning, but the planner must terminal or adapt when required fields remain +unknown. + +### 6.3 Quality policy + +Move target selection out of clients. Add a pure `QualityPolicy` that maps: + +- `original` to no user-requested reduction, while still permitting mandatory + compatibility adaptation; +- a fixed rung to a maximum height with no upscaling; and +- `auto` to the lowest of device maximum, bandwidth estimate with safety + margin, user cap, metered policy, and administrator constraints. + +Start with one server-owned ladder. Reconcile it with the existing web constants +in `web/src/player/hooks/useTranscodeQuality.ts`; do not maintain two unrelated +bitrate tables. The v3 result returns the actual height/bitrate and decision +reason. Multi-variant ABR is explicitly outside this work. + +### 6.4 Audio + +Evaluate the selected track, not `MediaFile.CodecAudio` alone: + +- exact sink passthrough codec plus channel count/layout permits audio copy and + a `passthrough=true` claim; +- client decode support permits copy with `passthrough=false`; +- otherwise use video-copy/audio adaptation when permitted; +- otherwise terminal with `transcoding_disabled`, + `audio_conversion_unsupported`, or policy/capacity reason. + +The effective recipe must report the actual output channels/layout. Do not call +AAC stereo output Atmos-preserving. DTS core extraction, E-AC-3 conversion, or +other lossy/core-only changes are named transformations with warnings. + +### 6.5 HDR and Dolby Vision + +Use `VideoTrack` fields already populated by the scanner: DV profile, BL +compatibility ID, EL presence, range type, color metadata, bit depth, profile, +level, and dimensions. + +Add an explicit enhancement-layer classification: + +```text +none | mel | fel | unknown +``` + +An analyzer interface may use an installed `dovi_tool`/libdovi implementation; +without it, Profile 7 with an EL remains `unknown`. Unknown must never be +reported as MEL, FEL, or validated native DV. + +Represent transformations in a registry with: + +- stable name and recipe version; +- required binary/filter/hardware capability; +- accepted source metadata; +- promised output range; +- argument builder; +- validation fixture; and +- failure-to-terminal mapping. + +Initial entries should cover only paths proven by fixtures, for example: + +- `dv_metadata_strip_to_hdr10` using a compatible HDR10 base layer; +- `dv_p7_to_p8_1` only when the selected toolchain and fixture prove the output; +- `hdr_to_sdr_tonemap` only when integrated and remote transcode paths both + implement the same declared result. + +FFmpeg's `dovi_rpu` bitstream filter can strip DV metadata without decoding, +and `dovi_split` can separate Profile 7 layers +([FFmpeg bitstream-filter documentation](https://ffmpeg.org/ffmpeg-bitstream-filters.html#dovi_005frpu)). +[`dovi_tool`](https://github.com/quietvoid/dovi_tool) can inspect RPU metadata +and convert compatible streams. Probe these capabilities at startup and expose +only registered, available transformations. Do not silently run the current +Profile 7 strip path and still claim Dolby Vision preservation. + +The current progressive remux path is not v3-safe unchanged: when `dovi_rpu` +is present it strips Profile 7 metadata automatically, while the no-filter path +leaves the copied bitstream without that named adaptation. Refactor remux input +to require an explicit transformation choice. A Profile 7 progressive remux is +eligible only when it either preserves a fixture-validated native P7 stream or +runs the registered `dv_metadata_strip_to_hdr10` transformation against a +validated HDR10-compatible base layer. When the required filter/tool is +missing, disqualify that remux candidate rather than emitting an ambiguous +stream. + +### 6.6 Subtitles + +Resolve exactly one mode: + +- `off`: no selected track; +- `render`: return the source artifact when Media3 can render it with required + fidelity; +- `convert`: return a v3 artifact URL, MIME, format and timing origin; +- `burn_in`: force video encoding and declare any HDR/range degradation; or +- terminal when the selected subtitle cannot meet policy. + +Map `preserve` to `require_authored_fidelity` and `compatible` to +`allow_simplified_rendering`, subject to administrator policy. ASS/SSA styling, +font attachments, PGS, VobSub and DVB must each have fixtures; do not infer all +bitmap paths from one codec test. + +Converted artifacts need session ownership, signed or authenticated fetches, +bounded cache lifetime, and timing relative to the returned stream origin. + +## 7. Transport orchestration refactor + +Extract the body of `HandleStartTranscode` into a server-callable operation that +accepts a validated normalized recipe and returns: + +- final manifest URL; +- effective local/remote node and hardware encoder; +- effective file ID; +- final codecs/resolution/bitrate/channels/range; +- player/source timeline mapping; +- seek-window semantics; and +- typed terminal/startup failure. + +Both the legacy endpoint and v3 service call this operation. The legacy handler +continues accepting its existing body and maps it to the normalized recipe; +there is no `/api/v1` breaking change. + +Jellyfin compatibility is behaviorally out of scope for protocol v3. Its +handlers currently start local/remote transcodes through parallel paths and a +shared `TranscodeManager`, not through `HandleStartTranscode`. Preserve those +wire and lifecycle paths during this work and keep their regression tests +green. The extracted lower-level starter may be adopted by jellycompat only in +a separately reviewed convergence change; do not half-migrate one of its local +or remote branches. + +Apply the same extraction to progressive remux startup where necessary. The v3 +planner chooses a recipe; the transport starter is not allowed to silently +change it. If safety logic must change copy to encode—for example seeked HEVC or +subtitle burn-in—the starter returns the effective normalized recipe so the +plan, claims, warnings, recipe card, and FFmpeg process remain identical. + +## 8. Route events and observability + +Add `playback_route_events` with bounded columns for: + +- playback/session/plan/plan-attempt identities; +- event and failure classification; +- fallback reason; +- output-route generation; +- sanitized diagnostics JSON; +- user/profile/client metadata derived from auth and headers; and +- received timestamp. + +Accept only a closed event set initially: + +```text +plan_selected +plan_invalidated +plan_failed +first_frame +terminal +stopped +``` + +Allowlist diagnostic keys and cap key count, key/value length, body size, and +events per user/attempt. Strip URLs, headers, tokens, file paths, free-form +stack traces, and unknown keys. Enqueue/batch writes so telemetry never blocks +playback planning. + +Add exact release queries for: + +- starts by delivery, recipe, dynamic range and client model; +- first-frame rate and latency; +- replan/failure rate by classification; +- terminal reasons; +- PCM retry and passthrough outcomes; +- repeated-key/loop prevention; and +- DV/HDR degradation transformations. + +Operational logs include IDs and reason codes but no stream tokens. + +## 9. Implementation sequence + +Each phase should be a separate reviewable PR and link the playback capability +epic/sub-issue. + +### Phase 0 — Freeze the contract + +- Add Go structs/enums mirroring the Kotlin v3 wire model. +- Split only the start-envelope dispatch: protocol `3` returns the disabled + no-session negotiation response, while absent/non-3 requests replay the + buffered body through the unchanged legacy decoder. +- Resolve the additive capability, effective-file, subtitle-policy and route- + diagnostic gaps from section 3.3 in a coordinated Android change. +- Add request/response JSON golden fixtures and canonical attempt-key fixtures. +- Add strict validation tests, legacy-envelope tests, and unknown-field tests. +- Add `GET /playback/capability` and the disabled negotiation response. +- Keep `playback.protocol_v3_enabled=false`. + +**Exit:** Go and Android decode the same fixtures and produce identical attempt +keys; Android populates the detailed capability fields; legacy start tests are +unchanged; disabled v3 starts allocate no legacy playback session. + +### Phase 1 — Source facts and transformation registry + +- Complete normalized source descriptors from `MediaFile` tracks. +- Add MEL/FEL/unknown metadata and conservative probe behavior. +- Add installed FFmpeg/dovi tool capability probes. +- Define transformation registry and terminal reasons. +- Add scanner/model migration or JSONB backfill handling as required. + +**Exit:** every source fact used by the planner is present or explicitly +unknown; no unavailable transformation is advertised. + +### Phase 2 — Pure planner + +- Implement capability normalization, stable tracks, quality policy, direct + eligibility, candidate ordering, recipe claims, deterministic plan IDs and + attempt-key rejection. +- Cover direct, progressive remux, HLS copy, audio adaptation, full transcode, + subtitles and terminal outcomes with table-driven tests. +- Do not start sessions or FFmpeg from planner tests. + +**Exit:** the full Phase 0 decision matrix passes as a pure deterministic +function. + +### Phase 3 — Start orchestration + +- Wire the Phase 0 dispatcher's protocol-3 branch to the completed v3 + orchestration service; keep the legacy branch unchanged. +- Extract reusable transcode/remux transport starters. +- Start sessions and final transports from the v3 service. +- Return complete plans with final URLs and timelines. +- Persist attempt/current-plan state. + +**Exit:** every playable fixture fetches and prepares its returned URL; a +terminal fixture allocates no leaked session/transcode. + +### Phase 4 — Replan and idempotency + +- Add route, handler, persistent idempotency records and per-session + serialization. +- Replan track/quality/output changes and classified runtime failures. +- Reuse session IDs when safe; close superseded transports exactly once. +- Reject attempted recipes and stale failed plan IDs. +- Add restart, duplicate request, concurrent request and crash-window tests. + +**Exit:** repeated identical replan requests return the same result without a +second FFmpeg process; a different payload under the same ID conflicts. + +### Phase 5 — Route events and rollout controls + +- Add sanitized asynchronous route-event ingestion and retention cleanup. +- Add release queries/dashboards and structured plan logs. +- Add shadow planning for legacy starts on validation servers: compute and log + v3 choice without advertising or executing it. +- Compare v3 shadow decisions with current production routes. + +**Exit:** telemetry is queryable, bounded, privacy-reviewed and cannot delay a +start/replan response. + +### Phase 6 — Hardware and multi-repository validation + +- Run original/remux/transcode fixtures locally and through proxy/transcode + nodes. +- Verify restart reconstruction, session limits, transcode-disabled policy, + alternate versions and output-route changes. +- Run Android Shield tests first on SDR/1080p, then the named 4K DV/AVR chain. +- Record plan, route events, decoder, TV range indicator and AVR format for the + same attempt. +- Publish minimum server revision and enable v3 on validation deployments. + +**Exit:** all Phase 0 fixtures pass on the deployed revision and Android no +longer reports `server_upgrade_required`. + +## 10. Test matrix + +At minimum, include: + +### Contract and compatibility + +- absent/2/3/unknown protocol versions; +- disabled and enabled feature negotiation; +- v3 nested capabilities are consumed, not silently dropped; +- legacy start response and HTTP status remain unchanged; +- old web/Apple transcode-start requests still work; +- malformed plan/terminal responses cannot be emitted. + +### Direct and adaptation + +- H.264/AAC MP4 direct; +- HEVC MKV original when the exact Media3 envelope supports it; +- container-only progressive and HLS remux; +- unsupported TrueHD/DTS layout with video copy and audio conversion; +- resolution/user-cap quality transcode; +- transcode disabled, user denied, capacity unavailable, missing tool and + policy denied terminal results; +- alternate lower-resolution file with stable track remapping. + +### HDR/DV + +- HDR10, HDR10+, HLG and SDR direct gates; +- DV profiles 5, 7 MEL, 7 FEL, 7 unknown and relevant profile 8 variants; +- BL compatibility ID and range mismatch; +- native DV, validated HDR10 base-layer fallback and SDR tone map; +- missing `dovi_rpu`/`dovi_split`/dovi-tool behavior; +- no transformation claims when output was not validated. + +### Audio and subtitles + +- local decode versus passthrough by exact channels/layout; +- E-AC-3 JOC, TrueHD, DTS-HD/core and PCM fallback replan; +- text sidecar/embedded, ASS with/without authored fidelity, PGS, VobSub, DVB, + conversion and burn-in; +- artifact timing after seeked remux/transcode; +- burn-in forces encode and reports HDR degradation. + +### Recovery and load + +- transport/decoder failures, track/quality/output invalidation; +- deterministic same-recipe loop rejection; +- idempotent duplicate and concurrent replans; +- API and transcode-node restart between plan and first manifest; +- session expiry, explicit stop and terminal cleanup; +- no double capacity accounting or orphan FFmpeg process; +- route-event flood/body/diagnostic limits. + +## 11. Verification commands + +Commands assume the repository root is the cwd. + +```text +GOWORK=off go test ./internal/playback ./internal/api/handlers ./internal/transcodenode ./internal/nodepool +GOWORK=off go test -race ./internal/playback ./internal/api/handlers +make verify-local-paths +make lint +make build +``` + +Add a focused integration target that starts PostgreSQL/Redis, creates a media +fixture, calls v3 start/replan, fetches the returned stream or manifest, submits +route events, and verifies cleanup. Contract fixtures must also run in Android +CI against its Kotlin serializers and attempt-key function. Android tests also +prove detailed codec/layout capability emission and a local PCM-recovery route +event. + +## 12. Rollout and rollback + +1. Merge schema/types with v3 disabled. +2. Deploy shadow planner and compare decisions without changing playback. +3. Enable v3 only on a validation server with the Android test build. +4. Complete SDR, HDR, DV, passthrough, subtitle, restart and node fixtures. +5. Publish the minimum server revision. +6. Enable v3 for production servers after the named soak and telemetry gates. + +Rollback is the dynamic `playback.protocol_v3_enabled` setting. Disabling it +stops advertising the feature and returns the no-session negotiation response +to new v3 starts. Existing v3 sessions remain playable until stop/expiry; do +not terminate active streams during rollback. Legacy web and Apple endpoints +remain available throughout. + +## 13. Definition of done + +- The three Android canonical endpoints and playback capability endpoint are + implemented with profile/session ownership checks. +- Every playable response is complete, final and fetchable without client-owned + recipe decisions. +- Every enabled delivery/transformation has a passing integration fixture. +- Direct decisions validate the selected track and full source/output facts. +- Android supplies the detailed per-codec and per-layout capability facts those + direct decisions consume; missing facts fail conservatively. +- Terminal reasons replace unsafe direct fallbacks. +- Replans are idempotent, loop-safe, restart-aware and capacity-safe. +- Route telemetry is bounded, sanitized and queryable. +- Existing legacy server, web, Apple, Jellyfin compatibility and audiobook + playback tests remain green. +- Android passes direct/remux/transcode/replan tests against the deployed server + revision, followed by the documented 4K Dolby Vision/AVR validation. diff --git a/docs/wiki/admin/media-folder-and-naming.md b/docs/wiki/admin/media-folder-and-naming.md index 962e3750..6e5d4786 100644 --- a/docs/wiki/admin/media-folder-and-naming.md +++ b/docs/wiki/admin/media-folder-and-naming.md @@ -199,6 +199,14 @@ Silo often sees sidecar files that mirror the media basename: These are common and expected. Episode naming guidance in this page refers to the media files themselves, but sidecars may legitimately reuse the same stem. +Some NFO sidecars are actively read: `movie.nfo`, `tvshow.nfo`, `.nfo`, +`season.nfo`, `.nfo`, and sidecar artwork (season posters, episode +thumbnails) feed the built-in NFO metadata provider — including `` provider IDs, +which act as trusted matching anchors just like `{tmdb-...}` folder tags. NFO files never +change how files are grouped; naming and layout still decide series/season/episode +structure. See [Local NFO Metadata](nfo-local-metadata.md) for the supported fields and +merge behavior. + Supplemental directories next to a movie (and directly under a series root) are scanned as **extras** attached to that item, following the Jellyfin/Plex folder convention: diff --git a/docs/wiki/admin/nfo-local-metadata.md b/docs/wiki/admin/nfo-local-metadata.md new file mode 100644 index 00000000..c8533f58 --- /dev/null +++ b/docs/wiki/admin/nfo-local-metadata.md @@ -0,0 +1,177 @@ +--- +title: Local NFO Metadata +description: How Silo reads Kodi/Jellyfin-style NFO sidecar files and merges them with online metadata providers. +summary: Supported NFO fields, merge and refresh semantics, and the naming-supplies-structure contract for curated libraries. +tags: + - silo + - docs + - wiki + - libraries + - metadata + - nfo +audience: + - operator + - end-user +last_reviewed: 2026-07-12 +related: + - ../index.md + - media-folder-and-naming.md +--- + +# Local NFO Metadata + +Silo ships a built-in **NFO Files** metadata provider that reads Kodi/Jellyfin-style `.nfo` +sidecar files. It runs inside the same provider chain as online providers (TMDB, TVDB, ...), +so one toggle and one priority per library control it. By default it sits at priority 1: the +NFO wins every field it declares, and online providers fill in whatever it leaves out. + +Clients can detect whether a library type includes the NFO provider via +`GET /api/v1/libraries/provider-defaults`. + +## Where Silo looks for NFO files + +For each movie or series, Silo checks (in order): + +1. `movie.nfo` / `tvshow.nfo` in the item's folder +2. `.nfo` next to the media file (e.g. `My Movie (2021).nfo`) + +For series, Silo additionally reads: + +- `season.nfo` inside each season directory (`` root: title, plot) +- `.nfo` next to each episode file (`` root: title, + plot, air date, runtime, ratings) +- Season posters: `poster.jpg` (or `folder`/`cover`) inside the season directory, or + `seasonNN-poster.jpg` in the series root +- Episode thumbnails: `-thumb.jpg` + +A sidecar only applies when its root element matches the item type: a stray `movie.nfo` in +a series folder is skipped in favor of `tvshow.nfo`, and a `tvshow.nfo` next to a movie +file is ignored. Files that fail to parse are skipped silently and the next candidate is +tried — a broken NFO never blocks matching. + +Season and episode numbers declared inside an NFO (``, ``, +``) are advisory: the directory name and the `SxxEyy` filename pattern decide +where metadata lands, and a mismatch is logged and resolved in the filename's favor. +Multi-episode NFO documents (several `` blocks in one file) are not +supported; the first block is used and a warning is logged. + +## Supported fields + +| NFO element | Applies to | Silo field | +|---|---|---| +| `` | movie, series | Title | +| `<originaltitle>` | movie, series | Original title | +| `<tagline>` | movie, series | Tagline | +| `<plot>` | movie, series | Overview | +| `<year>` | movie, series | Year (derived from the premiere date when omitted) | +| `<runtime>` | movie, series | Runtime in minutes (non-numeric values ignored) | +| `<premiered>` / `<releasedate>` | movie | Release date (`YYYY-MM-DD`) | +| `<premiered>` / `<aired>` | series | First air date (`YYYY-MM-DD`) | +| `<mpaa>` | movie, series | Content rating | +| `<genre>` (repeated) | movie, series | Genres | +| `<studio>` (repeated) | movie, series | Studios | +| `<country>` (repeated) | movie, series | Countries | +| `<tag>` (repeated) | movie, series | Keywords | +| `<ratings>` / `<rating>` | movie, series | Ratings (named sources: `imdb`, `tmdb`/`themoviedb`, `tomatometerallcritics`, `tomatometerallaudience`; a legacy bare `<rating>` fills the IMDB slot) | +| `<actor>` (`name`/`role`/`order`) | movie, series | Cast (actor `<thumb>` URLs are ignored) | +| `<director>`, `<credits>` | movie, series | Crew (directors and writers) | +| `<uniqueid type="tmdb|imdb|tvdb">` | movie, series | Trusted identity anchor for matching | +| `<title>`, `<plot>` (`season.nfo`) | season | Season name and overview | +| `<title>`, `<plot>`, `<aired>`, `<runtime>`, `<ratings>` (`<episodedetails>`) | episode | Episode title, overview, air date, runtime, ratings | + +Not yet supported: `<set>` (collections) — a planned follow-up. `<userrating>` is read +but not stored (Silo has no per-user rating field). Unknown elements are ignored, so +exports from Kodi, Jellyfin, or tinyMediaManager work as-is. + +## How NFO data merges with online providers + +- **NFO wins what it declares.** With the NFO provider at priority 1, every field the NFO + populates beats the same field from online providers. +- **Online providers backfill the rest.** Anything the NFO leaves out (cast, artwork, + runtime, ratings, ...) is filled by the next providers in the chain, keyed by the + `<uniqueid>` values when present. +- **Genres are all-or-nothing.** The first provider that supplies any genres supplies the + whole list; later providers do not append to it. An NFO with no `<genre>` entries leaves + genres entirely to online providers. +- **Field locks are respected.** Fields locked in the Edit Metadata dialog (including + Year and release/air dates) are never overwritten by any provider, NFO included. + +## Editing an NFO after the initial match + +Scheduled background refreshes only fill empty fields — they intentionally do **not** +re-apply NFO changes over existing values. To pick up edits: + +**Edit the NFO, then use Refresh metadata on the item.** A manual refresh re-reads the +sidecar and replaces all unlocked fields with the merged provider results, NFO first. + +## Naming supplies structure, NFO supplies presentation + +NFO files describe *what an item is called and about* — they do not change *how files are +grouped*. Folder layout and filenames still decide what is a series, a season, and an +episode (see [Supported Media Folder Structures and Naming](media-folder-and-naming.md)). +This makes fully local libraries with no online match work well, for example a fitness +library: + +``` +Fitness/ + P90X/ + tvshow.nfo # show title/plot (no <uniqueid> needed) + poster.jpg fanart.jpg + Season 01/ + season.nfo # "Course A: Classic" + poster.jpg # season poster + P90X S01E01 - Chest and Back.mkv + P90X S01E01 - Chest and Back.nfo # episode title/plot + P90X S01E01 - Chest and Back-thumb.jpg # episode thumbnail +``` + +The folder and episode naming build the series/season/episode tree; the NFO sidecars +supply every name, description, and image — even when no online provider knows the +content. Episodes without an `.nfo` still appear with their generic "Episode N" title, +so partially curated libraries degrade gracefully. + +## Mixed libraries (sports example) + +A single library of type **Mixed** can hold movie-shaped and show-shaped content side by +side — the common case is sports, where pay-per-view events are movies and the weekly +show is a series, and most of it has no TMDB/TVDB presence: + +``` +WWE/ + WrestleMania 41 (2025)/ + WrestleMania 41 (2025).mkv + movie.nfo # event title/plot (a <uniqueid> is optional) + poster.jpg + WWE SmackDown/ + tvshow.nfo + Season 27/ + season.nfo + poster.jpg + WWE SmackDown S27E15.mkv + WWE SmackDown S27E15.nfo + WWE SmackDown S27E15-thumb.jpg +``` + +How classification works in a mixed library: + +- **Naming decides the type, per file, before any metadata provider runs.** An `SxxEyy` + pattern or a `Season NN` folder routes a file to the series lane; everything else is a + movie. An NFO never flips the type: a `tvshow.nfo` sitting next to a movie-classified + file is ignored (the root element must match the item type), not applied. +- **NFOs supply metadata and identity only.** Once the type is fixed, the NFO provider + behaves exactly as in dedicated movie/series libraries — including season and episode + depth for the show-shaped content. An event whose `movie.nfo` carries a + `<uniqueid type="tmdb">` still gets full online enrichment, while ID-less events next + to it fall back to their local NFO and artwork. +- **Misclassified group?** Override its **Type** on the root in the library admin + (Auto → Movie/Series) and rescan — that is the correction path; fixing the NFO alone + will not re-route a file. + +Tools that generate sports libraries (sportarr-style managers) should therefore write +standard `SxxEyy` naming for show content and standalone movie naming (`Title (Year)/`) +for events, and place NFOs/posters exactly as in the dedicated-library layouts above. + +## Source References + +- `internal/metadata/nfo/` — parser and provider +- `internal/metadata/merge.go` — merge semantics and field locks diff --git a/docs/wiki/index.md b/docs/wiki/index.md index 92b01ce5..2e79245d 100644 --- a/docs/wiki/index.md +++ b/docs/wiki/index.md @@ -35,6 +35,8 @@ rewriting. reference for the folder layouts and filenames Silo can scan and match today. - [Collection Templates](admin/collection-templates.md) - Curated, one-click starting points for synced library collections sourced from TMDB, Trakt, and MDBList. +- [Local NFO Metadata](admin/nfo-local-metadata.md) - Supported NFO sidecar fields, how they merge + with online providers, and the naming-supplies-structure contract. ## Deployment diff --git a/go.mod b/go.mod index 4205c237..669174e9 100644 --- a/go.mod +++ b/go.mod @@ -3,9 +3,9 @@ module github.com/Silo-Server/silo-server go 1.26.4 require ( - github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/credentials v1.19.10 - github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 + github.com/aws/aws-sdk-go-v2 v1.42.1 + github.com/aws/aws-sdk-go-v2/credentials v1.19.29 + github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2 github.com/go-chi/chi/v5 v5.2.5 github.com/golang-jwt/jwt/v5 v5.3.1 github.com/google/uuid v1.6.0 @@ -22,6 +22,7 @@ require ( require ( github.com/SherClockHolmes/webpush-go v1.4.0 github.com/abadojack/whatlanggo v1.0.1 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 github.com/go-chi/cors v1.2.2 github.com/gorilla/websocket v1.5.3 github.com/h2non/bimg v1.1.9 @@ -109,15 +110,15 @@ require ( require ( github.com/Silo-Server/silo-plugin-sdk v0.10.0 - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect - github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/smithy-go v1.24.2 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 // indirect + github.com/aws/smithy-go v1.27.3 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect @@ -144,7 +145,7 @@ require ( golang.org/x/sync v0.21.0 golang.org/x/text v0.38.0 golang.org/x/time v0.15.0 - google.golang.org/grpc v1.81.1 + google.golang.org/grpc v1.82.1 google.golang.org/protobuf v1.36.11 ) diff --git a/go.sum b/go.sum index 80bffd6f..5c5c4ac2 100644 --- a/go.sum +++ b/go.sum @@ -16,30 +16,44 @@ github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x0 github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q= github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE= -github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= -github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= -github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/credentials v1.19.10 h1:EEhmEUFCE1Yhl7vDhNOI5OCL/iKMdkkYFTRpZXNw7m8= -github.com/aws/aws-sdk-go-v2/credentials v1.19.10/go.mod h1:RnnlFCAlxQCkN2Q379B67USkBMu1PipEEiibzYN5UTE= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22 h1:rWyie/PxDRIdhNf4DzRk0lvjVOqFJuNnO8WwaIRVxzQ= -github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.22/go.mod h1:zd/JsJ4P7oGfUhXn1VyLqaRZwPmZwg44Jf2dS84Dm3Y= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= -github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 h1:JRaIgADQS/U6uXDqlPiefP32yXTda7Kqfx+LgspooZM= -github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13/go.mod h1:CEuVn5WqOMilYl+tbccq8+N2ieCy0gVn3OtRb0vBNNM= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 h1:c31//R3xgIJMSC8S6hEVq+38DcvUlgFY0FM6mSI5oto= -github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21/go.mod h1:r6+pf23ouCB718FUxaqzZdbpYFyDtehyZcmP5KL9FkA= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWURB8avufQq9gFsheUgjVD9536obIknfM= -github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= -github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= -github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/aws/aws-sdk-go-v2 v1.42.1 h1:9eOTgu1z/dVtYpNZ3/8/XbbaX0x/BqE3HUzAzs6K0ek= +github.com/aws/aws-sdk-go-v2 v1.42.1/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E= +github.com/aws/aws-sdk-go-v2/config v1.32.30 h1:XwsEzpTJfQYJbFicz/QMLwAZdyeNVVoOEkbF7R3gPJk= +github.com/aws/aws-sdk-go-v2/config v1.32.30/go.mod h1:Ud32SuMc+/9BGxfpSVld7HrE2o05JwKmXY4M3jOQNZU= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29 h1:WHZGssHH887cO0ox07SIQZsFx3MKD4ps6w0xUEmnKYQ= +github.com/aws/aws-sdk-go-v2/credentials v1.19.29/go.mod h1:Mhl0xR6zjguiuj00XRx2wMx22sAltk7oya39sT7fdg8= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30 h1:/hi1JADLEW9YYryEz1w4GQu0EtP23pP553Cf9KgsDV4= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.30/go.mod h1:/3AOgy4K17Dm4ucMZVC/MJkzy5kmfKUcINRHZyo0koQ= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34 h1:Pn7OsMwBLbkZ6OnCxWHAjf0L/22H8cnhxZC0uPwtMtg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.22.34/go.mod h1:eToXR/Gk1uqpn04eSmdgVXwfS0WvH8aG4eBFr8ygbpU= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30 h1:xM/Is9cKMHa8Jj8zkvWhvrFkZsXJV9E+BB4g0HW0duQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.30/go.mod h1:WueJeNDZvK1fMYEWJIkcivBfEzUkTpBhzlrUKKY8EuA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30 h1:jn46zC9LdsVR/ZpMIJqMqb8hHv31BlLx3ulVqNspUOk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.30/go.mod h1:1hTMsAgbdS/AtUi4bw8+gUuh1pceo+eXRLfpSuSQj3M= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31 h1:3GUprIsfmGcC5SACIyB0e7E0BM1O1b3Erl5CePYIAeQ= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.31/go.mod h1:7PuV1yl5e2xnUbm+RqvVg5i2iBM8EyijZNoI9wsOoOc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23 h1:9Fjh6fi/U5JEStVZijmaMpUwE/gvBJj7x2B/PjbO9To= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.23/go.mod h1:iMoT2f1tClxrWAAnKCXjZQ6LOmfLrMG14wmnWpM+F14= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30 h1:/Z5jmNrKsSD7EmDjzAPsm/3L9IuOkzaynklJZ1qX7S4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.30/go.mod h1:lEzEZnOosE7zi8Z6royW1cFJTD9fpab4Ul1SBrllewk= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31 h1:uao4A3QZ5UmB326V6KF+qRpv9Tjz7IlnlnTbbANntlU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.31/go.mod h1:I/1+z0VwL1GhQyLgkoHDlygpUZ+iTAwOQ/NsftiUL2I= +github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2 h1:5C00eQYpTrgQXnp6V3P6P7zPElna3AXvlukbANE6nJI= +github.com/aws/aws-sdk-go-v2/service/s3 v1.105.2/go.mod h1:zdmCoFO/dSI7GlrwsPqFJI+WlFnSU4Tc8TJnlXrM1Do= +github.com/aws/aws-sdk-go-v2/service/signin v1.4.1 h1:V7ZZ300WPXGjvkyore5DGe0ljVPOxCXie/thWdtSBXE= +github.com/aws/aws-sdk-go-v2/service/signin v1.4.1/go.mod h1:mxC0nT/C8wMMS97DemZPzvUZxvIt+2Iq+eS3JdFZGgg= +github.com/aws/aws-sdk-go-v2/service/sso v1.32.1 h1:gYFYh4iLLcAOJRLNPY2aD2g9DIhKn4eof8UkIrr1rTk= +github.com/aws/aws-sdk-go-v2/service/sso v1.32.1/go.mod h1:u8af9Nqkmqnr96f7v9nHqzZT9XBwbXEkTiqT4ROuJSE= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1 h1:arjT9Cm3/WYbGmD5TUZHk4UQn4Lle1fUNZs5FC6CtF0= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.37.1/go.mod h1:DMPWJBjYs6+3+f/qhBFEFPPlQ6NlhWjai3dJNvipJ84= +github.com/aws/aws-sdk-go-v2/service/sts v1.44.1 h1:RvfHDg+xvAeZ+5741vUEjpOVtYSIm93W2zhx10Xtydw= +github.com/aws/aws-sdk-go-v2/service/sts v1.44.1/go.mod h1:9gdl4RrflIdpDb2TlXshWgR1F9TeCkvqDx77Vpr4Z/Q= +github.com/aws/smithy-go v1.27.3 h1:F3Zb497UhhskkfpJmfkXswyo+t0sh9OTBnIHjogWbVY= +github.com/aws/smithy-go v1.27.3/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= @@ -442,8 +456,8 @@ google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa h1: google.golang.org/genproto/googleapis/api v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:q4lMZS6kskjT5HvCPrnnypcDPVJqT/f4nfxmkE7gryY= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa h1:mZHHdPZl0dbGHCflZgAq/Q468DWVFcU2whhB2KAo8fk= google.golang.org/genproto/googleapis/rpc v0.0.0-20260526163538-3dc84a4a5aaa/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= -google.golang.org/grpc v1.81.1 h1:VnnIIZ88UzOOKLukQi+ImGz8O1Wdp8nAGGnvOfEIWQQ= -google.golang.org/grpc v1.81.1/go.mod h1:xGH9GfzOyMTGIOXBJmXt+BX/V0kcdQbdcuwQ/zNw42I= +google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE= +google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/internal/access/resolver_test.go b/internal/access/resolver_test.go index 5b6c4bca..5a6448ec 100644 --- a/internal/access/resolver_test.go +++ b/internal/access/resolver_test.go @@ -124,7 +124,7 @@ func (s stubStore) DeleteHomeDismissal(context.Context, string, string, string) panic("unused") } func (s stubStore) AddFavorite(context.Context, string, string) error { panic("unused") } -func (s stubStore) AddFavoriteAt(context.Context, string, string, time.Time) error { +func (s stubStore) AddFavoriteAt(context.Context, string, string, time.Time) (bool, error) { panic("unused") } func (s stubStore) RemoveFavorite(context.Context, string, string) error { diff --git a/internal/api/handlers/admin.go b/internal/api/handlers/admin.go index f5631e54..979df400 100644 --- a/internal/api/handlers/admin.go +++ b/internal/api/handlers/admin.go @@ -30,6 +30,7 @@ import ( "github.com/Silo-Server/silo-server/internal/catalog" "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/diagnostics" evt "github.com/Silo-Server/silo-server/internal/events" "github.com/Silo-Server/silo-server/internal/markers" "github.com/Silo-Server/silo-server/internal/models" @@ -64,6 +65,12 @@ type ServerSettingsStore interface { GetAll(ctx context.Context) (map[string]string, error) } +type DiagnosticsEnablementStore interface { + PutStream(ctx context.Context, bucket, key string, r io.Reader, contentType string) error + DeleteObject(ctx context.Context, bucket, key string) error + Bucket() string +} + type AdminJobCreator interface { Create(ctx context.Context, input adminjob.CreateJobInput) (*models.AdminJob, error) CreateLibraryRefresh( @@ -97,6 +104,7 @@ type AdminHandler struct { EventBus cache.EventBus EventsHub *evt.Hub SettingsRepo ServerSettingsStore + DiagnosticsStore DiagnosticsEnablementStore JobRepo AdminJobCreator ItemRefreshResolver ItemRefreshScopeResolver ImpersonationService ImpersonationService @@ -2215,6 +2223,38 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques return } req.Value = strconv.FormatBool(enabled) + case diagnostics.KeyUploadsEnabled: + enabled, err := strconv.ParseBool(strings.TrimSpace(req.Value)) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", diagnostics.KeyUploadsEnabled+" must be true or false") + return + } + req.Value = strconv.FormatBool(enabled) + if enabled { + if err := h.validateDiagnosticsUploadsEnabled(r.Context()); err != nil { + writeError(w, http.StatusBadRequest, "storage_unavailable", err.Error()) + return + } + } + case diagnostics.KeyMaxBundleBytes, + diagnostics.KeyMaxUncompressedBytes, + diagnostics.KeyMaxReportsPerUserDay, + diagnostics.KeyRetentionDays, + diagnostics.KeyMaxBytesPerUser: + normalized, err := h.normalizeDiagnosticsNumericSetting(r.Context(), key, req.Value) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + req.Value = normalized + case diagnostics.KeyConsentNoticeVersion: + n, err := strconv.Atoi(strings.TrimSpace(req.Value)) + if err != nil || n < 1 { + writeError(w, http.StatusBadRequest, "bad_request", + diagnostics.KeyConsentNoticeVersion+" must be an integer greater than 0") + return + } + req.Value = strconv.Itoa(n) case policy.SettingDecisionLogScopeSampleRate: n, err := strconv.Atoi(strings.TrimSpace(req.Value)) if err != nil || n <= 0 { @@ -2243,10 +2283,11 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques "subtitle_ai.transcribe_quota_period must be day, week, or month") return } - case notifications.SettingApplePushDeliveryEnabled: + case notifications.SettingApplePushDeliveryEnabled, + notifications.SettingAndroidPushDeliveryEnabled: enabled, err := strconv.ParseBool(strings.TrimSpace(req.Value)) if err != nil { - writeError(w, http.StatusBadRequest, "bad_request", "notifications.apple_push_delivery_enabled must be true or false") + writeError(w, http.StatusBadRequest, "bad_request", key+" must be true or false") return } req.Value = strconv.FormatBool(enabled) @@ -2382,3 +2423,85 @@ func (h *AdminHandler) HandleUpdateSetting(w http.ResponseWriter, r *http.Reques } writeJSON(w, http.StatusOK, adminSettingResponse{Key: key, Value: req.Value, RestartRequired: restartRequired}) } + +func (h *AdminHandler) validateDiagnosticsUploadsEnabled(ctx context.Context) error { + if h.DiagnosticsStore == nil || strings.TrimSpace(h.DiagnosticsStore.Bucket()) == "" { + return errors.New("diagnostics uploads require configured private object storage") + } + const probeKey = "diagnostics/.probe" + if err := h.DiagnosticsStore.PutStream( + ctx, + h.DiagnosticsStore.Bucket(), + probeKey, + strings.NewReader("ok"), + "application/octet-stream", + ); err != nil { + return fmt.Errorf("diagnostics storage probe write failed: %w", err) + } + if err := h.DiagnosticsStore.DeleteObject(ctx, h.DiagnosticsStore.Bucket(), probeKey); err != nil { + return fmt.Errorf("diagnostics storage probe delete failed: %w", err) + } + return nil +} + +func (h *AdminHandler) normalizeDiagnosticsNumericSetting(ctx context.Context, key, raw string) (string, error) { + const ( + mib = int64(1024 * 1024) + gib = int64(1024 * mib) + ) + + value, err := strconv.ParseInt(strings.TrimSpace(raw), 10, 64) + if err != nil { + return "", fmt.Errorf("%s must be an integer", key) + } + + settings := diagnostics.DefaultSettings() + if h.SettingsRepo != nil { + loaded, loadErr := diagnostics.LoadSettings(ctx, h.SettingsRepo) + if loadErr != nil { + return "", fmt.Errorf("load diagnostics settings: %w", loadErr) + } + settings = loaded + } + + switch key { + case diagnostics.KeyMaxBundleBytes: + if value < mib || value > 256*mib { + return "", fmt.Errorf("%s must be between 1 MiB (%d bytes) and 256 MiB (%d bytes)", key, mib, 256*mib) + } + if value > settings.MaxUncompressedBytes { + return "", fmt.Errorf("%s must not exceed %s (%d bytes)", key, diagnostics.KeyMaxUncompressedBytes, settings.MaxUncompressedBytes) + } + // A single bundle can never exceed the per-user byte cap, or every upload + // at this size would fail quota; keep the two bounds consistent. + if value > settings.MaxBytesPerUser { + return "", fmt.Errorf("%s must not exceed %s (%d bytes)", key, diagnostics.KeyMaxBytesPerUser, settings.MaxBytesPerUser) + } + case diagnostics.KeyMaxUncompressedBytes: + if value < settings.MaxBundleBytes || value > gib { + return "", fmt.Errorf("%s must be between %s (%d bytes) and 1 GiB (%d bytes)", key, diagnostics.KeyMaxBundleBytes, settings.MaxBundleBytes, gib) + } + case diagnostics.KeyMaxReportsPerUserDay: + if value < 1 || value > 1000 { + return "", fmt.Errorf("%s must be between 1 and 1000", key) + } + case diagnostics.KeyRetentionDays: + if value < 1 || value > 365 { + return "", fmt.Errorf("%s must be between 1 and 365", key) + } + case diagnostics.KeyMaxBytesPerUser: + if value < 10*mib || value > 10*gib { + return "", fmt.Errorf("%s must be between 10 MiB (%d bytes) and 10 GiB (%d bytes)", key, 10*mib, 10*gib) + } + // The per-user cap must leave room for at least one max-size bundle, or + // /diagnostics/status would advertise a bundle size InsertReceiving always + // rejects as quota_exceeded. + if value < settings.MaxBundleBytes { + return "", fmt.Errorf("%s must be at least %s (%d bytes)", key, diagnostics.KeyMaxBundleBytes, settings.MaxBundleBytes) + } + default: + return "", fmt.Errorf("unsupported diagnostics numeric setting %s", key) + } + + return strconv.FormatInt(value, 10), nil +} diff --git a/internal/api/handlers/admin_apple_push.go b/internal/api/handlers/admin_apple_push.go index f5b858a5..241175a5 100644 --- a/internal/api/handlers/admin_apple_push.go +++ b/internal/api/handlers/admin_apple_push.go @@ -1,6 +1,7 @@ package handlers import ( + "context" "encoding/json" "errors" "math" @@ -64,8 +65,21 @@ type adminPushRelayRegisterResponse struct { // HandleTest handles POST /admin/notifications/push/apple/test. func (h *AdminApplePushHandler) HandleTest(w http.ResponseWriter, r *http.Request) { + h.handleTest(w, r, "Apple", func(ctx context.Context, profileID, serverDeviceID string) (*notifications.ApplePushTestResult, error) { + return h.system.SendApplePushTest(ctx, profileID, serverDeviceID) + }) +} + +// HandleTestAndroid handles POST /admin/notifications/push/fcm/test. +func (h *AdminApplePushHandler) HandleTestAndroid(w http.ResponseWriter, r *http.Request) { + h.handleTest(w, r, "Android", func(ctx context.Context, profileID, serverDeviceID string) (*notifications.ApplePushTestResult, error) { + return h.system.SendAndroidPushTest(ctx, profileID, serverDeviceID) + }) +} + +func (h *AdminApplePushHandler) handleTest(w http.ResponseWriter, r *http.Request, platformLabel string, send func(context.Context, string, string) (*notifications.ApplePushTestResult, error)) { if h == nil || h.system == nil { - writeError(w, http.StatusServiceUnavailable, "unavailable", "Apple push delivery is not available") + writeError(w, http.StatusServiceUnavailable, "unavailable", platformLabel+" push delivery is not available") return } var req adminApplePushTestRequest @@ -73,17 +87,17 @@ func (h *AdminApplePushHandler) HandleTest(w http.ResponseWriter, r *http.Reques writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body") return } - result, err := h.system.SendApplePushTest(r.Context(), req.ProfileID, req.ServerDeviceID) + result, err := send(r.Context(), req.ProfileID, req.ServerDeviceID) if err != nil { switch { case errors.Is(err, notifications.ErrPushDeliveryInvalid): writeError(w, http.StatusBadRequest, "bad_request", err.Error()) case errors.Is(err, notifications.ErrPushDeliveryNotFound): - writeError(w, http.StatusNotFound, "not_found", "Apple push device not found") + writeError(w, http.StatusNotFound, "not_found", platformLabel+" push device not found") case errors.Is(err, notifications.ErrPushDeliveryUnavailable): - writeError(w, http.StatusServiceUnavailable, "unavailable", "Apple push delivery is not available") + writeError(w, http.StatusServiceUnavailable, "unavailable", platformLabel+" push delivery is not available") default: - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to send Apple push test") + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to send "+platformLabel+" push test") } return } diff --git a/internal/api/handlers/admin_diagnostics.go b/internal/api/handlers/admin_diagnostics.go new file mode 100644 index 00000000..0ce035f3 --- /dev/null +++ b/internal/api/handlers/admin_diagnostics.go @@ -0,0 +1,265 @@ +package handlers + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "strconv" + "strings" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/diagnostics" +) + +const diagnosticsDownloadExpiry = 15 * time.Minute + +type AdminDiagnosticsService interface { + ListForAdmin(ctx context.Context, filters diagnostics.ListFilters) (diagnostics.ListResult, error) + GetReport(ctx context.Context, id string) (*diagnostics.Report, error) + PresignReportDownload(ctx context.Context, report *diagnostics.Report, expiry time.Duration) (string, error) + OpenReportDownload(ctx context.Context, report *diagnostics.Report) (io.ReadCloser, error) + DeleteReport(ctx context.Context, id string) (*diagnostics.Report, error) +} + +type adminDiagnosticsEffectiveTTL interface { + EffectiveReportDownloadTTL(requested time.Duration) time.Duration +} + +type diagnosticsDownloadURLResponse struct { + DownloadURL string `json:"download_url"` + ExpiresAt time.Time `json:"expires_at"` +} + +func RegisterAdminDiagnosticsRoutes(r chi.Router, h *DiagnosticsHandler) { + if h == nil { + return + } + r.Route("/diagnostics/reports", func(r chi.Router) { + r.Get("/", h.HandleAdminListReports) + r.Get("/{id}", h.HandleAdminGetReport) + r.Get("/{id}/download", h.HandleAdminDownloadReport) + r.Delete("/{id}", h.HandleAdminDeleteReport) + }) +} + +func (h *DiagnosticsHandler) HandleAdminListReports(w http.ResponseWriter, r *http.Request) { + admin, ok := h.admin() + if !ok { + writeDiagnosticsAdminUnavailable(w) + return + } + filters, err := parseDiagnosticsReportListFilters(r) + if err != nil { + writeDiagnosticsAdminError(w, err) + return + } + result, err := admin.ListForAdmin(r.Context(), filters) + if err != nil { + writeDiagnosticsAdminError(w, err) + return + } + writeJSON(w, http.StatusOK, result) +} + +func (h *DiagnosticsHandler) HandleAdminGetReport(w http.ResponseWriter, r *http.Request) { + admin, ok := h.admin() + if !ok { + writeDiagnosticsAdminUnavailable(w) + return + } + report, err := admin.GetReport(r.Context(), chi.URLParam(r, "id")) + if err != nil { + writeDiagnosticsAdminError(w, err) + return + } + writeJSON(w, http.StatusOK, report) +} + +func (h *DiagnosticsHandler) HandleAdminDownloadReport(w http.ResponseWriter, r *http.Request) { + admin, ok := h.admin() + if !ok { + writeDiagnosticsAdminUnavailable(w) + return + } + reportID := chi.URLParam(r, "id") + report, err := admin.GetReport(r.Context(), reportID) + if err != nil { + writeDiagnosticsAdminError(w, err) + return + } + if report.State != diagnostics.StateReady { + writeDiagnosticsAdminError(w, diagnostics.ErrReportNotReady) + return + } + + ttl := diagnosticsEffectiveDownloadTTL(admin, diagnosticsDownloadExpiry) + if !diagnosticsProxyDownloadRequested(r) { + if url, err := admin.PresignReportDownload(r.Context(), report, ttl); err == nil && strings.TrimSpace(url) != "" { + h.auditDownload(r, report.ID) + writeJSON(w, http.StatusOK, diagnosticsDownloadURLResponse{ + DownloadURL: url, + ExpiresAt: time.Now().UTC().Add(ttl), + }) + return + } + } + + body, err := admin.OpenReportDownload(r.Context(), report) + if err != nil { + writeDiagnosticsAdminError(w, err) + return + } + defer body.Close() + + h.auditDownload(r, report.ID) + w.Header().Set("Content-Type", diagnostics.ReportDownloadContentType) + w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=%q", diagnosticsReportFilename(report))) + if report.BlobBytes != nil && *report.BlobBytes >= 0 { + w.Header().Set("Content-Length", strconv.FormatInt(*report.BlobBytes, 10)) + } + w.WriteHeader(http.StatusOK) + if _, err := io.Copy(w, body); err != nil { + h.diagnosticsLogger().WarnContext(r.Context(), "diagnostic report download stream failed", + "component", "diagnostics", + "admin_user_id", currentAdminUserID(r), + "report_id", report.ID, + "error", err, + ) + } +} + +func (h *DiagnosticsHandler) HandleAdminDeleteReport(w http.ResponseWriter, r *http.Request) { + admin, ok := h.admin() + if !ok { + writeDiagnosticsAdminUnavailable(w) + return + } + reportID := chi.URLParam(r, "id") + report, err := admin.DeleteReport(r.Context(), reportID) + if err != nil { + writeDiagnosticsAdminError(w, err) + return + } + h.diagnosticsLogger().InfoContext(r.Context(), "diagnostic report deleted", + "component", "diagnostics", + "admin_user_id", currentAdminUserID(r), + "report_id", report.ID, + ) + w.WriteHeader(http.StatusNoContent) +} + +func (h *DiagnosticsHandler) admin() (AdminDiagnosticsService, bool) { + if h.adminService == nil { + return nil, false + } + return h.adminService, true +} + +func (h *DiagnosticsHandler) diagnosticsLogger() *slog.Logger { + if h.logger == nil { + return slog.Default() + } + return h.logger +} + +func (h *DiagnosticsHandler) auditDownload(r *http.Request, reportID string) { + h.diagnosticsLogger().InfoContext(r.Context(), "diagnostic report downloaded", + "component", "diagnostics", + "admin_user_id", currentAdminUserID(r), + "report_id", reportID, + ) +} + +func parseDiagnosticsReportListFilters(r *http.Request) (diagnostics.ListFilters, error) { + filters := diagnostics.ListFilters{ + Platform: strings.TrimSpace(r.URL.Query().Get("platform")), + ReportType: strings.TrimSpace(r.URL.Query().Get("report_type")), + Cursor: strings.TrimSpace(r.URL.Query().Get("cursor")), + Limit: parseLimit(r, 50), + } + + userID, err := parseOptionalIntQuery(r, "user_id") + if err != nil { + return diagnostics.ListFilters{}, err + } + filters.UserID = userID + + if from, err := parseTimeQuery(r, "from"); err != nil { + return diagnostics.ListFilters{}, err + } else { + filters.From = from + } + if to, err := parseTimeQuery(r, "to"); err != nil { + return diagnostics.ListFilters{}, err + } else { + filters.To = to + } + + shortID := strings.TrimSpace(r.URL.Query().Get("short_id")) + if shortID != "" { + normalized, err := diagnostics.ParseShortID(shortID) + if err != nil { + return diagnostics.ListFilters{}, err + } + filters.ShortID = normalized + } + return filters, nil +} + +func diagnosticsReportFilename(report *diagnostics.Report) string { + name := strings.TrimSpace(report.ShortID) + if name == "" { + name = strings.TrimSpace(report.ID) + } + if name == "" { + name = "report" + } + return "silo-diagnostics-" + name + ".tar.gz" +} + +func diagnosticsProxyDownloadRequested(r *http.Request) bool { + raw := strings.TrimSpace(r.URL.Query().Get("proxy")) + return raw == "1" || strings.EqualFold(raw, "true") || strings.EqualFold(raw, "yes") +} + +func diagnosticsEffectiveDownloadTTL(admin AdminDiagnosticsService, requested time.Duration) time.Duration { + ttl := requested + if effective, ok := admin.(adminDiagnosticsEffectiveTTL); ok { + ttl = effective.EffectiveReportDownloadTTL(requested) + } + if ttl <= 0 { + return requested + } + return ttl +} + +func writeDiagnosticsAdminUnavailable(w http.ResponseWriter) { + writeError(w, http.StatusServiceUnavailable, "service_unavailable", "Diagnostics reports are not configured") +} + +func writeDiagnosticsAdminError(w http.ResponseWriter, err error) { + var parseErr *requestParseError + switch { + case errors.As(err, &parseErr): + writeError(w, http.StatusBadRequest, "bad_request", parseErr.Error()) + case errors.Is(err, diagnostics.ErrInvalidShortID): + writeError(w, http.StatusBadRequest, "bad_request", "Invalid short_id") + case errors.Is(err, diagnostics.ErrInvalidCursor): + writeError(w, http.StatusBadRequest, "bad_request", "Invalid cursor") + case errors.Is(err, diagnostics.ErrNotFound): + writeError(w, http.StatusNotFound, "not_found", "Diagnostic report not found") + case errors.Is(err, diagnostics.ErrReportNotReady): + writeError(w, http.StatusConflict, "not_ready", "Diagnostic report is not ready") + case errors.Is(err, diagnostics.ErrStorageUnavailable): + writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "Diagnostics storage is not available") + case diagnostics.IsObjectNotFound(err): + writeError(w, http.StatusNotFound, "not_found", "Diagnostic report bundle not found") + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Diagnostics report operation failed") + } +} diff --git a/internal/api/handlers/admin_images.go b/internal/api/handlers/admin_images.go index 4a941f15..5905cac9 100644 --- a/internal/api/handlers/admin_images.go +++ b/internal/api/handlers/admin_images.go @@ -7,10 +7,12 @@ import ( "fmt" "log/slog" "net/http" + "time" "github.com/go-chi/chi/v5" "github.com/Silo-Server/silo-server/internal/catalog" + evt "github.com/Silo-Server/silo-server/internal/events" "github.com/Silo-Server/silo-server/internal/metadata" "github.com/Silo-Server/silo-server/internal/models" ) @@ -52,6 +54,7 @@ type AdminImageHandler struct { imageSvc ImageService imageResolver ImageURLResolver detailSvc *catalog.DetailService + EventsHub *evt.Hub } // NewAdminImageHandler creates a handler for admin image selection endpoints. @@ -110,6 +113,8 @@ type applyItemImageResponse struct { ContentID string `json:"content_id"` StoredPath string `json:"stored_path"` Thumbhash string `json:"thumbhash"` + ImageURL string `json:"image_url,omitempty"` + Revision string `json:"revision,omitempty"` } // resolvedItem holds the result of looking up a content ID across all three tables. @@ -304,6 +309,18 @@ func (h *AdminImageHandler) HandleApplyItemImage(w http.ResponseWriter, r *http. } imageType := metadata.ImageTypeFromString(req.Type) + // Episodes only have stills. Clients historically sent "poster" here (the + // old flow silently dropped it), so coerce rather than reject. + if resolved.contentType == "episode" { + imageType = metadata.ImageStill + } + // Reject unsupported target/image combinations before spending a download + // and an S3 upload on an image that can never be published. + if err := catalog.ValidateArtworkSelectionTarget(resolved.contentType, metadata.ImageTypeToString(imageType)); err != nil { + writeError(w, http.StatusBadRequest, "unsupported_image_type", + fmt.Sprintf("%s items do not accept %s images", resolved.contentType, metadata.ImageTypeToString(imageType))) + return + } providerID := req.ProviderID if providerID == "" { providerID = primaryProvider(resolved.parentItem) @@ -344,52 +361,54 @@ func (h *AdminImageHandler) HandleApplyItemImage(w http.ResponseWriter, r *http. return } - // Build the MetadataUpdate targeting only the relevant image field. - upd := buildImageUpdate(imageType, result.StoredPath, result.Thumbhash) - - // Lock FieldImages on the parent media_items row to prevent - // automatic refreshes from overwriting the manual selection. - if resolved.contentType == "movie" || resolved.contentType == "series" { - locked := mergeLockedField(resolved.parentItem.LockedFields, int(metadata.FieldImages)) - upd.LockedFields = &locked - } - - // Persist to the correct table. - if err := h.persistImageUpdate(r.Context(), contentID, resolved, &upd); err != nil { + err = h.detailSvc.PublishArtworkSelection(r.Context(), catalog.ArtworkSelection{ + TargetType: resolved.contentType, + TargetContentID: contentID, + ParentContentID: resolved.parentItem.ContentID, + ImageType: metadata.ImageTypeToString(imageType), + StoredPath: result.StoredPath, + SourcePath: req.OriginalURL, + Thumbhash: result.Thumbhash, + LockField: int(metadata.FieldImages), + }) + if err != nil { slog.ErrorContext(r.Context(), "admin images: persist failed", "component", "api", "content_id", contentID, "error", err) - writeError(w, http.StatusInternalServerError, "internal_error", "Image cached but failed to update item") + if cleanupErr := h.detailSvc.QueueArtworkRevisionGC( + r.Context(), result.StoredPath, metadata.ImageTypeToString(imageType), time.Now().Add(time.Hour), + ); cleanupErr != nil { + slog.ErrorContext(r.Context(), "admin images: failed to queue unpublished artwork cleanup", "component", "api", + "content_id", contentID, "stored_path", result.StoredPath, "error", cleanupErr) + } + switch { + case errors.Is(err, catalog.ErrItemNotFound), + errors.Is(err, catalog.ErrSeasonNotFound), + errors.Is(err, catalog.ErrEpisodeNotFound): + writeError(w, http.StatusNotFound, "not_found", "Item not found") + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Image cached but failed to update item") + } return } - // Also lock FieldImages on the parent series for seasons/episodes. - if resolved.contentType == "season" || resolved.contentType == "episode" { - parentLocked := mergeLockedField(resolved.parentItem.LockedFields, int(metadata.FieldImages)) - parentUpd := catalog.MetadataUpdate{LockedFields: &parentLocked} - if lockErr := h.detailSvc.UpdateMediaItemMetadata(r.Context(), resolved.parentItem.ContentID, &parentUpd); lockErr != nil { - slog.ErrorContext(r.Context(), "admin images: failed to lock FieldImages on parent series", "component", "api", - "content_id", contentID, "parent_id", resolved.parentItem.ContentID, "error", lockErr) - } + publishEventMetadataUpdate(r.Context(), h.EventsHub, 0, contentID) + if resolved.parentItem.ContentID != contentID { + publishEventMetadataUpdate(r.Context(), h.EventsHub, 0, resolved.parentItem.ContentID) + } + + imageURL := "" + if h.imageResolver != nil { + imageURL = h.imageResolver.ResolveImageURL(r.Context(), result.StoredPath, "original") } writeJSON(w, http.StatusOK, applyItemImageResponse{ ContentID: contentID, StoredPath: result.StoredPath, Thumbhash: result.Thumbhash, + ImageURL: imageURL, + Revision: result.Revision, }) } -// persistImageUpdate writes the image path update to the correct table. -func (h *AdminImageHandler) persistImageUpdate(ctx context.Context, contentID string, resolved *resolvedItem, upd *catalog.MetadataUpdate) error { - switch resolved.contentType { - case "season": - return h.detailSvc.UpdateSeasonMetadata(ctx, contentID, upd) - case "episode": - return h.detailSvc.UpdateEpisodeMetadata(ctx, contentID, upd) - default: - return h.detailSvc.UpdateMediaItemMetadata(ctx, contentID, upd) - } -} - // resolveImageFolderID finds the primary library folder for a content ID. func (h *AdminImageHandler) resolveImageFolderID(ctx context.Context, contentID string) (int, error) { if h.folders == nil { @@ -417,38 +436,6 @@ func buildProviderIDs(item *models.MediaItem) map[string]string { return ids } -// buildImageUpdate constructs a MetadataUpdate targeting a single image field. -func buildImageUpdate(imageType metadata.ImageType, storedPath, thumbhash string) catalog.MetadataUpdate { - var upd catalog.MetadataUpdate - switch imageType { - case metadata.ImageBackdrop: - upd.BackdropPath = &storedPath - upd.BackdropThumbhash = &thumbhash - case metadata.ImageLogo: - upd.LogoPath = &storedPath - case metadata.ImageStill: - upd.StillPath = &storedPath - upd.StillThumbhash = &thumbhash - default: // poster - upd.PosterPath = &storedPath - upd.PosterThumbhash = &thumbhash - } - return upd -} - -// mergeLockedField adds a field value to the locked fields slice if not already present. -func mergeLockedField(existing []int, field int) []int { - for _, f := range existing { - if f == field { - return existing - } - } - result := make([]int, len(existing)+1) - copy(result, existing) - result[len(existing)] = field - return result -} - // primaryProvider returns the primary provider slug for a media item. func primaryProvider(item *models.MediaItem) string { if item.TmdbID != "" { diff --git a/internal/api/handlers/diagnostics.go b/internal/api/handlers/diagnostics.go new file mode 100644 index 00000000..2c3db0ea --- /dev/null +++ b/internal/api/handlers/diagnostics.go @@ -0,0 +1,373 @@ +package handlers + +import ( + "context" + "errors" + "io" + "log/slog" + "mime" + "mime/multipart" + "net/http" + "strings" + "sync" + "time" + + apimw "github.com/Silo-Server/silo-server/internal/api/middleware" + "github.com/Silo-Server/silo-server/internal/auth" + "github.com/Silo-Server/silo-server/internal/diagnostics" +) + +const ( + diagnosticsMultipartOverheadBytes = int64(128 * 1024) + diagnosticsBusyRetryAfter = "5" + diagnosticsQuotaRetryAfter = "60" + // diagnosticsUploadReadTimeout replaces the shared 30s server ReadTimeout + // for this route only. Bundles are up to 10 MiB (admin-tunable higher) and + // mobile crash uploads over slow uplinks routinely exceed 30s; without this + // net/http aborts the read mid-stream before Ingest can return a retryable + // error. The bound stays generous but finite so a stalled upload still can't + // hold the connection open indefinitely. + diagnosticsUploadReadTimeout = 10 * time.Minute +) + +var ( + errDiagnosticsPartTooLarge = errors.New("diagnostics multipart part too large") + errDiagnosticsUnexpectedPart = errors.New("diagnostics multipart contains unexpected part") +) + +type DiagnosticsService interface { + Status(ctx context.Context, userID int) (diagnostics.Status, error) + Ingest(ctx context.Context, userID int, profileID *string, manifestJSON []byte, bundle io.Reader) (diagnostics.IngestResult, error) +} + +type DiagnosticsHandler struct { + service DiagnosticsService + adminService AdminDiagnosticsService + inflight *diagnosticsInFlightLimiter + logger *slog.Logger +} + +func NewDiagnosticsHandler(service DiagnosticsService) *DiagnosticsHandler { + handler := &DiagnosticsHandler{ + service: service, + inflight: newDiagnosticsInFlightLimiter(4), + logger: slog.Default(), + } + if adminService, ok := service.(AdminDiagnosticsService); ok { + handler.adminService = adminService + } + return handler +} + +func (h *DiagnosticsHandler) HandleStatus(w http.ResponseWriter, r *http.Request) { + userID, ok := diagnosticsUserID(w, r) + if !ok { + return + } + status, err := h.service.Status(r.Context(), userID) + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load diagnostics status") + return + } + writeJSON(w, http.StatusOK, status) +} + +func (h *DiagnosticsHandler) HandleUpload(w http.ResponseWriter, r *http.Request) { + // Extend the read deadline for this request only; the server-wide + // ReadTimeout is too short for slow, large bundle uploads (see the constant). + rc := http.NewResponseController(w) + if err := rc.SetReadDeadline(time.Now().Add(diagnosticsUploadReadTimeout)); err != nil { + h.diagnosticsLogger().WarnContext(r.Context(), "diagnostics upload read deadline not extended", + "component", "diagnostics", + "error", err, + ) + } + // Extend the write deadline the same way. cmd/silo's integrated server sets + // WriteTimeout 120s, but a slow upload can take longer than that within the + // read window above; Ingest may store and mark the report ready and then the + // final writeJSON would miss the 120s write deadline, so the client sees a + // timeout and retries a report that already succeeded. + if err := rc.SetWriteDeadline(time.Now().Add(diagnosticsUploadReadTimeout)); err != nil { + h.diagnosticsLogger().WarnContext(r.Context(), "diagnostics upload write deadline not extended", + "component", "diagnostics", + "error", err, + ) + } + + userID, ok := diagnosticsUserID(w, r) + if !ok { + claims := apimw.GetClaims(r.Context()) + if claims != nil && claims.TokenType == auth.TokenTypeAPIKey { + h.logRejected(r.Context(), claims.UserID, "api_key_not_allowed") + } + return + } + + status, err := h.service.Status(r.Context(), userID) + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load diagnostics status") + return + } + maxBundleBytes := status.MaxBundleBytes + if maxBundleBytes <= 0 { + maxBundleBytes = diagnostics.DefaultMaxBundleBytes + } + r.Body = http.MaxBytesReader(w, r.Body, maxBundleBytes+diagnosticsMultipartOverheadBytes) + + switch status.Status { + case diagnostics.StatusDisabled: + writeError(w, http.StatusForbidden, "disabled", "Diagnostics uploads are disabled") + return + case diagnostics.StatusStorageUnavailable: + writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "Diagnostics storage is not configured") + return + case diagnostics.StatusAvailable: + default: + writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "Diagnostics storage is not available") + return + } + + release, acquired := h.inflight.acquire(userID) + if !acquired { + h.logRejected(r.Context(), userID, "busy") + w.Header().Set("Retry-After", diagnosticsBusyRetryAfter) + writeError(w, http.StatusServiceUnavailable, "busy", "Diagnostics upload capacity is busy") + return + } + defer release() + + mr, err := r.MultipartReader() + if err != nil { + h.writeDiagnosticsMultipartError(r.Context(), userID, w, err) + return + } + + manifestPart, err := nextDiagnosticsPart(mr, "manifest", "application/json") + if err != nil { + h.writeDiagnosticsMultipartError(r.Context(), userID, w, err) + return + } + manifestJSON, err := readDiagnosticsPart(manifestPart, diagnostics.MaxManifestBytes) + _ = manifestPart.Close() + if err != nil { + h.writeDiagnosticsMultipartError(r.Context(), userID, w, err) + return + } + + bundlePart, err := nextDiagnosticsPart(mr, "bundle", diagnostics.BundleContentType) + if err != nil { + h.writeDiagnosticsMultipartError(r.Context(), userID, w, err) + return + } + defer bundlePart.Close() + + profileID := strings.TrimSpace(r.Header.Get("X-Profile-Id")) + var profileIDPtr *string + if profileID != "" { + profileIDPtr = &profileID + } + + result, err := h.service.Ingest( + r.Context(), + userID, + profileIDPtr, + manifestJSON, + &exactlyTwoPartBundleReader{part: bundlePart, mr: mr}, + ) + if err != nil { + writeDiagnosticsServiceError(w, err) + return + } + writeJSON(w, http.StatusCreated, result) +} + +func diagnosticsUserID(w http.ResponseWriter, r *http.Request) (int, bool) { + if !hasBearerAuthorizationHeader(r) { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authorization bearer token required") + return 0, false + } + claims := apimw.GetClaims(r.Context()) + if claims == nil { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return 0, false + } + if claims.TokenType == auth.TokenTypeAPIKey { + writeError(w, http.StatusForbidden, "api_key_not_allowed", "API keys cannot upload diagnostics") + return 0, false + } + if claims.TokenType != auth.TokenTypeAccess { + writeError(w, http.StatusForbidden, "forbidden", "Diagnostics require a user access token") + return 0, false + } + if claims.UserID <= 0 { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return 0, false + } + return claims.UserID, true +} + +func hasBearerAuthorizationHeader(r *http.Request) bool { + header := r.Header.Get("Authorization") + if header == "" { + return false + } + parts := strings.SplitN(header, " ", 2) + return len(parts) == 2 && strings.EqualFold(parts[0], "bearer") && strings.TrimSpace(parts[1]) != "" +} + +func nextDiagnosticsPart(mr *multipart.Reader, expectedName, expectedContentType string) (*multipart.Part, error) { + part, err := mr.NextPart() + if err != nil { + return nil, err + } + // Reject a mismatched part without calling part.Close(): Close drains the + // unread part body first, so a max-size wrongly-named/typed first part would + // stream up to the bundle limit — holding the per-user/global in-flight slot — + // before we return 400. Abandoning the part returns immediately; net/http then + // discards only a small bounded prefix of the request before closing the + // connection, so malformed uploads fail promptly under load. + if part.FormName() != expectedName { + return nil, errDiagnosticsUnexpectedPart + } + if !diagnosticsContentTypeMatches(part.Header.Get("Content-Type"), expectedContentType) { + return nil, errDiagnosticsUnexpectedPart + } + return part, nil +} + +func diagnosticsContentTypeMatches(raw, expected string) bool { + mediaType, _, err := mime.ParseMediaType(raw) + if err != nil { + return false + } + return strings.EqualFold(mediaType, expected) +} + +func readDiagnosticsPart(part *multipart.Part, limit int64) ([]byte, error) { + data, err := io.ReadAll(io.LimitReader(part, limit+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > limit { + return nil, errDiagnosticsPartTooLarge + } + return data, nil +} + +type exactlyTwoPartBundleReader struct { + part *multipart.Part + mr *multipart.Reader + checked bool +} + +func (r *exactlyTwoPartBundleReader) Read(p []byte) (int, error) { + n, err := r.part.Read(p) + if errors.Is(err, io.EOF) && !r.checked { + r.checked = true + next, nextErr := r.mr.NextPart() + if errors.Is(nextErr, io.EOF) { + return n, io.EOF + } + if nextErr != nil { + return n, nextErr + } + _ = next.Close() + return n, errDiagnosticsUnexpectedPart + } + return n, err +} + +func (h *DiagnosticsHandler) writeDiagnosticsMultipartError(ctx context.Context, userID int, w http.ResponseWriter, err error) { + var maxBytesErr *http.MaxBytesError + switch { + case errors.As(err, &maxBytesErr), errors.Is(err, errDiagnosticsPartTooLarge): + h.logRejected(ctx, userID, "too_large") + writeError(w, http.StatusRequestEntityTooLarge, "too_large", "Diagnostics upload is too large") + default: + h.logRejected(ctx, userID, "invalid_bundle") + writeError(w, http.StatusBadRequest, "invalid_bundle", "Invalid diagnostics upload") + } +} + +func (h *DiagnosticsHandler) logRejected(ctx context.Context, userID int, reason string) { + logger := h.logger + if logger == nil { + logger = slog.Default() + } + args := []any{ + "component", "diagnostics", + "result", "rejected", + "reason", reason, + } + if userID > 0 { + args = append(args, "user_id", userID) + } + logger.InfoContext(ctx, "diagnostic report rejected", args...) +} + +func writeDiagnosticsServiceError(w http.ResponseWriter, err error) { + var maxBytesErr *http.MaxBytesError + switch { + case errors.As(err, &maxBytesErr), errors.Is(err, diagnostics.ErrTooLarge): + writeError(w, http.StatusRequestEntityTooLarge, "too_large", "Diagnostics upload is too large") + case errors.Is(err, diagnostics.ErrDisabled): + writeError(w, http.StatusForbidden, "disabled", "Diagnostics uploads are disabled") + case errors.Is(err, diagnostics.ErrStorageUnavailable): + writeError(w, http.StatusServiceUnavailable, "storage_unavailable", "Diagnostics storage is not configured") + case errors.Is(err, diagnostics.ErrQuotaExceeded): + w.Header().Set("Retry-After", diagnosticsQuotaRetryAfter) + writeError(w, http.StatusTooManyRequests, "quota_exceeded", "Diagnostics upload quota exceeded") + case errors.Is(err, diagnostics.ErrUnsupportedSchema): + writeError(w, http.StatusBadRequest, "unsupported_schema", "Diagnostics schema version is not supported") + case errors.Is(err, diagnostics.ErrDestinationMismatch): + writeError(w, http.StatusBadRequest, "destination_mismatch", "Diagnostics destination does not match this server") + case errors.Is(err, diagnostics.ErrStaleConsent): + writeError(w, http.StatusBadRequest, "stale_consent", "Diagnostics consent notice is stale") + case errors.Is(err, diagnostics.ErrArchiveMismatch): + writeError(w, http.StatusBadRequest, "archive_mismatch", "Diagnostics archive metadata does not match") + case errors.Is(err, diagnostics.ErrProfileMismatch): + writeError(w, http.StatusBadRequest, "profile_mismatch", "Diagnostics profile does not match the captured report") + case errors.Is(err, diagnostics.ErrChildProfileForbidden): + writeError(w, http.StatusForbidden, "child_profile_forbidden", "Diagnostics cannot be attributed to a child profile") + case errors.Is(err, diagnostics.ErrInvalidBundle): + writeError(w, http.StatusBadRequest, "invalid_bundle", "Invalid diagnostics bundle") + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Diagnostics upload failed") + } +} + +type diagnosticsInFlightLimiter struct { + mu sync.Mutex + active map[int]struct{} + global chan struct{} +} + +func newDiagnosticsInFlightLimiter(globalLimit int) *diagnosticsInFlightLimiter { + if globalLimit <= 0 { + globalLimit = 1 + } + return &diagnosticsInFlightLimiter{ + active: make(map[int]struct{}), + global: make(chan struct{}, globalLimit), + } +} + +func (l *diagnosticsInFlightLimiter) acquire(userID int) (func(), bool) { + l.mu.Lock() + defer l.mu.Unlock() + if _, ok := l.active[userID]; ok { + return nil, false + } + select { + case l.global <- struct{}{}: + l.active[userID] = struct{}{} + default: + return nil, false + } + return func() { + l.mu.Lock() + delete(l.active, userID) + l.mu.Unlock() + <-l.global + }, true +} diff --git a/internal/api/handlers/diagnostics_test.go b/internal/api/handlers/diagnostics_test.go new file mode 100644 index 00000000..df01ba9c --- /dev/null +++ b/internal/api/handlers/diagnostics_test.go @@ -0,0 +1,876 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/textproto" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + apimw "github.com/Silo-Server/silo-server/internal/api/middleware" + "github.com/Silo-Server/silo-server/internal/auth" + "github.com/Silo-Server/silo-server/internal/diagnostics" +) + +func TestDiagnosticsUploadHappyPath(t *testing.T) { + service := newFakeDiagnosticsService() + handler := NewDiagnosticsHandler(service) + + rec := httptest.NewRecorder() + handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, []diagnosticsPart{ + {name: "manifest", contentType: "application/json", body: []byte(`{"ok":true}`)}, + {name: "bundle", contentType: "application/gzip", body: []byte("bundle")}, + }, accessClaims())) + + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusCreated, rec.Body.String()) + } + var resp diagnostics.IngestResult + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.ShortID != "SILO-ABCDEF123456" { + t.Fatalf("short_id = %q, want SILO-ABCDEF123456", resp.ShortID) + } + if service.ingestCalls != 1 { + t.Fatalf("ingest calls = %d, want 1", service.ingestCalls) + } +} + +func TestDiagnosticsUploadManifestTooBig(t *testing.T) { + service := newFakeDiagnosticsService() + handler := NewDiagnosticsHandler(service) + logs := captureDiagnosticsLogs(handler) + + rec := httptest.NewRecorder() + handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, []diagnosticsPart{ + {name: "manifest", contentType: "application/json", body: bytes.Repeat([]byte("x"), int(diagnostics.MaxManifestBytes)+1)}, + {name: "bundle", contentType: "application/gzip", body: []byte("bundle")}, + }, accessClaims())) + + assertDiagnosticsError(t, rec, http.StatusRequestEntityTooLarge, "too_large") + if service.ingestCalls != 0 { + t.Fatalf("ingest calls = %d, want 0", service.ingestCalls) + } + assertDiagnosticsRejectionLog(t, logs, "too_large", 42) +} + +func TestDiagnosticsUploadWrongPartOrder(t *testing.T) { + service := newFakeDiagnosticsService() + handler := NewDiagnosticsHandler(service) + logs := captureDiagnosticsLogs(handler) + + rec := httptest.NewRecorder() + handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, []diagnosticsPart{ + {name: "bundle", contentType: "application/gzip", body: []byte("bundle")}, + {name: "manifest", contentType: "application/json", body: []byte(`{"ok":true}`)}, + }, accessClaims())) + + assertDiagnosticsError(t, rec, http.StatusBadRequest, "invalid_bundle") + if service.ingestCalls != 0 { + t.Fatalf("ingest calls = %d, want 0", service.ingestCalls) + } + assertDiagnosticsRejectionLog(t, logs, "invalid_bundle", 42) +} + +func TestDiagnosticsUploadWrongFirstPartNotDrained(t *testing.T) { + service := newFakeDiagnosticsService() + handler := NewDiagnosticsHandler(service) + + // A large, wrongly-named first part must be rejected without draining its + // body: an invalid upload must not be allowed to stream its whole payload + // (holding the in-flight slot) before it gets a 400. + var body bytes.Buffer + writer := multipart.NewWriter(&body) + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", `form-data; name="bundle"`) + header.Set("Content-Type", "application/gzip") + w, err := writer.CreatePart(header) + if err != nil { + t.Fatalf("create multipart part: %v", err) + } + if _, err := w.Write(bytes.Repeat([]byte("x"), 1<<20)); err != nil { + t.Fatalf("write multipart part: %v", err) + } + if err := writer.Close(); err != nil { + t.Fatalf("close multipart writer: %v", err) + } + + counter := &countingReader{r: &body} + req := httptest.NewRequest(http.MethodPost, "/api/v1/diagnostics/reports", counter) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Authorization", "Bearer token") + req = req.WithContext(apimw.SetClaims(req.Context(), accessClaims())) + + rec := httptest.NewRecorder() + handler.HandleUpload(rec, req) + + assertDiagnosticsError(t, rec, http.StatusBadRequest, "invalid_bundle") + if service.ingestCalls != 0 { + t.Fatalf("ingest calls = %d, want 0", service.ingestCalls) + } + // Only the small multipart framing/headers should have been read; the 1 MiB + // wrong-first-part body must not have been drained. + if counter.n > 128*1024 { + t.Fatalf("read %d bytes from request body, want the wrong first part rejected without draining (< 128 KiB)", counter.n) + } +} + +type countingReader struct { + r io.Reader + n int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n += int64(n) + return n, err +} + +func TestDiagnosticsUploadBodyTooLargeLogsRejection(t *testing.T) { + service := newFakeDiagnosticsService() + service.status.MaxBundleBytes = 1 + handler := NewDiagnosticsHandler(service) + logs := captureDiagnosticsLogs(handler) + + body := strings.NewReader(strings.Repeat("x\n", int(diagnosticsMultipartOverheadBytes/2)+2)) + req := httptest.NewRequest(http.MethodPost, "/api/v1/diagnostics/reports", body) + req.Header.Set("Content-Type", "multipart/form-data; boundary=diagnostics-boundary") + req.Header.Set("Authorization", "Bearer token") + req = req.WithContext(apimw.SetClaims(req.Context(), accessClaims())) + rec := httptest.NewRecorder() + handler.HandleUpload(rec, req) + + assertDiagnosticsError(t, rec, http.StatusRequestEntityTooLarge, "too_large") + if service.ingestCalls != 0 { + t.Fatalf("ingest calls = %d, want 0", service.ingestCalls) + } + assertDiagnosticsRejectionLog(t, logs, "too_large", 42) +} + +func TestDiagnosticsUploadMissingPart(t *testing.T) { + service := newFakeDiagnosticsService() + handler := NewDiagnosticsHandler(service) + + rec := httptest.NewRecorder() + handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, []diagnosticsPart{ + {name: "manifest", contentType: "application/json", body: []byte(`{"ok":true}`)}, + }, accessClaims())) + + assertDiagnosticsError(t, rec, http.StatusBadRequest, "invalid_bundle") + if service.ingestCalls != 0 { + t.Fatalf("ingest calls = %d, want 0", service.ingestCalls) + } +} + +func TestDiagnosticsUploadDisabled(t *testing.T) { + service := newFakeDiagnosticsService() + service.status.Status = diagnostics.StatusDisabled + handler := NewDiagnosticsHandler(service) + + rec := httptest.NewRecorder() + handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, validDiagnosticsParts(), accessClaims())) + + assertDiagnosticsError(t, rec, http.StatusForbidden, "disabled") + if service.ingestCalls != 0 { + t.Fatalf("ingest calls = %d, want 0", service.ingestCalls) + } +} + +func TestDiagnosticsUploadStorageUnavailable(t *testing.T) { + service := newFakeDiagnosticsService() + service.status.Status = diagnostics.StatusStorageUnavailable + handler := NewDiagnosticsHandler(service) + + rec := httptest.NewRecorder() + handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, validDiagnosticsParts(), accessClaims())) + + assertDiagnosticsError(t, rec, http.StatusServiceUnavailable, "storage_unavailable") + if service.ingestCalls != 0 { + t.Fatalf("ingest calls = %d, want 0", service.ingestCalls) + } +} + +func TestDiagnosticsUploadQuotaExceededSetsRetryAfter(t *testing.T) { + service := newFakeDiagnosticsService() + service.ingestErr = &diagnostics.QuotaError{Kind: diagnostics.QuotaKindReportsPerDay, Limit: 20} + handler := NewDiagnosticsHandler(service) + + rec := httptest.NewRecorder() + handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, validDiagnosticsParts(), accessClaims())) + + assertDiagnosticsError(t, rec, http.StatusTooManyRequests, "quota_exceeded") + if got := rec.Header().Get("Retry-After"); got != diagnosticsQuotaRetryAfter { + t.Fatalf("Retry-After = %q, want %q", got, diagnosticsQuotaRetryAfter) + } +} + +func TestDiagnosticsUploadRejectsAPIKey(t *testing.T) { + service := newFakeDiagnosticsService() + handler := NewDiagnosticsHandler(service) + logs := captureDiagnosticsLogs(handler) + + rec := httptest.NewRecorder() + req := newDiagnosticsUploadRequest(t, validDiagnosticsParts(), &auth.Claims{ + UserID: 42, + TokenType: auth.TokenTypeAPIKey, + }) + req.Header.Set("Authorization", "Bearer sa_test") + handler.HandleUpload(rec, req) + + assertDiagnosticsError(t, rec, http.StatusForbidden, "api_key_not_allowed") + if service.statusCalls != 0 { + t.Fatalf("status calls = %d, want 0", service.statusCalls) + } + assertDiagnosticsRejectionLog(t, logs, "api_key_not_allowed", 42) +} + +func TestDiagnosticsUploadBusy(t *testing.T) { + service := newFakeDiagnosticsService() + service.started = make(chan struct{}) + service.block = make(chan struct{}) + handler := NewDiagnosticsHandler(service) + logs := captureDiagnosticsLogs(handler) + + firstReq := newDiagnosticsUploadRequest(t, validDiagnosticsParts(), accessClaims()) + firstDone := make(chan *httptest.ResponseRecorder, 1) + go func() { + rec := httptest.NewRecorder() + handler.HandleUpload(rec, firstReq) + firstDone <- rec + }() + <-service.started + + rec := httptest.NewRecorder() + handler.HandleUpload(rec, newDiagnosticsUploadRequest(t, validDiagnosticsParts(), accessClaims())) + + assertDiagnosticsError(t, rec, http.StatusServiceUnavailable, "busy") + if got := rec.Header().Get("Retry-After"); got != diagnosticsBusyRetryAfter { + t.Fatalf("Retry-After = %q, want %q", got, diagnosticsBusyRetryAfter) + } + assertDiagnosticsRejectionLog(t, logs, "busy", 42) + + close(service.block) + first := <-firstDone + if first.Code != http.StatusCreated { + t.Fatalf("first status = %d, want %d; body=%s", first.Code, http.StatusCreated, first.Body.String()) + } +} + +func TestDiagnosticsAdminListReportsParsesFilters(t *testing.T) { + service := newFakeDiagnosticsService() + handler := NewDiagnosticsHandler(service) + req := httptest.NewRequest( + http.MethodGet, + "/api/v1/admin/diagnostics/reports?user_id=42&platform=ios&report_type=crash&from=2026-07-19T10:00:00Z&to=2026-07-20T10:00:00Z&short_id=abcdef123456&limit=25&cursor=next", + nil, + ) + rec := httptest.NewRecorder() + + handler.HandleAdminListReports(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + if service.listCalls != 1 { + t.Fatalf("list calls = %d, want 1", service.listCalls) + } + filters := service.listFilters + if filters.UserID == nil || *filters.UserID != 42 { + t.Fatalf("UserID = %v, want 42", filters.UserID) + } + if filters.Platform != "ios" || filters.ReportType != "crash" { + t.Fatalf("filters = %#v, want platform ios report_type crash", filters) + } + if filters.From == nil || !filters.From.Equal(time.Date(2026, 7, 19, 10, 0, 0, 0, time.UTC)) { + t.Fatalf("From = %v, want expected timestamp", filters.From) + } + if filters.To == nil || !filters.To.Equal(time.Date(2026, 7, 20, 10, 0, 0, 0, time.UTC)) { + t.Fatalf("To = %v, want expected timestamp", filters.To) + } + if filters.ShortID != "SILO-ABCDEF123456" || filters.Limit != 25 || filters.Cursor != "next" { + t.Fatalf("filters = %#v, want normalized short id, limit, cursor", filters) + } +} + +func TestDiagnosticsAdminDownloadReturnsPresignedURLWhenAvailable(t *testing.T) { + service := newFakeDiagnosticsService() + service.getReport = readyDiagnosticsReport() + service.presignURL = "https://storage.example.test/report" + service.effectiveTTL = 5 * time.Minute + handler := NewDiagnosticsHandler(service) + + before := time.Now().UTC() + rec := httptest.NewRecorder() + adminDiagnosticsRouter(handler).ServeHTTP(rec, adminDiagnosticsRequest(http.MethodGet, "/diagnostics/reports/report-1/download", adminClaims())) + after := time.Now().UTC() + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + var resp diagnosticsDownloadURLResponse + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.DownloadURL != service.presignURL { + t.Fatalf("download_url = %q, want %q", resp.DownloadURL, service.presignURL) + } + if service.presignExpiry != service.effectiveTTL { + t.Fatalf("presign expiry = %s, want %s", service.presignExpiry, service.effectiveTTL) + } + if resp.ExpiresAt.Before(before.Add(service.effectiveTTL)) || resp.ExpiresAt.After(after.Add(service.effectiveTTL)) { + t.Fatalf("expires_at = %s, want within effective TTL window", resp.ExpiresAt) + } + if service.openCalls != 0 { + t.Fatalf("open calls = %d, want 0", service.openCalls) + } +} + +func TestDiagnosticsAdminDownloadStreamsWhenProxyRequested(t *testing.T) { + service := newFakeDiagnosticsService() + service.getReport = readyDiagnosticsReport() + service.presignURL = "https://storage.example.test/report" + service.openData = []byte("gzipped report") + handler := NewDiagnosticsHandler(service) + + rec := httptest.NewRecorder() + adminDiagnosticsRouter(handler).ServeHTTP(rec, adminDiagnosticsRequest(http.MethodGet, "/diagnostics/reports/report-1/download?proxy=1", adminClaims())) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + if service.presignCalls != 0 { + t.Fatalf("presign calls = %d, want 0", service.presignCalls) + } + if service.openCalls != 1 { + t.Fatalf("open calls = %d, want 1", service.openCalls) + } + if rec.Body.String() != "gzipped report" { + t.Fatalf("body = %q, want streamed data", rec.Body.String()) + } +} + +func TestDiagnosticsAdminDownloadStreamsWhenPresignUnavailable(t *testing.T) { + service := newFakeDiagnosticsService() + service.getReport = readyDiagnosticsReport() + service.presignErr = errors.New("presign unavailable") + service.openData = []byte("gzipped report") + handler := NewDiagnosticsHandler(service) + + rec := httptest.NewRecorder() + adminDiagnosticsRouter(handler).ServeHTTP(rec, adminDiagnosticsRequest(http.MethodGet, "/diagnostics/reports/report-1/download", adminClaims())) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + if got := rec.Header().Get("Content-Type"); got != diagnostics.ReportDownloadContentType { + t.Fatalf("Content-Type = %q, want %q", got, diagnostics.ReportDownloadContentType) + } + if !strings.Contains(rec.Header().Get("Content-Disposition"), "silo-diagnostics-SILO-ABCDEF123456.tar.gz") { + t.Fatalf("Content-Disposition = %q, want diagnostic filename", rec.Header().Get("Content-Disposition")) + } + if rec.Body.String() != "gzipped report" { + t.Fatalf("body = %q, want streamed data", rec.Body.String()) + } +} + +func TestDiagnosticsAdminListReportsRejectsMalformedQuery(t *testing.T) { + service := newFakeDiagnosticsService() + handler := NewDiagnosticsHandler(service) + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics/reports?user_id=abc", nil) + rec := httptest.NewRecorder() + + handler.HandleAdminListReports(rec, req) + + assertDiagnosticsError(t, rec, http.StatusBadRequest, "bad_request") + if service.listCalls != 0 { + t.Fatalf("list calls = %d, want 0", service.listCalls) + } +} + +func TestDiagnosticsAdminListReportsRejectsMalformedCursor(t *testing.T) { + service := newFakeDiagnosticsService() + service.listErr = diagnostics.ErrInvalidCursor + handler := NewDiagnosticsHandler(service) + req := httptest.NewRequest(http.MethodGet, "/api/v1/admin/diagnostics/reports?cursor=bad", nil) + rec := httptest.NewRecorder() + + handler.HandleAdminListReports(rec, req) + + assertDiagnosticsError(t, rec, http.StatusBadRequest, "bad_request") +} + +func TestDiagnosticsAdminDeleteEmitsAuditEvent(t *testing.T) { + service := newFakeDiagnosticsService() + service.deleteReport = readyDiagnosticsReport() + var logs bytes.Buffer + handler := NewDiagnosticsHandler(service) + handler.logger = slog.New(slog.NewJSONHandler(&logs, nil)) + + rec := httptest.NewRecorder() + adminDiagnosticsRouter(handler).ServeHTTP(rec, adminDiagnosticsRequest(http.MethodDelete, "/diagnostics/reports/report-1", adminClaims())) + + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusNoContent, rec.Body.String()) + } + logLine := logs.String() + if !strings.Contains(logLine, `"msg":"diagnostic report deleted"`) || + !strings.Contains(logLine, `"admin_user_id":7`) || + !strings.Contains(logLine, `"report_id":"report-1"`) { + t.Fatalf("audit log = %s", logLine) + } +} + +func TestDiagnosticsAdminRoutesRejectNonActingAdmin(t *testing.T) { + service := newFakeDiagnosticsService() + handler := NewDiagnosticsHandler(service) + router := chi.NewRouter() + router.Use(apimw.RequireActingAdmin(nil)) + RegisterAdminDiagnosticsRoutes(router, handler) + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, adminDiagnosticsRequest(http.MethodGet, "/diagnostics/reports", &auth.Claims{ + UserID: 8, + Role: "user", + TokenType: auth.TokenTypeAccess, + })) + + if rec.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusForbidden, rec.Body.String()) + } + if service.listCalls != 0 { + t.Fatalf("list calls = %d, want 0", service.listCalls) + } +} + +func TestDiagnosticsUploadsEnablementRequiresStorageProbe(t *testing.T) { + settings := &fakeServerSettingsStore{values: map[string]string{}} + handler := &AdminHandler{SettingsRepo: settings} + router := chi.NewRouter() + router.Put("/admin/settings/{key}", handler.HandleUpdateSetting) + + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest( + http.MethodPut, + "/admin/settings/"+diagnostics.KeyUploadsEnabled, + strings.NewReader(`{"value":"true"}`), + )) + + assertDiagnosticsError(t, rec, http.StatusBadRequest, "storage_unavailable") + if settings.values[diagnostics.KeyUploadsEnabled] != "" { + t.Fatalf("setting persisted = %q, want empty", settings.values[diagnostics.KeyUploadsEnabled]) + } + + store := &fakeDiagnosticsEnablementStore{bucket: "private"} + handler.DiagnosticsStore = store + rec = httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest( + http.MethodPut, + "/admin/settings/"+diagnostics.KeyUploadsEnabled, + strings.NewReader(`{"value":"true"}`), + )) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + if settings.values[diagnostics.KeyUploadsEnabled] != "true" { + t.Fatalf("setting persisted = %q, want true", settings.values[diagnostics.KeyUploadsEnabled]) + } + if !sameStringSlice(store.ops, []string{"put:diagnostics/.probe", "delete:diagnostics/.probe"}) { + t.Fatalf("probe ops = %v, want put/delete probe", store.ops) + } +} + +func TestDiagnosticsNumericSettingsAcceptBoundaries(t *testing.T) { + cases := []struct { + name string + key string + value string + initial map[string]string + }{ + {name: "bundle minimum", key: diagnostics.KeyMaxBundleBytes, value: "1048576", initial: map[string]string{diagnostics.KeyMaxUncompressedBytes: "1073741824"}}, + {name: "bundle maximum", key: diagnostics.KeyMaxBundleBytes, value: "268435456", initial: map[string]string{diagnostics.KeyMaxUncompressedBytes: "1073741824", diagnostics.KeyMaxBytesPerUser: "10737418240"}}, + {name: "uncompressed bundle floor", key: diagnostics.KeyMaxUncompressedBytes, value: "1048576", initial: map[string]string{diagnostics.KeyMaxBundleBytes: "1048576"}}, + {name: "uncompressed maximum", key: diagnostics.KeyMaxUncompressedBytes, value: "1073741824", initial: map[string]string{diagnostics.KeyMaxBundleBytes: "1048576"}}, + {name: "reports minimum", key: diagnostics.KeyMaxReportsPerUserDay, value: "1"}, + {name: "reports maximum", key: diagnostics.KeyMaxReportsPerUserDay, value: "1000"}, + {name: "retention minimum", key: diagnostics.KeyRetentionDays, value: "1"}, + {name: "retention maximum", key: diagnostics.KeyRetentionDays, value: "365"}, + {name: "user bytes minimum", key: diagnostics.KeyMaxBytesPerUser, value: "10485760"}, + {name: "user bytes maximum", key: diagnostics.KeyMaxBytesPerUser, value: "10737418240"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + settings := &fakeServerSettingsStore{values: tc.initial} + rec := updateDiagnosticsSetting(t, &AdminHandler{SettingsRepo: settings}, tc.key, tc.value) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, http.StatusOK, rec.Body.String()) + } + if settings.values[tc.key] != tc.value { + t.Fatalf("stored value = %q, want %q", settings.values[tc.key], tc.value) + } + }) + } +} + +func TestDiagnosticsNumericSettingsRejectOutOfRangeValues(t *testing.T) { + cases := []struct { + name string + key string + value string + initial map[string]string + }{ + {name: "bundle below minimum", key: diagnostics.KeyMaxBundleBytes, value: "1048575"}, + {name: "bundle above maximum", key: diagnostics.KeyMaxBundleBytes, value: "268435457", initial: map[string]string{diagnostics.KeyMaxUncompressedBytes: "1073741824"}}, + {name: "bundle above uncompressed", key: diagnostics.KeyMaxBundleBytes, value: "67108865", initial: map[string]string{diagnostics.KeyMaxUncompressedBytes: "67108864"}}, + {name: "uncompressed below bundle", key: diagnostics.KeyMaxUncompressedBytes, value: "10485759", initial: map[string]string{diagnostics.KeyMaxBundleBytes: "10485760"}}, + {name: "bundle above per-user cap", key: diagnostics.KeyMaxBundleBytes, value: "104857600", initial: map[string]string{diagnostics.KeyMaxUncompressedBytes: "1073741824", diagnostics.KeyMaxBytesPerUser: "10485760"}}, + {name: "per-user below bundle", key: diagnostics.KeyMaxBytesPerUser, value: "10485760", initial: map[string]string{diagnostics.KeyMaxBundleBytes: "104857600", diagnostics.KeyMaxUncompressedBytes: "1073741824"}}, + {name: "uncompressed above maximum", key: diagnostics.KeyMaxUncompressedBytes, value: "1073741825"}, + {name: "reports below minimum", key: diagnostics.KeyMaxReportsPerUserDay, value: "0"}, + {name: "reports above maximum", key: diagnostics.KeyMaxReportsPerUserDay, value: "1001"}, + {name: "retention below minimum", key: diagnostics.KeyRetentionDays, value: "0"}, + {name: "retention above maximum", key: diagnostics.KeyRetentionDays, value: "366"}, + {name: "user bytes below minimum", key: diagnostics.KeyMaxBytesPerUser, value: "10485759"}, + {name: "user bytes above maximum", key: diagnostics.KeyMaxBytesPerUser, value: "10737418241"}, + {name: "not an integer", key: diagnostics.KeyRetentionDays, value: "thirty"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + settings := &fakeServerSettingsStore{values: tc.initial} + rec := updateDiagnosticsSetting(t, &AdminHandler{SettingsRepo: settings}, tc.key, tc.value) + body := rec.Body.String() + assertDiagnosticsError(t, rec, http.StatusBadRequest, "bad_request") + if _, stored := settings.values[tc.key]; stored { + t.Fatalf("invalid value was stored: %#v", settings.values) + } + if !strings.Contains(body, tc.key) { + t.Fatalf("error body = %s, want clear error naming %s", body, tc.key) + } + }) + } +} + +type diagnosticsPart struct { + name string + contentType string + body []byte +} + +func validDiagnosticsParts() []diagnosticsPart { + return []diagnosticsPart{ + {name: "manifest", contentType: "application/json", body: []byte(`{"ok":true}`)}, + {name: "bundle", contentType: "application/gzip", body: []byte("bundle")}, + } +} + +func newDiagnosticsUploadRequest(t *testing.T, parts []diagnosticsPart, claims *auth.Claims) *http.Request { + t.Helper() + var body bytes.Buffer + writer := multipart.NewWriter(&body) + for _, part := range parts { + header := make(textproto.MIMEHeader) + header.Set("Content-Disposition", `form-data; name="`+part.name+`"`) + header.Set("Content-Type", part.contentType) + w, err := writer.CreatePart(header) + if err != nil { + t.Fatalf("create multipart part: %v", err) + } + if _, err := w.Write(part.body); err != nil { + t.Fatalf("write multipart part: %v", err) + } + } + if err := writer.Close(); err != nil { + t.Fatalf("close multipart writer: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/api/v1/diagnostics/reports", &body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("Authorization", "Bearer token") + if claims != nil { + req = req.WithContext(apimw.SetClaims(req.Context(), claims)) + } + return req +} + +func accessClaims() *auth.Claims { + return &auth.Claims{ + UserID: 42, + TokenType: auth.TokenTypeAccess, + } +} + +func assertDiagnosticsError(t *testing.T, rec *httptest.ResponseRecorder, status int, code string) { + t.Helper() + if rec.Code != status { + t.Fatalf("status = %d, want %d; body=%s", rec.Code, status, rec.Body.String()) + } + var resp struct { + Error string `json:"error"` + } + if err := json.NewDecoder(rec.Body).Decode(&resp); err != nil { + t.Fatalf("decode error response: %v", err) + } + if resp.Error != code { + t.Fatalf("error = %q, want %q", resp.Error, code) + } +} + +func captureDiagnosticsLogs(handler *DiagnosticsHandler) *bytes.Buffer { + var logs bytes.Buffer + handler.logger = slog.New(slog.NewJSONHandler(&logs, nil)) + return &logs +} + +func assertDiagnosticsRejectionLog(t *testing.T, logs *bytes.Buffer, reason string, userID int) { + t.Helper() + line := logs.String() + for _, want := range []string{ + `"msg":"diagnostic report rejected"`, + `"component":"diagnostics"`, + `"result":"rejected"`, + `"reason":"` + reason + `"`, + `"user_id":` + strconv.Itoa(userID), + } { + if !strings.Contains(line, want) { + t.Fatalf("rejection log = %s, want %s", line, want) + } + } +} + +func updateDiagnosticsSetting(t *testing.T, handler *AdminHandler, key, value string) *httptest.ResponseRecorder { + t.Helper() + router := chi.NewRouter() + router.Put("/admin/settings/{key}", handler.HandleUpdateSetting) + rec := httptest.NewRecorder() + router.ServeHTTP(rec, httptest.NewRequest( + http.MethodPut, + "/admin/settings/"+key, + strings.NewReader(`{"value":"`+value+`"}`), + )) + return rec +} + +func readyDiagnosticsReport() *diagnostics.Report { + blobBucket := "private" + blobKey := "diagnostics/7/report-1.tar.gz" + blobBytes := int64(13) + return &diagnostics.Report{ + ID: "report-1", + ShortID: "SILO-ABCDEF123456", + UserID: 7, + State: diagnostics.StateReady, + ReportType: "crash", + Platform: "android-tv", + BlobBucket: &blobBucket, + BlobKey: &blobKey, + BlobBytes: &blobBytes, + } +} + +func adminDiagnosticsRouter(handler *DiagnosticsHandler) chi.Router { + router := chi.NewRouter() + RegisterAdminDiagnosticsRoutes(router, handler) + return router +} + +func adminDiagnosticsRequest(method, target string, claims *auth.Claims) *http.Request { + req := httptest.NewRequest(method, target, nil) + if claims != nil { + req = req.WithContext(apimw.SetClaims(req.Context(), claims)) + } + return req +} + +func adminClaims() *auth.Claims { + return &auth.Claims{ + UserID: 7, + Role: "admin", + TokenType: auth.TokenTypeAccess, + } +} + +type fakeDiagnosticsService struct { + mu sync.Mutex + status diagnostics.Status + statusCalls int + ingestCalls int + ingestErr error + started chan struct{} + block chan struct{} + listCalls int + listFilters diagnostics.ListFilters + listResult diagnostics.ListResult + listErr error + getReport *diagnostics.Report + getErr error + presignURL string + presignErr error + presignCalls int + presignExpiry time.Duration + effectiveTTL time.Duration + openCalls int + openData []byte + openErr error + deleteReport *diagnostics.Report + deleteErr error +} + +func newFakeDiagnosticsService() *fakeDiagnosticsService { + return &fakeDiagnosticsService{ + status: diagnostics.Status{ + Status: diagnostics.StatusAvailable, + ServerInstanceID: "server-1", + AcceptedSchemaVersions: []int{1}, + MaxBundleBytes: diagnostics.DefaultMaxBundleBytes, + MaxManifestBytes: diagnostics.MaxManifestBytes, + RetentionDays: diagnostics.DefaultRetentionDays, + ConsentNoticeVersion: diagnostics.DefaultConsentNoticeVer, + }, + } +} + +func (f *fakeDiagnosticsService) Status(context.Context, int) (diagnostics.Status, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.statusCalls++ + return f.status, nil +} + +func (f *fakeDiagnosticsService) Ingest(_ context.Context, _ int, _ *string, _ []byte, bundle io.Reader) (diagnostics.IngestResult, error) { + f.mu.Lock() + f.ingestCalls++ + started := f.started + block := f.block + err := f.ingestErr + f.mu.Unlock() + + if started != nil { + close(started) + } + if block != nil { + <-block + } + if err != nil { + return diagnostics.IngestResult{}, err + } + if _, err := io.ReadAll(bundle); err != nil { + return diagnostics.IngestResult{}, err + } + return diagnostics.IngestResult{ + ReportID: "11111111-1111-1111-1111-111111111111", + ShortID: "SILO-ABCDEF123456", + }, nil +} + +func (f *fakeDiagnosticsService) ListForAdmin(_ context.Context, filters diagnostics.ListFilters) (diagnostics.ListResult, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.listCalls++ + f.listFilters = filters + if f.listErr != nil { + return diagnostics.ListResult{}, f.listErr + } + return f.listResult, nil +} + +func (f *fakeDiagnosticsService) GetReport(context.Context, string) (*diagnostics.Report, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.getErr != nil { + return nil, f.getErr + } + if f.getReport != nil { + return f.getReport, nil + } + return readyDiagnosticsReport(), nil +} + +func (f *fakeDiagnosticsService) PresignReportDownload(_ context.Context, _ *diagnostics.Report, expiry time.Duration) (string, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.presignCalls++ + f.presignExpiry = expiry + if f.presignErr != nil { + return "", f.presignErr + } + return f.presignURL, nil +} + +func (f *fakeDiagnosticsService) EffectiveReportDownloadTTL(requested time.Duration) time.Duration { + f.mu.Lock() + defer f.mu.Unlock() + if f.effectiveTTL > 0 { + return f.effectiveTTL + } + return requested +} + +func (f *fakeDiagnosticsService) OpenReportDownload(context.Context, *diagnostics.Report) (io.ReadCloser, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.openCalls++ + if f.openErr != nil { + return nil, f.openErr + } + return io.NopCloser(bytes.NewReader(f.openData)), nil +} + +func (f *fakeDiagnosticsService) DeleteReport(context.Context, string) (*diagnostics.Report, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.deleteErr != nil { + return nil, f.deleteErr + } + if f.deleteReport != nil { + return f.deleteReport, nil + } + return readyDiagnosticsReport(), nil +} + +type fakeDiagnosticsEnablementStore struct { + bucket string + ops []string +} + +func (f *fakeDiagnosticsEnablementStore) PutStream(_ context.Context, _ string, key string, r io.Reader, _ string) error { + if _, err := io.ReadAll(r); err != nil { + return err + } + f.ops = append(f.ops, "put:"+key) + return nil +} + +func (f *fakeDiagnosticsEnablementStore) DeleteObject(_ context.Context, _ string, key string) error { + f.ops = append(f.ops, "delete:"+key) + return nil +} + +func (f *fakeDiagnosticsEnablementStore) Bucket() string { + return f.bucket +} + +func sameStringSlice(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/api/handlers/events_ws.go b/internal/api/handlers/events_ws.go index c771abf8..e22789a5 100644 --- a/internal/api/handlers/events_ws.go +++ b/internal/api/handlers/events_ws.go @@ -108,6 +108,21 @@ func (h *EventsHandler) HandleWebSocket(w http.ResponseWriter, r *http.Request) conn, err := wsUpgrader.Upgrade(w, r, nil) if err != nil { + // The Android/KMP client has a long history of silent handshake + // failures here (gorilla writes the 4xx itself); log the exact + // upgrade-relevant request shape so a failing client is diagnosable + // from the server alone. + slog.WarnContext(r.Context(), "events websocket upgrade failed", "component", "api", + "error", err, + "user_id", claims.UserID, + "proto", r.Proto, + "method", r.Method, + "connection_header", r.Header.Get("Connection"), + "upgrade_header", r.Header.Get("Upgrade"), + "ws_version", r.Header.Get("Sec-Websocket-Version"), + "ws_key_present", r.Header.Get("Sec-Websocket-Key") != "", + "user_agent", r.UserAgent(), + ) return } defer conn.Close() diff --git a/internal/api/handlers/items.go b/internal/api/handlers/items.go index 3a5cb5a4..2b8b543d 100644 --- a/internal/api/handlers/items.go +++ b/internal/api/handlers/items.go @@ -344,6 +344,7 @@ type episodeResponse struct { StillThumbhash string `json:"still_thumbhash,omitempty"` UserData *catalog.SeasonUserData `json:"user_data,omitempty"` Files []episodeFileResponse `json:"files,omitempty"` + OverlaySummary *models.OverlaySummary `json:"overlay_summary,omitempty"` } type episodeImageFallback struct { @@ -959,8 +960,10 @@ func (h *ItemsHandler) buildEpisodeResponses(r *http.Request, episodes []*models } for i := range resp { resp[i].StillURL = stillURLs[stillPaths[i]].URL - resp[i].Files = episodeFileResponses(filesByEpisode[resp[i].ContentID], filter) + files := catalog.FilterMediaFilesByAccess(filesByEpisode[resp[i].ContentID], filter) + resp[i].Files = episodeFileResponses(files, filter) resp[i].UserData = userData[resp[i].ContentID] + resp[i].OverlaySummary = overlays.BuildSummary(files) } return resp } diff --git a/internal/api/handlers/libraries.go b/internal/api/handlers/libraries.go index d2dfe344..6013831d 100644 --- a/internal/api/handlers/libraries.go +++ b/internal/api/handlers/libraries.go @@ -8,7 +8,6 @@ import ( "io" "log/slog" "net/http" - "os" "path/filepath" "slices" "sort" @@ -31,6 +30,7 @@ import ( "github.com/Silo-Server/silo-server/internal/metadata" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/plugins" + "github.com/Silo-Server/silo-server/internal/rootcheck" "github.com/Silo-Server/silo-server/internal/scanner" "github.com/Silo-Server/silo-server/internal/scantrigger" "github.com/Silo-Server/silo-server/internal/sections" @@ -251,6 +251,11 @@ type libraryMountCheckRootResponse struct { Reachable bool `json:"reachable"` ErrorCode *string `json:"error_code"` ErrorMessage *string `json:"error_message"` + // SuspectEmpty is set when the root is reachable but the library holds + // only missing-marked files under it — the signature of a lost mount + // exposing an empty mountpoint directory, which a reachability probe + // alone cannot detect. Additive field; absent/false for healthy roots. + SuspectEmpty bool `json:"suspect_empty"` } type libraryMountCheckResponse struct { @@ -847,8 +852,9 @@ func (h *LibraryHandler) HandleCheckLibraryMount(w http.ResponseWriter, r *http. return } - resp := checkLibraryMount(folder) - if resp.Healthy && folder.ScanWarningCode != nil && *folder.ScanWarningCode == "empty_root" { + resp := h.checkLibraryMount(r.Context(), folder) + if resp.Healthy && folder.ScanWarningCode != nil && + (*folder.ScanWarningCode == "empty_root" || *folder.ScanWarningCode == "dead_root") { if err := h.folderRepo.ClearScanWarning(r.Context(), folder.ID); err != nil { if errors.Is(err, catalog.ErrFolderNotFound) { writeError(w, http.StatusNotFound, "not_found", "Library not found") @@ -1256,7 +1262,7 @@ func scanBoolMetric(result *libraryingest.Result, pick func(*scanner.ScanResult) return pick(result.ScanResult) } -func checkLibraryMount(folder *models.MediaFolder) libraryMountCheckResponse { +func (h *LibraryHandler) checkLibraryMount(ctx context.Context, folder *models.MediaFolder) libraryMountCheckResponse { resp := libraryMountCheckResponse{ Status: "ok", LibraryID: folder.ID, @@ -1273,40 +1279,58 @@ func checkLibraryMount(folder *models.MediaFolder) libraryMountCheckResponse { } unreachable := 0 - for _, path := range folder.Paths { + emptyPaths := make([]string, 0, len(folder.Paths)) + probes := rootcheck.ProbeManyWithTimeout(ctx, folder.Paths, rootcheck.DefaultProbeTimeout) + for i, path := range folder.Paths { + probe := probes[i] root := libraryMountCheckRootResponse{ Path: path, - Reachable: true, + Reachable: probe.Reachable, } - - info, err := os.Stat(path) - if err != nil { - code, message := classifyMountCheckError(err, false) - root.Reachable = false - root.ErrorCode = stringPtr(code) - root.ErrorMessage = stringPtr(message) - } else if !info.IsDir() { - root.Reachable = false - root.ErrorCode = stringPtr("not_directory") - root.ErrorMessage = stringPtr("Path is not a directory") - } else if _, err := os.ReadDir(path); err != nil { - code, message := classifyMountCheckError(err, true) - root.Reachable = false - root.ErrorCode = stringPtr(code) - root.ErrorMessage = stringPtr(message) - } - - if !root.Reachable { + if !probe.Reachable { + root.ErrorCode = stringPtr(probe.ErrorCode) + root.ErrorMessage = stringPtr(probe.ErrorMessage) unreachable++ resp.Healthy = false + } else if probe.Empty { + emptyPaths = append(emptyPaths, path) } resp.Roots = append(resp.Roots, root) } - switch unreachable { - case 0: + // A reachable but literally empty root can be a lost mount that left its + // bare mountpoint behind. Cross-check against the catalog: an empty root + // holding only missing-marked files is suspect, and reporting it healthy + // would both mislead the operator and wrongly clear the dead_root + // warning. Best-effort — probe results stand on a query error. + suspect := 0 + if h.pool != nil && len(emptyPaths) > 0 { + suspectRoots, err := scanner.NewFileRepository(h.pool).ListRootsWithOnlyMissingFiles(ctx, folder.ID, emptyPaths) + if err != nil { + slog.WarnContext(ctx, "mount check: suspect-empty root query failed", "component", "api", "library_id", folder.ID, "error", err) + } else if len(suspectRoots) > 0 { + suspectSet := make(map[string]bool, len(suspectRoots)) + for _, root := range suspectRoots { + suspectSet[root] = true + } + for i := range resp.Roots { + if resp.Roots[i].Reachable && suspectSet[resp.Roots[i].Path] { + resp.Roots[i].SuspectEmpty = true + suspect++ + resp.Healthy = false + } + } + } + } + + switch { + case unreachable == 0 && suspect == 0: resp.Summary = "All configured roots are reachable" - case 1: + case unreachable == 0: + resp.Summary = fmt.Sprintf("%d of %d roots reachable but empty while the library still has cataloged files (lost mount?)", suspect, len(folder.Paths)) + case suspect > 0: + resp.Summary = fmt.Sprintf("%d of %d roots unreachable; %d more reachable but empty while the library still has cataloged files", unreachable, len(folder.Paths), suspect) + case unreachable == 1: resp.Summary = fmt.Sprintf("1 of %d roots unreachable", len(folder.Paths)) default: resp.Summary = fmt.Sprintf("%d of %d roots unreachable", unreachable, len(folder.Paths)) @@ -1315,19 +1339,6 @@ func checkLibraryMount(folder *models.MediaFolder) libraryMountCheckResponse { return resp } -func classifyMountCheckError(err error, isRead bool) (string, string) { - switch { - case errors.Is(err, os.ErrNotExist): - return "not_found", "Path does not exist" - case errors.Is(err, os.ErrPermission): - return "permission_denied", "Permission denied" - case isRead: - return "read_failed", "Failed to read directory" - default: - return "stat_failed", "Failed to stat path" - } -} - func stringPtr(v string) *string { return &v } @@ -2191,22 +2202,7 @@ func buildSeededChainEntries(level string, candidates []seedCandidate) []metadat } func metadataContentLevelsForLibraryType(libraryType string) []string { - switch libraryType { - case "series": - return []string{"series", "season", "episode"} - case "movies", "movie": - return []string{"movie"} - case "audiobooks", "audiobook": - return []string{"audiobook"} - case "ebooks", "ebook": - return []string{"ebook"} - case "manga": - return []string{"manga"} - case "mixed": - return []string{"movie", "series", "season", "episode", "audiobook", "ebook"} - default: - return nil - } + return metadata.ContentLevelsForLibraryType(libraryType) } // HandleListStaleIDs handles GET /libraries/stale-ids. diff --git a/internal/api/handlers/notifications.go b/internal/api/handlers/notifications.go index f062f22d..1134b257 100644 --- a/internal/api/handlers/notifications.go +++ b/internal/api/handlers/notifications.go @@ -388,21 +388,30 @@ func (h *NotificationsHandler) HandleCapability(w http.ResponseWriter, r *http.R } } applePush := capabilityPush{Available: false, Provider: "off", SupportedModes: []string{"in_app_only"}} + androidPush := capabilityPush{Available: false, Provider: "off", SupportedModes: []string{"in_app_only"}} // Like web push above, availability requires both the wiring (cipher/store) // and the admin delivery toggle: Available must mean "setup will actually // deliver", not "the server could store a token". - if h.system.PushDevices != nil && h.system.PushDevices.Available() && - h.system.Settings.ApplePushDeliveryEnabled(r.Context()) { - applePush = capabilityPush{ - Available: true, - Provider: notifications.PushProviderSiloRelay, - SupportedModes: []string{notifications.PushModePrivatePush, notifications.PushModeInAppOnly}, + if h.system.PushDevices != nil && h.system.PushDevices.Available() { + if h.system.Settings.ApplePushDeliveryEnabled(r.Context()) { + applePush = capabilityPush{ + Available: true, + Provider: notifications.PushProviderSiloRelay, + SupportedModes: []string{notifications.PushModePrivatePush, notifications.PushModeInAppOnly}, + } + } + if h.system.Settings.AndroidPushDeliveryEnabled(r.Context()) { + androidPush = capabilityPush{ + Available: true, + Provider: notifications.PushProviderSiloRelay, + SupportedModes: []string{notifications.PushModePrivatePush, notifications.PushModeInAppOnly}, + } } } writeJSON(w, http.StatusOK, capabilityResponse{ InApp: capabilityInApp{Enabled: h.system.Settings.UIEnabled(r.Context())}, ApplePush: applePush, - AndroidPush: capabilityPush{Available: false, Provider: "off", SupportedModes: []string{"in_app_only"}}, + AndroidPush: androidPush, WebPush: webPush, Webhooks: webhooks, Email: email, diff --git a/internal/api/handlers/notifications_push_devices.go b/internal/api/handlers/notifications_push_devices.go index 8780646f..e03f9001 100644 --- a/internal/api/handlers/notifications_push_devices.go +++ b/internal/api/handlers/notifications_push_devices.go @@ -7,6 +7,7 @@ import ( apimw "github.com/Silo-Server/silo-server/internal/api/middleware" "github.com/Silo-Server/silo-server/internal/notifications" + "github.com/go-chi/chi/v5" ) type applePushRegisterRequest struct { @@ -73,3 +74,80 @@ func (h *NotificationsHandler) HandleRegisterApplePushDevice(w http.ResponseWrit PushMode: device.PushMode, }) } + +type pushDeviceRegisterRequest struct { + Platform string `json:"platform"` + Token string `json:"token"` + DeviceID string `json:"device_id"` + PushMode string `json:"push_mode"` +} + +// HandleRegisterPushDevice handles POST /notifications/push/devices, the +// platform-generic registration used by the Android client. Apple installs +// keep the dedicated /devices/push/apple route because their registration +// carries APNs-specific fields (environment, topic). +func (h *NotificationsHandler) HandleRegisterPushDevice(w http.ResponseWriter, r *http.Request) { + service := h.pushDevices() + if service == nil || !service.Available() { + writeError(w, http.StatusServiceUnavailable, "unavailable", "Push registration is not available") + return + } + + var req pushDeviceRegisterRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body") + return + } + if req.Platform != notifications.PushPlatformAndroid { + writeError(w, http.StatusUnprocessableEntity, "unsupported_push_device", "platform is not supported on this endpoint") + return + } + + device, err := service.RegisterFCM(r.Context(), apimw.GetUserID(r.Context()), apimw.GetProfileID(r.Context()), notifications.FCMPushRegistrationInput{ + DeviceID: req.DeviceID, + FCMToken: req.Token, + PushMode: req.PushMode, + }) + if err != nil { + switch { + case errors.Is(err, notifications.ErrPushDeviceInvalid): + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + case errors.Is(err, notifications.ErrPushDeviceUnsupported): + writeError(w, http.StatusUnprocessableEntity, "unsupported_push_device", err.Error()) + case errors.Is(err, notifications.ErrPushDeviceUnavailable): + writeError(w, http.StatusServiceUnavailable, "unavailable", "Push registration is not available") + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to register push device") + } + return + } + + writeJSON(w, http.StatusOK, applePushRegisterResponse{ + ID: device.ID, + ServerDeviceID: device.ServerDeviceID, + Enabled: device.Enabled, + PushMode: device.PushMode, + }) +} + +// HandleUnregisterPushDevice handles DELETE /notifications/push/devices/{device_id}. +func (h *NotificationsHandler) HandleUnregisterPushDevice(w http.ResponseWriter, r *http.Request) { + service := h.pushDevices() + if service == nil || !service.Available() { + writeError(w, http.StatusServiceUnavailable, "unavailable", "Push registration is not available") + return + } + err := service.Unregister(r.Context(), apimw.GetProfileID(r.Context()), chi.URLParam(r, "device_id")) + if err != nil { + switch { + case errors.Is(err, notifications.ErrPushDeviceInvalid): + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + case errors.Is(err, notifications.ErrPushDeviceUnavailable): + writeError(w, http.StatusServiceUnavailable, "unavailable", "Push registration is not available") + default: + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to unregister push device") + } + return + } + w.WriteHeader(http.StatusNoContent) +} diff --git a/internal/api/handlers/notifications_push_devices_test.go b/internal/api/handlers/notifications_push_devices_test.go index 1cac7070..39930adc 100644 --- a/internal/api/handlers/notifications_push_devices_test.go +++ b/internal/api/handlers/notifications_push_devices_test.go @@ -15,10 +15,12 @@ import ( ) type handlerPushStore struct { - got notifications.ApplePushDeviceRegistration - calls int - device *notifications.PushDevice - err error + got notifications.ApplePushDeviceRegistration + gotFCM notifications.FCMPushDeviceRegistration + calls int + device *notifications.PushDevice + err error + deleted []string } func (f *handlerPushStore) UpsertApple(ctx context.Context, registration notifications.ApplePushDeviceRegistration, cipher *secret.Cipher) (*notifications.PushDevice, error) { @@ -38,6 +40,29 @@ func (f *handlerPushStore) UpsertApple(ctx context.Context, registration notific }, nil } +func (f *handlerPushStore) UpsertFCM(ctx context.Context, registration notifications.FCMPushDeviceRegistration, cipher *secret.Cipher) (*notifications.PushDevice, error) { + f.calls++ + f.gotFCM = registration + if f.err != nil { + return nil, f.err + } + if f.device != nil { + return f.device, nil + } + return ¬ifications.PushDevice{ + ID: "push-row-android", + Platform: notifications.PushPlatformAndroid, + ServerDeviceID: "server-device-android", + Enabled: true, + PushMode: registration.PushMode, + }, nil +} + +func (f *handlerPushStore) DeleteByProfileDevice(ctx context.Context, profileID, deviceID string) error { + f.deleted = append(f.deleted, profileID+"/"+deviceID) + return f.err +} + func handlerPushCipher(t *testing.T) *secret.Cipher { t.Helper() cipher, err := secret.New([]byte("01234567890123456789012345678901")) @@ -88,6 +113,68 @@ func TestHandleRegisterApplePushDevice(t *testing.T) { } } +func newPushDevicesRequest(method, target, body string) *http.Request { + req := httptest.NewRequest(method, target, strings.NewReader(body)) + ctx := apimw.SetClaims(req.Context(), &auth.Claims{UserID: 42, Role: "user", TokenType: auth.TokenTypeAccess}) + ctx = apimw.SetProfileID(ctx, "profile-1") + return req.WithContext(ctx) +} + +func TestHandleRegisterPushDeviceAndroid(t *testing.T) { + store := &handlerPushStore{} + handler := NewNotificationsHandler(¬ifications.System{ + PushDevices: notifications.NewPushDeviceService(store, handlerPushCipher(t)), + }, nil) + + token := strings.Repeat("F", 140) + body := `{ + "platform":"android", + "token":"` + token + `", + "device_id":"local-device", + "push_mode":"private_push" + }` + rr := httptest.NewRecorder() + handler.HandleRegisterPushDevice(rr, newPushDevicesRequest(http.MethodPost, "/api/v1/notifications/push/devices", body)) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + var response applePushRegisterResponse + if err := json.NewDecoder(rr.Body).Decode(&response); err != nil { + t.Fatalf("decode response: %v", err) + } + if response.ID != "push-row-android" || response.PushMode != notifications.PushModePrivatePush { + t.Fatalf("unexpected response: %+v", response) + } + if store.gotFCM.UserID != 42 || store.gotFCM.ProfileID != "profile-1" || store.gotFCM.FCMToken != token { + t.Fatalf("unexpected stored registration: %+v", store.gotFCM) + } +} + +func TestHandleRegisterPushDeviceRejectsUnknownPlatformAndBadToken(t *testing.T) { + newHandler := func() *NotificationsHandler { + return NewNotificationsHandler(¬ifications.System{ + PushDevices: notifications.NewPushDeviceService(&handlerPushStore{}, handlerPushCipher(t)), + }, nil) + } + + rr := httptest.NewRecorder() + newHandler().HandleRegisterPushDevice(rr, newPushDevicesRequest(http.MethodPost, + "/api/v1/notifications/push/devices", + `{"platform":"apple","token":"`+strings.Repeat("F", 140)+`","device_id":"local-device"}`)) + if rr.Code != http.StatusUnprocessableEntity { + t.Fatalf("apple platform status = %d, want 422", rr.Code) + } + + rr = httptest.NewRecorder() + newHandler().HandleRegisterPushDevice(rr, newPushDevicesRequest(http.MethodPost, + "/api/v1/notifications/push/devices", + `{"platform":"android","token":"short","device_id":"local-device"}`)) + if rr.Code != http.StatusBadRequest { + t.Fatalf("bad token status = %d, want 400", rr.Code) + } +} + func TestHandleRegisterApplePushDeviceErrors(t *testing.T) { tests := []struct { name string diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index 3dd61e28..2d01acea 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -18,6 +18,7 @@ import ( "time" "github.com/go-chi/chi/v5" + "github.com/google/uuid" apimw "github.com/Silo-Server/silo-server/internal/api/middleware" "github.com/Silo-Server/silo-server/internal/catalog" @@ -50,6 +51,10 @@ type SessionManagerInterface interface { EndTransport(sessionID string) error SetEffectiveMediaFileID(sessionID string, fileID int) error SetTranscodeNodeURL(sessionID, url string) error + SetTranscodeRoute(sessionID string, route playback.TranscodeRoute) error + ApplyReplacement(sessionID string, replacement playback.SessionReplacement) (playback.SessionReplacementRollback, error) + ApplyReplacementIfRoute(sessionID string, expected playback.TranscodeRoute, replacement playback.SessionReplacement) (playback.SessionReplacementRollback, bool, error) + RollbackReplacement(sessionID string, rollback playback.SessionReplacementRollback) error SetWebSocket(sessionID string, connected bool) error SetRealtimeConnection(sessionID string, connected bool) error SetProgressPersistenceDisabled(sessionID string, disabled bool) error @@ -128,6 +133,14 @@ type PlaybackOriginalLanguageLookup interface { GetOriginalLanguage(ctx context.Context, contentID string) (string, error) } +type copySeekAnchorResolver func( + ctx context.Context, + ffmpegPath string, + inputPath string, + requestedSeekSeconds float64, + segmentDuration int, +) (float64, int, error) + // PlaybackHandler handles playback session HTTP endpoints. type PlaybackHandler struct { sessionMgr SessionManagerInterface @@ -170,12 +183,30 @@ type PlaybackHandler struct { // playbackConfig(), which falls back to defaults when unset. PlaybackConfig func() config.PlaybackConfig FFmpegLogSink playback.FFmpegLogSink + copySeekAnchor copySeekAnchorResolver realtimeCommandMu sync.Mutex realtimeCommands map[string]playbackCommandRecord // tm owns the transcode-session lifecycle (live map, recipe cards, and // restart reconstruct) shared with the jellycompat handler. The handler // delegates all transcode-session and recipe operations to it. tm *playback.TranscodeManager + // PlanStoreV3 owns the short-lived protocol-v3 control-plane state. Router + // wiring replaces the in-memory default with PostgreSQL in integrated mode. + PlanStoreV3 playback.PlanStoreV3 + v3RegistryOnce sync.Once + v3Registry *playback.TransformationRegistryV3 + v3NodeCapabilitiesMu sync.Mutex + v3NodeCapabilities map[string]v3NodeCapabilityCache + v3EventOnce sync.Once + v3EventQueue chan playback.RouteEventRecordV3 + v3ReplanMu sync.Mutex + v3ReplanLocks map[string]*v3ReplanLock + v3ReplanSlotsOnce sync.Once + v3ReplanSlots chan struct{} + v3EventRateMu sync.Mutex + v3EventRates map[string]v3EventRate + v3FlagMu sync.Mutex + v3Flags map[string]v3FlagCacheEntry } type PlaybackWatchScrobbler interface { @@ -196,6 +227,7 @@ func NewPlaybackHandler(sessionMgr SessionManagerInterface, opts ...FilePathReso sessionMgr: sessionMgr, realtimeCommands: make(map[string]playbackCommandRecord), tm: playback.NewTranscodeManager(), + PlanStoreV3: playback.NewMemoryPlanStoreV3(), } if len(opts) > 0 { h.fileResolver = opts[0] @@ -416,11 +448,24 @@ type changeAudioRequest struct { // changeAudioResponse represents the JSON response for PATCH /playback/{session_id}/audio. type changeAudioResponse struct { - AudioTrackIndex int `json:"audio_track_index"` - PlayMethod string `json:"play_method"` - StreamURL string `json:"stream_url"` - SwitchMode string `json:"switch_mode"` - PlaybackInfo *playbackInfoResult `json:"playback_info,omitempty"` + AudioTrackIndex int `json:"audio_track_index"` + PlayMethod string `json:"play_method"` + StreamURL string `json:"stream_url"` + SwitchMode string `json:"switch_mode"` + PlayerStartSeconds *float64 `json:"player_start_seconds,omitempty"` + StreamOriginSeconds *float64 `json:"stream_origin_seconds,omitempty"` + TimelineOffsetSeconds *float64 `json:"timeline_offset_seconds,omitempty"` + CanSeekAnywhere *bool `json:"can_seek_anywhere,omitempty"` + PlaybackInfo *playbackInfoResult `json:"playback_info,omitempty"` +} + +func (resp *changeAudioResponse) setCopyTimeline(position, origin float64) { + playerStart := max(0, position-origin) + canSeekAnywhere := false + resp.PlayerStartSeconds = &playerStart + resp.StreamOriginSeconds = &origin + resp.TimelineOffsetSeconds = &origin + resp.CanSeekAnywhere = &canSeekAnywhere } type transcodeStartRequest struct { @@ -496,6 +541,7 @@ func buildTranscodeStartResponse( file *models.MediaFile, switchedFileID *int, manifestURL string, + streamOriginSeconds float64, ) transcodeStartResponse { resp := transcodeStartResponse{ SessionID: req.SessionID, @@ -511,13 +557,29 @@ func buildTranscodeStartResponse( resp.CanSeekAnywhere = true return resp } - resp.PlayerStartSeconds = 0 - resp.StreamOriginSeconds = req.SeekSeconds - resp.TimelineOffsetSeconds = req.SeekSeconds + resp.PlayerStartSeconds = max(0, req.SeekSeconds-streamOriginSeconds) + resp.StreamOriginSeconds = streamOriginSeconds + resp.TimelineOffsetSeconds = streamOriginSeconds resp.CanSeekAnywhere = false return resp } +func (h *PlaybackHandler) resolveLegacyCopySeekAnchor( + ctx context.Context, + ffmpegPath string, + inputPath string, + requestedSeekSeconds float64, + segmentDuration int, +) (float64, int, error) { + resolver := h.copySeekAnchor + if resolver == nil { + resolver = playback.ResolveCopySeekAnchor + } + probeCtx, cancel := context.WithTimeout(ctx, 8*time.Second) + defer cancel() + return resolver(probeCtx, ffmpegPath, inputPath, requestedSeekSeconds, segmentDuration) +} + func (h *PlaybackHandler) ensurePlaybackProbe(ctx context.Context, file *models.MediaFile) *models.MediaFile { if h == nil || h.ProbeEnsurer == nil || file == nil { return file @@ -649,7 +711,7 @@ func (h *PlaybackHandler) playbackStreamURL(s *playback.Session) string { func identityRecipeCard(s *playback.Session) playback.RecipeCard { switch s.PlayMethod { case playback.PlayRemux: - return playback.NewRemuxRecipeCard(s.ID, s.UserID, s.ProfileID, s.MediaFileID, s.TranscodeAudio, s.AudioTrackIndex) + return playback.NewRemuxRecipeCard(s.ID, s.UserID, s.ProfileID, s.MediaFileID, s.TranscodeAudio, s.AudioTrackIndex, s.RemuxDVMode) default: return playback.NewDirectRecipeCard(s.ID, s.UserID, s.ProfileID, s.MediaFileID) } @@ -713,6 +775,209 @@ func requestedMediaFileID(session *playback.Session) int { return session.MediaFileID } +func remoteTransportID(session *playback.Session) string { + if session != nil && session.TranscodeTransportID != "" { + return session.TranscodeTransportID + } + if session == nil { + return "" + } + return session.ID +} + +func sessionTranscodeRoute(session *playback.Session) playback.TranscodeRoute { + if session == nil { + return playback.TranscodeRoute{} + } + return playback.TranscodeRoute{ + NodeURL: session.TranscodeNodeURL, + TransportID: session.TranscodeTransportID, + } +} + +const legacyTransportMarker = "-legacy-" + +func isLegacyTransportSession(session *playback.Session) bool { + if session == nil { + return false + } + if session.TranscodeTransportID == "" { + return true + } + return strings.HasPrefix(session.TranscodeTransportID, session.ID+legacyTransportMarker) +} + +func newLegacyTransportID(sessionID string) string { + return sessionID + legacyTransportMarker + uuid.NewString() +} + +func (h *PlaybackHandler) transcodeRouteMatches( + sessionID string, + expectedLocal *playback.TranscodeSession, + route playback.TranscodeRoute, +) bool { + unlock := h.tm.LockSessionLifecycle(sessionID) + defer unlock() + session, err := h.sessionMgr.GetSession(sessionID) + if err != nil || session == nil { + return false + } + return h.tm.GetTranscodeSession(sessionID) == expectedLocal && + sessionTranscodeRoute(session) == route +} + +func (h *PlaybackHandler) commitLegacyRemoteReplacement( + sessionID string, + previousLocal *playback.TranscodeSession, + previousRoute playback.TranscodeRoute, + replacement playback.SessionReplacement, +) error { + unlock := h.tm.LockSessionLifecycle(sessionID) + if h.tm.GetTranscodeSession(sessionID) != previousLocal { + unlock() + return playback.ErrSessionSuperseded + } + + _, published, err := h.sessionMgr.ApplyReplacementIfRoute(sessionID, previousRoute, replacement) + if err != nil { + unlock() + return err + } + if !published { + unlock() + return playback.ErrSessionSuperseded + } + if previousLocal != nil { + // A crash monitor can remove the predecessor while the remote successor is + // preparing. A false result is harmless: the new route is already complete, + // and CloseTranscodeSessionIf never touches a different local successor. + h.tm.CloseTranscodeSessionIf(sessionID, previousLocal, "") + } + unlock() + + if previousRoute.NodeURL != "" { + previousProcessID := previousRoute.TransportID + if previousProcessID == "" { + previousProcessID = sessionID + } + h.tm.StopRemoteTranscode(previousProcessID, previousRoute.NodeURL) + } + return nil +} + +func (h *PlaybackHandler) commitLegacyRemoteLastWriter( + sessionID string, + successorRoute playback.TranscodeRoute, + replacement playback.SessionReplacement, +) error { + unlock := h.tm.LockSessionLifecycle(sessionID) + current, err := h.sessionMgr.GetSession(sessionID) + if err != nil { + unlock() + return err + } + previousRoute := sessionTranscodeRoute(current) + previousLocal := h.tm.GetTranscodeSession(sessionID) + if _, err := h.sessionMgr.ApplyReplacement(sessionID, replacement); err != nil { + unlock() + return err + } + if previousLocal != nil { + h.tm.CloseTranscodeSessionIf(sessionID, previousLocal, "") + } + unlock() + + if previousRoute.NodeURL != "" && previousRoute != successorRoute { + previousProcessID := previousRoute.TransportID + if previousProcessID == "" { + previousProcessID = sessionID + } + h.tm.StopRemoteTranscode(previousProcessID, previousRoute.NodeURL) + } + return nil +} + +func (h *PlaybackHandler) commitLegacyLocalReplacement( + ctx context.Context, + sessionID string, + previousLocal *playback.TranscodeSession, + previousRoute playback.TranscodeRoute, + opts playback.TranscodeOpts, + replacement func(*playback.TranscodeSession) playback.SessionReplacement, +) (*playback.TranscodeSession, error) { + if opts.OutputSubdir == "" { + return nil, errors.New("transactional local replacement requires a generation output directory") + } + + unlock := h.tm.LockSessionLifecycle(sessionID) + if h.tm.GetTranscodeSession(sessionID) != previousLocal { + unlock() + return nil, playback.ErrSessionSuperseded + } + current, err := h.sessionMgr.GetSession(sessionID) + if err != nil { + unlock() + return nil, err + } + if sessionTranscodeRoute(current) != previousRoute { + unlock() + return nil, playback.ErrSessionSuperseded + } + + successor, err := h.startLocalPlaybackTransport(ctx, opts) + if err != nil { + unlock() + return nil, err + } + if _, err := successor.WaitForManifest(8 * time.Second); err != nil { + _ = successor.Close() + unlock() + return nil, fmt.Errorf("local successor not ready: %w", err) + } + + rollback, published, err := h.sessionMgr.ApplyReplacementIfRoute(sessionID, previousRoute, replacement(successor)) + if err != nil || !published { + _ = successor.Close() + unlock() + if err != nil { + return nil, err + } + return nil, playback.ErrSessionSuperseded + } + if !h.tm.SwapTranscodeSessionIf(sessionID, previousLocal, successor) { + rollbackErr := h.sessionMgr.RollbackReplacement(sessionID, rollback) + _ = successor.Close() + unlock() + return nil, errors.Join(playback.ErrSessionSuperseded, rollbackErr) + } + unlock() + + if previousLocal != nil { + _ = previousLocal.Close() + } + if previousRoute.NodeURL != "" { + previousProcessID := previousRoute.TransportID + if previousProcessID == "" { + previousProcessID = sessionID + } + h.tm.StopRemoteTranscode(previousProcessID, previousRoute.NodeURL) + } + return successor, nil +} + +func (h *PlaybackHandler) closeTranscodeForSession(session *playback.Session) { + if session == nil { + return + } + // Local sessions remain keyed by the public playback session. Remote v3 + // processes use a plan-scoped transport identity so a prepared successor can + // coexist with its predecessor until commit. + h.tm.CloseTranscodeSession(session.ID, "") + if session.TranscodeNodeURL != "" { + h.tm.StopRemoteTranscode(remoteTransportID(session), session.TranscodeNodeURL) + } +} + func (h *PlaybackHandler) loadFileByPreferredID( ctx context.Context, preferredID int, @@ -1083,35 +1348,41 @@ func (h *PlaybackHandler) scrobbleEventForSession(ctx context.Context, session * DurationSeconds: duration, OccurredAt: time.Now().UTC(), } - if h.StableIdentityResolver == nil { - event.Kind = "movie" - return event - } - identity := h.StableIdentityResolver.ResolveHistoryIdentity(ctx, mediaItemID) - event.Kind = identity.StableType - if event.Kind == "" { - event.Kind = "movie" - } - event.SeasonNumber = intPtrValue(identity.Season) - event.EpisodeNumber = intPtrValue(identity.Episode) - if identity.ProviderIDs != nil { - event.IMDbID = identity.ProviderIDs["imdb"] - event.TMDBID = identity.ProviderIDs["tmdb"] - event.TVDBID = identity.ProviderIDs["tvdb"] - } - if identity.SeriesProviderIDs != nil { - event.SeriesIMDbID = identity.SeriesProviderIDs["imdb"] - event.SeriesTMDBID = identity.SeriesProviderIDs["tmdb"] - event.SeriesTVDBID = identity.SeriesProviderIDs["tvdb"] - } - return event + return watchsync.ResolveScrobbleIdentity(ctx, h.StableIdentityResolver, event) } -func intPtrValue(value *int) int { - if value == nil { - return 0 +func (h *PlaybackHandler) scrobbleEventForStoppedSession( + ctx context.Context, + session *playback.Session, + stopResult watchstate.PlaybackStopResult, +) (watchsync.ScrobbleEvent, bool) { + if session == nil || session.DisableProgressPersistence { + return watchsync.ScrobbleEvent{}, false } - return *value + + mediaItemID := stopResult.MediaItemID + duration := stopResult.DurationSeconds + position := stopResult.FinalPositionSeconds + if mediaItemID == "" { + if h.fileResolver == nil { + return watchsync.ScrobbleEvent{}, false + } + file, err := h.loadFileByPreferredID(ctx, requestedMediaFileID(session), session.MediaFileID) + if err != nil || file == nil { + return watchsync.ScrobbleEvent{}, false + } + mediaItemID = playbackProgressTarget(file) + if mediaItemID == "" { + return watchsync.ScrobbleEvent{}, false + } + duration = float64(file.Duration) + position = session.Position + } + + event := h.scrobbleEventForSession(ctx, session, mediaItemID, duration, position) + event.HistoryID = stopResult.HistoryID + event.Completed = stopResult.Completed + return event, true } func (h *PlaybackHandler) buildAdminHistoryEntry( @@ -1198,15 +1469,12 @@ func (h *PlaybackHandler) finalizeSessionStop(ctx context.Context, session *play } stopResult := h.persistStopAndHistory(ctx, session) - if h.WatchScrobbler != nil && stopResult.MediaItemID != "" { - event := h.scrobbleEventForSession(ctx, session, stopResult.MediaItemID, stopResult.DurationSeconds, stopResult.FinalPositionSeconds) - event.HistoryID = stopResult.HistoryID - event.Completed = stopResult.Completed - if stopResult.Completed { + if h.WatchScrobbler != nil { + if event, ok := h.scrobbleEventForStoppedSession(ctx, session, stopResult); ok && (userInitiated || stopResult.Completed) { if err := h.WatchScrobbler.ScrobbleStop(ctx, event); err != nil { slog.WarnContext(ctx, "failed to queue watch provider stop scrobble", "component", "api", "session", session.ID, "error", err) } - } else if !stopResult.SkippedBelowMinResume { + } else if ok { if err := h.WatchScrobbler.ScrobblePause(ctx, event); err != nil { slog.WarnContext(ctx, "failed to queue watch provider pause scrobble", "component", "api", "session", session.ID, "error", err) } @@ -1226,7 +1494,7 @@ func (h *PlaybackHandler) finalizeSessionStop(ctx context.Context, session *play } } - h.tm.CloseTranscodeSession(session.ID, session.TranscodeNodeURL) + h.closeTranscodeForSession(session) if syncNow { h.syncSessionsNow(ctx, syncReason) } @@ -1260,7 +1528,7 @@ func (h *PlaybackHandler) finalizeSessionAbort(ctx context.Context, session *pla // Abort is a connection drop / non-terminal teardown — keep the recipe card // so the client can reconstruct on reconnect. - h.tm.CloseTranscodeSession(session.ID, session.TranscodeNodeURL) + h.closeTranscodeForSession(session) if syncNow { h.syncSessionsNow(ctx, syncReason) } @@ -1358,8 +1626,36 @@ func (h *PlaybackHandler) persistAudioPreference( // --- Handler methods --- -// HandleStartPlayback handles POST /playback/start. +// HandleStartPlayback dispatches the shared start endpoint by protocol +// envelope while preserving the exact legacy request decoder and behavior. func (h *PlaybackHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Request) { + if apimw.GetUserID(r.Context()) == 0 { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return + } + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxPlaybackV3BodyBytes)) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body") + return + } + var envelope struct { + ProtocolVersion *int `json:"protocol_version"` + } + if err := json.NewDecoder(bytes.NewReader(body)).Decode(&envelope); err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body") + return + } + if envelope.ProtocolVersion != nil && *envelope.ProtocolVersion == playback.ProtocolV3 { + h.handleStartPlaybackV3(w, r, body) + return + } + r.Body = io.NopCloser(bytes.NewReader(body)) + h.handleStartPlaybackLegacy(w, r) +} + +// handleStartPlaybackLegacy is the pre-v3 start implementation. Keep changes +// to this function independent from protocol-v3 routing. +func (h *PlaybackHandler) handleStartPlaybackLegacy(w http.ResponseWriter, r *http.Request) { userID := apimw.GetUserID(r.Context()) if userID == 0 { writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") @@ -1703,6 +1999,11 @@ func (h *PlaybackHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Req } } + if h.protocolV3ShadowEnabled(r.Context()) { + shadowReq := req + shadowReq.ProfileID = profileID + go h.shadowLegacyPlaybackV3(context.WithoutCancel(r.Context()), shadowReq, requestedFile, effectiveFile, audioTrackIndex, session.PlayMethod, session.TranscodeAudio, session.ID) + } h.syncSessionsNow(r.Context(), "start") writeJSON(w, http.StatusCreated, resp) } @@ -1958,6 +2259,7 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return } + previousRoute := sessionTranscodeRoute(session) var req changeAudioRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { @@ -2021,6 +2323,60 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. } else if transcodeAudio { targetAudioCodec = "aac" } + + // A legacy copy-video restart needs a fresh keyframe origin for the new + // position. Resolve it before mutating the durable session or stopping the + // current transport so a probe failure leaves the active stream intact. + restartSegmentDuration := session.SegmentDuration + if restartSegmentDuration <= 0 { + restartSegmentDuration = playback.DefaultSegmentDuration + } + if ts := h.tm.GetTranscodeSession(sessionID); ts != nil { + if liveDuration := ts.Opts().SegmentDuration; liveDuration > 0 { + restartSegmentDuration = liveDuration + } + } + restartSeekSeconds := alignedSeekSeconds(req.Position, restartSegmentDuration, targetVideoCodec) + restartStartSegment := computeStartSegment(restartSeekSeconds, restartSegmentDuration) + restartStreamOriginSeconds := 0.0 + restartCopyAnchorResolved := false + legacyCopyRestart := session.PlayMethod == playback.PlayTranscode && + strings.EqualFold(targetVideoCodec, "copy") && isLegacyTransportSession(session) + if legacyCopyRestart { + restartCopyAnchorResolved = true + if req.Position > 0 { + anchor, anchorSegment, anchorErr := h.resolveLegacyCopySeekAnchor( + r.Context(), + h.playbackConfig().FFmpegPath, + file.FilePath, + req.Position, + restartSegmentDuration, + ) + if anchorErr != nil { + slog.ErrorContext(r.Context(), "failed to resolve copy-video audio-switch seek anchor", "component", "api", + "playback_session_id", sessionID, + "requested_seek_seconds", req.Position, + "error", anchorErr, + ) + writeError(w, http.StatusInternalServerError, "remux_seek_anchor_failed", "Failed to resolve remux seek position") + return + } + restartStreamOriginSeconds = anchor + restartStartSegment = anchorSegment + } + } + var deferredRemoteCopyPlan *nodepool.Plan + if legacyCopyRestart && strings.TrimSpace(session.TranscodeNodeURL) != "" && + h.NodePlanner != nil && h.JWTSecret != "" { + estKbps := targetBitrateKbps + if estKbps <= 0 { + estKbps = fileBitrateKbps(file) + } + plan := h.NodePlanner.PlanSession(sessionID, session.TranscodeNodeURL, true, estKbps) + if plan.ProxyNode != nil && plan.TranscodeNode != nil { + deferredRemoteCopyPlan = &plan + } + } slog.InfoContext(r.Context(), "audio switch computed playback state", "component", "api", "playback_session_id", sessionID, "previous_base_play_method", baseMethod, @@ -2030,46 +2386,6 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. "audio_codec", newTrack.Codec, "transcode_audio", transcodeAudio, ) - if err := h.sessionMgr.UpdateAudioTrack(sessionID, req.AudioTrackIndex, newMethod); err != nil { - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update audio track") - return - } - if err := h.sessionMgr.UpdateStreamState(sessionID, playback.SessionStreamState{ - PlayMethod: session.PlayMethod, - BasePlayMethod: newMethod, - AudioTrackIndex: req.AudioTrackIndex, - TranscodeAudio: transcodeAudio, - ClientIP: session.ClientIP, - StreamBitrateKbps: streamBitrateKbps, - TargetResolution: targetResolution, - TargetVideoCodec: targetVideoCodec, - TargetAudioCodec: targetAudioCodec, - TargetBitrateKbps: targetBitrateKbps, - // Carry the byte-affecting recipe forward: an audio switch changes only the - // audio selection, so subtitles and cadence must survive the state update - // (UpdateStreamState overwrites these fields unconditionally). - SubtitleTrackIndex: session.SubtitleTrackIndex, - SubtitleBurnIn: session.SubtitleBurnIn, - SegmentDuration: session.SegmentDuration, - }); err != nil { - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update stream state") - return - } - - // Handle transcode restart. - if session.PlayMethod == playback.PlayTranscode { - if ts := h.tm.GetTranscodeSession(sessionID); ts != nil { - ts.SetAudioTrackIndex(req.AudioTrackIndex) - tsOpts := ts.Opts() - startSegment := computeStartSegment(req.Position, tsOpts.SegmentDuration) - seekSeconds := alignedSeekSeconds(req.Position, tsOpts.SegmentDuration, tsOpts.TargetCodecVideo) - // Throttler + exit monitor re-arm via the session's restart hook. - if restartErr := h.tm.RestartSessionLocked(context.WithoutCancel(r.Context()), sessionID, ts, seekSeconds, startSegment); restartErr != nil { - slog.ErrorContext(r.Context(), "failed to restart transcode for audio switch", "component", "api", "session", sessionID, "error", restartErr) - } - } - } - updatedSession := *session updatedSession.AudioTrackIndex = req.AudioTrackIndex updatedSession.BasePlayMethod = newMethod @@ -2081,13 +2397,110 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. updatedSession.TargetVideoCodec = targetVideoCodec updatedSession.TargetAudioCodec = targetAudioCodec updatedSession.TargetBitrateKbps = targetBitrateKbps + updatedSession.SegmentDuration = restartSegmentDuration + + audioSwitchReplacement := func(route playback.TranscodeRoute) playback.SessionReplacement { + return playback.SessionReplacement{ + EffectiveMediaFileID: session.MediaFileID, + StreamState: playback.SessionStreamState{ + PlayMethod: updatedSession.PlayMethod, + BasePlayMethod: newMethod, + AudioTrackIndex: req.AudioTrackIndex, + TranscodeAudio: transcodeAudio, + RemuxDVMode: session.RemuxDVMode, + ClientIP: session.ClientIP, + ClientName: session.ClientName, + ClientVersion: session.ClientVersion, + ClientUserAgent: session.ClientUserAgent, + StreamBitrateKbps: streamBitrateKbps, + TargetResolution: targetResolution, + TargetVideoCodec: targetVideoCodec, + TargetAudioCodec: targetAudioCodec, + TargetBitrateKbps: targetBitrateKbps, + TranscodeHWAccel: updatedSession.TranscodeHWAccel, + TranscodeNodeURL: route.NodeURL, + TranscodeTransportID: route.TransportID, + TranscodeRouteSet: true, + SubtitleTrackIndex: session.SubtitleTrackIndex, + SubtitleBurnIn: session.SubtitleBurnIn, + SegmentDuration: restartSegmentDuration, + }, + } + } + audioStatePublished := false + publishAudioSwitch := func(route playback.TranscodeRoute) error { + if _, err := h.sessionMgr.ApplyReplacement(sessionID, audioSwitchReplacement(route)); err != nil { + return err + } + audioStatePublished = true + updatedSession.TranscodeNodeURL = route.NodeURL + updatedSession.TranscodeTransportID = route.TransportID + return nil + } + + // Local audio switches stage a successor in a generation-scoped directory. + // The predecessor keeps serving until the successor has a manifest and the + // full session replacement has been published atomically. + if session.PlayMethod == playback.PlayTranscode && deferredRemoteCopyPlan == nil { + if previousLocal := h.tm.GetTranscodeSession(sessionID); previousLocal != nil { + opts := previousLocal.Opts() + outputSubdir := newLegacyTransportID(sessionID) + opts.OutputSubdir = outputSubdir + opts.OutputDir = filepath.Join(h.playbackConfig().TranscodeDir, outputSubdir) + opts.TranscodeTransportID = "" + opts.AudioTrackIndex = req.AudioTrackIndex + opts.SeekSeconds = restartSeekSeconds + opts.StartSegmentNumber = restartStartSegment + opts.StreamOriginSeconds = restartStreamOriginSeconds + opts.CopySeekAnchorResolved = restartCopyAnchorResolved + opts.FastStart = true + successor, restartErr := h.commitLegacyLocalReplacement( + context.WithoutCancel(r.Context()), + sessionID, + previousLocal, + previousRoute, + opts, + func(successor *playback.TranscodeSession) playback.SessionReplacement { + updatedSession.TranscodeHWAccel = successor.Opts().HWAccel + return audioSwitchReplacement(playback.TranscodeRoute{}) + }, + ) + if restartErr != nil { + if errors.Is(restartErr, playback.ErrSessionSuperseded) { + writeError(w, http.StatusConflict, "transcode_replaced", "A newer playback transport replaced this request") + return + } + slog.ErrorContext(r.Context(), "failed to prepare transcode for audio switch", "component", "api", "session", sessionID, "error", restartErr) + writeError(w, http.StatusInternalServerError, "transcode_start_failed", "Failed to restart transcode session") + return + } + audioStatePublished = true + updatedSession.TranscodeNodeURL = "" + updatedSession.TranscodeTransportID = "" + successor.SetRestartHook(func(ctx context.Context) { + h.maybeStartThrottler(ctx, successor) + h.tm.MonitorLocalTranscodeExit(sessionID, successor) + }) + h.maybeStartThrottler(r.Context(), successor) + h.tm.MonitorLocalTranscodeExit(sessionID, successor) + } + } + + if session.PlayMethod != playback.PlayTranscode { + if err := publishAudioSwitch(playback.TranscodeRoute{}); err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update audio track") + return + } + } // The switched recipe travels in the freshly minted stream token on the new // serve URL below, so a post-restart reconstruct resumes with the switched // audio/method. For transcode the full-recipe manifest URL is rebuilt further // down (proxy or local); for direct/remux the identity token on StreamURL // carries the new audio selection. - h.persistAudioPreference(r.Context(), userID, session.ProfileID, file, req.AudioTrackIndex) + if audioStatePublished { + h.persistAudioPreference(r.Context(), userID, session.ProfileID, file, req.AudioTrackIndex) + } // For a local transcode, playbackStreamURL returns the bare manifest URL // without the full-recipe ?st= token, so a post-restart reconstruct would @@ -2119,22 +2532,28 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. if estKbps <= 0 { estKbps = fileBitrateKbps(file) } - plan := h.NodePlanner.PlanSession(sessionID, session.TranscodeNodeURL, needsTranscode, estKbps) + var plan nodepool.Plan + if deferredRemoteCopyPlan != nil { + plan = *deferredRemoteCopyPlan + } else { + plan = h.NodePlanner.PlanSession(sessionID, session.TranscodeNodeURL, needsTranscode, estKbps) + } if proxyNode := plan.ProxyNode; proxyNode != nil && (!needsTranscode || plan.TranscodeNode != nil) { // Remote (offloaded) transcode: the API server owns no local // TranscodeSession (the LOCAL restart block above was a no-op), so // the node's ffmpeg is still serving the OLD audio track. POST a - // fresh /transcode/start with the new AudioTrackIndex — the node - // tears down the existing session for this ID and restarts ffmpeg - // (handleStart in internal/transcodenode/server.go), which IS the - // remote restart mechanism — then mint the replacement proxy URL + // fresh /transcode/start with the new AudioTrackIndex. Encoded legacy + // streams retain the node's same-ID replacement behavior; copy-video + // streams prepare a distinct successor and retire the predecessor only + // after readiness and route publication. Then mint the proxy URL // from a FULL recipe card so a later node restart reconstructs with // the switched audio (the lean identity-only claims used for remux // below omit the byte-affecting encode fields and would 404). isOffloaded := strings.TrimSpace(session.TranscodeNodeURL) != "" if needsTranscode && plan.TranscodeNode != nil && isOffloaded { nodeURL := plan.TranscodeNode.URL - _ = h.sessionMgr.SetTranscodeNodeURL(sessionID, nodeURL) + atomicLegacyReplacement := legacyCopyRestart + previousLocalTranscode := h.tm.GetTranscodeSession(sessionID) // Restart from the FULL live recipe, not a partial re-derivation. // An audio switch alters only audio selection — subtitle burn-in and @@ -2145,97 +2564,144 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. // post-restart reconstruct, so recover them here. Embed a concrete // segment duration (not 0): the node's recipe token treats // SegmentDuration<=0 as "incomplete" and would 404 on a node restart. - segmentDuration := session.SegmentDuration - if segmentDuration <= 0 { - segmentDuration = playback.DefaultSegmentDuration - } - seekSeconds := alignedSeekSeconds(req.Position, segmentDuration, updatedSession.TargetVideoCodec) + segmentDuration := restartSegmentDuration subtitleTrackIndex := session.SubtitleTrackIndex subtitleBurnIn := session.SubtitleBurnIn subtitleCodec := "" if subtitleBurnIn && subtitleTrackIndex >= 0 { subtitleCodec = embeddedSubtitleCodec(file, subtitleTrackIndex) } - startSegment := computeStartSegment(seekSeconds, segmentDuration) - // Derive the encode recipe the same way HandleStartTranscode // does — from the durable session target fields plus the file — // changing only the audio track. SourceVideoCodec/TotalDuration // come from the file; the resolution/codec/bitrate targets and // hwaccel come from the session's persisted stream state. + // A v3 session's node job runs under its generation-scoped + // transport ID; restarting under the bare session ID would + // spawn a duplicate job beside it. + restartTransportID := remoteTransportID(&updatedSession) + if atomicLegacyReplacement { + restartTransportID = newLegacyTransportID(sessionID) + } nodeReq := transcodenode.TranscodeStartRequest{ - SessionID: sessionID, - InputPath: file.FilePath, - SourceVideoCodec: file.CodecVideo, - SeekSeconds: seekSeconds, - StartSegmentNumber: startSegment, - TargetResolution: updatedSession.TargetResolution, - TargetCodecVideo: updatedSession.TargetVideoCodec, - TargetCodecAudio: updatedSession.TargetAudioCodec, - TargetBitrateKbps: updatedSession.TargetBitrateKbps, - SegmentDuration: segmentDuration, - HWAccel: session.TranscodeHWAccel, - AudioTrackIndex: req.AudioTrackIndex, - SubtitleTrackIndex: subtitleTrackIndex, - SubtitleBurnIn: subtitleBurnIn, - SubtitleCodec: subtitleCodec, - TotalDuration: float64(file.Duration), + SessionID: restartTransportID, + InputPath: file.FilePath, + SourceVideoCodec: file.CodecVideo, + SeekSeconds: restartSeekSeconds, + StreamOriginSeconds: restartStreamOriginSeconds, + CopySeekAnchorResolved: restartCopyAnchorResolved, + StartSegmentNumber: restartStartSegment, + TargetResolution: updatedSession.TargetResolution, + TargetCodecVideo: updatedSession.TargetVideoCodec, + TargetCodecAudio: updatedSession.TargetAudioCodec, + TargetBitrateKbps: updatedSession.TargetBitrateKbps, + SegmentDuration: segmentDuration, + HWAccel: session.TranscodeHWAccel, + AudioTrackIndex: req.AudioTrackIndex, + SubtitleTrackIndex: subtitleTrackIndex, + SubtitleBurnIn: subtitleBurnIn, + SubtitleCodec: subtitleCodec, + TotalDuration: float64(file.Duration), + RequireReady: atomicLegacyReplacement, } if strings.TrimSpace(nodeReq.HWAccel) == "" { nodeReq.HWAccel = h.playbackConfig().HWAccel } - - body, _ := json.Marshal(nodeReq) - ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 10*time.Second) - defer cancel() - httpReq, reqErr := http.NewRequestWithContext(ctx, http.MethodPost, nodeURL+"/transcode/start", bytes.NewReader(body)) - if reqErr != nil { - slog.ErrorContext(r.Context(), "failed to build remote transcode restart for audio switch", "component", "api", "session", sessionID, "node", nodeURL, "error", reqErr) - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to build transcode request") - return + // A v3 DV strip remux carries its bitstream filter in the + // durable session route; dropping it here would hand the node + // a DV7 copy recipe that leaves dangling RPUs. + if updatedSession.RemuxDVMode == playback.RemuxDVStripToHDR10V3 && strings.EqualFold(nodeReq.TargetCodecVideo, "copy") { + nodeReq.VideoBitstreamFilter = playback.DV7ToHDR10BitstreamFilter } - httpReq.Header.Set("Content-Type", "application/json") - httpReq.Header.Set("Authorization", "Bearer "+h.JWTSecret) - nodeResp, doErr := http.DefaultClient.Do(httpReq) - if doErr != nil { - slog.ErrorContext(r.Context(), "remote transcode restart for audio switch failed", "component", "api", "session", sessionID, "node", nodeURL, "error", doErr) + startResp, status, startErr := h.startRemotePlaybackTransport( + context.WithoutCancel(r.Context()), + nodeURL, + nodeReq, + ) + if startErr != nil { + if atomicLegacyReplacement { + h.tm.StopRemoteTranscode(restartTransportID, nodeURL) + } + slog.ErrorContext(r.Context(), "remote transcode restart for audio switch failed", "component", "api", "session", sessionID, "node", nodeURL, "error", startErr) writeError(w, http.StatusBadGateway, "transcode_node_unavailable", "Transcode node is unavailable") return } - defer nodeResp.Body.Close() - if nodeResp.StatusCode != http.StatusAccepted { - slog.ErrorContext(r.Context(), "remote transcode restart for audio switch rejected", "component", "api", "session", sessionID, "node", nodeURL, "status", nodeResp.StatusCode) + if status != http.StatusAccepted { + if atomicLegacyReplacement { + h.tm.StopRemoteTranscode(restartTransportID, nodeURL) + } + slog.ErrorContext(r.Context(), "remote transcode restart for audio switch rejected", "component", "api", "session", sessionID, "node", nodeURL, "status", status) writeError(w, http.StatusBadGateway, "transcode_start_failed", "Transcode node rejected the request") return } - - var startResp transcodenode.TranscodeStartResponse - if decErr := json.NewDecoder(nodeResp.Body).Decode(&startResp); decErr != nil { - slog.WarnContext(r.Context(), "remote transcode restart response decode failed", "component", "api", "session", sessionID, "node", nodeURL, "error", decErr) + effectiveHWAccel := effectiveRemoteHWAccel(startResp, nodeReq) + updatedSession.TranscodeHWAccel = effectiveHWAccel + successorTransportID := restartTransportID + if !atomicLegacyReplacement { + // Empty is the durable legacy representation for a node process + // running under the public playback session ID. + successorTransportID = session.TranscodeTransportID } - effectiveHWAccel := strings.TrimSpace(startResp.HWAccel) - if effectiveHWAccel == "" { - effectiveHWAccel = strings.TrimSpace(nodeReq.HWAccel) + successorRoute := playback.TranscodeRoute{NodeURL: nodeURL, TransportID: successorTransportID} + if atomicLegacyReplacement { + if routeErr := h.commitLegacyRemoteReplacement( + sessionID, + previousLocalTranscode, + previousRoute, + audioSwitchReplacement(successorRoute), + ); routeErr != nil { + h.tm.StopRemoteTranscode(restartTransportID, nodeURL) + if errors.Is(routeErr, playback.ErrSessionSuperseded) { + writeError(w, http.StatusConflict, "transcode_replaced", "A newer playback transport replaced this request") + return + } + slog.ErrorContext(r.Context(), "publish remote audio-switch transcode route", "component", "api", "session", sessionID, "node", nodeURL, "error", routeErr) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to publish transcode session") + return + } + audioStatePublished = true + if !h.transcodeRouteMatches(sessionID, nil, successorRoute) { + writeError(w, http.StatusConflict, "transcode_replaced", "A newer playback transport replaced this request") + return + } + } else { + if routeErr := h.commitLegacyRemoteLastWriter( + sessionID, + successorRoute, + audioSwitchReplacement(successorRoute), + ); routeErr != nil { + slog.ErrorContext(r.Context(), "publish remote audio-switch transcode route", "component", "api", "session", sessionID, "node", nodeURL, "error", routeErr) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to publish transcode session") + return + } + audioStatePublished = true } + updatedSession.TranscodeNodeURL = successorRoute.NodeURL + updatedSession.TranscodeTransportID = successorRoute.TransportID + h.persistAudioPreference(r.Context(), userID, session.ProfileID, file, req.AudioTrackIndex) card := playback.NewRecipeCard(updatedSession.UserID, updatedSession.ProfileID, updatedSession.MediaFileID, nodeURL, playback.TranscodeOpts{ - InputPath: nodeReq.InputPath, - SessionID: nodeReq.SessionID, - SourceVideoCodec: nodeReq.SourceVideoCodec, - SeekSeconds: nodeReq.SeekSeconds, - StartSegmentNumber: nodeReq.StartSegmentNumber, - TargetResolution: nodeReq.TargetResolution, - TargetCodecVideo: nodeReq.TargetCodecVideo, - TargetCodecAudio: nodeReq.TargetCodecAudio, - TargetBitrateKbps: nodeReq.TargetBitrateKbps, - SegmentDuration: nodeReq.SegmentDuration, - HWAccel: effectiveHWAccel, - AudioTrackIndex: nodeReq.AudioTrackIndex, - SubtitleTrackIndex: nodeReq.SubtitleTrackIndex, - SubtitleBurnIn: nodeReq.SubtitleBurnIn, - SubtitleCodec: nodeReq.SubtitleCodec, - TotalDuration: nodeReq.TotalDuration, + InputPath: nodeReq.InputPath, + SessionID: sessionID, + TranscodeTransportID: restartTransportID, + VideoBitstreamFilter: nodeReq.VideoBitstreamFilter, + SourceVideoCodec: nodeReq.SourceVideoCodec, + SeekSeconds: nodeReq.SeekSeconds, + StreamOriginSeconds: nodeReq.StreamOriginSeconds, + CopySeekAnchorResolved: nodeReq.CopySeekAnchorResolved, + StartSegmentNumber: nodeReq.StartSegmentNumber, + TargetResolution: nodeReq.TargetResolution, + TargetCodecVideo: nodeReq.TargetCodecVideo, + TargetCodecAudio: nodeReq.TargetCodecAudio, + TargetBitrateKbps: nodeReq.TargetBitrateKbps, + SegmentDuration: nodeReq.SegmentDuration, + HWAccel: effectiveHWAccel, + AudioTrackIndex: nodeReq.AudioTrackIndex, + SubtitleTrackIndex: nodeReq.SubtitleTrackIndex, + SubtitleBurnIn: nodeReq.SubtitleBurnIn, + SubtitleCodec: nodeReq.SubtitleCodec, + TotalDuration: nodeReq.TotalDuration, }) resp.StreamURL = h.buildProxyManifestURL(card, proxyNode) } else { @@ -2254,10 +2720,13 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. UserID: updatedSession.UserID, ProfileID: updatedSession.ProfileID, MediaFileID: updatedSession.MediaFileID, + // v3 sessions route by transport ID and pin an explicit DV + // mode; a re-minted token must not silently shed either. + TranscodeTransportID: updatedSession.TranscodeTransportID, + RemuxDVMode: string(updatedSession.RemuxDVMode), } if plan.TranscodeNode != nil { tokenClaims.TranscodeNode = plan.TranscodeNode.URL - _ = h.sessionMgr.SetTranscodeNodeURL(sessionID, plan.TranscodeNode.URL) } if token, signErr := streamtoken.Sign(tokenClaims, h.JWTSecret, playback.MaxTokenTTL); signErr == nil { switch updatedSession.PlayMethod { @@ -2270,6 +2739,16 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. } } } + if !audioStatePublished { + if err := publishAudioSwitch(previousRoute); err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update audio track") + return + } + h.persistAudioPreference(r.Context(), userID, session.ProfileID, file, req.AudioTrackIndex) + } + if legacyCopyRestart { + resp.setCopyTimeline(req.Position, restartStreamOriginSeconds) + } h.syncSessionsNow(r.Context(), "audio_change") writeJSON(w, http.StatusOK, resp) @@ -2421,33 +2900,24 @@ type transcodeStartState struct { hwAccel string } -// finalizeTranscodeStart updates the playback session state after a transcode -// has been started (either locally or on a remote node). -func (h *PlaybackHandler) finalizeTranscodeStart(r *http.Request, st transcodeStartState) { - if st.switchedFileID != nil { - if err := h.sessionMgr.SetEffectiveMediaFileID(st.req.SessionID, *st.switchedFileID); err != nil { - slog.ErrorContext(r.Context(), "failed to update effective media file", "component", "api", "session", st.req.SessionID, "error", err, "playback_session_id", st.req.SessionID) - } +func effectiveRemoteHWAccel( + response transcodenode.TranscodeStartResponse, + request transcodenode.TranscodeStartRequest, +) string { + hwAccel := strings.TrimSpace(response.HWAccel) + if hwAccel == "" { + hwAccel = strings.TrimSpace(request.HWAccel) } + return hwAccel +} +func transcodeStartReplacement(st transcodeStartState, route playback.TranscodeRoute) playback.SessionReplacement { streamBitrateKbps := st.req.TargetBitrateKbps if streamBitrateKbps <= 0 { streamBitrateKbps = st.file.Bitrate } - transcodeAudio := st.req.TargetCodecAudio != "" && !strings.EqualFold(st.req.TargetCodecAudio, "copy") + transcodeAudio := playback.TranscodesAudio(st.req.TargetCodecAudio) baseMethod := semanticPlayMethod(st.session) - - slog.InfoContext(r.Context(), "transcode start preserved base playback state", "component", "api", - "playback_session_id", st.req.SessionID, - "base_play_method", baseMethod, - "transport_play_method", playback.PlayTranscode, - "audio_track_index", st.session.AudioTrackIndex, - "target_codec_video", st.req.TargetCodecVideo, - "target_codec_audio", st.req.TargetCodecAudio, - "copy_video_original", strings.EqualFold(st.req.TargetCodecVideo, "copy"), - "transcode_audio", transcodeAudio, - ) - // Persist the byte-affecting recipe (subtitles + segment cadence) so a later // offloaded audio switch can rebuild the exact same stream. The session is the // only recovery source for offloaded transcodes (no local ts.Opts()). Normalize @@ -2457,26 +2927,49 @@ func (h *PlaybackHandler) finalizeTranscodeStart(r *http.Request, st transcodeSt segmentDuration = playback.DefaultSegmentDuration } - if err := h.sessionMgr.UpdateStreamState(st.req.SessionID, playback.SessionStreamState{ - PlayMethod: playback.PlayTranscode, - BasePlayMethod: baseMethod, - AudioTrackIndex: st.session.AudioTrackIndex, - TranscodeAudio: transcodeAudio, - ClientIP: st.session.ClientIP, - StreamBitrateKbps: streamBitrateKbps, - TargetResolution: st.req.TargetResolution, - TargetVideoCodec: st.req.TargetCodecVideo, - TargetAudioCodec: st.req.TargetCodecAudio, - TargetBitrateKbps: st.req.TargetBitrateKbps, - TranscodeHWAccel: st.hwAccel, - SubtitleTrackIndex: st.req.SubtitleTrackIndex, - SubtitleBurnIn: st.req.SubtitleBurnIn, - SegmentDuration: segmentDuration, - }); err != nil { - slog.ErrorContext(r.Context(), "failed to update transcode stream state", "component", "api", "session", st.req.SessionID, "error", err, "playback_session_id", st.req.SessionID) + effectiveFileID := st.file.ID + if st.switchedFileID != nil { + effectiveFileID = *st.switchedFileID } + return playback.SessionReplacement{ + EffectiveMediaFileID: effectiveFileID, + StreamState: playback.SessionStreamState{ + PlayMethod: playback.PlayTranscode, + BasePlayMethod: baseMethod, + AudioTrackIndex: st.session.AudioTrackIndex, + TranscodeAudio: transcodeAudio, + RemuxDVMode: st.session.RemuxDVMode, + ClientIP: st.session.ClientIP, + ClientName: st.session.ClientName, + ClientVersion: st.session.ClientVersion, + ClientUserAgent: st.session.ClientUserAgent, + StreamBitrateKbps: streamBitrateKbps, + TargetResolution: st.req.TargetResolution, + TargetVideoCodec: st.req.TargetCodecVideo, + TargetAudioCodec: st.req.TargetCodecAudio, + TargetBitrateKbps: st.req.TargetBitrateKbps, + TranscodeHWAccel: st.hwAccel, + TranscodeNodeURL: route.NodeURL, + TranscodeTransportID: route.TransportID, + TranscodeRouteSet: true, + SubtitleTrackIndex: st.req.SubtitleTrackIndex, + SubtitleBurnIn: st.req.SubtitleBurnIn, + SegmentDuration: segmentDuration, + }, + } +} - h.syncSessionsNow(r.Context(), "transcode_start") +func logTranscodeStartState(r *http.Request, st transcodeStartState) { + slog.InfoContext(r.Context(), "transcode start preserved base playback state", "component", "api", + "playback_session_id", st.req.SessionID, + "base_play_method", semanticPlayMethod(st.session), + "transport_play_method", playback.PlayTranscode, + "audio_track_index", st.session.AudioTrackIndex, + "target_codec_video", st.req.TargetCodecVideo, + "target_codec_audio", st.req.TargetCodecAudio, + "copy_video_original", strings.EqualFold(st.req.TargetCodecVideo, "copy"), + "transcode_audio", st.req.TargetCodecAudio != "" && !strings.EqualFold(st.req.TargetCodecAudio, "copy"), + ) } // HandleStartTranscode handles POST /playback/transcode/start. @@ -2515,15 +3008,12 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re if !h.ensureUserTranscodingAllowed(w, r, userID, requiresVideoTranscode) { return } - // Close any existing transcode so a new one can start at different quality. - // Check both local sessions AND remote node assignments — without the - // remote check, switching quality on a transcode node never sends DELETE, - // leaving the old ffmpeg running and its segments on disk. - if h.tm.GetTranscodeSession(req.SessionID) != nil || session.TranscodeNodeURL != "" { - // Restarting the transcode under the SAME session id (quality/seek - // change) — keep the card; it is re-saved with the new opts below. - h.tm.CloseTranscodeSession(req.SessionID, session.TranscodeNodeURL) - } + // Keep the active transport alive through every fallible preflight. A local + // replacement is retired under the lifecycle lock immediately before spawn; + // a legacy remote replacement is prepared under a distinct process identity + // and retired only after the node accepts and the session publishes its successor. + previousLocalTranscode := h.tm.GetTranscodeSession(req.SessionID) + previousRoute := sessionTranscodeRoute(session) abortCurrentSession := func(reason string, cause error) { if abortErr := h.abortPlaybackSession(r.Context(), session); abortErr != nil && !errors.Is(abortErr, playback.ErrSessionNotFound) { slog.ErrorContext(r.Context(), "failed to abort playback session", "component", "api", @@ -2586,22 +3076,6 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re } } - // Resume and seek-start requests generally cannot safely stream-copy video - // into HLS output. Arbitrary HEVC seek points often land on non-keyframes, - // which can leave Chromium stuck on a frozen frame while audio continues - // advancing. MPEG-2 compatibility HLS is allowed to keep copy-video so Apple - // devices can avoid a full video transcode for those files. - if req.SeekSeconds > 0 && strings.EqualFold(req.TargetCodecVideo, "copy") && !playback.IsMPEG2VideoCodec(file.CodecVideo) { - slog.InfoContext(r.Context(), "forcing video transcode for seeked copy request", "component", "api", - "playback_session_id", req.SessionID, - "seek_seconds", req.SeekSeconds, - "source_video_codec", file.CodecVideo, - "requested_target_codec_video", req.TargetCodecVideo, - "effective_target_codec_video", "h264", - ) - req.TargetCodecVideo = "h264" - } - // Subtitle burn-in composites subtitles into the video frames, which is // impossible with -c:v copy. If the requested recipe would stream-copy // video (e.g. a remux "original" restart that adds burn-in), force an @@ -2617,9 +3091,24 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re req.TargetCodecVideo = "h264" } + // A copy-video HLS output of a Dolby Vision Profile 7 source must strip the + // RPU metadata (the enhancement layer is dropped by stream mapping): raw P7 + // NALs presented as plain HEVC stall hardware decoders. The V3 start path + // derives this from the plan (videoBitstreamFilterForPlanV3) and the + // audio-switch restart derives it from the durable session route, but this + // client-driven restart endpoint historically dropped it — the client asking + // for "copy" has no way to know the source needs the strip. Derived after + // the burn-in guard above so a copy request it rewrites to h264 never carries + // a copy-only bitstream filter. + videoBitstreamFilter := "" + if strings.EqualFold(req.TargetCodecVideo, "copy") && + (session.RemuxDVMode == playback.RemuxDVStripToHDR10V3 || file.PrimaryDVProfile() == 7) { + videoBitstreamFilter = playback.DV7ToHDR10BitstreamFilter + } + // The request-level permission check above intentionally runs before the - // existing transcode is closed. Recheck when server-side normalization has - // upgraded an allowed copy-video request into actual video encoding. + // existing transcode is closed. Recheck when subtitle burn-in normalization + // has upgraded an allowed copy-video request into actual video encoding. if !requiresVideoTranscode && !strings.EqualFold(req.TargetCodecVideo, "copy") && !h.ensureUserTranscodingAllowed(w, r, userID, true) { return @@ -2687,6 +3176,44 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re subtitleCodec = resolvedCodec } + // A copy-video input seek starts at the demuxer's preceding keyframe. Keep + // the requested position as FFmpeg's -ss input, but resolve that keyframe + // before selecting local/offloaded transport so filenames, reconstruction, + // and the client timeline all describe the media that is actually emitted. + playbackCfg := h.playbackConfig() + transportSeekSeconds := alignedSeekSeconds(req.SeekSeconds, req.SegmentDuration, req.TargetCodecVideo) + startSegmentNumber := computeStartSegment(transportSeekSeconds, req.SegmentDuration) + streamOriginSeconds := 0.0 + if videoCopy { + streamOriginSeconds = req.SeekSeconds + if req.SeekSeconds > 0 { + anchor, anchorSegment, anchorErr := h.resolveLegacyCopySeekAnchor( + r.Context(), + playbackCfg.FFmpegPath, + file.FilePath, + req.SeekSeconds, + req.SegmentDuration, + ) + if anchorErr != nil { + slog.ErrorContext(r.Context(), "failed to resolve copy-video seek anchor", "component", "api", + "playback_session_id", req.SessionID, + "requested_seek_seconds", req.SeekSeconds, + "error", anchorErr, + ) + writeError(w, http.StatusInternalServerError, "remux_seek_anchor_failed", "Failed to resolve remux seek position") + return + } + streamOriginSeconds = anchor + startSegmentNumber = anchorSegment + slog.DebugContext(r.Context(), "resolved copy-video seek anchor", "component", "api", + "playback_session_id", req.SessionID, + "requested_seek_seconds", req.SeekSeconds, + "stream_origin_seconds", streamOriginSeconds, + "start_segment_number", startSegmentNumber, + ) + } + } + // Determine whether to run locally or forward to a remote transcode node. var plan nodepool.Plan if h.NodePlanner != nil { @@ -2700,66 +3227,99 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re if tcNode != nil { // Remote transcode: forward to the assigned node. - if err := h.sessionMgr.SetTranscodeNodeURL(req.SessionID, tcNode.URL); err != nil { - slog.ErrorContext(r.Context(), "set transcode node URL", "component", "api", "error", err, "session", req.SessionID, "playback_session_id", req.SessionID) + replacementTransportID := req.SessionID + reconstructionTransportID := "" + atomicLegacyReplacement := videoCopy && isLegacyTransportSession(session) + if atomicLegacyReplacement { + replacementTransportID = newLegacyTransportID(req.SessionID) + reconstructionTransportID = replacementTransportID } - nodeReq := transcodenode.TranscodeStartRequest{ - SessionID: req.SessionID, - InputPath: file.FilePath, - SourceVideoCodec: file.CodecVideo, - SeekSeconds: alignedSeekSeconds(req.SeekSeconds, req.SegmentDuration, req.TargetCodecVideo), - StartSegmentNumber: computeStartSegment(req.SeekSeconds, req.SegmentDuration), - TargetResolution: req.TargetResolution, - TargetCodecVideo: req.TargetCodecVideo, - TargetCodecAudio: req.TargetCodecAudio, - TargetBitrateKbps: req.TargetBitrateKbps, - SegmentDuration: req.SegmentDuration, - HWAccel: h.playbackConfig().HWAccel, - AudioTrackIndex: session.AudioTrackIndex, - SubtitleTrackIndex: req.SubtitleTrackIndex, - SubtitleBurnIn: req.SubtitleBurnIn, - SubtitleCodec: subtitleCodec, - TotalDuration: float64(file.Duration), + SessionID: replacementTransportID, + InputPath: file.FilePath, + SourceVideoCodec: file.CodecVideo, + VideoBitstreamFilter: videoBitstreamFilter, + SeekSeconds: transportSeekSeconds, + StreamOriginSeconds: streamOriginSeconds, + CopySeekAnchorResolved: videoCopy, + StartSegmentNumber: startSegmentNumber, + TargetResolution: req.TargetResolution, + TargetCodecVideo: req.TargetCodecVideo, + TargetCodecAudio: req.TargetCodecAudio, + TargetBitrateKbps: req.TargetBitrateKbps, + SegmentDuration: req.SegmentDuration, + HWAccel: playbackCfg.HWAccel, + AudioTrackIndex: session.AudioTrackIndex, + SubtitleTrackIndex: req.SubtitleTrackIndex, + SubtitleBurnIn: req.SubtitleBurnIn, + SubtitleCodec: subtitleCodec, + TotalDuration: float64(file.Duration), + RequireReady: atomicLegacyReplacement, } - body, _ := json.Marshal(nodeReq) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - - httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, tcNode.URL+"/transcode/start", bytes.NewReader(body)) - if err != nil { - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to build transcode request") - return - } - httpReq.Header.Set("Content-Type", "application/json") - httpReq.Header.Set("Authorization", "Bearer "+h.JWTSecret) - - resp, err := http.DefaultClient.Do(httpReq) + nodeResp, status, err := h.startRemotePlaybackTransport(context.Background(), tcNode.URL, nodeReq) if err != nil { + if atomicLegacyReplacement { + h.tm.StopRemoteTranscode(replacementTransportID, tcNode.URL) + } slog.ErrorContext(r.Context(), "remote transcode start failed", "component", "api", "error", err, "node", tcNode.URL, "session", req.SessionID, "playback_session_id", req.SessionID) writeError(w, http.StatusBadGateway, "transcode_node_unavailable", "Transcode node is unavailable") return } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusAccepted { - slog.ErrorContext(r.Context(), "remote transcode start rejected", "component", "api", "status", resp.StatusCode, "node", tcNode.URL) + if status != http.StatusAccepted { + if atomicLegacyReplacement { + h.tm.StopRemoteTranscode(replacementTransportID, tcNode.URL) + } + slog.ErrorContext(r.Context(), "remote transcode start rejected", "component", "api", "status", status, "node", tcNode.URL) writeError(w, http.StatusBadGateway, "transcode_start_failed", "Transcode node rejected the request") return } - var nodeResp transcodenode.TranscodeStartResponse - if err := json.NewDecoder(resp.Body).Decode(&nodeResp); err != nil { - slog.WarnContext(r.Context(), "remote transcode start response decode failed", "component", "api", - "error", err, - "node", tcNode.URL, - "session", req.SessionID, - "playback_session_id", req.SessionID, - ) + effectiveHWAccel := effectiveRemoteHWAccel(nodeResp, nodeReq) + startState := transcodeStartState{ + req: req, + file: file, + session: session, + switchedFileID: switchedFileID, + hwAccel: effectiveHWAccel, } - effectiveHWAccel := strings.TrimSpace(nodeResp.HWAccel) - if effectiveHWAccel == "" { - effectiveHWAccel = strings.TrimSpace(nodeReq.HWAccel) + successorRoute := playback.TranscodeRoute{ + NodeURL: tcNode.URL, + TransportID: reconstructionTransportID, + } + if atomicLegacyReplacement { + if err := h.commitLegacyRemoteReplacement( + req.SessionID, + previousLocalTranscode, + previousRoute, + transcodeStartReplacement(startState, successorRoute), + ); err != nil { + h.tm.StopRemoteTranscode(replacementTransportID, tcNode.URL) + if errors.Is(err, playback.ErrSessionSuperseded) { + writeError(w, http.StatusConflict, "transcode_replaced", "A newer playback transport replaced this request") + return + } + slog.ErrorContext(r.Context(), "publish legacy transcode route", "component", "api", "error", err, "session", req.SessionID, "playback_session_id", req.SessionID) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to publish transcode session") + return + } + if !h.transcodeRouteMatches(req.SessionID, nil, successorRoute) { + writeError(w, http.StatusConflict, "transcode_replaced", "A newer playback transport replaced this request") + return + } + logTranscodeStartState(r, startState) + h.syncSessionsNow(r.Context(), "transcode_start") + } else { + if err := h.commitLegacyRemoteLastWriter( + req.SessionID, + successorRoute, + transcodeStartReplacement(startState, successorRoute), + ); err != nil { + slog.ErrorContext(r.Context(), "publish remote transcode route", "component", "api", "error", err, "session", req.SessionID, "playback_session_id", req.SessionID) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to publish transcode session") + return + } + logTranscodeStartState(r, startState) + h.syncSessionsNow(r.Context(), "transcode_start") } // The remote transcode's full recipe rides the proxy manifest token so the @@ -2767,32 +3327,29 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re // (and a node could someday self-reconstruct from it). Node-side segment // reconstruction is a follow-up (see spec multi-node section). card := playback.NewRecipeCard(session.UserID, session.ProfileID, session.MediaFileID, tcNode.URL, playback.TranscodeOpts{ - InputPath: nodeReq.InputPath, - SessionID: nodeReq.SessionID, - SourceVideoCodec: nodeReq.SourceVideoCodec, - SeekSeconds: nodeReq.SeekSeconds, - StartSegmentNumber: nodeReq.StartSegmentNumber, - TargetResolution: nodeReq.TargetResolution, - TargetCodecVideo: nodeReq.TargetCodecVideo, - TargetCodecAudio: nodeReq.TargetCodecAudio, - TargetBitrateKbps: nodeReq.TargetBitrateKbps, - SegmentDuration: nodeReq.SegmentDuration, - HWAccel: effectiveHWAccel, - AudioTrackIndex: nodeReq.AudioTrackIndex, - SubtitleTrackIndex: nodeReq.SubtitleTrackIndex, - SubtitleBurnIn: nodeReq.SubtitleBurnIn, - SubtitleCodec: nodeReq.SubtitleCodec, - TotalDuration: nodeReq.TotalDuration, + InputPath: nodeReq.InputPath, + SessionID: req.SessionID, + TranscodeTransportID: reconstructionTransportID, + SourceVideoCodec: nodeReq.SourceVideoCodec, + VideoBitstreamFilter: nodeReq.VideoBitstreamFilter, + SeekSeconds: nodeReq.SeekSeconds, + StreamOriginSeconds: nodeReq.StreamOriginSeconds, + CopySeekAnchorResolved: nodeReq.CopySeekAnchorResolved, + StartSegmentNumber: nodeReq.StartSegmentNumber, + TargetResolution: nodeReq.TargetResolution, + TargetCodecVideo: nodeReq.TargetCodecVideo, + TargetCodecAudio: nodeReq.TargetCodecAudio, + TargetBitrateKbps: nodeReq.TargetBitrateKbps, + SegmentDuration: nodeReq.SegmentDuration, + HWAccel: effectiveHWAccel, + AudioTrackIndex: nodeReq.AudioTrackIndex, + SubtitleTrackIndex: nodeReq.SubtitleTrackIndex, + SubtitleBurnIn: nodeReq.SubtitleBurnIn, + SubtitleCodec: nodeReq.SubtitleCodec, + TotalDuration: nodeReq.TotalDuration, }) manifestURL := h.buildProxyManifestURL(card, plan.ProxyNode) - h.finalizeTranscodeStart(r, transcodeStartState{ - req: req, - file: file, - session: session, - switchedFileID: switchedFileID, - hwAccel: effectiveHWAccel, - }) - writeJSON(w, http.StatusAccepted, buildTranscodeStartResponse(req, file, switchedFileID, manifestURL)) + writeJSON(w, http.StatusAccepted, buildTranscodeStartResponse(req, file, switchedFileID, manifestURL, streamOriginSeconds)) return } @@ -2806,52 +3363,112 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re } // Snapshot once so the directory, ffmpeg path, and hwaccel of this // session stay consistent even if the config reloads mid-start. - playbackCfg := h.playbackConfig() if err := os.MkdirAll(playbackCfg.TranscodeDir, 0o755); err != nil { writeError(w, http.StatusInternalServerError, "internal_error", "Failed to prepare transcode directory") return } - // Hold the per-session lifecycle lock across teardown → spawn → register so a - // concurrent reconstruct (or another fresh start) cannot run a second ffmpeg - // writer against this session's output directory. The in-lock close tears down - // any session a reconstruct rebuilt between the earlier close and here so the - // fresh ffmpeg is the sole writer. - unlock := h.tm.LockSessionLifecycle(req.SessionID) - h.tm.CloseTranscodeSession(req.SessionID, "") - transcodeSession, err := playback.StartTranscode(context.WithoutCancel(r.Context()), playback.TranscodeOpts{ - InputPath: file.FilePath, - OutputDir: filepath.Join(playbackCfg.TranscodeDir, req.SessionID), - SessionID: req.SessionID, - SourceVideoCodec: file.CodecVideo, - SeekSeconds: alignedSeekSeconds(req.SeekSeconds, req.SegmentDuration, req.TargetCodecVideo), - StartSegmentNumber: computeStartSegment(req.SeekSeconds, req.SegmentDuration), - TargetResolution: req.TargetResolution, - TargetCodecVideo: req.TargetCodecVideo, - TargetCodecAudio: req.TargetCodecAudio, - TargetBitrateKbps: req.TargetBitrateKbps, - SegmentDuration: req.SegmentDuration, - FFmpegPath: playbackCfg.FFmpegPath, - HWAccel: playbackCfg.HWAccel, - HWDevice: playbackCfg.HWDevice, - AudioTrackIndex: session.AudioTrackIndex, - SubtitleTrackIndex: req.SubtitleTrackIndex, - SubtitleBurnIn: req.SubtitleBurnIn, - SubtitleCodec: subtitleCodec, - TotalDuration: float64(file.Duration), - FastStart: true, - NodeType: "integrated", - ExecutionMode: "integrated", - FFmpegLogSink: h.FFmpegLogSink, - }) - if err != nil { - unlock() - writeError(w, http.StatusInternalServerError, "internal_error", "Failed to start transcode session") - return + localOpts := playback.TranscodeOpts{ + InputPath: file.FilePath, + OutputDir: filepath.Join(playbackCfg.TranscodeDir, req.SessionID), + SessionID: req.SessionID, + SourceVideoCodec: file.CodecVideo, + VideoBitstreamFilter: videoBitstreamFilter, + SeekSeconds: transportSeekSeconds, + StreamOriginSeconds: streamOriginSeconds, + CopySeekAnchorResolved: videoCopy, + StartSegmentNumber: startSegmentNumber, + TargetResolution: req.TargetResolution, + TargetCodecVideo: req.TargetCodecVideo, + TargetCodecAudio: req.TargetCodecAudio, + TargetBitrateKbps: req.TargetBitrateKbps, + SegmentDuration: req.SegmentDuration, + FFmpegPath: playbackCfg.FFmpegPath, + HWAccel: playbackCfg.HWAccel, + HWDevice: playbackCfg.HWDevice, + AudioTrackIndex: session.AudioTrackIndex, + SubtitleTrackIndex: req.SubtitleTrackIndex, + SubtitleBurnIn: req.SubtitleBurnIn, + SubtitleCodec: subtitleCodec, + TotalDuration: float64(file.Duration), + FastStart: true, + NodeType: "integrated", + ExecutionMode: "integrated", + FFmpegLogSink: h.FFmpegLogSink, } - - h.tm.RegisterTranscodeSession(req.SessionID, transcodeSession) - unlock() + startState := transcodeStartState{ + req: req, + file: file, + session: session, + switchedFileID: switchedFileID, + } + localRoute := playback.TranscodeRoute{} + var transcodeSession *playback.TranscodeSession + if videoCopy { + // Copy-mode successors must coexist with the predecessor until readiness. + // A generation-scoped directory prevents two ffmpeg processes from writing + // the same manifest and segments during that overlap. + outputSubdir := newLegacyTransportID(req.SessionID) + localOpts.OutputSubdir = outputSubdir + localOpts.OutputDir = filepath.Join(playbackCfg.TranscodeDir, outputSubdir) + transcodeSession, err = h.commitLegacyLocalReplacement( + context.WithoutCancel(r.Context()), + req.SessionID, + previousLocalTranscode, + previousRoute, + localOpts, + func(successor *playback.TranscodeSession) playback.SessionReplacement { + startState.hwAccel = successor.Opts().HWAccel + return transcodeStartReplacement(startState, localRoute) + }, + ) + if err != nil { + if errors.Is(err, playback.ErrSessionSuperseded) { + writeError(w, http.StatusConflict, "transcode_replaced", "A newer playback transport replaced this request") + return + } + slog.ErrorContext(r.Context(), "prepare local copy transcode replacement", "component", "api", "error", err, "session", req.SessionID, "playback_session_id", req.SessionID) + writeError(w, http.StatusInternalServerError, "transcode_start_failed", "Failed to start transcode session") + return + } + if !h.transcodeRouteMatches(req.SessionID, transcodeSession, localRoute) { + writeError(w, http.StatusConflict, "transcode_replaced", "A newer playback transport replaced this request") + return + } + } else { + // Encoded legacy starts keep their historical serialized last-writer-wins + // behavior. Re-read the predecessor under the lifecycle lock instead of + // rejecting this request because another overlapping start finished first. + unlock := h.tm.LockSessionLifecycle(req.SessionID) + currentSession, currentErr := h.sessionMgr.GetSession(req.SessionID) + if currentErr != nil { + unlock() + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to verify playback transport") + return + } + currentRoute := sessionTranscodeRoute(currentSession) + currentProcessID := remoteTransportID(currentSession) + h.tm.CloseTranscodeSession(req.SessionID, "") + transcodeSession, err = h.startLocalPlaybackTransport(r.Context(), localOpts) + if err != nil { + unlock() + writeError(w, http.StatusInternalServerError, "transcode_start_failed", "Failed to start transcode session") + return + } + h.tm.RegisterTranscodeSession(req.SessionID, transcodeSession) + startState.hwAccel = transcodeSession.Opts().HWAccel + if _, err := h.sessionMgr.ApplyReplacement(req.SessionID, transcodeStartReplacement(startState, localRoute)); err != nil { + h.tm.CloseTranscodeSessionIf(req.SessionID, transcodeSession, "") + unlock() + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to publish transcode session") + return + } + unlock() + if currentRoute.NodeURL != "" { + h.tm.StopRemoteTranscode(currentProcessID, currentRoute.NodeURL) + } + } + logTranscodeStartState(r, startState) // Re-arm the throttler and exit monitor after every Restart of this // handler-created session, regardless of which code path triggers it @@ -2875,24 +3492,18 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re fmt.Sprintf("/playback/transcode/%s/master.m3u8", req.SessionID), h.signSessionToken(card), ) - h.finalizeTranscodeStart(r, transcodeStartState{ - req: req, - file: file, - session: session, - switchedFileID: switchedFileID, - hwAccel: transcodeSession.Opts().HWAccel, - }) - writeJSON(w, http.StatusAccepted, buildTranscodeStartResponse(req, file, switchedFileID, manifestURL)) + h.syncSessionsNow(r.Context(), "transcode_start") + writeJSON(w, http.StatusAccepted, buildTranscodeStartResponse(req, file, switchedFileID, manifestURL, streamOriginSeconds)) } // HandleGetTranscodeManifest handles GET /playback/transcode/{session_id}/master.m3u8. // Auth is optional — the session UUID serves as an access token (same pattern // as /stream/{session_id}). When auth context is present, ownership is verified. // -// Known-duration sessions expose a synthetic full VOD manifest so the player -// can seek immediately. Copy-mode seeks that would start mid-GOP are forced to -// encoded HLS earlier in HandleStartTranscode; otherwise BuildPlaybackManifest -// still uses the same synthetic VOD path when the session duration is known. +// Known-duration encoded sessions expose a synthetic full VOD manifest so the +// player can seek immediately. Copy-video sessions expose FFmpeg's real +// keyframe-aligned manifest and use the resolved stream origin returned by +// HandleStartTranscode. func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") session, status, card := h.loadTranscodeServeSession(r, sessionID) @@ -2914,7 +3525,7 @@ func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *h if session.TranscodeNodeURL != "" { h.touchSessionActivity(sessionID) h.proxyToTranscodeNode(w, r, session.TranscodeNodeURL, - "/transcode/"+sessionID+"/master.m3u8") + "/transcode/"+remoteTransportID(session)+"/master.m3u8") return } // Local transcode whose process state was lost: reconstruct it from the @@ -2969,7 +3580,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht h.touchSessionActivity(sessionID) segmentName := chi.URLParam(r, "name") h.proxyToTranscodeNode(w, r, session.TranscodeNodeURL, - "/transcode/"+sessionID+"/segment/"+segmentName) + "/transcode/"+remoteTransportID(session)+"/segment/"+segmentName) return } // Resume near the segment the client is fetching so reconstruct does not diff --git a/internal/api/handlers/playback_sessions.go b/internal/api/handlers/playback_sessions.go index 4e2ea7d1..e0d202ac 100644 --- a/internal/api/handlers/playback_sessions.go +++ b/internal/api/handlers/playback_sessions.go @@ -72,6 +72,32 @@ type playbackSessionRow struct { RequestedVideoResolution string `json:"requested_video_resolution,omitempty"` VideoDecision string `json:"video_decision,omitempty"` AudioDecision string `json:"audio_decision,omitempty"` + EffectivePlayMethod string `json:"effective_play_method,omitempty"` + IsJellyfinClient bool `json:"is_jellyfin_client,omitempty"` +} + +// playbackSessionsCapabilitiesResponse advertises the additive fields of the +// live admin session payload so independently deployed clients (Android, +// Apple) can feature-detect them. Both fields are omitempty on the wire, so +// absence on a row is otherwise indistinguishable from an older server. +type playbackSessionsCapabilitiesResponse struct { + // EffectivePlayMethod reports that rows carry effective_play_method. + EffectivePlayMethod bool `json:"effective_play_method"` + // EffectivePlayMethodValues is the closed bucket vocabulary a supported + // server emits (absent field = unknown). + EffectivePlayMethodValues []string `json:"effective_play_method_values"` + // IsJellyfinClient reports that rows carry is_jellyfin_client. + IsJellyfinClient bool `json:"is_jellyfin_client"` +} + +// HandleGetSessionsCapabilities exposes additive feature support for the live +// admin session payload (GET /admin/sessions/capabilities). +func (h *AdminHandler) HandleGetSessionsCapabilities(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, http.StatusOK, playbackSessionsCapabilitiesResponse{ + EffectivePlayMethod: true, + EffectivePlayMethodValues: []string{"direct", "remux", "transcode", "audio"}, + IsJellyfinClient: true, + }) } // PlaybackSessionsQuery scopes live session listing. @@ -248,6 +274,8 @@ func enrichPlaybackSessionRow(row *playbackSessionRow, audioTracksJSON []byte) { } row.VideoDecision, row.AudioDecision = sessionComponentDecision(row.PlayMethod, row.TranscodeAudio, row.TargetVideoCodec) + row.EffectivePlayMethod = effectivePlayMethod(row.VideoDecision, row.AudioDecision) + row.IsJellyfinClient = isJellyfinEcosystemClient(row.ClientName, row.ClientUserAgent) var audioTracks []models.AudioTrack if len(audioTracksJSON) > 0 { @@ -335,6 +363,79 @@ func sessionComponentDecision(playMethod string, transcodeAudio bool, targetVide } } +// effectivePlayMethod reduces the per-stream decisions to the single bucket +// the admin activity views aggregate on. Raw play_method is misleading there: +// an HLS repackage reports play_method "transcode" with a copied video stream, +// and an audio-only re-encode reports "remux" — the decisions carry what +// actually costs CPU. +// - video re-encoded -> "transcode" +// - only audio re-encoded -> "audio" +// - streams only repackaged -> "remux" +// - nothing touched -> "direct" +// +// Returns "" when the decisions are unknown (empty or unrecognized +// play_method, e.g. a stale row from an older node), so consumers can +// distinguish "unknown" from a definite bucket instead of inventing one. +func effectivePlayMethod(videoDecision, audioDecision string) string { + switch { + case videoDecision == "" && audioDecision == "": + return "" + case videoDecision == "transcode": + return "transcode" + case audioDecision == "transcode": + return "audio" + case videoDecision == "direct" && audioDecision == "direct": + return "direct" + default: + return "remux" + } +} + +// jellyfinClientTokens identifies Jellyfin-ecosystem clients — sessions served +// through the Jellyfin compatibility surface — by client name (parsed from the +// MediaBrowser auth header) or user agent. This is the single source of truth +// behind is_jellyfin_client, which the admin UIs surface as the "JF" pill; +// keep it in step with the display-labeling rules in +// playbackClientDisplayName so a client that gets a label also gets the flag. +// Kodi and mpv reach Silo only through the Jellyfin compat surface today +// (jellyfin-kodi/JellyCon, jellyfin-mpv-shim). Generic browser tokens are +// deliberately excluded: the native web player shares those user agents. +var jellyfinClientTokens = []string{ + "jellyfin", + "findroid", + "streamyfin", + "swiftfin", + "jellycon", + "wholphin", + "fladder", + "vidhub", + "senplayer", + "infuse", + "delfin", + "finamp", + "kodi", + "mpv", +} + +// isJellyfinEcosystemClient reports whether the session's client metadata +// matches a known Jellyfin-ecosystem client. This is a heuristic: an +// unrecognized fork simply gets no JF pill (cosmetic). Stamping a compat-origin +// flag on the session at creation would be exact and is the eventual fix. +func isJellyfinEcosystemClient(clientName, userAgent string) bool { + for _, value := range []string{clientName, userAgent} { + value = strings.ToLower(strings.TrimSpace(value)) + if value == "" { + continue + } + for _, token := range jellyfinClientTokens { + if strings.Contains(value, token) { + return true + } + } + } + return false +} + func firstNonEmptyValue(values ...string) string { for _, value := range values { if trimmed := strings.TrimSpace(value); trimmed != "" { diff --git a/internal/api/handlers/playback_sessions_test.go b/internal/api/handlers/playback_sessions_test.go index e28e795c..d41e89f0 100644 --- a/internal/api/handlers/playback_sessions_test.go +++ b/internal/api/handlers/playback_sessions_test.go @@ -1,6 +1,11 @@ package handlers -import "testing" +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) func TestSessionComponentDecisionLabelsCopiedAudioDuringHLSAsRemux(t *testing.T) { videoDecision, audioDecision := sessionComponentDecision("transcode", false, "copy") @@ -12,3 +17,89 @@ func TestSessionComponentDecisionLabelsCopiedAudioDuringHLSAsRemux(t *testing.T) t.Fatalf("audioDecision = %q, want remux", audioDecision) } } + +// TestEffectivePlayMethodBuckets pins the bucket for every decision pair +// sessionComponentDecision can produce, plus the unknown case. +func TestEffectivePlayMethodBuckets(t *testing.T) { + cases := []struct { + name string + playMethod string + transcodeAudio bool + targetVideo string + want string + }{ + {"direct play", "direct", false, "", "direct"}, + {"plain remux", "remux", false, "", "remux"}, + {"audio-only re-encode via remux", "remux", true, "", "audio"}, + {"full video transcode", "transcode", true, "h264", "transcode"}, + {"video transcode with copied audio", "transcode", false, "h264", "transcode"}, + {"video-copy HLS repackage", "transcode", false, "copy", "remux"}, + {"video-copy HLS with audio re-encode", "transcode", true, "copy", "audio"}, + // Unknown play_method (stale row from an older node): the bucket must + // stay empty rather than inventing a method from transcode_audio. + {"unknown method with transcode_audio set", "hls", true, "", ""}, + {"empty method", "", false, "", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + video, audio := sessionComponentDecision(tc.playMethod, tc.transcodeAudio, tc.targetVideo) + if got := effectivePlayMethod(video, audio); got != tc.want { + t.Fatalf("effectivePlayMethod(%q, %q) = %q, want %q", video, audio, got, tc.want) + } + }) + } +} + +// TestSessionsCapabilitiesAdvertisesActivityFields pins the feature-detection +// contract: the additive session fields are omitempty on the wire, so this +// endpoint is how independently deployed clients distinguish an older server +// from a supported one reporting an unknown method / non-Jellyfin session. +func TestSessionsCapabilitiesAdvertisesActivityFields(t *testing.T) { + rr := httptest.NewRecorder() + (&AdminHandler{}).HandleGetSessionsCapabilities(rr, httptest.NewRequest(http.MethodGet, "/admin/sessions/capabilities", nil)) + + if rr.Code != http.StatusOK { + t.Fatalf("status = %d", rr.Code) + } + var resp playbackSessionsCapabilitiesResponse + if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode capabilities: %v", err) + } + if !resp.EffectivePlayMethod || !resp.IsJellyfinClient { + t.Fatalf("capabilities must advertise both fields: %+v", resp) + } + want := []string{"direct", "remux", "transcode", "audio"} + if len(resp.EffectivePlayMethodValues) != len(want) { + t.Fatalf("bucket vocabulary = %v, want %v", resp.EffectivePlayMethodValues, want) + } + for i, v := range want { + if resp.EffectivePlayMethodValues[i] != v { + t.Fatalf("bucket vocabulary = %v, want %v", resp.EffectivePlayMethodValues, want) + } + } +} + +func TestIsJellyfinEcosystemClient(t *testing.T) { + cases := []struct { + name string + clientName string + userAgent string + want bool + }{ + {"jellyfin web by name", "Jellyfin Web", "", true}, + {"findroid by name", "Findroid", "Findroid/0.15", true}, + {"infuse by user agent only", "", "Infuse-Direct/8.4.6", true}, + {"kodi addon by name", "Kodi", "Kodi/21.0", true}, + {"mpv shim by user agent", "", "mpv 0.38.0", true}, + {"native android client", "Silo Android", "okhttp/4.12", false}, + {"generic browser", "", "Mozilla/5.0 (X11) Chrome/120.0 Safari/537.36", false}, + {"no metadata", "", "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isJellyfinEcosystemClient(tc.clientName, tc.userAgent); got != tc.want { + t.Fatalf("isJellyfinEcosystemClient(%q, %q) = %v, want %v", tc.clientName, tc.userAgent, got, tc.want) + } + }) + } +} diff --git a/internal/api/handlers/playback_test.go b/internal/api/handlers/playback_test.go index 582c5e7c..48f93e71 100644 --- a/internal/api/handlers/playback_test.go +++ b/internal/api/handlers/playback_test.go @@ -5,11 +5,17 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" + "math" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" + "sort" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -26,6 +32,7 @@ import ( "github.com/Silo-Server/silo-server/internal/transcodenode" "github.com/Silo-Server/silo-server/internal/userdb" "github.com/Silo-Server/silo-server/internal/userstore" + "github.com/Silo-Server/silo-server/internal/watchsync" ) type testUserStoreProvider struct { @@ -42,6 +49,40 @@ type testPlaybackFileResolver struct { file *models.MediaFile } +type firstBlockingSessionManager struct { + *playback.SessionManager + blocked atomic.Bool + entered chan struct{} + release chan struct{} +} + +func (m *firstBlockingSessionManager) UpdateStreamState(sessionID string, state playback.SessionStreamState) error { + return m.SessionManager.UpdateStreamState(sessionID, state) +} + +func (m *firstBlockingSessionManager) ApplyReplacementIfRoute( + sessionID string, + expected playback.TranscodeRoute, + replacement playback.SessionReplacement, +) (playback.SessionReplacementRollback, bool, error) { + if m.blocked.CompareAndSwap(false, true) { + close(m.entered) + <-m.release + } + return m.SessionManager.ApplyReplacementIfRoute(sessionID, expected, replacement) +} + +func (m *firstBlockingSessionManager) ApplyReplacement( + sessionID string, + replacement playback.SessionReplacement, +) (playback.SessionReplacementRollback, error) { + if m.blocked.CompareAndSwap(false, true) { + close(m.entered) + <-m.release + } + return m.SessionManager.ApplyReplacement(sessionID, replacement) +} + func (r testPlaybackFileResolver) GetByID(context.Context, int) (*models.MediaFile, error) { return r.file, nil } @@ -93,6 +134,27 @@ func (noopPlaybackAdminStore) RecordHistory(context.Context, AdminPlaybackHistor func (noopPlaybackAdminStore) DeleteSession(context.Context, string) error { return nil } +type recordingPlaybackWatchScrobbler struct { + starts []watchsync.ScrobbleEvent + pauses []watchsync.ScrobbleEvent + stops []watchsync.ScrobbleEvent +} + +func (s *recordingPlaybackWatchScrobbler) ScrobbleStart(_ context.Context, event watchsync.ScrobbleEvent) error { + s.starts = append(s.starts, event) + return nil +} + +func (s *recordingPlaybackWatchScrobbler) ScrobblePause(_ context.Context, event watchsync.ScrobbleEvent) error { + s.pauses = append(s.pauses, event) + return nil +} + +func (s *recordingPlaybackWatchScrobbler) ScrobbleStop(_ context.Context, event watchsync.ScrobbleEvent) error { + s.stops = append(s.stops, event) + return nil +} + type testEpisodeLookup struct { episode *models.Episode } @@ -128,6 +190,18 @@ func (failingSessionManager) EndTransport(string) error { return nil } func (failingSessionManager) SetEffectiveMediaFileID(string, int) error { return nil } func (failingSessionManager) SetTranscodeNodeURL(string, string) error { return nil } +func (failingSessionManager) SetTranscodeRoute(string, playback.TranscodeRoute) error { + return nil +} +func (failingSessionManager) ApplyReplacement(string, playback.SessionReplacement) (playback.SessionReplacementRollback, error) { + return playback.SessionReplacementRollback{}, nil +} +func (failingSessionManager) ApplyReplacementIfRoute(string, playback.TranscodeRoute, playback.SessionReplacement) (playback.SessionReplacementRollback, bool, error) { + return playback.SessionReplacementRollback{}, true, nil +} +func (failingSessionManager) RollbackReplacement(string, playback.SessionReplacementRollback) error { + return nil +} func (failingSessionManager) SetWebSocket(string, bool) error { return nil } @@ -178,12 +252,50 @@ func writePlaybackTestFFmpeg(t *testing.T) string { t.Helper() path := filepath.Join(t.TempDir(), "fake-ffmpeg.sh") - if err := os.WriteFile(path, []byte("#!/bin/sh\nsleep 30\n"), 0o755); err != nil { + script := "#!/bin/sh\n" + + "last=\"\"\n" + + "for arg in \"$@\"; do last=\"$arg\"; done\n" + + "case \"$last\" in\n" + + " *.m3u8) out=\"$(dirname \"$last\")\"; mkdir -p \"$out\"; " + + "printf x > \"$out/init.mp4\"; printf x > \"$out/seg_0.m4s\"; " + + "printf x > \"$out/seg_1.m4s\"; printf x > \"$out/seg_2.m4s\"; " + + "printf '#EXTM3U\\n#EXT-X-VERSION:7\\n#EXT-X-TARGETDURATION:2\\n" + + "#EXT-X-MEDIA-SEQUENCE:0\\n#EXT-X-MAP:URI=\"init.mp4\"\\n" + + "#EXTINF:2.0,\\nseg_0.m4s\\n#EXTINF:2.0,\\nseg_1.m4s\\n" + + "#EXTINF:2.0,\\nseg_2.m4s\\n' > \"$last\" ;;\n" + + "esac\n" + + "sleep 30\n" + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { t.Fatalf("write fake ffmpeg: %v", err) } return path } +func writePlaybackTestFFmpegFailingAfterFirstStart(t *testing.T) string { + t.Helper() + + dir := t.TempDir() + path := filepath.Join(dir, "fake-ffmpeg.sh") + countPath := filepath.Join(dir, "started") + script := "#!/bin/sh\n" + + "if [ -e \"" + countPath + "\" ]; then echo 'intentional successor failure' >&2; exit 1; fi\n" + + ": > \"" + countPath + "\"\n" + + "last=\"\"\n" + + "for arg in \"$@\"; do last=\"$arg\"; done\n" + + "out=\"$(dirname \"$last\")\"; mkdir -p \"$out\"\n" + + "printf x > \"$out/init.mp4\"; printf x > \"$out/seg_0.m4s\"; " + + "printf x > \"$out/seg_1.m4s\"; printf x > \"$out/seg_2.m4s\"\n" + + "printf '#EXTM3U\\n#EXT-X-VERSION:7\\n#EXT-X-TARGETDURATION:2\\n" + + "#EXT-X-MEDIA-SEQUENCE:0\\n#EXT-X-MAP:URI=\"init.mp4\"\\n" + + "#EXTINF:2.0,\\nseg_0.m4s\\n#EXTINF:2.0,\\nseg_1.m4s\\n" + + "#EXTINF:2.0,\\nseg_2.m4s\\n' > \"$last\"\n" + + "sleep 30\n" + if err := os.WriteFile(path, []byte(script), 0o755); err != nil { + t.Fatalf("write fail-after-first fake ffmpeg: %v", err) + } + return path +} + func playbackTestConfig(ffmpegPath, transcodeDir string) func() config.PlaybackConfig { return func() config.PlaybackConfig { return config.PlaybackConfig{ @@ -357,6 +469,101 @@ func TestPlaybackSessionProgressPersistenceCanBeDisabled(t *testing.T) { } } +func TestFinalizeSessionStop_UsesProviderLifecycleSemantics(t *testing.T) { + tests := []struct { + name string + position float64 + userInitiated bool + disablePersistence bool + wantPauseCalls int + wantStopCalls int + wantEventPosition float64 + }{ + { + name: "user exit stops below-resume-threshold scrobble", + position: 120, + userInitiated: true, + wantStopCalls: 1, + wantEventPosition: 120, + }, + { + name: "system teardown pauses reconstructable scrobble", + position: 120, + wantPauseCalls: 1, + wantEventPosition: 120, + }, + { + name: "system teardown pauses persisted incomplete scrobble", + position: 600, + wantPauseCalls: 1, + wantEventPosition: 600, + }, + { + name: "completed system teardown stops scrobble", + position: 3500, + wantStopCalls: 1, + wantEventPosition: 3500, + }, + { + name: "user exit at zero still stops scrobble", + userInitiated: true, + wantStopCalls: 1, + wantEventPosition: 0, + }, + { + name: "disabled persistence does not scrobble", + position: 120, + userInitiated: true, + disablePersistence: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := newPlaybackTestStore(t) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + Duration: 3600, + } + scrobbler := &recordingPlaybackWatchScrobbler{} + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0), testPlaybackFileResolver{file: file}) + handler.StoreProvider = testUserStoreProvider{store: store} + handler.WatchScrobbler = scrobbler + + session := &playback.Session{ + ID: "session-1", + UserID: 1, + ProfileID: "profile-1", + MediaFileID: file.ID, + Position: tt.position, + DisableProgressPersistence: tt.disablePersistence, + StartedAt: time.Now().Add(-time.Minute), + } + handler.finalizeSessionStop(context.Background(), session, false, "", tt.userInitiated) + + if got := len(scrobbler.pauses); got != tt.wantPauseCalls { + t.Fatalf("pause calls = %d, want %d", got, tt.wantPauseCalls) + } + if got := len(scrobbler.stops); got != tt.wantStopCalls { + t.Fatalf("stop calls = %d, want %d", got, tt.wantStopCalls) + } + if len(scrobbler.pauses)+len(scrobbler.stops) == 1 { + event := scrobbler.pauses + if len(event) == 0 { + event = scrobbler.stops + } + if event[0].MediaItemID != file.ContentID { + t.Fatalf("media item ID = %q, want %q", event[0].MediaItemID, file.ContentID) + } + if event[0].PositionSeconds != tt.wantEventPosition { + t.Fatalf("position = %v, want %v", event[0].PositionSeconds, tt.wantEventPosition) + } + } + }) + } +} + func TestBuildTranscodeStartResponse_UnifiedSeekAnywhere(t *testing.T) { file := &models.MediaFile{Duration: 3600} @@ -369,18 +576,19 @@ func TestBuildTranscodeStartResponse_UnifiedSeekAnywhere(t *testing.T) { file, nil, "/playback/transcode/session-copy/master.m3u8", + 16, ) if copyResp.CanSeekAnywhere { t.Fatal("copy-mode response should require explicit restart seeks") } - if copyResp.PlayerStartSeconds != 0 { - t.Fatalf("copy-mode PlayerStartSeconds = %v, want 0", copyResp.PlayerStartSeconds) + if math.Abs(copyResp.PlayerStartSeconds-2.261) > 0.0001 { + t.Fatalf("copy-mode PlayerStartSeconds = %v, want 2.261", copyResp.PlayerStartSeconds) } - if copyResp.StreamOriginSeconds != 18.261 { - t.Fatalf("copy-mode StreamOriginSeconds = %v, want 18.261", copyResp.StreamOriginSeconds) + if copyResp.StreamOriginSeconds != 16 { + t.Fatalf("copy-mode StreamOriginSeconds = %v, want 16", copyResp.StreamOriginSeconds) } - if copyResp.TimelineOffsetSeconds != 18.261 { - t.Fatalf("copy-mode TimelineOffsetSeconds = %v, want 18.261", copyResp.TimelineOffsetSeconds) + if copyResp.TimelineOffsetSeconds != 16 { + t.Fatalf("copy-mode TimelineOffsetSeconds = %v, want 16", copyResp.TimelineOffsetSeconds) } encodedResp := buildTranscodeStartResponse( @@ -392,6 +600,7 @@ func TestBuildTranscodeStartResponse_UnifiedSeekAnywhere(t *testing.T) { file, nil, "/playback/transcode/session-encoded/master.m3u8", + 0, ) if !encodedResp.CanSeekAnywhere { t.Fatal("encoded response should advertise seek-anywhere") @@ -1320,22 +1529,27 @@ func TestHandleChangeAudioTrack_RemoteTranscodeRestartsNodeAndMintsFullRecipe(t var remoteStartRequests int var remoteStartReq transcodenode.TranscodeStartRequest + var remoteDeletes []string remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost || r.URL.Path != "/transcode/start" { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + remoteStartRequests++ + if err := json.NewDecoder(r.Body).Decode(&remoteStartReq); err != nil { + t.Errorf("decode remote start request: %v", err) + } + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(transcodenode.TranscodeStartResponse{ + SessionID: remoteStartReq.SessionID, + Status: "accepted", + HWAccel: "none", + }) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/transcode/"): + remoteDeletes = append(remoteDeletes, r.URL.Path) + w.WriteHeader(http.StatusNoContent) + default: t.Errorf("unexpected remote request %s %s", r.Method, r.URL.Path) http.Error(w, "unexpected", http.StatusBadRequest) - return } - remoteStartRequests++ - if err := json.NewDecoder(r.Body).Decode(&remoteStartReq); err != nil { - t.Errorf("decode remote start request: %v", err) - } - w.WriteHeader(http.StatusAccepted) - _ = json.NewEncoder(w).Encode(transcodenode.TranscodeStartResponse{ - SessionID: remoteStartReq.SessionID, - Status: "accepted", - HWAccel: "none", - }) })) defer remote.Close() @@ -1380,7 +1594,13 @@ func TestHandleChangeAudioTrack_RemoteTranscodeRestartsNodeAndMintsFullRecipe(t t.Fatalf("remote AudioTrackIndex = %d, want 1", remoteStartReq.AudioTrackIndex) } if remoteStartReq.SessionID != sessionID { - t.Fatalf("remote SessionID = %q, want %q", remoteStartReq.SessionID, sessionID) + t.Fatalf("remote SessionID = %q, want existing encoded transport %q", remoteStartReq.SessionID, sessionID) + } + if remoteStartReq.RequireReady { + t.Fatal("encoded audio switch unexpectedly changed to legacy copy replacement lifecycle") + } + if len(remoteDeletes) != 0 { + t.Fatalf("remote DELETEs = %v, want node-managed same-ID replacement", remoteDeletes) } // Seek 121.5s with the node-default 2s segments => align to 120s / segment 60. if remoteStartReq.StartSegmentNumber != 60 { @@ -1443,6 +1663,248 @@ func TestHandleChangeAudioTrack_RemoteTranscodeRestartsNodeAndMintsFullRecipe(t if claims.TranscodeNode != remote.URL { t.Fatalf("token TranscodeNode = %q, want %q", claims.TranscodeNode, remote.URL) } + if claims.TranscodeTransportID != remoteStartReq.SessionID { + t.Fatalf("token TranscodeTransportID = %q, want %q", claims.TranscodeTransportID, remoteStartReq.SessionID) + } + updated, err := sessionMgr.GetSession(sessionID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if updated.TranscodeNodeURL != remote.URL || updated.TranscodeTransportID != "" { + t.Fatalf("published transcode route = %q/%q", updated.TranscodeNodeURL, updated.TranscodeTransportID) + } +} + +func TestHandleChangeAudioTrack_RemoteCopyRestartUsesFreshSeekAnchor(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-copy.mkv"), + Resolution: "1080p", + CodecVideo: "hevc", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{ + {Codec: "ac3", Default: true}, + {Codec: "dts"}, + }, + } + + var remoteStartReq transcodenode.TranscodeStartRequest + var remoteDeletes []string + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + if err := json.NewDecoder(r.Body).Decode(&remoteStartReq); err != nil { + t.Errorf("decode remote start request: %v", err) + } + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(transcodenode.TranscodeStartResponse{ + SessionID: remoteStartReq.SessionID, + Status: "accepted", + HWAccel: "none", + }) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/transcode/"): + remoteDeletes = append(remoteDeletes, r.URL.Path) + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected remote request %s %s", r.Method, r.URL.Path) + http.Error(w, "unexpected", http.StatusBadRequest) + } + })) + defer remote.Close() + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.JWTSecret = "test-secret" + handler.copySeekAnchor = func(context.Context, string, string, float64, int) (float64, int, error) { + return 96, 48, nil + } + group := "rack-a" + proxies := nodepool.NewProxyPool() + proxies.SetNodes([]*nodepool.Node{{ + ID: 1, Name: "proxy-1", Type: nodepool.NodeTypeProxy, URL: "http://proxy-1", + Enabled: true, Healthy: true, Group: &group, + }}) + transcodes := nodepool.NewTranscodePool() + transcodes.SetNodes([]*nodepool.Node{{ + ID: 2, Name: "transcode-1", Type: nodepool.NodeTypeTranscode, URL: remote.URL, + Enabled: true, Healthy: true, Group: &group, + }}) + handler.NodePlanner = nodepool.NewPlanner(proxies, transcodes) + + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if err := sessionMgr.UpdateStreamState(session.ID, playback.SessionStreamState{ + PlayMethod: playback.PlayTranscode, + BasePlayMethod: playback.PlayRemux, + AudioTrackIndex: 0, + TranscodeAudio: true, + TargetVideoCodec: "copy", + TargetAudioCodec: "aac", + TranscodeHWAccel: "none", + SegmentDuration: 2, + }); err != nil { + t.Fatalf("UpdateStreamState: %v", err) + } + if err := sessionMgr.SetTranscodeNodeURL(session.ID, remote.URL); err != nil { + t.Fatalf("SetTranscodeNodeURL: %v", err) + } + + changeReq := httptest.NewRequest( + http.MethodPatch, + "/api/v1/playback/"+session.ID+"/audio", + strings.NewReader(`{"audio_track_index":1,"position":100}`), + ).WithContext(newAuthorizedPlaybackContext()) + changeReq = withPlaybackRouteParam(changeReq, "session_id", session.ID) + + rr := httptest.NewRecorder() + handler.HandleChangeAudioTrack(rr, changeReq) + if rr.Code != http.StatusOK { + t.Fatalf("change status = %d, body = %s", rr.Code, rr.Body.String()) + } + if remoteStartReq.TargetCodecVideo != "copy" || remoteStartReq.SeekSeconds != 100 || + remoteStartReq.StreamOriginSeconds != 96 || !remoteStartReq.CopySeekAnchorResolved || + remoteStartReq.StartSegmentNumber != 48 { + t.Fatalf("remote copy restart recipe = %+v", remoteStartReq) + } + if !strings.HasPrefix(remoteStartReq.SessionID, session.ID+legacyTransportMarker) { + t.Fatalf("remote SessionID = %q, want legacy replacement for %q", remoteStartReq.SessionID, session.ID) + } + if !remoteStartReq.RequireReady { + t.Fatal("remote copy replacement did not require readiness before predecessor retirement") + } + if len(remoteDeletes) != 1 || remoteDeletes[0] != "/transcode/"+session.ID { + t.Fatalf("remote DELETEs = %v, want predecessor %q", remoteDeletes, session.ID) + } + + var resp changeAudioResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode change response: %v", err) + } + token := strings.TrimSuffix(strings.TrimPrefix(resp.StreamURL, "http://proxy-1/stream/transcode/"), "/master.m3u8") + claims, err := streamtoken.Verify(token, handler.JWTSecret) + if err != nil { + t.Fatalf("verify stream token: %v", err) + } + if claims.TargetCodec != "copy" || claims.SeekSeconds != 100 || + claims.StreamOriginSeconds != 96 || !claims.CopySeekAnchorResolved || + claims.StartSegmentNumber != 48 { + t.Fatalf("remote copy reconstruction claims = %+v", claims) + } + if claims.TranscodeTransportID != remoteStartReq.SessionID { + t.Fatalf("token TranscodeTransportID = %q, want %q", claims.TranscodeTransportID, remoteStartReq.SessionID) + } +} + +func TestHandleChangeAudioTrack_RemoteCopyRejectionPreservesPredecessorState(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-copy-rejected.mkv"), + Resolution: "1080p", + CodecVideo: "hevc", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "ac3", Default: true}, {Codec: "dts"}}, + } + + var replacementID string + var deletedIDs []string + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + var startReq transcodenode.TranscodeStartRequest + if err := json.NewDecoder(r.Body).Decode(&startReq); err != nil { + t.Errorf("decode remote start request: %v", err) + } + replacementID = startReq.SessionID + http.Error(w, "rejected", http.StatusInternalServerError) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/transcode/"): + deletedIDs = append(deletedIDs, strings.TrimPrefix(r.URL.Path, "/transcode/")) + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected remote request %s %s", r.Method, r.URL.Path) + http.Error(w, "unexpected", http.StatusBadRequest) + } + })) + defer remote.Close() + + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if err := sessionMgr.UpdateStreamState(session.ID, playback.SessionStreamState{ + PlayMethod: playback.PlayTranscode, + BasePlayMethod: playback.PlayRemux, + AudioTrackIndex: 0, + TranscodeAudio: true, + TargetVideoCodec: "copy", + TargetAudioCodec: "aac", + TranscodeHWAccel: "none", + SegmentDuration: 2, + }); err != nil { + t.Fatalf("UpdateStreamState: %v", err) + } + if err := sessionMgr.SetTranscodeNodeURL(session.ID, remote.URL); err != nil { + t.Fatalf("SetTranscodeNodeURL: %v", err) + } + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.JWTSecret = "test-secret" + handler.copySeekAnchor = func(context.Context, string, string, float64, int) (float64, int, error) { + return 96, 48, nil + } + group := "rack-a" + proxies := nodepool.NewProxyPool() + proxies.SetNodes([]*nodepool.Node{{ + ID: 1, Name: "proxy-1", Type: nodepool.NodeTypeProxy, URL: "http://proxy-1", + Enabled: true, Healthy: true, Group: &group, + }}) + transcodes := nodepool.NewTranscodePool() + transcodes.SetNodes([]*nodepool.Node{{ + ID: 2, Name: "transcode-1", Type: nodepool.NodeTypeTranscode, URL: remote.URL, + Enabled: true, Healthy: true, Group: &group, + }}) + handler.NodePlanner = nodepool.NewPlanner(proxies, transcodes) + + request := httptest.NewRequest( + http.MethodPatch, + "/api/v1/playback/"+session.ID+"/audio", + strings.NewReader(`{"audio_track_index":1,"position":100}`), + ).WithContext(newAuthorizedPlaybackContext()) + request = withPlaybackRouteParam(request, "session_id", session.ID) + response := httptest.NewRecorder() + handler.HandleChangeAudioTrack(response, request) + if response.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d, body = %s", response.Code, http.StatusBadGateway, response.Body.String()) + } + if !strings.HasPrefix(replacementID, session.ID+legacyTransportMarker) { + t.Fatalf("replacement ID = %q, want distinct legacy transport for %q", replacementID, session.ID) + } + if len(deletedIDs) != 1 || deletedIDs[0] != replacementID { + t.Fatalf("deleted transports = %v, want only rejected replacement %q", deletedIDs, replacementID) + } + persisted, err := sessionMgr.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if persisted.TranscodeNodeURL != remote.URL || persisted.TranscodeTransportID != "" { + t.Fatalf("predecessor route changed after rejection: %q/%q", persisted.TranscodeNodeURL, persisted.TranscodeTransportID) + } + if persisted.AudioTrackIndex != 0 || persisted.BasePlayMethod != playback.PlayRemux || + persisted.TargetVideoCodec != "copy" || persisted.TargetAudioCodec != "aac" { + t.Fatalf("predecessor state changed after rejection: %+v", persisted) + } } // TestHandleChangeAudioTrack_RemoteTranscodeNodeUnreachableSurfacesError @@ -1570,22 +2032,848 @@ func TestHandleStartTranscode_LocalPathPropagatesSelectedAudioTrack(t *testing.T } } -func TestHandleStartTranscode_MPEG2SeekedCopyRemainsCopyVideo(t *testing.T) { +func TestHandleChangeAudioTrack_LocalCopyRestartUsesFreshSeekAnchor(t *testing.T) { sessionMgr := playback.NewSessionManager(0, 0) - filePath := writePlaybackTestMediaFile(t, "movie-mpeg2.mkv") file := &models.MediaFile{ - ID: 42, - ContentID: "movie-1", - FilePath: filePath, - Resolution: "1080p", - CodecVideo: "mpeg2video", - CodecAudio: "dts", - Container: "mkv", - Bitrate: 25000, - Duration: 3600, - AudioTracks: []models.AudioTrack{ - {Codec: "dts", Default: true}, - }, + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-local-copy.mkv"), + Resolution: "1080p", + CodecVideo: "hevc", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "ac3", Default: true}, {Codec: "dts"}}, + } + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.JWTSecret = "test-secret" + handler.PlaybackConfig = playbackTestConfig(writePlaybackTestFFmpeg(t), t.TempDir()) + handler.copySeekAnchor = func(_ context.Context, _, _ string, requested float64, _ int) (float64, int, error) { + switch requested { + case 18: + return 16, 8, nil + case 100: + return 96, 48, nil + default: + return 0, 0, fmt.Errorf("unexpected seek %v", requested) + } + } + + startReq := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":18,"target_codec_video":"copy","target_codec_audio":"aac","segment_duration":2,"subtitle_track_index":-1}`), + ).WithContext(newAuthorizedPlaybackContext()) + startRR := httptest.NewRecorder() + handler.HandleStartTranscode(startRR, startReq) + if startRR.Code != http.StatusAccepted { + t.Fatalf("start status = %d, body = %s", startRR.Code, startRR.Body.String()) + } + + predecessor := handler.tm.GetTranscodeSession(session.ID) + if predecessor == nil { + t.Fatal("expected local copy-video session") + } + t.Cleanup(func() { handler.tm.CloseTranscodeSession(session.ID, "") }) + + changeReq := httptest.NewRequest( + http.MethodPatch, + "/api/v1/playback/"+session.ID+"/audio", + strings.NewReader(`{"audio_track_index":1,"position":100}`), + ).WithContext(newAuthorizedPlaybackContext()) + changeReq = withPlaybackRouteParam(changeReq, "session_id", session.ID) + changeRR := httptest.NewRecorder() + handler.HandleChangeAudioTrack(changeRR, changeReq) + if changeRR.Code != http.StatusOK { + t.Fatalf("change status = %d, body = %s", changeRR.Code, changeRR.Body.String()) + } + + successor := handler.tm.GetTranscodeSession(session.ID) + if successor == nil || successor == predecessor { + t.Fatal("audio switch did not publish a prepared successor") + } + if predecessor.IsRunning() { + t.Fatal("audio switch predecessor is still running after commit") + } + opts := successor.Opts() + if opts.TargetCodecVideo != "copy" || opts.SeekSeconds != 100 || + opts.StreamOriginSeconds != 96 || !opts.CopySeekAnchorResolved || + opts.StartSegmentNumber != 48 || opts.AudioTrackIndex != 1 { + t.Fatalf("local copy restart opts = %+v", opts) + } + + var resp changeAudioResponse + if err := json.NewDecoder(changeRR.Body).Decode(&resp); err != nil { + t.Fatalf("decode change response: %v", err) + } + manifestURL, err := url.Parse(resp.StreamURL) + if err != nil { + t.Fatalf("parse stream URL: %v", err) + } + claims, err := streamtoken.Verify(manifestURL.Query().Get(streamTokenParam), handler.JWTSecret) + if err != nil { + t.Fatalf("verify stream token: %v", err) + } + if claims.TargetCodec != "copy" || claims.SeekSeconds != 100 || + claims.StreamOriginSeconds != 96 || !claims.CopySeekAnchorResolved || + claims.StartSegmentNumber != 48 || claims.AudioTrackIndex != 1 { + t.Fatalf("local copy reconstruction claims = %+v", claims) + } + if resp.PlayerStartSeconds == nil || *resp.PlayerStartSeconds != 4 || + resp.StreamOriginSeconds == nil || *resp.StreamOriginSeconds != 96 || + resp.TimelineOffsetSeconds == nil || *resp.TimelineOffsetSeconds != 96 || + resp.CanSeekAnywhere == nil || *resp.CanSeekAnywhere { + t.Fatalf("local copy response timeline = %+v", resp) + } +} + +func TestHandleChangeAudioTrack_LocalCopyFailurePreservesPredecessor(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-local-copy-failure.mkv"), + Resolution: "1080p", + CodecVideo: "hevc", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "ac3", Default: true}, {Codec: "dts"}}, + } + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.PlaybackConfig = playbackTestConfig( + writePlaybackTestFFmpegFailingAfterFirstStart(t), + t.TempDir(), + ) + handler.copySeekAnchor = func(_ context.Context, _, _ string, requested float64, _ int) (float64, int, error) { + return requested - 4, int((requested - 4) / 2), nil + } + + startReq := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":18,"target_codec_video":"copy","target_codec_audio":"aac","segment_duration":2,"subtitle_track_index":-1}`), + ).WithContext(newAuthorizedPlaybackContext()) + startRR := httptest.NewRecorder() + handler.HandleStartTranscode(startRR, startReq) + if startRR.Code != http.StatusAccepted { + t.Fatalf("start status = %d, body = %s", startRR.Code, startRR.Body.String()) + } + predecessor := handler.tm.GetTranscodeSession(session.ID) + if predecessor == nil || !predecessor.IsRunning() { + t.Fatal("expected running predecessor") + } + t.Cleanup(func() { handler.tm.CloseTranscodeSession(session.ID, "") }) + + changeReq := httptest.NewRequest( + http.MethodPatch, + "/api/v1/playback/"+session.ID+"/audio", + strings.NewReader(`{"audio_track_index":1,"position":100}`), + ).WithContext(newAuthorizedPlaybackContext()) + changeReq = withPlaybackRouteParam(changeReq, "session_id", session.ID) + changeRR := httptest.NewRecorder() + handler.HandleChangeAudioTrack(changeRR, changeReq) + if changeRR.Code != http.StatusInternalServerError { + t.Fatalf("change status = %d, body = %s", changeRR.Code, changeRR.Body.String()) + } + if current := handler.tm.GetTranscodeSession(session.ID); current != predecessor { + t.Fatal("failed successor replaced the predecessor") + } + if !predecessor.IsRunning() { + t.Fatal("failed successor stopped the predecessor") + } + persisted, err := sessionMgr.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if persisted.AudioTrackIndex != 0 { + t.Fatalf("persisted audio track = %d, want predecessor track 0", persisted.AudioTrackIndex) + } +} + +func TestHandleStartTranscode_SeekedCopyRemainsCopyVideo(t *testing.T) { + for _, tc := range []struct { + name string + videoCodec string + }{ + {name: "h264", videoCodec: "h264"}, + {name: "hevc", videoCodec: "hevc"}, + {name: "mpeg2", videoCodec: "mpeg2video"}, + } { + t.Run(tc.name, func(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-"+tc.name+".mkv"), + Resolution: "1080p", + CodecVideo: tc.videoCodec, + CodecAudio: "dts", + Container: "mkv", + Bitrate: 25000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "dts", Default: true}}, + } + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.PlaybackConfig = playbackTestConfig(writePlaybackTestFFmpeg(t), t.TempDir()) + handler.copySeekAnchor = func(context.Context, string, string, float64, int) (float64, int, error) { + return 16, 8, nil + } + + transcodeReq := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":18.261,"target_resolution":"","target_codec_video":"copy","target_codec_audio":"aac","target_bitrate_kbps":0,"segment_duration":2,"subtitle_track_index":-1,"subtitle_burn_in":false}`), + ).WithContext(newAuthorizedPlaybackContext()) + + transcodeRR := httptest.NewRecorder() + handler.HandleStartTranscode(transcodeRR, transcodeReq) + if transcodeRR.Code != http.StatusAccepted { + t.Fatalf("transcode status = %d, body = %s", transcodeRR.Code, transcodeRR.Body.String()) + } + + var response transcodeStartResponse + if err := json.NewDecoder(transcodeRR.Body).Decode(&response); err != nil { + t.Fatalf("decode transcode response: %v", err) + } + if math.Abs(response.PlayerStartSeconds-2.261) > 0.0001 || response.StreamOriginSeconds != 16 || + response.TimelineOffsetSeconds != 16 || response.CanSeekAnywhere { + t.Fatalf("copy response timeline = %+v", response) + } + + transcodeSession := handler.tm.GetTranscodeSession(session.ID) + if transcodeSession == nil { + t.Fatal("expected local transcode session") + } + t.Cleanup(func() { _ = transcodeSession.Close() }) + opts := transcodeSession.Opts() + if opts.TargetCodecVideo != "copy" || opts.SourceVideoCodec != tc.videoCodec || opts.TargetCodecAudio != "aac" { + t.Fatalf("copy recipe = video %q source %q audio %q", opts.TargetCodecVideo, opts.SourceVideoCodec, opts.TargetCodecAudio) + } + if opts.SeekSeconds != 18.261 || opts.StreamOriginSeconds != 16 || !opts.CopySeekAnchorResolved || opts.StartSegmentNumber != 8 || opts.TargetResolution != "" { + t.Fatalf("copy seek recipe = seek %v origin %v segment %d resolution %q", opts.SeekSeconds, opts.StreamOriginSeconds, opts.StartSegmentNumber, opts.TargetResolution) + } + + persisted, err := sessionMgr.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if persisted.TargetVideoCodec != "copy" || semanticPlayMethod(persisted) != playback.PlayRemux { + t.Fatalf("persisted recipe = target %q semantic method %q", persisted.TargetVideoCodec, semanticPlayMethod(persisted)) + } + }) + } +} + +func TestHandleStartTranscode_CopyAnchorFailureKeepsActiveTransport(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-anchor-failure.mkv"), + Resolution: "1080p", + CodecVideo: "h264", + CodecAudio: "aac", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "aac", Default: true}}, + } + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayTranscode, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.PlaybackConfig = playbackTestConfig(writePlaybackTestFFmpeg(t), t.TempDir()) + + initialReq := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":0,"target_resolution":"720p","target_codec_video":"h264","target_codec_audio":"aac","target_bitrate_kbps":2000,"segment_duration":2,"subtitle_track_index":-1}`), + ).WithContext(newAuthorizedPlaybackContext()) + initialRR := httptest.NewRecorder() + handler.HandleStartTranscode(initialRR, initialReq) + if initialRR.Code != http.StatusAccepted { + t.Fatalf("initial start status = %d, body = %s", initialRR.Code, initialRR.Body.String()) + } + active := handler.tm.GetTranscodeSession(session.ID) + if active == nil || !active.IsRunning() { + t.Fatal("expected initial transcode to be running") + } + t.Cleanup(func() { _ = active.Close() }) + + handler.copySeekAnchor = func(context.Context, string, string, float64, int) (float64, int, error) { + return 0, 0, errors.New("probe failed") + } + replacementReq := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":100,"target_codec_video":"copy","target_codec_audio":"aac","segment_duration":2,"subtitle_track_index":-1}`), + ).WithContext(newAuthorizedPlaybackContext()) + replacementRR := httptest.NewRecorder() + handler.HandleStartTranscode(replacementRR, replacementReq) + if replacementRR.Code != http.StatusInternalServerError { + t.Fatalf("replacement status = %d, body = %s", replacementRR.Code, replacementRR.Body.String()) + } + if got := handler.tm.GetTranscodeSession(session.ID); got != active { + t.Fatalf("active transcode was replaced after failed preflight: got %p, want %p", got, active) + } + if !active.IsRunning() { + t.Fatal("active transcode was stopped before copy seek anchor preflight completed") + } +} + +func TestHandleStartTranscode_SeekedCopyPreservedOnRemoteNode(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-remote.mkv"), + Resolution: "1080p", + CodecVideo: "h264", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "ac3", Default: true}}, + } + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + var remoteStartReq transcodenode.TranscodeStartRequest + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/transcode/start" { + t.Errorf("unexpected remote request %s %s", r.Method, r.URL.Path) + http.Error(w, "unexpected", http.StatusBadRequest) + return + } + if err := json.NewDecoder(r.Body).Decode(&remoteStartReq); err != nil { + t.Errorf("decode remote start request: %v", err) + } + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(transcodenode.TranscodeStartResponse{ + SessionID: remoteStartReq.SessionID, + Status: "accepted", + HWAccel: "none", + }) + })) + defer remote.Close() + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.JWTSecret = "test-secret" + handler.PlaybackConfig = playbackTestConfig(writePlaybackTestFFmpeg(t), t.TempDir()) + handler.copySeekAnchor = func(context.Context, string, string, float64, int) (float64, int, error) { + return 16, 8, nil + } + pool := nodepool.NewTranscodePool() + pool.SetNodes([]*nodepool.Node{{ + Name: "transcode-1", Type: nodepool.NodeTypeTranscode, URL: remote.URL, + Enabled: true, Healthy: true, + }}) + handler.NodePlanner = nodepool.NewPlanner(nodepool.NewProxyPool(), pool) + + transcodeReq := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":18.261,"target_resolution":"","target_codec_video":"copy","target_codec_audio":"aac","target_bitrate_kbps":0,"segment_duration":2,"subtitle_track_index":-1,"subtitle_burn_in":false}`), + ).WithContext(newAuthorizedPlaybackContext()) + transcodeRR := httptest.NewRecorder() + handler.HandleStartTranscode(transcodeRR, transcodeReq) + if transcodeRR.Code != http.StatusAccepted { + t.Fatalf("transcode status = %d, body = %s", transcodeRR.Code, transcodeRR.Body.String()) + } + if remoteStartReq.TargetCodecVideo != "copy" || remoteStartReq.SeekSeconds != 18.261 || + remoteStartReq.StreamOriginSeconds != 16 || !remoteStartReq.CopySeekAnchorResolved || remoteStartReq.StartSegmentNumber != 8 { + t.Fatalf("remote copy recipe = codec %q seek %v origin %v segment %d", remoteStartReq.TargetCodecVideo, remoteStartReq.SeekSeconds, remoteStartReq.StreamOriginSeconds, remoteStartReq.StartSegmentNumber) + } + if !strings.HasPrefix(remoteStartReq.SessionID, session.ID+legacyTransportMarker) { + t.Fatalf("remote SessionID = %q, want legacy transport for %q", remoteStartReq.SessionID, session.ID) + } + if !remoteStartReq.RequireReady { + t.Fatal("remote copy start did not require readiness before route publication") + } + + var response transcodeStartResponse + if err := json.NewDecoder(transcodeRR.Body).Decode(&response); err != nil { + t.Fatalf("decode transcode response: %v", err) + } + if math.Abs(response.PlayerStartSeconds-2.261) > 0.0001 || response.StreamOriginSeconds != 16 || + response.TimelineOffsetSeconds != 16 || response.CanSeekAnywhere { + t.Fatalf("remote copy response timeline = %+v", response) + } + manifestURL, err := url.Parse(response.ManifestURL) + if err != nil { + t.Fatalf("parse manifest URL: %v", err) + } + claims, err := streamtoken.Verify(manifestURL.Query().Get(streamTokenParam), handler.JWTSecret) + if err != nil { + t.Fatalf("verify stream token: %v", err) + } + if claims.TargetCodec != "copy" || claims.SeekSeconds != 18.261 || claims.StreamOriginSeconds != 16 || !claims.CopySeekAnchorResolved || + claims.StartSegmentNumber != 8 || claims.TranscodeNode != remote.URL { + t.Fatalf("remote reconstruction claims = codec %q seek %v origin %v segment %d node %q", claims.TargetCodec, claims.SeekSeconds, claims.StreamOriginSeconds, claims.StartSegmentNumber, claims.TranscodeNode) + } + if claims.TranscodeTransportID != remoteStartReq.SessionID { + t.Fatalf("remote reconstruction transport = %q, want %q", claims.TranscodeTransportID, remoteStartReq.SessionID) + } + persisted, err := sessionMgr.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if persisted.TranscodeNodeURL != remote.URL || persisted.TranscodeTransportID != remoteStartReq.SessionID { + t.Fatalf("published remote route = %q/%q", persisted.TranscodeNodeURL, persisted.TranscodeTransportID) + } +} + +func TestHandleStartTranscode_RemoteRejectionPreservesLegacyPredecessor(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-remote-reject.mkv"), + Resolution: "1080p", + CodecVideo: "h264", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "ac3", Default: true}}, + } + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + var replacementID string + var deletedIDs []string + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + var startReq transcodenode.TranscodeStartRequest + if err := json.NewDecoder(r.Body).Decode(&startReq); err != nil { + t.Errorf("decode remote start request: %v", err) + } + replacementID = startReq.SessionID + http.Error(w, "rejected", http.StatusInternalServerError) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/transcode/"): + deletedIDs = append(deletedIDs, strings.TrimPrefix(r.URL.Path, "/transcode/")) + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected remote request %s %s", r.Method, r.URL.Path) + http.Error(w, "unexpected", http.StatusBadRequest) + } + })) + defer remote.Close() + if err := sessionMgr.SetTranscodeNodeURL(session.ID, remote.URL); err != nil { + t.Fatalf("SetTranscodeNodeURL: %v", err) + } + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.JWTSecret = "test-secret" + handler.PlaybackConfig = playbackTestConfig(writePlaybackTestFFmpeg(t), t.TempDir()) + handler.copySeekAnchor = func(context.Context, string, string, float64, int) (float64, int, error) { + return 16, 8, nil + } + pool := nodepool.NewTranscodePool() + pool.SetNodes([]*nodepool.Node{{ + Name: "transcode-1", Type: nodepool.NodeTypeTranscode, URL: remote.URL, + Enabled: true, Healthy: true, + }}) + handler.NodePlanner = nodepool.NewPlanner(nodepool.NewProxyPool(), pool) + + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":18,"target_codec_video":"copy","target_codec_audio":"aac","segment_duration":2,"subtitle_track_index":-1}`), + ).WithContext(newAuthorizedPlaybackContext()) + response := httptest.NewRecorder() + handler.HandleStartTranscode(response, request) + if response.Code != http.StatusBadGateway { + t.Fatalf("status = %d, want %d, body = %s", response.Code, http.StatusBadGateway, response.Body.String()) + } + if !strings.HasPrefix(replacementID, session.ID+legacyTransportMarker) { + t.Fatalf("replacement ID = %q, want distinct legacy transport for %q", replacementID, session.ID) + } + if len(deletedIDs) != 1 || deletedIDs[0] != replacementID { + t.Fatalf("deleted transports = %v, want only rejected replacement %q", deletedIDs, replacementID) + } + persisted, err := sessionMgr.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if persisted.TranscodeNodeURL != remote.URL || persisted.TranscodeTransportID != "" { + t.Fatalf("predecessor route changed after rejection: %q/%q", persisted.TranscodeNodeURL, persisted.TranscodeTransportID) + } +} + +func TestHandleStartTranscode_ConcurrentRemoteReplacementsPublishOneWinner(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-remote-concurrent.mkv"), + Resolution: "1080p", + CodecVideo: "h264", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "ac3", Default: true}}, + } + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + var remoteMu sync.Mutex + var preparedIDs []string + var deletedIDs []string + prepared := make(chan struct{}, 2) + release := make(chan struct{}) + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + var startReq transcodenode.TranscodeStartRequest + if err := json.NewDecoder(r.Body).Decode(&startReq); err != nil { + t.Errorf("decode remote start request: %v", err) + } + remoteMu.Lock() + preparedIDs = append(preparedIDs, startReq.SessionID) + remoteMu.Unlock() + prepared <- struct{}{} + <-release + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(transcodenode.TranscodeStartResponse{ + SessionID: startReq.SessionID, + Status: "accepted", + HWAccel: "none", + }) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/transcode/"): + remoteMu.Lock() + deletedIDs = append(deletedIDs, strings.TrimPrefix(r.URL.Path, "/transcode/")) + remoteMu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected remote request %s %s", r.Method, r.URL.Path) + http.Error(w, "unexpected", http.StatusBadRequest) + } + })) + defer remote.Close() + if err := sessionMgr.SetTranscodeNodeURL(session.ID, remote.URL); err != nil { + t.Fatalf("SetTranscodeNodeURL: %v", err) + } + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.JWTSecret = "test-secret" + handler.PlaybackConfig = playbackTestConfig(writePlaybackTestFFmpeg(t), t.TempDir()) + handler.copySeekAnchor = func(context.Context, string, string, float64, int) (float64, int, error) { + return 16, 8, nil + } + pool := nodepool.NewTranscodePool() + pool.SetNodes([]*nodepool.Node{{ + Name: "transcode-1", Type: nodepool.NodeTypeTranscode, URL: remote.URL, + Enabled: true, Healthy: true, + }}) + handler.NodePlanner = nodepool.NewPlanner(nodepool.NewProxyPool(), pool) + + responses := make(chan int, 2) + start := func() { + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":18,"target_codec_video":"copy","target_codec_audio":"aac","segment_duration":2,"subtitle_track_index":-1}`), + ).WithContext(newAuthorizedPlaybackContext()) + response := httptest.NewRecorder() + handler.HandleStartTranscode(response, request) + responses <- response.Code + } + go start() + go start() + <-prepared + <-prepared + close(release) + + statuses := []int{<-responses, <-responses} + sort.Ints(statuses) + if statuses[0] != http.StatusAccepted || statuses[1] != http.StatusConflict { + t.Fatalf("replacement statuses = %v, want [%d %d]", statuses, http.StatusAccepted, http.StatusConflict) + } + persisted, err := sessionMgr.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + remoteMu.Lock() + preparedSnapshot := append([]string(nil), preparedIDs...) + deletedSnapshot := append([]string(nil), deletedIDs...) + remoteMu.Unlock() + if len(preparedSnapshot) != 2 || preparedSnapshot[0] == preparedSnapshot[1] { + t.Fatalf("prepared transport IDs = %v, want two distinct successors", preparedSnapshot) + } + winner := persisted.TranscodeTransportID + if winner != preparedSnapshot[0] && winner != preparedSnapshot[1] { + t.Fatalf("published transport = %q, not one of %v", winner, preparedSnapshot) + } + loser := preparedSnapshot[0] + if loser == winner { + loser = preparedSnapshot[1] + } + sort.Strings(deletedSnapshot) + wantDeletes := []string{session.ID, loser} + sort.Strings(wantDeletes) + if len(deletedSnapshot) != 2 || deletedSnapshot[0] != wantDeletes[0] || deletedSnapshot[1] != wantDeletes[1] { + t.Fatalf("deleted transports = %v, want predecessor and loser %v", deletedSnapshot, wantDeletes) + } +} + +func TestHandleStartTranscode_RemoteSuccessorCannotBeOverwrittenByStaleFinalization(t *testing.T) { + baseSessionMgr := playback.NewSessionManager(0, 0) + sessionMgr := &firstBlockingSessionManager{ + SessionManager: baseSessionMgr, + entered: make(chan struct{}), + release: make(chan struct{}), + } + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-remote-state-order.mkv"), + Resolution: "1080p", + CodecVideo: "h264", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "ac3", Default: true}}, + } + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + var remoteMu sync.Mutex + preparedBitrates := make(map[string]int) + var deletedIDs []string + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + var startReq transcodenode.TranscodeStartRequest + if err := json.NewDecoder(r.Body).Decode(&startReq); err != nil { + t.Errorf("decode remote start request: %v", err) + } + remoteMu.Lock() + preparedBitrates[startReq.SessionID] = startReq.TargetBitrateKbps + remoteMu.Unlock() + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(transcodenode.TranscodeStartResponse{ + SessionID: startReq.SessionID, + Status: "accepted", + HWAccel: "none", + }) + case r.Method == http.MethodDelete && strings.HasPrefix(r.URL.Path, "/transcode/"): + deletedID := strings.TrimPrefix(r.URL.Path, "/transcode/") + remoteMu.Lock() + deletedIDs = append(deletedIDs, deletedID) + remoteMu.Unlock() + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected remote request %s %s", r.Method, r.URL.Path) + http.Error(w, "unexpected", http.StatusBadRequest) + } + })) + defer remote.Close() + if err := sessionMgr.SetTranscodeNodeURL(session.ID, remote.URL); err != nil { + t.Fatalf("SetTranscodeNodeURL: %v", err) + } + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.JWTSecret = "test-secret" + handler.PlaybackConfig = playbackTestConfig(writePlaybackTestFFmpeg(t), t.TempDir()) + handler.copySeekAnchor = func(context.Context, string, string, float64, int) (float64, int, error) { + return 16, 8, nil + } + pool := nodepool.NewTranscodePool() + pool.SetNodes([]*nodepool.Node{{ + Name: "transcode-1", Type: nodepool.NodeTypeTranscode, URL: remote.URL, + Enabled: true, Healthy: true, + }}) + handler.NodePlanner = nodepool.NewPlanner(nodepool.NewProxyPool(), pool) + + start := func(bitrate int) <-chan int { + result := make(chan int, 1) + go func() { + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(fmt.Sprintf( + `{"session_id":%q,"seek_seconds":18,"target_codec_video":"copy","target_codec_audio":"aac","target_bitrate_kbps":%d,"segment_duration":2,"subtitle_track_index":-1}`, + session.ID, + bitrate, + )), + ).WithContext(newAuthorizedPlaybackContext()) + response := httptest.NewRecorder() + handler.HandleStartTranscode(response, request) + result <- response.Code + }() + return result + } + + firstResult := start(1111) + <-sessionMgr.entered + secondResult := start(2222) + + // Give the second request time to prepare its successor and contend on the + // lifecycle commit while the first request's durable state publication is + // deliberately paused. + deadline := time.Now().Add(2 * time.Second) + for { + remoteMu.Lock() + preparedCount := len(preparedBitrates) + remoteMu.Unlock() + if preparedCount == 2 { + break + } + if time.Now().After(deadline) { + t.Fatal("second replacement was not prepared") + } + time.Sleep(time.Millisecond) + } + close(sessionMgr.release) + + if status := <-firstResult; status != http.StatusAccepted { + t.Fatalf("first status = %d, want %d", status, http.StatusAccepted) + } + if status := <-secondResult; status != http.StatusConflict { + t.Fatalf("superseded second status = %d, want %d", status, http.StatusConflict) + } + persisted, err := sessionMgr.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + remoteMu.Lock() + winnerBitrate := preparedBitrates[persisted.TranscodeTransportID] + deletedSnapshot := append([]string(nil), deletedIDs...) + remoteMu.Unlock() + if winnerBitrate != 1111 { + t.Fatalf("published transport bitrate = %d, want committed successor bitrate 1111", winnerBitrate) + } + if persisted.TargetBitrateKbps != winnerBitrate { + t.Fatalf("persisted bitrate = %d, want route winner bitrate %d", persisted.TargetBitrateKbps, winnerBitrate) + } + if len(deletedSnapshot) != 2 { + t.Fatalf("deleted transports = %v, want predecessor and first successor", deletedSnapshot) + } +} + +func TestHandleStartTranscode_ConcurrentLocalEncodedStartsRemainLastWriterWins(t *testing.T) { + baseSessionMgr := playback.NewSessionManager(0, 0) + sessionMgr := &firstBlockingSessionManager{ + SessionManager: baseSessionMgr, + entered: make(chan struct{}), + release: make(chan struct{}), + } + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-local-encoded-overlap.mkv"), + Resolution: "2160p", + CodecVideo: "hevc", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 25000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "ac3", Default: true}}, + } + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayTranscode, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.PlaybackConfig = playbackTestConfig(writePlaybackTestFFmpeg(t), t.TempDir()) + t.Cleanup(func() { handler.tm.CloseTranscodeSession(session.ID, "") }) + + start := func(bitrate int) <-chan int { + result := make(chan int, 1) + go func() { + request := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(fmt.Sprintf( + `{"session_id":%q,"target_resolution":"1080p","target_codec_video":"h264","target_codec_audio":"aac","target_bitrate_kbps":%d,"segment_duration":2,"subtitle_track_index":-1}`, + session.ID, + bitrate, + )), + ).WithContext(newAuthorizedPlaybackContext()) + response := httptest.NewRecorder() + handler.HandleStartTranscode(response, request) + result <- response.Code + }() + return result + } + + firstResult := start(1111) + <-sessionMgr.entered + secondResult := start(2222) + close(sessionMgr.release) + if status := <-firstResult; status != http.StatusAccepted { + t.Fatalf("first status = %d, want %d", status, http.StatusAccepted) + } + if status := <-secondResult; status != http.StatusAccepted { + t.Fatalf("second status = %d, want %d", status, http.StatusAccepted) + } + persisted, err := sessionMgr.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if persisted.TargetBitrateKbps != 2222 { + t.Fatalf("persisted bitrate = %d, want last writer bitrate 2222", persisted.TargetBitrateKbps) + } +} + +func TestHandleStartTranscode_SeekedCopyAllowedWhenVideoTranscodingDisabled(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + sessionMgr.SetLimitProvider(func(context.Context, int) (playback.SessionLimits, error) { + return playback.SessionLimits{TranscodingDisabled: true}, nil + }) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-copy-permission.mkv"), + Resolution: "1080p", + CodecVideo: "h264", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "ac3", Default: true}}, } session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) if err != nil { @@ -1595,111 +2883,76 @@ func TestHandleStartTranscode_MPEG2SeekedCopyRemainsCopyVideo(t *testing.T) { handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) handler.ItemAccess = allowAllPlaybackItemAccess{} handler.PlaybackConfig = playbackTestConfig(writePlaybackTestFFmpeg(t), t.TempDir()) - + handler.copySeekAnchor = func(context.Context, string, string, float64, int) (float64, int, error) { + return 16, 8, nil + } transcodeReq := httptest.NewRequest( - "POST", + http.MethodPost, "/api/v1/playback/transcode/start", - strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":18.261,"target_resolution":"1080p","target_codec_video":"copy","target_codec_audio":"aac","target_bitrate_kbps":0,"segment_duration":2,"subtitle_track_index":-1,"subtitle_burn_in":false}`), - ) - transcodeReq = transcodeReq.WithContext(newAuthorizedPlaybackContext()) - + strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":18.261,"target_resolution":"","target_codec_video":"copy","target_codec_audio":"aac","target_bitrate_kbps":0,"segment_duration":2,"subtitle_track_index":-1,"subtitle_burn_in":false}`), + ).WithContext(newAuthorizedPlaybackContext()) transcodeRR := httptest.NewRecorder() handler.HandleStartTranscode(transcodeRR, transcodeReq) - if transcodeRR.Code != 202 { + if transcodeRR.Code != http.StatusAccepted { t.Fatalf("transcode status = %d, body = %s", transcodeRR.Code, transcodeRR.Body.String()) } - transcodeSession := handler.tm.GetTranscodeSession(session.ID) if transcodeSession == nil { - t.Fatal("expected local transcode session") + t.Fatal("expected local copy-video session") } - t.Cleanup(func() { - _ = transcodeSession.Close() - }) - opts := transcodeSession.Opts() - if got := opts.TargetCodecVideo; got != "copy" { - t.Fatalf("mpeg2 seeked copy target video codec = %q, want copy", got) - } - if got := opts.SourceVideoCodec; got != "mpeg2video" { - t.Fatalf("mpeg2 seeked copy source video codec = %q, want mpeg2video", got) - } - if got := opts.TargetCodecAudio; got != "aac" { - t.Fatalf("mpeg2 seeked copy target audio codec = %q, want aac", got) + t.Cleanup(func() { _ = transcodeSession.Close() }) + if got := transcodeSession.Opts().TargetCodecVideo; got != "copy" { + t.Fatalf("target video codec = %q, want copy", got) } } -func TestHandleStartTranscode_ForcedVideoEncodingRechecksPermission(t *testing.T) { - for _, tc := range []struct { - name string - requestBody func(sessionID string) string - }{ - { - name: "seeked copy video", - requestBody: func(sessionID string) string { - return `{"session_id":"` + sessionID + `","seek_seconds":18.261,"target_resolution":"1080p","target_codec_video":"copy","target_codec_audio":"aac","target_bitrate_kbps":0,"segment_duration":2,"subtitle_track_index":-1,"subtitle_burn_in":false}` - }, +func TestHandleStartTranscode_SubtitleBurnInRechecksVideoTranscodePermission(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + sessionMgr.SetLimitProvider(func(context.Context, int) (playback.SessionLimits, error) { + return playback.SessionLimits{TranscodingDisabled: true}, nil + }) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie-burn-in.mkv"), + Resolution: "1080p", + CodecVideo: "h264", + CodecAudio: "ac3", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{{Codec: "ac3", Default: true}}, + SubtitleTracks: []models.SubtitleTrack{ + {Index: 0, Language: "en", Codec: "subrip"}, }, - { - name: "subtitle burn in", - requestBody: func(sessionID string) string { - return `{"session_id":"` + sessionID + `","seek_seconds":0,"target_resolution":"1080p","target_codec_video":"copy","target_codec_audio":"aac","target_bitrate_kbps":0,"segment_duration":2,"subtitle_track_index":0,"subtitle_burn_in":true}` - }, - }, - } { - t.Run(tc.name, func(t *testing.T) { - sessionMgr := playback.NewSessionManager(0, 0) - sessionMgr.SetLimitProvider(func(context.Context, int) (playback.SessionLimits, error) { - return playback.SessionLimits{TranscodingDisabled: true}, nil - }) - file := &models.MediaFile{ - ID: 42, - ContentID: "movie-1", - FilePath: writePlaybackTestMediaFile(t, "movie.mkv"), - Resolution: "1080p", - CodecVideo: "h264", - CodecAudio: "ac3", - Container: "mkv", - Bitrate: 8000, - Duration: 3600, - AudioTracks: []models.AudioTrack{ - {Codec: "ac3", Default: true}, - }, - SubtitleTracks: []models.SubtitleTrack{ - {Index: 0, Language: "en", Codec: "subrip"}, - }, - } - session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) - if err != nil { - t.Fatalf("StartSession: %v", err) - } + } + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } - handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) - handler.ItemAccess = allowAllPlaybackItemAccess{} + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + transcodeReq := httptest.NewRequest( + http.MethodPost, + "/api/v1/playback/transcode/start", + strings.NewReader(`{"session_id":"`+session.ID+`","seek_seconds":0,"target_resolution":"1080p","target_codec_video":"copy","target_codec_audio":"aac","target_bitrate_kbps":0,"segment_duration":2,"subtitle_track_index":0,"subtitle_burn_in":true}`), + ).WithContext(newAuthorizedPlaybackContext()) + transcodeRR := httptest.NewRecorder() + handler.HandleStartTranscode(transcodeRR, transcodeReq) - transcodeReq := httptest.NewRequest( - http.MethodPost, - "/api/v1/playback/transcode/start", - strings.NewReader(tc.requestBody(session.ID)), - ) - transcodeReq = transcodeReq.WithContext(newAuthorizedPlaybackContext()) - - transcodeRR := httptest.NewRecorder() - handler.HandleStartTranscode(transcodeRR, transcodeReq) - - if transcodeRR.Code != http.StatusForbidden { - t.Fatalf("status = %d, want %d, body = %s", transcodeRR.Code, http.StatusForbidden, transcodeRR.Body.String()) - } - var response errorResponse - if err := json.NewDecoder(transcodeRR.Body).Decode(&response); err != nil { - t.Fatalf("decode response: %v", err) - } - if response.Error != "transcoding_disabled" { - t.Fatalf("error = %q, want transcoding_disabled", response.Error) - } - if transcodeSession := handler.tm.GetTranscodeSession(session.ID); transcodeSession != nil { - t.Fatal("transcode session started despite disabled video transcoding") - } - }) + if transcodeRR.Code != http.StatusForbidden { + t.Fatalf("status = %d, want %d, body = %s", transcodeRR.Code, http.StatusForbidden, transcodeRR.Body.String()) + } + var response errorResponse + if err := json.NewDecoder(transcodeRR.Body).Decode(&response); err != nil { + t.Fatalf("decode response: %v", err) + } + if response.Error != "transcoding_disabled" { + t.Fatalf("error = %q, want transcoding_disabled", response.Error) + } + if transcodeSession := handler.tm.GetTranscodeSession(session.ID); transcodeSession != nil { + t.Fatal("transcode session started despite disabled video transcoding") } } diff --git a/internal/api/handlers/playback_transport.go b/internal/api/handlers/playback_transport.go new file mode 100644 index 00000000..1cd3e4b5 --- /dev/null +++ b/internal/api/handlers/playback_transport.go @@ -0,0 +1,84 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "strings" + "time" + + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +// startLocalPlaybackTransport is the shared local ffmpeg launch primitive for +// legacy and protocol-v3 orchestration. Callers retain ownership of lifecycle +// locking and decide whether registration is immediate or transactionally +// staged. +func (h *PlaybackHandler) startLocalPlaybackTransport(ctx context.Context, opts playback.TranscodeOpts) (*playback.TranscodeSession, error) { + return playback.StartTranscode(context.WithoutCancel(ctx), opts) +} + +// startRemotePlaybackTransport is the shared remote-node launch primitive. +// It returns the node's HTTP status separately so legacy and v3 can preserve +// their existing public error envelopes while executing identical transport +// startup and response parsing. +func (h *PlaybackHandler) startRemotePlaybackTransport(ctx context.Context, nodeURL string, request transcodenode.TranscodeStartRequest) (transcodenode.TranscodeStartResponse, int, error) { + body, err := json.Marshal(request) + if err != nil { + return transcodenode.TranscodeStartResponse{}, 0, err + } + requestCtx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + httpRequest, err := http.NewRequestWithContext(requestCtx, http.MethodPost, nodeURL+"/transcode/start", bytes.NewReader(body)) + if err != nil { + return transcodenode.TranscodeStartResponse{}, 0, err + } + httpRequest.Header.Set("Content-Type", "application/json") + httpRequest.Header.Set("Authorization", "Bearer "+h.JWTSecret) + response, err := http.DefaultClient.Do(httpRequest) + if err != nil { + return transcodenode.TranscodeStartResponse{}, 0, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusAccepted { + // Drain the (small) error body so the transport can reuse the + // connection instead of tearing it down on every failed start. + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) + return transcodenode.TranscodeStartResponse{}, response.StatusCode, nil + } + var result transcodenode.TranscodeStartResponse + if err := json.NewDecoder(response.Body).Decode(&result); err != nil { + slog.WarnContext(ctx, "remote transcode start response decode failed", "component", "api", "node", nodeURL, "error", err) + } + return result, response.StatusCode, nil +} + +func fetchRemoteTranscodeCapabilities(ctx context.Context, nodeURL, jwtSecret string) (playback.HWAccelInfo, error) { + requestCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + request, err := http.NewRequestWithContext(requestCtx, http.MethodGet, strings.TrimRight(nodeURL, "/")+"/hw-capabilities", nil) + if err != nil { + return playback.HWAccelInfo{}, err + } + request.Header.Set("Authorization", "Bearer "+jwtSecret) + response, err := http.DefaultClient.Do(request) + if err != nil { + return playback.HWAccelInfo{}, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return playback.HWAccelInfo{}, fmt.Errorf("node returned %d", response.StatusCode) + } + var info playback.HWAccelInfo + if err := json.NewDecoder(response.Body).Decode(&info); err != nil { + return playback.HWAccelInfo{}, err + } + info.Source = "transcode_node" + info.NodeURL = nodeURL + return info, nil +} diff --git a/internal/api/handlers/playback_v3.go b/internal/api/handlers/playback_v3.go new file mode 100644 index 00000000..851c0c45 --- /dev/null +++ b/internal/api/handlers/playback_v3.go @@ -0,0 +1,2109 @@ +package handlers + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + "github.com/go-chi/chi/v5" + "github.com/google/uuid" + + apimw "github.com/Silo-Server/silo-server/internal/api/middleware" + "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/clientip" + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/subtitles" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +const ( + maxPlaybackV3BodyBytes = 256 << 10 + maxPlaybackV3EventBodyBytes = 32 << 10 + replanLeaseDurationV3 = 15 * time.Second + v3NodeCapabilityTTL = time.Minute + // Failed capability fetches are memoized briefly so an unreachable node + // costs one timeout per window instead of one per planning request. + v3NodeCapabilityErrorTTL = 15 * time.Second + // Capability fetches on the planning path run under a deadline well below + // the fetch helper's own 10s timeout: planning happens on the start + // request path, where a slow node must degrade the union, not the user. + v3NodeCapabilityPlanTimeout = 3 * time.Second +) + +type v3NodeCapabilityCache struct { + transformations []playback.TransformationV3 + err error + expiresAt time.Time +} + +type preparedTransportV3 struct { + url string + nodeURL string + transportID string + commit func() + rollback func() + applySession func() (func() error, error) + afterDurableCommit func() +} + +type transportErrorV3 struct { + reason string + message string + retryable bool + cause error +} + +type v3ReplanLock struct { + mu sync.Mutex + refs int +} + +type v3EventRate struct { + windowStart time.Time + count int +} + +type replacementAdmissionCheckerV3 interface { + CheckReplacementAllowed(context.Context, string, playback.PlayMethod, bool) error +} + +type replacementReservationCancellerV3 interface { + CancelReplacementReservation(string) +} + +type replacementStateManagerV3 interface { + ApplyReplacement(string, playback.SessionReplacement) (playback.SessionReplacementRollback, error) + RollbackReplacement(string, playback.SessionReplacementRollback) error +} + +type sessionReservationReleaserV3 interface { + ReleaseSession(string) +} + +func (e *transportErrorV3) Error() string { + if e.cause != nil { + return e.reason + ": " + e.cause.Error() + } + return e.reason +} + +// v3FlagCacheTTL bounds how stale a v3 feature-flag read may be. Flags stay +// DB-backed so enabling or rolling back needs no process restart, but reading +// them once per start/replan/event would put an uncached settings SELECT on +// every latency-sensitive playback request. +const v3FlagCacheTTL = 5 * time.Second + +type v3FlagCacheEntry struct { + value bool + expiresAt time.Time +} + +func (h *PlaybackHandler) settingFlagCachedV3(ctx context.Context, key string) bool { + if h == nil || h.SettingsRepo == nil { + return false + } + now := time.Now() + h.v3FlagMu.Lock() + entry, ok := h.v3Flags[key] + h.v3FlagMu.Unlock() + if ok && now.Before(entry.expiresAt) { + return entry.value + } + value, err := h.SettingsRepo.Get(ctx, key) + enabled := err == nil && strings.EqualFold(strings.TrimSpace(value), "true") + if err == nil { + h.v3FlagMu.Lock() + if h.v3Flags == nil { + h.v3Flags = make(map[string]v3FlagCacheEntry) + } + h.v3Flags[key] = v3FlagCacheEntry{value: enabled, expiresAt: now.Add(v3FlagCacheTTL)} + h.v3FlagMu.Unlock() + } + return enabled +} + +func (h *PlaybackHandler) protocolV3Enabled(ctx context.Context) bool { + return h.settingFlagCachedV3(ctx, "playback.protocol_v3_enabled") +} + +func (h *PlaybackHandler) transformationRegistryV3(ctx context.Context) *playback.TransformationRegistryV3 { + h.v3RegistryOnce.Do(func() { + h.v3Registry = playback.ProbeTransformationRegistryV3(context.WithoutCancel(ctx), h.playbackConfig().FFmpegPath) + }) + return h.v3Registry +} + +// remoteTransformationsV3 is the transport-time capability lookup for a +// selected node. It never trusts memoized failures: those may be planning +// deadlines far shorter than this path's fetch budget, and rejecting the +// already-selected node on a stale planning timeout would fail a start the +// fetch could still validate. +func (h *PlaybackHandler) remoteTransformationsV3(ctx context.Context, nodeURL string) ([]playback.TransformationV3, error) { + return h.lookupRemoteTransformationsV3(ctx, nodeURL, false) +} + +// remoteTransformationsPlanningV3 is the planning-time variant: it honors +// negatively-cached fetch failures so an unreachable node costs one timeout +// per error-TTL window instead of one per playback start. +func (h *PlaybackHandler) remoteTransformationsPlanningV3(ctx context.Context, nodeURL string) ([]playback.TransformationV3, error) { + return h.lookupRemoteTransformationsV3(ctx, nodeURL, true) +} + +func (h *PlaybackHandler) lookupRemoteTransformationsV3(ctx context.Context, nodeURL string, honorCachedFailure bool) ([]playback.TransformationV3, error) { + now := time.Now() + h.v3NodeCapabilitiesMu.Lock() + entry, ok := h.v3NodeCapabilities[nodeURL] + h.v3NodeCapabilitiesMu.Unlock() + if ok && now.Before(entry.expiresAt) { + if entry.err == nil { + return append([]playback.TransformationV3(nil), entry.transformations...), nil + } + if honorCachedFailure { + return nil, entry.err + } + } + + info, err := fetchRemoteTranscodeCapabilities(ctx, nodeURL, h.JWTSecret) + if err != nil { + h.v3NodeCapabilitiesMu.Lock() + if h.v3NodeCapabilities == nil { + h.v3NodeCapabilities = make(map[string]v3NodeCapabilityCache) + } + h.v3NodeCapabilities[nodeURL] = v3NodeCapabilityCache{err: err, expiresAt: now.Add(v3NodeCapabilityErrorTTL)} + h.v3NodeCapabilitiesMu.Unlock() + return nil, err + } + entry = v3NodeCapabilityCache{ + transformations: append([]playback.TransformationV3(nil), info.Transformations...), + expiresAt: now.Add(v3NodeCapabilityTTL), + } + h.v3NodeCapabilitiesMu.Lock() + if h.v3NodeCapabilities == nil { + h.v3NodeCapabilities = make(map[string]v3NodeCapabilityCache) + } + h.v3NodeCapabilities[nodeURL] = entry + h.v3NodeCapabilitiesMu.Unlock() + return append([]playback.TransformationV3(nil), entry.transformations...), nil +} + +// transcodeNodeEnumeratorV3 exposes the pooled transcode nodes whose +// advertised transformations widen HLS planning; *nodepool.Planner implements +// it. +type transcodeNodeEnumeratorV3 interface { + TranscodeNodeURLs() []string +} + +// hlsPlanningRegistryV3 returns the registry HLS deliveries plan against: the +// local probe plus every pooled transcode node's advertised transformations. +// Only availability of locally-defined specs widens (name and recipe version +// pinned by this server), so any plan built from it passes the per-node +// advertisement validation when that node is selected, and the local-fallback +// validation in prepareTransportV3 rejects recipes only nodes can run. +// Without pooled nodes this is exactly the local registry. +func (h *PlaybackHandler) hlsPlanningRegistryV3(ctx context.Context) *playback.TransformationRegistryV3 { + local := h.transformationRegistryV3(ctx) + enumerator, ok := h.NodePlanner.(transcodeNodeEnumeratorV3) + if !ok { + return local + } + nodeURLs := enumerator.TranscodeNodeURLs() + if len(nodeURLs) == 0 { + return local + } + var merged []playback.TransformationV3 + for _, transformations := range h.pooledNodeTransformationsV3(ctx, nodeURLs) { + merged = append(merged, transformations...) + } + return local.WithAdvertised(merged) +} + +// lazyHLSPlanningRegistryV3 defers (and memoizes) the widened-registry build +// so the planner only pays for node capability lookups when a route decision +// actually depends on them; direct-play and other source-preserving starts +// never touch the pool. +func (h *PlaybackHandler) lazyHLSPlanningRegistryV3(ctx context.Context) func() *playback.TransformationRegistryV3 { + var once sync.Once + var registry *playback.TransformationRegistryV3 + return func() *playback.TransformationRegistryV3 { + once.Do(func() { registry = h.hlsPlanningRegistryV3(ctx) }) + return registry + } +} + +// pooledNodeTransformationsV3 collects the advertised transformations of the +// given transcode nodes, keyed by node URL. Stale cache entries are refreshed +// concurrently under a short planning deadline; nodes that cannot be reached +// contribute nothing (their failures are negatively cached), so planning +// degrades toward the local registry instead of blocking the start path. +func (h *PlaybackHandler) pooledNodeTransformationsV3(ctx context.Context, nodeURLs []string) map[string][]playback.TransformationV3 { + fetchCtx, cancel := context.WithTimeout(ctx, v3NodeCapabilityPlanTimeout) + defer cancel() + results := make([][]playback.TransformationV3, len(nodeURLs)) + var wg sync.WaitGroup + for i, nodeURL := range nodeURLs { + wg.Add(1) + go func(i int, nodeURL string) { + defer wg.Done() + transformations, err := h.remoteTransformationsPlanningV3(fetchCtx, nodeURL) + if err != nil { + slog.DebugContext(ctx, "protocol v3 node capability unavailable for planning", "component", "api", "node", nodeURL, "error", err) + return + } + results[i] = transformations + }(i, nodeURL) + } + wg.Wait() + byURL := make(map[string][]playback.TransformationV3, len(nodeURLs)) + for i, transformations := range results { + if transformations != nil { + byURL[nodeURLs[i]] = transformations + } + } + return byURL +} + +// capabilitySessionPlannerV3 is implemented by *nodepool.Planner; it lets the +// transport layer restrict node selection to nodes that can execute the +// plan's server transformations. +type capabilitySessionPlannerV3 interface { + PlanSessionWith(sessionID, currentTranscodeURL string, needsTranscode bool, estBitrateKbps int, eligible func(*nodepool.Node) bool) nodepool.Plan +} + +// planNodeSessionV3 selects transcode/proxy nodes for the session. Plans that +// carry server transformations restrict selection to nodes whose advertised +// capabilities validate against the plan, so load balancing in a +// heterogeneous pool cannot land a recipe on a node that would reject it when +// a capable sibling exists. Capability-blind selection remains for +// transformation-free plans and non-enumerating planners. +func (h *PlaybackHandler) planNodeSessionV3(ctx context.Context, session *playback.Session, result playback.PlannerResultV3) nodepool.Plan { + selector, selectable := h.NodePlanner.(capabilitySessionPlannerV3) + enumerator, enumerable := h.NodePlanner.(transcodeNodeEnumeratorV3) + if !selectable || !enumerable || !planRequiresServerTransformationsV3(result.Plan) { + return h.NodePlanner.PlanSession(session.ID, session.TranscodeNodeURL, true, result.TargetBitrateKbps) + } + capable := make(map[string]struct{}) + for nodeURL, advertised := range h.pooledNodeTransformationsV3(ctx, enumerator.TranscodeNodeURLs()) { + if validateAdvertisedTransformationsV3(result.Plan, advertised) == nil { + capable[nodeURL] = struct{}{} + } + } + // The predicate runs under the planner lock: a set lookup only. + return selector.PlanSessionWith(session.ID, session.TranscodeNodeURL, true, result.TargetBitrateKbps, func(node *nodepool.Node) bool { + if node == nil { + return false + } + _, ok := capable[node.URL] + return ok + }) +} + +// validateAdvertisedTransformationsV3 verifies that every server-executed +// transformation the plan requires is advertised — at the exact recipe +// version — by the executor under consideration (a pooled node's capability +// response or the local registry's Advertised set). +func validateAdvertisedTransformationsV3(plan *playback.PlanV3, advertised []playback.TransformationV3) error { + available := make(map[string]string, len(advertised)) + for _, transformation := range advertised { + available[strings.ToLower(strings.TrimSpace(transformation.Name))] = strings.TrimSpace(transformation.RecipeVersion) + } + if plan == nil { + return errors.New("playback plan is unavailable") + } + for _, required := range plan.Transformations { + if strings.EqualFold(required.Executor, "client") { + continue + } + version, ok := available[strings.ToLower(strings.TrimSpace(required.Name))] + if !ok || version != strings.TrimSpace(required.RecipeVersion) { + return fmt.Errorf("executor lacks transformation %s@%s", required.Name, required.RecipeVersion) + } + } + return nil +} + +// HandlePlaybackCapabilityV3 reports only transformations that the installed +// runtime has actually probed. The feature flag is read per request so rollback +// does not require a process restart. +func (h *PlaybackHandler) HandlePlaybackCapabilityV3(w http.ResponseWriter, r *http.Request) { + if apimw.GetUserID(r.Context()) == 0 { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required") + return + } + enabled := h.protocolV3Enabled(r.Context()) + response := playback.CapabilityResponseV3{Enabled: enabled, ProtocolVersions: []int{playback.ProtocolV3}} + if !enabled { + response.Features = []string{} + response.Deliveries = []playback.DeliveryV3{} + response.Transformations = []playback.TransformationV3{} + response.Reason = "disabled" + writeJSON(w, http.StatusOK, response) + return + } + response.Features = []string{playback.FeaturePlaybackPlanV3, playback.FeatureMedia3Only, playback.FeatureDetailedDecodeV3, playback.FeatureLayoutPassthrough, playback.FeatureRouteDiagnostics, playback.FeatureDeviceQuirksV3, playback.FeatureSeekReanchorV3} + response.Deliveries = []playback.DeliveryV3{playback.DeliveryOriginalHTTPV3, playback.DeliveryRemuxProgressiveV3, playback.DeliveryRemuxHLSV3, playback.DeliveryTranscodeHLSV3} + response.Transformations = h.transformationRegistryV3(r.Context()).Advertised() + writeJSON(w, http.StatusOK, response) +} + +func (h *PlaybackHandler) handleStartPlaybackV3(w http.ResponseWriter, r *http.Request, body []byte) { + var req playback.StartRequestV3 + if err := json.Unmarshal(body, &req); err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "Invalid protocol v3 request body") + return + } + warnings, err := req.NormalizeAndValidate() + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + profileID := apimw.GetProfileID(r.Context()) + if profileID == "" { + writeError(w, http.StatusBadRequest, "bad_request", "X-Profile-Id header is required") + return + } + if req.ProfileID != profileID { + writeError(w, http.StatusBadRequest, "bad_request", "profile_id must match X-Profile-Id") + return + } + if !h.protocolV3Enabled(r.Context()) { + writeJSON(w, http.StatusCreated, playback.DisabledResponseV3()) + return + } + userID := apimw.GetUserID(r.Context()) + digestBytes := sha256.Sum256(body) + requestDigest := hex.EncodeToString(digestBytes[:]) + if existing, lookupErr := h.PlanStoreV3.GetAttemptByPlaybackAttemptID(r.Context(), req.PlaybackAttemptID); lookupErr == nil { + if existing.UserID != userID || existing.ProfileID != profileID || existing.RequestedMediaFileID != req.FileID || + (existing.RequestDigest != "" && existing.RequestDigest != requestDigest) { + writeError(w, http.StatusConflict, "playback_attempt_reused", "The playback attempt ID belongs to a different request") + return + } + // The replayed plan is only usable while its session is alive; a dead + // session must surface as a retryable terminal so the client mints a + // fresh attempt instead of replaying a plan it can never stream. + if _, sessionErr := h.sessionMgr.GetSession(existing.SessionID); sessionErr != nil { + writeJSON(w, http.StatusCreated, playback.NewTerminalResponseV3("session_expired", "The playback session for this attempt has ended.", true)) + return + } + writeJSON(w, http.StatusCreated, decisionResponseFromAttemptV3(existing)) + return + } else if !errors.Is(lookupErr, playback.ErrSessionNotFound) { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to check playback attempt idempotency") + return + } + requestedFile, err := h.loadAuthorizedFile(r, req.FileID) + if err != nil { + writeV3FileError(w, err) + return + } + requestedFile = h.ensurePlaybackProbe(r.Context(), requestedFile) + audioIndex, err := resolveV3AudioIndex(requestedFile, req.AudioTrackID, req.AudioTrackIndex) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + effectiveFile := requestedFile + settings := h.plannerSettingsV3(r.Context()) + if err := preflightPlaybackFile(r.Context(), effectiveFile, h.MissingMarker, h.EventsHub); err != nil { + writePlaybackFilePreflightError(w, err) + return + } + result := playback.PlanPlaybackV3(playback.PlannerInputV3{ + Request: req, RequestedFile: requestedFile, EffectiveFile: effectiveFile, + AudioTrackIndex: audioIndex, Settings: settings, + Registry: h.transformationRegistryV3(r.Context()), HLSRegistry: h.lazyHLSPlanningRegistryV3(r.Context()), Now: time.Now(), + AdditionalSubtitles: h.downloadedSubtitleInventoryV3(r.Context(), effectiveFile), + }) + if result.Terminal != nil && result.Terminal.Reason == "no_alternate_version" && shouldTryAlternateFileV3(req.QualityPreference) { + if alternate, alternateErr := h.findAlternateFile(r.Context(), requestedFile); alternateErr == nil && alternate != nil { + effectiveFile = h.ensurePlaybackProbe(r.Context(), alternate) + audioIndex = remapAudioIndexV3(requestedFile, effectiveFile, audioIndex) + if err := h.remapSubtitleSelectionV3(r.Context(), requestedFile, effectiveFile, &req); err != nil { + writeJSON(w, http.StatusCreated, playback.NewTerminalResponseV3("subtitle_unavailable_in_version", err.Error(), false)) + return + } + if err := preflightPlaybackFile(r.Context(), effectiveFile, h.MissingMarker, h.EventsHub); err != nil { + writePlaybackFilePreflightError(w, err) + return + } + result = playback.PlanPlaybackV3(playback.PlannerInputV3{Request: req, RequestedFile: requestedFile, EffectiveFile: effectiveFile, AudioTrackIndex: audioIndex, Settings: settings, Registry: h.transformationRegistryV3(r.Context()), HLSRegistry: h.lazyHLSPlanningRegistryV3(r.Context()), Now: time.Now(), AdditionalSubtitles: h.downloadedSubtitleInventoryV3(r.Context(), effectiveFile)}) + } + } + if result.Terminal != nil { + slog.InfoContext(r.Context(), "playback plan decided", "component", "playback", + "outcome", "terminal", + "reason", result.Terminal.Reason, + "file_id", effectiveFile.ID, + "quality_preference", req.QualityPreference, + ) + response := playback.NewTerminalResponseV3(result.Terminal.Reason, result.Terminal.Message, result.Terminal.Retryable) + h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: playback.RouteEventV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: req.PlaybackAttemptID, Event: playback.RouteEventTerminalV3, FallbackReason: result.Terminal.Reason, OutputRouteGeneration: req.OutputRouteGeneration}, UserID: userID, ProfileID: profileID, ClientName: playbackClientInfoFromRequest(r).Name, ClientVersion: playbackClientInfoFromRequest(r).Version, ClientModel: req.ClientPlaybackContext.Device.Model}) + writeJSON(w, http.StatusCreated, response) + return + } + // One line per plan decision so route selection is reconstructible from + // server logs alone (finding a mis-planned route previously required + // correlating client logcat, ffmpeg commands, and session rows). + slog.InfoContext(r.Context(), "playback plan decided", "component", "playback", + "outcome", "plan", + "decision_reason", result.Plan.DecisionReason, + "delivery", result.Plan.Delivery, + "play_method", string(result.PlayMethod), + "requested_file_id", requestedFile.ID, + "effective_file_id", effectiveFile.ID, + "dv_profile", result.Plan.Source.DVProfile, + "dynamic_range", result.Plan.Source.DynamicRange, + "target_resolution", result.TargetResolution, + "target_bitrate_kbps", result.TargetBitrateKbps, + "quality_preference", req.QualityPreference, + "bandwidth_estimate_kbps", intOrZeroHandlerV3(req.BandwidthEstimateKbps), + ) + result.Plan.DegradationWarnings = append(result.Plan.DegradationWarnings, warnings...) + response, statusErr := h.startPlannedPlaybackV3(r, userID, profileID, req, requestDigest, requestedFile, effectiveFile, audioIndex, result) + if statusErr != nil { + if statusErr.reason == "internal_error" { + slog.ErrorContext(r.Context(), "protocol v3 start failed", "component", "api", "reason", statusErr.reason, "error", statusErr.cause) + } + writeJSON(w, http.StatusCreated, playback.NewTerminalResponseV3(statusErr.reason, statusErr.message, statusErr.retryable)) + return + } + writeJSON(w, http.StatusCreated, response) +} + +func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, profileID string, req playback.StartRequestV3, requestDigest string, requestedFile, effectiveFile *models.MediaFile, audioIndex int, result playback.PlannerResultV3) (playback.DecisionResponseV3, *transportErrorV3) { + if result.Plan == nil { + return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "The server produced no playback plan."} + } + if checker, ok := h.sessionMgr.(transcodePermissionChecker); ok && (result.PlayMethod == playback.PlayTranscode || result.TranscodeAudio) { + if err := checker.CheckTranscodingAllowed(r.Context(), userID, result.PlayMethod == playback.PlayTranscode); err != nil { + reason := "transcoding_disabled" + if errors.Is(err, playback.ErrAudioTranscodingDisabled) { + reason = "audio_transcoding_disabled" + } + return playback.DecisionResponseV3{}, &transportErrorV3{reason: reason, message: "The selected server adaptation is disabled for this user."} + } + } + clientInfo := playbackClientInfoFromRequest(r) + ctx := playback.WithClientInfo(r.Context(), clientInfo) + var session *playback.Session + var err error + if starter, ok := h.sessionMgr.(sessionStarterWithFilesContext); ok { + session, err = starter.StartSessionWithFilesContext(ctx, userID, profileID, effectiveFile.ID, requestedFile.ID, result.PlayMethod, result.TranscodeAudio) + } else { + session, err = h.sessionMgr.StartSessionWithFiles(userID, profileID, effectiveFile.ID, requestedFile.ID, result.PlayMethod, result.TranscodeAudio) + } + if err != nil { + return playback.DecisionResponseV3{}, sessionStartErrorV3(err) + } + abort := func() { _ = h.stopPlaybackSessionByID(context.WithoutCancel(r.Context()), session.ID, false) } + if err := h.sessionMgr.UpdateAudioTrack(session.ID, audioIndex, result.PlayMethod); err != nil { + abort() + return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to select the playback audio track.", cause: err} + } + position := floatOrZeroHandlerV3(req.StartPosition) + if err := h.sessionMgr.UpdateProgress(session.ID, position, false); err != nil { + abort() + return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to initialize the playback timeline.", cause: err} + } + session, err = h.sessionMgr.GetSession(session.ID) + if err != nil { + abort() + return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to load the initialized playback session.", cause: err} + } + result.Plan.SessionID = session.ID + transport, transportErr := h.prepareTransportV3(r, session, effectiveFile, result) + if transportErr != nil { + abort() + return playback.DecisionResponseV3{}, transportErr + } + result.Plan.Stream.URL = transport.url + if err := h.attachSubtitleArtifactV3(r.Context(), session.ID, effectiveFile, result.Plan, result.SubtitleTrackIndex); err != nil { + transport.rollback() + abort() + return playback.DecisionResponseV3{}, &transportErrorV3{reason: "subtitle_artifact_unavailable", message: "Failed to prepare the selected subtitle artifact.", cause: err} + } + response := playback.DecisionResponseV3{ProtocolVersion: playback.ProtocolV3, ServerFeatures: []string{playback.FeaturePlaybackPlanV3, playback.FeatureMedia3Only, playback.FeatureRouteDiagnostics, playback.FeatureDeviceQuirksV3, playback.FeatureSeekReanchorV3}, Outcome: playback.OutcomePlayableV3, SessionID: session.ID, PlaybackPlan: result.Plan} + record := playback.AttemptRecordV3{PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, UserID: userID, ProfileID: profileID, RequestedMediaFileID: requestedFile.ID, EffectiveMediaFileID: effectiveFile.ID, CurrentPlanID: result.Plan.PlanID, CurrentPlan: *result.Plan, NormalizedRequest: req, RequestDigest: requestDigest, ExpiresAt: time.Now().Add(playback.MaxTokenTTL)} + if err := h.updateV3SessionState(r.Context(), session, effectiveFile, result, transport); err != nil { + transport.rollback() + abort() + return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to commit the live playback session.", cause: err} + } + if err := h.PlanStoreV3.SaveAttempt(r.Context(), record); err != nil { + transport.rollback() + abort() + if errors.Is(err, playback.ErrIdempotencyKeyReusedV3) { + return playback.DecisionResponseV3{}, &transportErrorV3{reason: "playback_attempt_reused", message: "The playback attempt ID was reused with different input."} + } + if errors.Is(err, playback.ErrPlaybackAttemptExistsV3) { + existing, lookupErr := h.PlanStoreV3.GetAttemptByPlaybackAttemptID(r.Context(), req.PlaybackAttemptID) + if lookupErr == nil && existing.UserID == userID && existing.ProfileID == profileID && existing.RequestedMediaFileID == req.FileID { + // Replaying a concurrent duplicate is only valid while its + // session is alive; otherwise tell the client to mint a new + // attempt rather than hand it a plan it can never stream. + if _, sessionErr := h.sessionMgr.GetSession(existing.SessionID); sessionErr != nil { + return playback.DecisionResponseV3{}, &transportErrorV3{reason: "session_expired", message: "The playback session for this attempt has ended.", retryable: true} + } + return decisionResponseFromAttemptV3(existing), nil + } + } + return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to persist the playback plan.", cause: err} + } + transport.commit() + h.syncSessionsNow(r.Context(), "v3_start") + h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: playback.RouteEventV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, PlanID: result.Plan.PlanID, Event: playback.RouteEventPlanSelectedV3, AppliedQuirkIDs: appliedQuirkIDsV3(result.Plan), QuirkRegistryRevision: appliedQuirkRevisionV3(result.Plan), OutputRouteGeneration: req.OutputRouteGeneration}, UserID: userID, ProfileID: profileID, ClientName: clientInfo.Name, ClientVersion: clientInfo.Version, ClientModel: req.ClientPlaybackContext.Device.Model}) + return response, nil +} + +func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3) (preparedTransportV3, *transportErrorV3) { + if result.Plan.Delivery != playback.DeliveryTranscodeHLSV3 && result.Plan.Delivery != playback.DeliveryRemuxHLSV3 { + return h.prepareIdentityTransportV3(session, result), nil + } + if h.NodePlanner != nil { + plan := h.planNodeSessionV3(r.Context(), session, result) + if plan.TranscodeNode != nil { + transformations, err := h.remoteTransformationsV3(r.Context(), plan.TranscodeNode.URL) + if err == nil { + err = validateAdvertisedTransformationsV3(result.Plan, transformations) + } + if err == nil { + transport, transportErr := h.prepareRemoteTransportV3(r, session, file, result, plan) + if transportErr != nil { + if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok { + releaser.ReleaseSession(session.ID) + } + } + return transport, transportErr + } + slog.WarnContext(r.Context(), "protocol v3 transcode node capability mismatch", "node", plan.TranscodeNode.URL, "error", err) + if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok { + releaser.ReleaseSession(session.ID) + } + if !nodepool.LocalTranscodeFallbackAllowed(r.Context(), h.SettingsRepo) { + return preparedTransportV3{}, &transportErrorV3{reason: "transcode_node_capability_unavailable", message: "No transcode node can execute the selected playback recipe.", retryable: true, cause: err} + } + } + if !nodepool.LocalTranscodeFallbackAllowed(r.Context(), h.SettingsRepo) { + return preparedTransportV3{}, &transportErrorV3{reason: "capacity_unavailable", message: "No transcode node is available and local fallback is disabled.", retryable: true} + } + } + // Capability-union planning may select transformations only pooled nodes + // can execute; the local binary must prove it carries the recipe before + // this fallback spawns an ffmpeg that would fail at runtime. Retryable: + // a capable node freeing up satisfies the same plan. Transformation-free + // plans skip the check (and the local probe behind it) entirely. + if planRequiresServerTransformationsV3(result.Plan) { + if err := validateAdvertisedTransformationsV3(result.Plan, h.transformationRegistryV3(r.Context()).Advertised()); err != nil { + return preparedTransportV3{}, &transportErrorV3{reason: "transcode_node_capability_unavailable", message: "No available transcode executor can run the selected playback recipe.", retryable: true, cause: err} + } + } + return h.prepareLocalTransportV3(r, session, file, result) +} + +// planRequiresServerTransformationsV3 reports whether the plan carries any +// transformation the serving executor (local binary or transcode node) must +// perform, as opposed to client-executed ones. +func planRequiresServerTransformationsV3(plan *playback.PlanV3) bool { + if plan == nil { + return false + } + for _, transformation := range plan.Transformations { + if !strings.EqualFold(transformation.Executor, "client") { + return true + } + } + return false +} + +func (h *PlaybackHandler) prepareIdentityTransportV3(session *playback.Session, result playback.PlannerResultV3) preparedTransportV3 { + routeSession := *session + routeSession.PlayMethod = result.PlayMethod + routeSession.BasePlayMethod = result.PlayMethod + routeSession.MediaFileID = result.Plan.EffectiveMediaFileID + routeSession.AudioTrackIndex = plannedAudioTrackIndexV3(result, session.AudioTrackIndex) + routeSession.TranscodeAudio = result.TranscodeAudio + routeSession.RemuxDVMode = remuxDVModeForPlanV3(result.Plan) + previousNodeURL := session.TranscodeNodeURL + previousTransportID := remoteTransportID(session) + unlock := h.tm.LockSessionLifecycle(session.ID) + committed := false + streamURL := h.playbackStreamURL(&routeSession) + if result.Plan != nil && result.Plan.Delivery == playback.DeliveryRemuxProgressiveV3 { + configureProgressiveRemuxTimelineV3(result.Plan) + if seek := result.Plan.Timeline.StreamOriginSeconds; seek > 0 { + streamURL = appendPlaybackQueryV3(streamURL, "seek", strconv.FormatFloat(seek, 'f', -1, 64)) + } + } + return preparedTransportV3{ + url: streamURL, + commit: func() { + if committed { + return + } + committed = true + h.tm.CloseTranscodeSession(session.ID, "") + if previousNodeURL != "" { + h.tm.StopRemoteTranscode(previousTransportID, previousNodeURL) + } + unlock() + }, + rollback: func() { + if committed { + return + } + committed = true + unlock() + }, + } +} + +// A progressive remux is a freshly generated, chunked MP4 response and does +// not implement byte ranges. Its player clock therefore begins at zero at the +// requested source origin, and arbitrary seeks must request another server +// reanchor rather than issuing a Range request against the remux pipe. +func configureProgressiveRemuxTimelineV3(plan *playback.PlanV3) { + if plan == nil { + return + } + origin := plan.Timeline.SourceStartSeconds + plan.Timeline.PlayerStartSeconds = 0 + plan.Timeline.StreamOriginSeconds = origin + plan.Timeline.TimelineOffsetSeconds = origin + plan.Timeline.SeekWindowStartSeconds = &origin + plan.Timeline.SeekWindowEndSeconds = nil + plan.Timeline.CanSeekAnywhere = false + plan.Timeline.SeekRestoration = "source_position" +} + +func appendPlaybackQueryV3(rawURL, key, value string) string { + separator := "?" + if strings.ContainsRune(rawURL, '?') { + separator = "&" + } + return rawURL + separator + key + "=" + value +} + +func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3) (preparedTransportV3, *transportErrorV3) { + cfg := h.playbackConfig() + if err := os.MkdirAll(cfg.TranscodeDir, 0o755); err != nil { + return preparedTransportV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to prepare the transcode directory.", cause: err} + } + outputSubdir := transportGenerationV3(session.ID, result.Plan.PlanID) + outputDir := filepath.Join(cfg.TranscodeDir, outputSubdir) + videoCodec := result.TargetVideoCodec + if result.Plan.Delivery == playback.DeliveryRemuxHLSV3 { + videoCodec = "copy" + } + seekSeconds, startSegment := configureHLSTimelineV3(result.Plan, videoCodec, 2, float64(file.Duration)) + unlock := h.tm.LockSessionLifecycle(session.ID) + ts, err := h.startLocalPlaybackTransport(r.Context(), playback.TranscodeOpts{InputPath: file.FilePath, OutputDir: outputDir, OutputSubdir: outputSubdir, SessionID: session.ID, SourceVideoCodec: file.CodecVideo, VideoBitstreamFilter: videoBitstreamFilterForPlanV3(result.Plan), SeekSeconds: seekSeconds, StartSegmentNumber: startSegment, TargetResolution: result.TargetResolution, TargetCodecVideo: videoCodec, TargetCodecAudio: result.TargetAudioCodec, TargetAudioChannels: result.TargetAudioChannels, TargetBitrateKbps: result.TargetBitrateKbps, SegmentDuration: 2, FFmpegPath: cfg.FFmpegPath, HWAccel: cfg.HWAccel, HWDevice: cfg.HWDevice, AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex), SubtitleTrackIndex: result.SubtitleTransportTrackIndex, SubtitleBurnIn: result.SubtitleBurnIn, SubtitleCodec: result.SubtitleCodec, TotalDuration: float64(file.Duration), FastStart: true, NodeType: "integrated", ExecutionMode: "integrated", FFmpegLogSink: h.FFmpegLogSink}) + if err != nil { + unlock() + return preparedTransportV3{}, &transportErrorV3{reason: "transcode_start_failed", message: "Failed to start the playback transport.", retryable: true, cause: err} + } + if !ts.IsRunning() { + _ = ts.Close() + unlock() + return preparedTransportV3{}, &transportErrorV3{reason: "transcode_start_failed", message: "The playback transport exited during startup.", retryable: true} + } + card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, "", ts.Opts()) + url := appendStreamToken(fmt.Sprintf("/playback/transcode/%s/master.m3u8", session.ID), h.signSessionToken(card)) + committed := false + previousNodeURL := session.TranscodeNodeURL + previousTransportID := remoteTransportID(session) + return preparedTransportV3{ + url: url, + commit: func() { + if committed { + return + } + committed = true + previous := h.tm.SwapTranscodeSession(session.ID, ts) + unlock() + if previous != nil && previous != ts { + _ = previous.Close() + } + if previousNodeURL != "" { + h.tm.StopRemoteTranscode(previousTransportID, previousNodeURL) + } + ts.SetRestartHook(func(ctx context.Context) { + h.maybeStartThrottler(ctx, ts) + h.tm.MonitorLocalTranscodeExit(session.ID, ts) + }) + h.maybeStartThrottler(r.Context(), ts) + h.tm.MonitorLocalTranscodeExit(session.ID, ts) + }, + rollback: func() { + if committed { + return + } + committed = true + _ = ts.Close() + unlock() + }, + }, nil +} + +func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, nodePlan nodepool.Plan) (preparedTransportV3, *transportErrorV3) { + node := nodePlan.TranscodeNode + transportID := transportGenerationV3(session.ID, result.Plan.PlanID) + videoCodec := result.TargetVideoCodec + if result.Plan.Delivery == playback.DeliveryRemuxHLSV3 { + videoCodec = "copy" + } + seekSeconds, startSegment := configureHLSTimelineV3(result.Plan, videoCodec, 2, float64(file.Duration)) + req := transcodenode.TranscodeStartRequest{SessionID: transportID, InputPath: file.FilePath, SourceVideoCodec: file.CodecVideo, VideoBitstreamFilter: videoBitstreamFilterForPlanV3(result.Plan), SeekSeconds: seekSeconds, StartSegmentNumber: startSegment, TargetResolution: result.TargetResolution, TargetCodecVideo: videoCodec, TargetCodecAudio: result.TargetAudioCodec, TargetAudioChannels: result.TargetAudioChannels, TargetBitrateKbps: result.TargetBitrateKbps, SegmentDuration: 2, HWAccel: h.playbackConfig().HWAccel, AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex), SubtitleTrackIndex: result.SubtitleTransportTrackIndex, SubtitleBurnIn: result.SubtitleBurnIn, SubtitleCodec: result.SubtitleCodec, TotalDuration: float64(file.Duration), RequireReady: true} + nodeResp, status, err := h.startRemotePlaybackTransport(r.Context(), node.URL, req) + if err != nil { + // A timeout can fire after the node actually started the job; the + // stop is a harmless 404 when it never did, and reaps an orphan + // full-length transcode when it did. + h.tm.StopRemoteTranscode(transportID, node.URL) + return preparedTransportV3{}, &transportErrorV3{reason: "transcode_node_unavailable", message: "The selected transcode node is unavailable.", retryable: true, cause: err} + } + if status != http.StatusAccepted { + h.tm.StopRemoteTranscode(transportID, node.URL) + return preparedTransportV3{}, &transportErrorV3{reason: "transcode_start_failed", message: "The selected transcode node rejected the playback transport.", retryable: true} + } + hw := firstNonEmptyHandlerV3(strings.TrimSpace(nodeResp.HWAccel), strings.TrimSpace(req.HWAccel)) + card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, node.URL, playback.TranscodeOpts{InputPath: req.InputPath, SessionID: session.ID, TranscodeTransportID: transportID, SourceVideoCodec: req.SourceVideoCodec, VideoBitstreamFilter: req.VideoBitstreamFilter, SeekSeconds: req.SeekSeconds, StartSegmentNumber: req.StartSegmentNumber, TargetResolution: req.TargetResolution, TargetCodecVideo: req.TargetCodecVideo, TargetCodecAudio: req.TargetCodecAudio, TargetBitrateKbps: req.TargetBitrateKbps, SegmentDuration: req.SegmentDuration, HWAccel: hw, AudioTrackIndex: req.AudioTrackIndex, SubtitleTrackIndex: req.SubtitleTrackIndex, SubtitleBurnIn: req.SubtitleBurnIn, SubtitleCodec: req.SubtitleCodec, TotalDuration: req.TotalDuration}) + url := h.buildProxyManifestURL(card, nodePlan.ProxyNode) + committed := false + previousNodeURL := session.TranscodeNodeURL + previousTransportID := remoteTransportID(session) + unlock := h.tm.LockSessionLifecycle(session.ID) + return preparedTransportV3{url: url, nodeURL: node.URL, transportID: transportID, commit: func() { + if committed { + return + } + committed = true + h.tm.CloseTranscodeSession(session.ID, "") + if previousNodeURL != "" { + h.tm.StopRemoteTranscode(previousTransportID, previousNodeURL) + } + unlock() + }, rollback: func() { + if committed { + return + } + committed = true + h.tm.StopRemoteTranscode(transportID, node.URL) + // The accepted node job is gone; drop the planner reservation too so + // repeated failed starts cannot pin the node's max-job or bandwidth + // budget until the reservation ages out. + if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok { + releaser.ReleaseSession(session.ID) + } + unlock() + }}, nil +} + +func (h *PlaybackHandler) v3SessionStreamState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3) playback.SessionStreamState { + state := playback.SessionStreamState{PlayMethod: result.PlayMethod, BasePlayMethod: result.PlayMethod, AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex), TranscodeAudio: result.TranscodeAudio, RemuxDVMode: remuxDVModeForPlanV3(result.Plan), TranscodeNodeURL: transport.nodeURL, TranscodeTransportID: transport.transportID, TranscodeRouteSet: true, ClientIP: clientip.FromContext(ctx), ClientName: session.ClientName, ClientVersion: session.ClientVersion, ClientUserAgent: session.ClientUserAgent, StreamBitrateKbps: result.TargetBitrateKbps, TargetVideoCodec: result.TargetVideoCodec, TargetAudioCodec: result.TargetAudioCodec, TargetResolution: result.TargetResolution, SubtitleTrackIndex: result.SubtitleTransportTrackIndex, SubtitleBurnIn: result.SubtitleBurnIn} + if result.Plan != nil && (result.Plan.Delivery == playback.DeliveryTranscodeHLSV3 || result.Plan.Delivery == playback.DeliveryRemuxHLSV3) { + state.SegmentDuration = 2 + } + if state.StreamBitrateKbps <= 0 { + state.StreamBitrateKbps = fileBitrateKbps(file) + } + return state +} + +func (h *PlaybackHandler) updateV3SessionState(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, transport preparedTransportV3) error { + return h.sessionMgr.UpdateStreamState(session.ID, h.v3SessionStreamState(ctx, session, file, result, transport)) +} + +func plannedAudioTrackIndexV3(result playback.PlannerResultV3, fallback int) int { + if result.Plan != nil && result.Plan.SelectedTracks.Audio != nil && result.Plan.SelectedTracks.Audio.Index != nil { + return *result.Plan.SelectedTracks.Audio.Index + } + return fallback +} + +func transportGenerationV3(sessionID, planID string) string { + planSuffix := strings.TrimPrefix(planID, "plan:") + if len(planSuffix) > 12 { + planSuffix = planSuffix[:12] + } + return sessionID + "-" + planSuffix + "-" + uuid.NewString()[:8] +} + +func (h *PlaybackHandler) attachSubtitleArtifactV3(ctx context.Context, sessionID string, file *models.MediaFile, plan *playback.PlanV3, selectedIndex int) error { + if plan == nil || file == nil || selectedIndex < 0 || (plan.Subtitle.Mode != playback.SubtitleRenderV3 && plan.Subtitle.Mode != playback.SubtitleConvertV3) { + return nil + } + var downloaded []subtitles.DownloadedSubtitle + if h.SubtitleRepo != nil { + var err error + downloaded, err = h.SubtitleRepo.ListDownloadedSubtitles(ctx, file.ID) + if err != nil { + return err + } + } + for _, value := range buildSubtitleURLs(sessionID, file, downloaded, true) { + if value.Index != selectedIndex { + continue + } + format := strings.ToLower(value.Codec) + mime := subtitleMIMEV3(format) + url := value.URL + if plan.Subtitle.Mode == playback.SubtitleConvertV3 { + format = "vtt" + mime = "text/vtt" + url = forceSubtitleExtensionV3(value.URL, ".vtt") + } + plan.Subtitle.Artifact = &playback.SubtitleArtifactV3{URL: url, MIMEType: mime, Format: format, TimingOriginSeconds: plan.Timeline.StreamOriginSeconds} + return nil + } + return errors.New("selected subtitle artifact is absent from the frozen inventory") +} + +func (h *PlaybackHandler) downloadedSubtitleInventoryV3(ctx context.Context, file *models.MediaFile) []playback.SubtitleInventoryEntryV3 { + if h == nil || h.SubtitleRepo == nil || file == nil { + return nil + } + downloaded, err := h.SubtitleRepo.ListDownloadedSubtitles(ctx, file.ID) + if err != nil { + return nil + } + base := len(file.ExternalSubtitles) + len(file.SubtitleTracks) + result := make([]playback.SubtitleInventoryEntryV3, 0, len(downloaded)) + for index, value := range downloaded { + result = append(result, playback.SubtitleInventoryEntryV3{CombinedIndex: base + index, Codec: string(value.Format), Source: "downloaded"}) + } + return result +} + +// HandleReplanPlaybackV3 provides persistent idempotency and preserves the old +// transport until a successor has entered its startup state and the new plan is +// durably committed. +func (h *PlaybackHandler) HandleReplanPlaybackV3(w http.ResponseWriter, r *http.Request) { + userID := apimw.GetUserID(r.Context()) + profileID := apimw.GetProfileID(r.Context()) + if userID == 0 || profileID == "" { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication and profile are required") + return + } + if !h.protocolV3Enabled(r.Context()) { + writeJSON(w, http.StatusOK, playback.DisabledResponseV3()) + return + } + body, err := readBoundedV3Body(w, r, maxPlaybackV3BodyBytes) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body") + return + } + var req playback.ReplanRequestV3 + if err := json.Unmarshal(body, &req); err != nil || req.Validate() != nil { + writeError(w, http.StatusBadRequest, "bad_request", "Invalid replan request") + return + } + sessionID := chiURLParamV3(r, "session_id") + releaseSlot, err := h.acquireReplanSlotV3(r.Context()) + if err != nil { + writeError(w, http.StatusServiceUnavailable, "replan_capacity_exhausted", "The server is replanning too many sessions; retry shortly") + return + } + defer releaseSlot() + unlockReplan := h.lockReplanV3(sessionID) + defer unlockReplan() + unlockStore, err := h.PlanStoreV3.AcquireSessionLock(r.Context(), sessionID) + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to serialize the replan request") + return + } + defer unlockStore() + record, err := h.PlanStoreV3.GetAttempt(r.Context(), sessionID) + if err != nil { + // A store outage must read as retryable, not as the session being + // gone: clients tear playback down on session_not_found. + if !errors.Is(err, playback.ErrSessionNotFound) { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load the playback attempt") + return + } + writePlaybackSessionNotFound(w) + return + } + if record.UserID != userID || record.ProfileID != profileID { + writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another profile") + return + } + if record.PlaybackAttemptID != req.PlaybackAttemptID { + writeError(w, http.StatusConflict, "stale_playback_plan", "The failed plan is no longer current") + return + } + if _, err := h.sessionMgr.GetSession(sessionID); err != nil { + writePlaybackSessionNotFound(w) + return + } + digestBytes := sha256.Sum256(body) + digest := hex.EncodeToString(digestBytes[:]) + lease, err := h.PlanStoreV3.BeginReplan( + r.Context(), + sessionID, + req.ReplanRequestID, + digest, + record.CurrentReplanRequestID, + time.Now().Add(replanLeaseDurationV3), + ) + if errors.Is(err, playback.ErrIdempotencyKeyReusedV3) { + writeError(w, http.StatusConflict, "idempotency_key_reused", "The replan request ID was reused with different input") + return + } + if errors.Is(err, playback.ErrStaleReplanLeaseV3) { + writeError(w, http.StatusConflict, "stale_playback_plan", "A newer replacement plan is already active") + return + } + if err != nil { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to reserve the replan request") + return + } + if lease.State == playback.ReplanLeaseInFlightV3 { + writeError(w, http.StatusConflict, "replan_in_progress", "An identical replan is still in progress") + return + } + if lease.State == playback.ReplanLeaseCompletedV3 { + if record.CurrentReplanRequestID != req.ReplanRequestID || !completedReplanResponseMatchesAttemptV3(lease.Response, record) { + writeError(w, http.StatusConflict, "stale_playback_plan", "A newer replacement plan is already active") + return + } + if _, err := h.sessionMgr.GetSession(sessionID); err != nil { + writePlaybackSessionNotFound(w) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(lease.Response) + return + } + if record.CurrentPlanID != req.FailedPlanID { + writeError(w, http.StatusConflict, "stale_playback_plan", "The failed plan is no longer current") + return + } + response, updated, transport, replanErr := h.executeReplanV3(r, record, req) + if replanErr != nil { + response = playback.NewTerminalResponseV3(replanErr.reason, replanErr.message, replanErr.retryable) + updated = *record + } + updated.CurrentReplanRequestID = req.ReplanRequestID + encoded, _ := json.Marshal(response) + var rollbackSession func() error + if transport != nil && transport.applySession != nil { + var err error + rollbackSession, err = transport.applySession() + if err != nil { + transport.rollback() + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to commit the live replacement session") + return + } + } + if err := h.PlanStoreV3.CompleteReplan(r.Context(), sessionID, req.ReplanRequestID, record.CurrentReplanRequestID, encoded, updated); err != nil { + rollbackFailed := false + if rollbackSession != nil { + if rollbackErr := rollbackSession(); rollbackErr != nil { + rollbackFailed = true + slog.ErrorContext(r.Context(), "protocol v3 replacement rollback failed", "session", sessionID, "error", rollbackErr) + } + } + if transport != nil { + transport.rollback() + } + if rollbackFailed { + _ = h.stopPlaybackSessionByID(context.WithoutCancel(r.Context()), sessionID, false) + } + if errors.Is(err, playback.ErrReplanSupersededV3) { + writeError(w, http.StatusConflict, "stale_playback_plan", "A newer replacement plan is already active") + return + } + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to commit the replacement plan") + return + } + if transport != nil { + transport.commit() + if transport.afterDurableCommit != nil { + transport.afterDurableCommit() + } + } + writeJSON(w, http.StatusOK, response) +} + +func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.AttemptRecordV3, req playback.ReplanRequestV3) (playback.DecisionResponseV3, playback.AttemptRecordV3, *preparedTransportV3, *transportErrorV3) { + reservationHeld := false + reservationHandedOff := false + cancelReservation := func() { + if reservationHeld { + if canceller, ok := h.sessionMgr.(replacementReservationCancellerV3); ok { + canceller.CancelReplacementReservation(record.SessionID) + } + reservationHeld = false + } + } + defer func() { + if !reservationHandedOff { + cancelReservation() + } + }() + start := record.NormalizedRequest + operation := req.EffectiveOperation() + seekReanchor := operation == playback.ReplanOperationSeekReanchorV3 + seekFailureRecovery := operation == playback.ReplanOperationSeekFailureRecoveryV3 + seekScopedRecovery := seekReanchor || seekFailureRecovery + intentChange := false + if seekScopedRecovery { + if err := validateSeekRecoveryRequestV3(record, req); err != nil { + reason := "seek_reanchor_intent_mismatch" + if seekFailureRecovery { + reason = "seek_failure_recovery_intent_mismatch" + } + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{ + reason: reason, + message: err.Error(), + } + } + // Reconstruct the complete route intent from the durable current attempt. + // A seek request is not an authority boundary for replacing capability or + // device evidence: accepting those fields here could make the same file + // select a materially different route based on request-only claims. + start.FileID = record.EffectiveMediaFileID + start.StartPosition = &req.PositionSeconds + applySelectedTracksToStartV3(&start, record.CurrentPlan.SelectedTracks) + } else { + // Failure replans may omit unchanged tracks. The durable current plan + // holds the authoritative effective-file selections; the normalized + // request can still carry requested-edition identities after an + // alternate-version fallback, and validating those against the + // effective file would reject an otherwise valid replan. Seed from + // the plan first, then overlay the request's explicit changes. + applySelectedTracksToStartV3(&start, record.CurrentPlan.SelectedTracks) + switch req.Failure.Classification { + case "quality_changed": + nextQuality, _ := playback.NormalizeQualityV3(req.QualityPreference) + intentChange = nextQuality != start.QualityPreference + case "audio_track_changed": + intentChange = req.SelectedTracks.Audio != nil && + (req.SelectedTracks.Audio.ID != start.AudioTrackID || !optionalIntEqualV3(req.SelectedTracks.Audio.Index, start.AudioTrackIndex)) + case "subtitle_track_changed": + intentChange = req.SelectedTracks.Subtitle == nil && start.SubtitleTrackIndex != nil || + req.SelectedTracks.Subtitle != nil && + (req.SelectedTracks.Subtitle.ID != start.SubtitleTrackID || !optionalIntEqualV3(req.SelectedTracks.Subtitle.Index, start.SubtitleTrackIndex)) + case "output_route_changed": + intentChange = req.OutputRouteGeneration != start.OutputRouteGeneration + } + // Failure replans use the current effective file. Explicit user/output + // intent changes restart source selection from the requested edition. + start.FileID = record.EffectiveMediaFileID + if intentChange { + start.FileID = record.RequestedMediaFileID + } + start.QualityPreference = req.QualityPreference + start.StartPosition = &req.PositionSeconds + start.OutputRouteGeneration = req.OutputRouteGeneration + start.Metered = req.Metered + start.BandwidthEstimateKbps = copyOptionalIntV3(req.BandwidthEstimateKbps) + start.BandwidthCapKbps = copyOptionalIntV3(req.BandwidthCapKbps) + start.Capabilities = req.Capabilities + start.ClientPlaybackContext = req.ClientPlaybackContext + applySelectedTracksToStartV3(&start, req.SelectedTracks) + } + requestedFallbackID := record.EffectiveMediaFileID + effectiveFallbackID := record.RequestedMediaFileID + if seekScopedRecovery { + // Edition fallback is useful for ordinary failure replans, but never for + // a seek operation: the caller asked to move within the currently mounted + // source, not to select another version when that source disappears. + requestedFallbackID = 0 + effectiveFallbackID = 0 + } + requestedFile, err := h.loadFileByPreferredID(r.Context(), record.RequestedMediaFileID, requestedFallbackID) + requestedEditionResolved := err == nil && requestedFile != nil && requestedFile.ID == record.RequestedMediaFileID + if err != nil || requestedFile == nil { + if !seekScopedRecovery { + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "source_unavailable", message: "The requested media source is unavailable."} + } + // The requested edition is identity-only once another effective edition + // is mounted. Seeking must depend on that effective file remaining + // available, not on an inactive original edition still resolving. + requestedFile = &models.MediaFile{ID: record.RequestedMediaFileID} + } + plannerRequestedFile := requestedFile + if requestedFile.ID != record.RequestedMediaFileID { + // The live loader may fall back to the current effective file when the + // original edition is gone. Keep that file for metadata/remapping while + // preserving the durable requested-edition identity in every new plan. + plannerRequestedFile = &models.MediaFile{ID: record.RequestedMediaFileID} + } + currentEffectiveFile, err := h.loadFileByPreferredID(r.Context(), record.EffectiveMediaFileID, effectiveFallbackID) + if err != nil || currentEffectiveFile == nil { + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "source_unavailable", message: "The effective media source is unavailable."} + } + effectiveFile := currentEffectiveFile + if intentChange { + // Prefer returning to the requested edition, but a quality/output/track + // change must not abandon a healthy active alternate merely because the + // inactive original has gone missing since playback started. + if requestedEditionResolved && preflightPlaybackFile(r.Context(), requestedFile, h.MissingMarker, h.EventsHub) == nil { + effectiveFile = requestedFile + } + // Track identities only need remapping when the effective edition + // actually changes. Remapping within the same file would degrade an + // exact selection to a best-match lookup — e.g. moving a listener + // from an eng/ac3 commentary track to the identically-shaped main + // track on a quality change. + if currentEffectiveFile.ID != effectiveFile.ID { + if err := remapAudioSelectionV3(currentEffectiveFile, effectiveFile, &start); err != nil { + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "track_unavailable", message: err.Error()} + } + if start.SubtitleTrackIndex != nil || start.SubtitleTrackID != "" { + if err := h.remapSubtitleSelectionV3(r.Context(), currentEffectiveFile, effectiveFile, &start); err != nil { + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "track_unavailable", message: err.Error()} + } + } + } + } + start.FileID = effectiveFile.ID + if err := preflightPlaybackFile(r.Context(), effectiveFile, h.MissingMarker, h.EventsHub); err != nil { + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{ + reason: "source_unavailable", + message: "The effective media source is unavailable.", + cause: err, + } + } + if seekScopedRecovery && effectiveFile.Duration > 0 && req.PositionSeconds > float64(effectiveFile.Duration) { + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{ + reason: "invalid_seek_position", + message: "The requested seek position is beyond the end of the selected media source.", + } + } + if _, err := start.NormalizeAndValidate(); err != nil { + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "invalid_replan", message: err.Error()} + } + audioIndex, err := resolveV3AudioIndex(effectiveFile, start.AudioTrackID, start.AudioTrackIndex) + if err != nil { + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "track_unavailable", message: err.Error()} + } + attemptedKeys := []string(nil) + if !intentChange && !seekReanchor { + attemptedKeys = append(attemptedKeys, req.AttemptedPlanKeys...) + if !containsStringExactV3(attemptedKeys, req.PlanAttemptKey) { + attemptedKeys = append(attemptedKeys, req.PlanAttemptKey) + } + } + if !seekReanchor && (!intentChange || seekFailureRecovery) { + // Client attempt keys may include local mutations that the server cannot + // reproduce. Always exclude the durable server recipe as well so stale or + // malformed client history cannot immediately re-select the route that + // just failed and ping-pong the session. + currentKey := playback.PlanAttemptKeyV3(record.CurrentPlan, record.NormalizedRequest.OutputRouteGeneration, nil) + if !containsStringExactV3(attemptedKeys, currentKey) { + attemptedKeys = append(attemptedKeys, currentKey) + } + } + result := playback.PlanPlaybackV3(playback.PlannerInputV3{Request: start, RequestedFile: plannerRequestedFile, EffectiveFile: effectiveFile, AudioTrackIndex: audioIndex, Settings: h.plannerSettingsV3(r.Context()), Registry: h.transformationRegistryV3(r.Context()), HLSRegistry: h.lazyHLSPlanningRegistryV3(r.Context()), Now: time.Now(), AttemptedKeys: attemptedKeys, AdditionalSubtitles: h.downloadedSubtitleInventoryV3(r.Context(), effectiveFile)}) + if result.Terminal != nil && result.Terminal.Reason == "no_alternate_version" && replanAllowsAlternateFileV3(operation, start.QualityPreference) { + if alternate, alternateErr := h.findAlternateFile(r.Context(), requestedFile); alternateErr == nil && alternate != nil { + alternate = h.ensurePlaybackProbe(r.Context(), alternate) + remappedAudio := remapAudioIndexV3(effectiveFile, alternate, audioIndex) + if err := h.remapSubtitleSelectionV3(r.Context(), effectiveFile, alternate, &start); err == nil { + start.FileID = alternate.ID + if err := preflightPlaybackFile(r.Context(), alternate, h.MissingMarker, h.EventsHub); err == nil { + effectiveFile = alternate + audioIndex = remappedAudio + result = playback.PlanPlaybackV3(playback.PlannerInputV3{Request: start, RequestedFile: plannerRequestedFile, EffectiveFile: effectiveFile, AudioTrackIndex: audioIndex, Settings: h.plannerSettingsV3(r.Context()), Registry: h.transformationRegistryV3(r.Context()), HLSRegistry: h.lazyHLSPlanningRegistryV3(r.Context()), Now: time.Now(), AttemptedKeys: attemptedKeys, AdditionalSubtitles: h.downloadedSubtitleInventoryV3(r.Context(), effectiveFile)}) + } + } + } + } + if result.Terminal != nil { + return playback.NewTerminalResponseV3(result.Terminal.Reason, result.Terminal.Message, result.Terminal.Retryable), *record, nil, nil + } + session, err := h.sessionMgr.GetSession(record.SessionID) + if err != nil { + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "session_expired", message: "The playback session has expired.", retryable: true} + } + replacementManager, ok := h.sessionMgr.(replacementStateManagerV3) + if !ok { + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "internal_error", message: "The live session manager does not support atomic replacement."} + } + if checker, ok := h.sessionMgr.(replacementAdmissionCheckerV3); ok { + if err := checker.CheckReplacementAllowed(r.Context(), session.ID, result.PlayMethod, result.TranscodeAudio); err != nil { + mapped := sessionStartErrorV3(err) + return playback.DecisionResponseV3{}, *record, nil, mapped + } + _, reservationHeld = h.sessionMgr.(replacementReservationCancellerV3) + } + if checker, ok := h.sessionMgr.(transcodePermissionChecker); ok && (result.PlayMethod == playback.PlayTranscode || result.TranscodeAudio) { + if err := checker.CheckTranscodingAllowed(r.Context(), session.UserID, result.PlayMethod == playback.PlayTranscode); err != nil { + mapped := sessionStartErrorV3(err) + return playback.DecisionResponseV3{}, *record, nil, mapped + } + } + result.Plan.SessionID = session.ID + transport, transportErr := h.prepareTransportV3(r, session, effectiveFile, result) + if transportErr != nil { + return playback.DecisionResponseV3{}, *record, nil, transportErr + } + result.Plan.Stream.URL = transport.url + if err := h.attachSubtitleArtifactV3(r.Context(), session.ID, effectiveFile, result.Plan, result.SubtitleTrackIndex); err != nil { + transport.rollback() + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "subtitle_artifact_unavailable", message: "Failed to prepare the selected subtitle artifact.", cause: err} + } + if seekReanchor { + if err := validateSeekReanchorPlanV3(record, result.Plan); err != nil { + transport.rollback() + return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{ + reason: "seek_reanchor_route_changed", + message: err.Error(), + } + } + } + response := playback.DecisionResponseV3{ProtocolVersion: playback.ProtocolV3, ServerFeatures: []string{playback.FeaturePlaybackPlanV3, playback.FeatureMedia3Only, playback.FeatureRouteDiagnostics, playback.FeatureDeviceQuirksV3, playback.FeatureSeekReanchorV3}, Outcome: playback.OutcomePlayableV3, SessionID: session.ID, PlaybackPlan: result.Plan} + updated := *record + updated.CurrentPlanID = result.Plan.PlanID + updated.CurrentPlan = *result.Plan + updated.NormalizedRequest = start + updated.EffectiveMediaFileID = effectiveFile.ID + updated.ExpiresAt = time.Now().Add(playback.MaxTokenTTL) + originalRollback := transport.rollback + replacement := playback.SessionReplacement{ + EffectiveMediaFileID: effectiveFile.ID, + StreamState: h.v3SessionStreamState(r.Context(), session, effectiveFile, result, transport), + } + if seekScopedRecovery { + replacement.PositionSeconds = &req.PositionSeconds + replacement.PreservePaused = true + } + transport.applySession = func() (func() error, error) { + rollback, err := replacementManager.ApplyReplacement(session.ID, replacement) + if err != nil { + return nil, err + } + return func() error { + return replacementManager.RollbackReplacement(session.ID, rollback) + }, nil + } + transport.afterDurableCommit = func() { + cancelReservation() + h.syncSessionsNow(r.Context(), "v3_replan") + event := playback.RouteEventPlanSelectedV3 + clientModel := req.ClientPlaybackContext.Device.Model + if seekReanchor { + event = playback.RouteEventRuntimeCorrectionSucceededV3 + clientModel = start.ClientPlaybackContext.Device.Model + } + h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: playback.RouteEventV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, PlanID: result.Plan.PlanID, PlanAttemptID: req.PlanAttemptID, PlanAttemptKey: playback.PlanAttemptKeyV3(*result.Plan, start.OutputRouteGeneration, nil), Event: event, FallbackReason: req.Failure.Classification, AppliedQuirkIDs: appliedQuirkIDsV3(result.Plan), QuirkRegistryRevision: appliedQuirkRevisionV3(result.Plan), OutputRouteGeneration: start.OutputRouteGeneration}, UserID: session.UserID, ProfileID: session.ProfileID, ClientName: session.ClientName, ClientVersion: session.ClientVersion, ClientModel: clientModel}) + } + transport.rollback = func() { + originalRollback() + cancelReservation() + } + reservationHandedOff = true + return response, updated, &transport, nil +} + +func validateSeekRecoveryRequestV3(record *playback.AttemptRecordV3, req playback.ReplanRequestV3) error { + if record == nil { + return errors.New("the current playback attempt is unavailable") + } + wantedQuality, _ := playback.NormalizeQualityV3(record.NormalizedRequest.QualityPreference) + requestedQuality, _ := playback.NormalizeQualityV3(req.QualityPreference) + if requestedQuality != wantedQuality { + return errors.New("seek recovery cannot change playback quality") + } + if req.OutputRouteGeneration != record.NormalizedRequest.OutputRouteGeneration { + return errors.New("seek recovery cannot change the output route") + } + if !sameSelectedTracksV3(req.SelectedTracks, record.CurrentPlan.SelectedTracks) { + return errors.New("seek recovery cannot change selected tracks") + } + return nil +} + +func validateSeekReanchorPlanV3(record *playback.AttemptRecordV3, candidate *playback.PlanV3) error { + if record == nil || candidate == nil { + return errors.New("seek reanchor produced no playback route") + } + current := record.CurrentPlan + if candidate.PlanID != record.CurrentPlanID || candidate.PlanID != current.PlanID { + return errors.New("seek reanchor changed the playback plan identity") + } + if candidate.RequestedMediaFileID != record.RequestedMediaFileID || candidate.EffectiveMediaFileID != record.EffectiveMediaFileID { + return errors.New("seek reanchor changed the selected media version") + } + if !sameSelectedTracksV3(candidate.SelectedTracks, current.SelectedTracks) { + return errors.New("seek reanchor changed selected tracks") + } + if candidate.Engine != current.Engine || + candidate.Stream.MIMEType != current.Stream.MIMEType || + candidate.Stream.HeaderRefresh != current.Stream.HeaderRefresh || + !sameEffectiveRecipeV3(candidate.EffectiveRecipe, current.EffectiveRecipe) || + candidate.Claims != current.Claims || + candidate.Subtitle.Mode != current.Subtitle.Mode || + candidate.Subtitle.TrackID != current.Subtitle.TrackID || + !sameSubtitleArtifactRouteV3(candidate.Subtitle.Artifact, current.Subtitle.Artifact) || + candidate.SubtitleFidelityPolicy != current.SubtitleFidelityPolicy || + !sameTransformationsV3(candidate.Transformations, current.Transformations) || + !sameAppliedQuirksV3(candidate.AppliedQuirks, current.AppliedQuirks) || + !sameStringMultisetV3(candidate.RuntimeCorrections, current.RuntimeCorrections) { + return errors.New("seek reanchor changed the playback route semantics") + } + generation := record.NormalizedRequest.OutputRouteGeneration + currentKey := playback.PlanAttemptKeyV3(current, generation, nil) + candidateKey := playback.PlanAttemptKeyV3(*candidate, generation, nil) + if candidateKey != currentKey { + return errors.New("seek reanchor changed the playback route recipe") + } + return nil +} + +func sameSubtitleArtifactRouteV3(left, right *playback.SubtitleArtifactV3) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + // Signed URLs and timing origins are allowed to rotate when a transport is + // reopened; the player-facing artifact representation is not. + return left.MIMEType == right.MIMEType && left.Format == right.Format +} + +func sameEffectiveRecipeV3(left, right playback.EffectiveRecipeV3) bool { + return left.VideoCodec == right.VideoCodec && + left.AudioCodec == right.AudioCodec && + optionalIntEqualV3(left.Width, right.Width) && + optionalIntEqualV3(left.Height, right.Height) && + optionalFloatEqualV3(left.FrameRate, right.FrameRate) && + optionalIntEqualV3(left.BitrateKbps, right.BitrateKbps) && + left.DynamicRange == right.DynamicRange && + optionalIntEqualV3(left.AudioChannels, right.AudioChannels) && + left.AudioLayout == right.AudioLayout +} + +func sameTransformationsV3(left, right []playback.TransformationV3) bool { + if len(left) != len(right) { + return false + } + matched := make([]bool, len(right)) + for _, candidate := range left { + found := false + for index, current := range right { + if !matched[index] && candidate.Name == current.Name && candidate.Executor == current.Executor && + candidate.RecipeVersion == current.RecipeVersion && sameStringMultisetV3(candidate.ValidatedClaims, current.ValidatedClaims) { + matched[index] = true + found = true + break + } + } + if !found { + return false + } + } + return true +} + +func sameAppliedQuirksV3(left, right []playback.AppliedQuirkV3) bool { + if len(left) != len(right) { + return false + } + matched := make([]bool, len(right)) + for _, candidate := range left { + found := false + for index, current := range right { + if !matched[index] && candidate == current { + matched[index] = true + found = true + break + } + } + if !found { + return false + } + } + return true +} + +func sameStringMultisetV3(left, right []string) bool { + if len(left) != len(right) { + return false + } + counts := make(map[string]int, len(left)) + for _, value := range left { + counts[value]++ + } + for _, value := range right { + counts[value]-- + if counts[value] < 0 { + return false + } + } + return true +} + +func applySelectedTracksToStartV3(start *playback.StartRequestV3, selected playback.SelectedTracksV3) { + if start == nil { + return + } + if selected.Audio != nil { + start.AudioTrackID = selected.Audio.ID + start.AudioTrackIndex = copyOptionalIntV3(selected.Audio.Index) + } + if selected.Subtitle != nil { + start.SubtitleTrackID = selected.Subtitle.ID + start.SubtitleTrackIndex = copyOptionalIntV3(selected.Subtitle.Index) + } else { + start.SubtitleTrackID = "" + start.SubtitleTrackIndex = nil + } +} + +func sameSelectedTracksV3(left, right playback.SelectedTracksV3) bool { + return sameTrackIdentityV3(left.Audio, right.Audio) && sameTrackIdentityV3(left.Subtitle, right.Subtitle) +} + +func sameTrackIdentityV3(left, right *playback.TrackIdentityV3) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + return left.ID == right.ID && optionalIntEqualV3(left.Index, right.Index) +} + +func copyOptionalIntV3(value *int) *int { + if value == nil { + return nil + } + copy := *value + return © +} + +func shouldTryAlternateFileV3(qualityPreference string) bool { + return !strings.EqualFold(strings.TrimSpace(qualityPreference), "original") +} + +func replanAllowsAlternateFileV3(operation playback.ReplanOperationV3, qualityPreference string) bool { + return operation == playback.ReplanOperationFailureRecoveryV3 && shouldTryAlternateFileV3(qualityPreference) +} + +func (h *PlaybackHandler) lockReplanV3(sessionID string) func() { + h.v3ReplanMu.Lock() + if h.v3ReplanLocks == nil { + h.v3ReplanLocks = make(map[string]*v3ReplanLock) + } + entry := h.v3ReplanLocks[sessionID] + if entry == nil { + entry = &v3ReplanLock{} + h.v3ReplanLocks[sessionID] = entry + } + entry.refs++ + h.v3ReplanMu.Unlock() + entry.mu.Lock() + return func() { + entry.mu.Unlock() + h.v3ReplanMu.Lock() + entry.refs-- + if entry.refs == 0 { + delete(h.v3ReplanLocks, sessionID) + } + h.v3ReplanMu.Unlock() + } +} + +// maxConcurrentReplansV3 bounds simultaneous replan executions. Each replan +// pins one pooled DB connection for its advisory session lock while issuing +// further store queries from the same pool; without a bound, a recovery storm +// (a transcode node dying with dozens of active sessions) turns every pool +// connection into a lock holder and the inner queries deadlock against them. +const maxConcurrentReplansV3 = 8 + +// sessionLockCapacityAdvisorV3 lets a plan store cap replan concurrency below +// the fixed default when its own connection budget is smaller; a pool sized at +// or below the default would otherwise let lock holders starve the inner +// store queries that must complete before any lock is released. +type sessionLockCapacityAdvisorV3 interface { + SessionLockCapacity() int +} + +// acquireReplanSlotV3 blocks until a replan slot frees or the request context +// is cancelled; excess replans queue here holding no DB resources at all. +func (h *PlaybackHandler) acquireReplanSlotV3(ctx context.Context) (func(), error) { + h.v3ReplanSlotsOnce.Do(func() { + capacity := maxConcurrentReplansV3 + if advisor, ok := h.PlanStoreV3.(sessionLockCapacityAdvisorV3); ok { + if advised := advisor.SessionLockCapacity(); advised > 0 && advised < capacity { + capacity = advised + } + } + h.v3ReplanSlots = make(chan struct{}, capacity) + }) + select { + case h.v3ReplanSlots <- struct{}{}: + var once sync.Once + return func() { once.Do(func() { <-h.v3ReplanSlots }) }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (h *PlaybackHandler) HandlePlaybackRouteEventV3(w http.ResponseWriter, r *http.Request) { + userID := apimw.GetUserID(r.Context()) + profileID := apimw.GetProfileID(r.Context()) + if userID == 0 || profileID == "" { + writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication and profile are required") + return + } + if !h.protocolV3Enabled(r.Context()) { + writeError(w, http.StatusConflict, "protocol_disabled", "Playback protocol v3 is disabled") + return + } + body, err := readBoundedV3Body(w, r, maxPlaybackV3EventBodyBytes) + if err != nil { + writeError(w, http.StatusBadRequest, "bad_request", "Invalid event body") + return + } + var event playback.RouteEventV3 + if err := json.Unmarshal(body, &event); err != nil || !validRouteEventV3(event) { + writeError(w, http.StatusBadRequest, "bad_request", "Invalid route event") + return + } + // The rate limiter runs before the ownership lookup so the per-minute + // budget bounds the store reads as well as the writes. + if !h.allowRouteEventV3(userID, event.PlaybackAttemptID) { + writeError(w, http.StatusTooManyRequests, "event_rate_limited", "Playback route event rate exceeded") + return + } + var identity *playback.AttemptIdentityV3 + var identityErr error + if event.SessionID != "" { + identity, identityErr = h.PlanStoreV3.GetAttemptIdentity(r.Context(), event.SessionID) + } else { + identity, identityErr = h.PlanStoreV3.GetAttemptIdentityByPlaybackAttemptID(r.Context(), event.PlaybackAttemptID) + } + if identityErr != nil { + // A store outage is not an ownership violation; keep 403 for genuine + // mismatches so clients stop sending events for foreign sessions. + if !errors.Is(identityErr, playback.ErrSessionNotFound) { + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize the route event") + return + } + writeError(w, http.StatusForbidden, "forbidden", "Route event does not belong to this profile") + return + } + if identity.UserID != userID || identity.ProfileID != profileID || + (event.SessionID != "" && identity.PlaybackAttemptID != event.PlaybackAttemptID) { + writeError(w, http.StatusForbidden, "forbidden", "Route event does not belong to this profile") + return + } + event.Diagnostics = sanitizeDiagnosticsV3(event.Diagnostics) + client := playbackClientInfoFromRequest(r) + h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: event, UserID: userID, ProfileID: profileID, ClientName: client.Name, ClientVersion: client.Version, ClientModel: event.Diagnostics["device_model"]}) + w.WriteHeader(http.StatusAccepted) +} + +// StartV3Maintenance expires cached signed responses and old telemetry on the +// application lifecycle rather than on latency-sensitive playback requests. +func (h *PlaybackHandler) StartV3Maintenance(ctx context.Context) { + if h == nil || h.PlanStoreV3 == nil || ctx == nil { + return + } + go func() { + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case now := <-ticker.C: + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + if _, err := h.PlanStoreV3.CleanupExpired(cleanupCtx, now); err != nil { + slog.Warn("playback v3 cleanup failed", "error", err) + } + cancel() + } + } + }() +} + +func (h *PlaybackHandler) allowRouteEventV3(userID int, attemptID string) bool { + attemptKey := fmt.Sprintf("attempt:%d:%s", userID, attemptID) + userKey := fmt.Sprintf("user:%d", userID) + now := time.Now() + h.v3EventRateMu.Lock() + defer h.v3EventRateMu.Unlock() + if h.v3EventRates == nil { + h.v3EventRates = make(map[string]v3EventRate) + } + attemptEntry := h.v3EventRates[attemptKey] + if attemptEntry.windowStart.IsZero() || now.Sub(attemptEntry.windowStart) >= time.Minute { + attemptEntry = v3EventRate{windowStart: now} + } + userEntry := h.v3EventRates[userKey] + if userEntry.windowStart.IsZero() || now.Sub(userEntry.windowStart) >= time.Minute { + userEntry = v3EventRate{windowStart: now} + } + if attemptEntry.count >= 120 || userEntry.count >= 600 { + return false + } + attemptEntry.count++ + userEntry.count++ + h.v3EventRates[attemptKey] = attemptEntry + h.v3EventRates[userKey] = userEntry + if len(h.v3EventRates) > 10_000 { + for candidate, value := range h.v3EventRates { + if now.Sub(value.windowStart) > 2*time.Minute { + delete(h.v3EventRates, candidate) + } + } + } + return true +} + +func (h *PlaybackHandler) enqueueRouteEventV3(event playback.RouteEventRecordV3) { + if h == nil || h.PlanStoreV3 == nil { + return + } + h.v3EventOnce.Do(func() { + h.v3EventQueue = make(chan playback.RouteEventRecordV3, 512) + go func() { + for value := range h.v3EventQueue { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + if err := h.PlanStoreV3.RecordRouteEvent(ctx, value); err != nil { + slog.Warn("playback route event write failed", "error", err, "event", value.Event) + } + cancel() + } + }() + }) + select { + case h.v3EventQueue <- event: + default: + slog.Warn("playback route event dropped", "event", event.Event, "playback_attempt_id", event.PlaybackAttemptID) + } +} + +func (h *PlaybackHandler) plannerSettingsV3(ctx context.Context) playback.PlannerSettingsV3 { + settings := playback.PlannerSettingsV3{TranscodeEnabled: h.playbackConfig().TranscodeEnabled} + if h.SettingsRepo != nil { + value, _ := h.SettingsRepo.Get(ctx, "allow_4k_transcode") + settings.Allow4KTranscode = strings.EqualFold(value, "true") + } + return settings +} + +func resolveV3AudioIndex(file *models.MediaFile, trackID string, fallback *int) (int, error) { + index := 0 + if trackID != "" { + fileID, kind, ordinal, ok := playback.ParseTrackIDV3(trackID) + if !ok || kind != "audio" || file == nil || fileID != file.ID { + return 0, errors.New("selected audio track identity is invalid") + } + index = ordinal + } else if fallback != nil { + index = *fallback + } + if file == nil || len(file.AudioTracks) == 0 { + if index == 0 { + return 0, nil + } + return 0, errors.New("selected audio track is unavailable") + } + if index < 0 || index >= len(file.AudioTracks) { + return 0, errors.New("selected audio track is unavailable") + } + return index, nil +} + +func remapAudioIndexV3(source, target *models.MediaFile, index int) int { + if source == nil || target == nil || index < 0 || index >= len(source.AudioTracks) { + return normalizeAudioTrackIndex(target, index) + } + wanted := source.AudioTracks[index] + for i, candidate := range target.AudioTracks { + if strings.EqualFold(candidate.Codec, wanted.Codec) && strings.EqualFold(candidate.Language, wanted.Language) && candidate.Channels == wanted.Channels { + return i + } + } + return normalizeAudioTrackIndex(target, index) +} + +// remapAudioSelectionV3 rebinds the request's audio selection when the +// effective media file changes. ID-only selections are equally file-bound: +// the stale ID would be rejected against the new file's track list +// downstream, so derive the source index from it and remap like any other. +func remapAudioSelectionV3(source, target *models.MediaFile, request *playback.StartRequestV3) error { + if request == nil || source == nil || target == nil || source.ID == target.ID { + return nil + } + if request.AudioTrackIndex == nil { + if request.AudioTrackID == "" { + return nil + } + fileID, kind, ordinal, ok := playback.ParseTrackIDV3(request.AudioTrackID) + if !ok || kind != "audio" || fileID != source.ID { + return errors.New("The selected audio track identity is invalid for the source file.") + } + request.AudioTrackIndex = &ordinal + } + remapped := remapAudioIndexV3(source, target, *request.AudioTrackIndex) + request.AudioTrackIndex = &remapped + request.AudioTrackID = playback.TrackIDV3(target.ID, "audio", remapped) + return nil +} + +func (h *PlaybackHandler) remapSubtitleSelectionV3(ctx context.Context, source, target *models.MediaFile, request *playback.StartRequestV3) error { + if request == nil || source == nil || target == nil || source.ID == target.ID { + return nil + } + if request.SubtitleTrackIndex == nil { + // ID-only selections are equally file-bound: the stale ID would be + // parsed against the alternate file's track list downstream, so + // derive the source index from it and remap like any other. + if request.SubtitleTrackID == "" { + return nil + } + fileID, kind, ordinal, ok := playback.ParseTrackIDV3(request.SubtitleTrackID) + if !ok || kind != "subtitle" || fileID != source.ID { + return errors.New("The selected subtitle track identity is invalid for the source file.") + } + request.SubtitleTrackIndex = &ordinal + } + index := *request.SubtitleTrackIndex + if index < 0 { + return errors.New("The selected subtitle track index is invalid.") + } + targetIndex := -1 + switch { + case index < len(source.ExternalSubtitles): + wanted := source.ExternalSubtitles[index] + for candidateIndex, candidate := range target.ExternalSubtitles { + if strings.EqualFold(candidate.Language, wanted.Language) && strings.EqualFold(candidate.Format, wanted.Format) && candidate.Forced == wanted.Forced { + targetIndex = candidateIndex + break + } + } + case index < len(source.ExternalSubtitles)+len(source.SubtitleTracks): + wanted := source.SubtitleTracks[index-len(source.ExternalSubtitles)] + for candidateIndex, candidate := range target.SubtitleTracks { + if strings.EqualFold(candidate.Language, wanted.Language) && strings.EqualFold(candidate.Codec, wanted.Codec) && candidate.Forced == wanted.Forced { + targetIndex = len(target.ExternalSubtitles) + candidateIndex + break + } + } + default: + if h.SubtitleRepo != nil { + sourceDownloaded, sourceErr := h.SubtitleRepo.ListDownloadedSubtitles(ctx, source.ID) + targetDownloaded, targetErr := h.SubtitleRepo.ListDownloadedSubtitles(ctx, target.ID) + downloadedIndex := index - len(source.ExternalSubtitles) - len(source.SubtitleTracks) + if sourceErr == nil && targetErr == nil && downloadedIndex >= 0 && downloadedIndex < len(sourceDownloaded) { + wanted := sourceDownloaded[downloadedIndex] + for candidateIndex, candidate := range targetDownloaded { + if strings.EqualFold(candidate.Language, wanted.Language) && strings.EqualFold(string(candidate.Format), string(wanted.Format)) && strings.EqualFold(candidate.ReleaseName, wanted.ReleaseName) { + targetIndex = len(target.ExternalSubtitles) + len(target.SubtitleTracks) + candidateIndex + break + } + } + } + } + } + if targetIndex < 0 { + return errors.New("The selected subtitle track is unavailable in the effective file version.") + } + request.SubtitleTrackIndex = &targetIndex + request.SubtitleTrackID = playback.TrackIDV3(target.ID, "subtitle", targetIndex) + return nil +} + +func sessionStartErrorV3(err error) *transportErrorV3 { + switch { + case errors.Is(err, playback.ErrTooManyStreams), errors.Is(err, playback.ErrTooManyTranscodes): + return &transportErrorV3{reason: "capacity_unavailable", message: "Playback capacity is currently unavailable.", retryable: true} + case errors.Is(err, playback.ErrTranscodingDisabled), errors.Is(err, playback.ErrAudioTranscodingDisabled): + return &transportErrorV3{reason: "transcoding_disabled", message: "The selected server adaptation is disabled."} + case errors.Is(err, playback.ErrPlaybackNotAllowed): + return &transportErrorV3{reason: "policy_denied", message: "Playback is denied by server policy."} + default: + return &transportErrorV3{reason: "internal_error", message: "Failed to start the playback session.", cause: err} + } +} + +func decisionResponseFromAttemptV3(record *playback.AttemptRecordV3) playback.DecisionResponseV3 { + if record == nil { + return playback.DecisionResponseV3{} + } + plan := record.CurrentPlan + if plan.AppliedQuirks == nil { + plan.AppliedQuirks = []playback.AppliedQuirkV3{} + } + if plan.RuntimeCorrections == nil { + plan.RuntimeCorrections = []string{} + } + return playback.DecisionResponseV3{ProtocolVersion: playback.ProtocolV3, ServerFeatures: []string{playback.FeaturePlaybackPlanV3, playback.FeatureMedia3Only, playback.FeatureRouteDiagnostics, playback.FeatureDeviceQuirksV3, playback.FeatureSeekReanchorV3}, Outcome: playback.OutcomePlayableV3, SessionID: record.SessionID, PlaybackPlan: &plan} +} + +func completedReplanResponseMatchesAttemptV3(raw json.RawMessage, record *playback.AttemptRecordV3) bool { + if record == nil { + return false + } + var response playback.DecisionResponseV3 + if len(raw) == 0 || json.Unmarshal(raw, &response) != nil { + return false + } + if response.PlaybackPlan == nil { + // Terminal responses deliberately leave the attempt plan untouched. Their + // freshness is carried by CurrentReplanRequestID (and its DB trigger). + return response.Terminal != nil + } + if response.SessionID != record.SessionID || response.PlaybackPlan.SessionID != record.SessionID { + return false + } + candidate, candidateErr := json.Marshal(response.PlaybackPlan) + current, currentErr := json.Marshal(record.CurrentPlan) + return candidateErr == nil && currentErr == nil && bytes.Equal(candidate, current) +} + +func appliedQuirkIDsV3(plan *playback.PlanV3) []string { + if plan == nil { + return nil + } + result := make([]string, 0, len(plan.AppliedQuirks)) + for _, quirk := range plan.AppliedQuirks { + result = append(result, quirk.ID) + } + return result +} + +func appliedQuirkRevisionV3(plan *playback.PlanV3) string { + if plan == nil || len(plan.AppliedQuirks) == 0 { + return "" + } + return plan.AppliedQuirks[0].RegistryRevision +} + +func writeV3FileError(w http.ResponseWriter, err error) { + if errors.Is(err, catalog.ErrItemNotFound) || errors.Is(err, catalog.ErrEpisodeNotFound) { + writeError(w, http.StatusNotFound, "not_found", "Media file not found") + return + } + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to authorize media file") +} +func readBoundedV3Body(w http.ResponseWriter, r *http.Request, limit int64) ([]byte, error) { + return ioReadAllV3(http.MaxBytesReader(w, r.Body, limit)) +} +func ioReadAllV3(reader interface{ Read([]byte) (int, error) }) ([]byte, error) { + var buffer bytes.Buffer + _, err := buffer.ReadFrom(reader) + return buffer.Bytes(), err +} +func chiURLParamV3(r *http.Request, key string) string { return chi.URLParam(r, key) } +func floatOrZeroHandlerV3(v *float64) float64 { + if v == nil { + return 0 + } + return *v +} + +func intOrZeroHandlerV3(v *int) int { + if v == nil { + return 0 + } + return *v +} +func firstNonEmptyHandlerV3(values ...string) string { + for _, value := range values { + if value != "" { + return value + } + } + return "" +} +func subtitleMIMEV3(format string) string { + switch strings.ToLower(format) { + case "ass", "ssa": + return "text/x-ssa" + case "srt", "subrip": + return "application/x-subrip" + case "pgs", "hdmv_pgs_subtitle": + return "application/octet-stream" + default: + return "text/vtt" + } +} + +func forceSubtitleExtensionV3(rawURL, extension string) string { + pathPart, query, hasQuery := strings.Cut(rawURL, "?") + if slash := strings.LastIndex(pathPart, "/"); slash >= 0 { + if dot := strings.LastIndex(pathPart[slash+1:], "."); dot >= 0 { + pathPart = pathPart[:slash+1+dot] + extension + } else { + pathPart += extension + } + } + if hasQuery { + return pathPart + "?" + query + } + return pathPart +} + +func remuxDVModeForPlanV3(plan *playback.PlanV3) playback.RemuxDVMode { + if plan == nil { + return "" + } + for _, transformation := range plan.Transformations { + if transformation.Name == "server_dv7_to_hdr10" { + return playback.RemuxDVStripToHDR10V3 + } + } + if plan.Source.DVProfile == 0 { + return "" + } + if plan.Source.DVProfile == 7 { + // Without the strip transformation a P7 remux would drop the + // enhancement layer and leave dangling RPUs. A P7 plan claiming Dolby + // Vision is a client-side transform of the original bytes, so any + // remux attempt against this session must still be rejected. + return playback.RemuxDVRejectP7V3 + } + if plan.Claims.Video.DolbyVision { + return playback.RemuxDVPreserveV3 + } + return "" +} + +func videoBitstreamFilterForPlanV3(plan *playback.PlanV3) string { + if plan == nil { + return "" + } + for _, transformation := range plan.Transformations { + if transformation.Executor == "server" && transformation.Name == "server_dv7_to_hdr10" && transformation.RecipeVersion == "1" { + return playback.DV7ToHDR10BitstreamFilter + } + } + return "" +} + +func configureHLSTimelineV3(plan *playback.PlanV3, videoCodec string, segmentDuration int, durationSeconds float64) (float64, int) { + if plan == nil { + return 0, 0 + } + requested := plan.Timeline.SourceStartSeconds + seek := alignedSeekSeconds(requested, segmentDuration, videoCodec) + startSegment := computeStartSegment(seek, segmentDuration) + plan.Timeline.SourceStartSeconds = requested + if strings.EqualFold(videoCodec, "copy") { + plan.Timeline.PlayerStartSeconds = 0 + plan.Timeline.StreamOriginSeconds = seek + plan.Timeline.TimelineOffsetSeconds = seek + windowStart := seek + plan.Timeline.SeekWindowStartSeconds = &windowStart + if durationSeconds > 0 { + windowEnd := durationSeconds + plan.Timeline.SeekWindowEndSeconds = &windowEnd + } else { + plan.Timeline.SeekWindowEndSeconds = nil + } + plan.Timeline.CanSeekAnywhere = false + plan.Timeline.SeekRestoration = "source_position" + } else { + plan.Timeline.PlayerStartSeconds = requested + plan.Timeline.StreamOriginSeconds = 0 + plan.Timeline.TimelineOffsetSeconds = 0 + plan.Timeline.SeekWindowStartSeconds = nil + plan.Timeline.SeekWindowEndSeconds = nil + plan.Timeline.CanSeekAnywhere = durationSeconds > 0 + plan.Timeline.SeekRestoration = "player_position" + } + return seek, startSegment +} + +var diagnosticKeysV3 = map[string]struct{}{ + "decoder_name": {}, "decoder_init_ms": {}, "first_frame_ms": {}, + "device_model": {}, "requested_quality": {}, "effective_quality": {}, + "pcm_recovery": {}, "retry_outcome": {}, "replan_request_id": {}, + "video_mime": {}, "video_codecs": {}, "video_width": {}, "video_height": {}, + "color_transfer": {}, "color_range": {}, + "error_code": {}, "error_code_name": {}, "error_cause": {}, + "transformation_name": {}, "transformation_version": {}, "transformation_stage": {}, + "input_dv_profile": {}, "output_dv_profile": {}, "rpu_converted_count": {}, + "rpu_failed_count": {}, "el_nal_dropped_count": {}, "sample_count": {}, + "transform_buffer_peak_bytes": {}, "requested_media_file_id": {}, "effective_media_file_id": {}, + "audio_output_mode": {}, "audio_mime": {}, "audio_channels": {}, "audio_decoder_name": {}, + "correction_id": {}, "correction_stage": {}, + "network_transport": {}, "network_metered": {}, "network_validated": {}, + "bandwidth_estimate_kbps": {}, "link_downstream_kbps": {}, + "target_source_position_seconds": {}, "reason": {}, +} + +func validRouteEventV3(event playback.RouteEventV3) bool { + if event.ProtocolVersion != playback.ProtocolV3 || len(event.PlaybackAttemptID) < 8 || len(event.PlaybackAttemptID) > 128 || event.OutputRouteGeneration < 0 || len(event.SessionID) > 128 || len(event.PlanID) > 128 || len(event.PlanAttemptID) > 128 || len(event.PlanAttemptKey) > 128 || len(event.FailureClassification) > 64 || len(event.FallbackReason) > 64 || len(event.AppliedQuirkIDs) > 16 || len(event.QuirkRegistryRevision) > 128 || len(event.Diagnostics) > 32 { + return false + } + for _, id := range event.AppliedQuirkIDs { + if len(id) == 0 || len(id) > 128 { + return false + } + } + return playback.ValidRouteEventNameV3(event.Event) +} +func sanitizeDiagnosticsV3(values map[string]string) map[string]string { + // Iterate the approved keys, not the client map: map iteration order is + // random, so a count-limited walk over client keys would keep an + // arbitrary subset and drop different diagnostics on identical retries. + result := make(map[string]string) + for key := range diagnosticKeysV3 { + value, ok := values[key] + if !ok { + continue + } + value = strings.TrimSpace(value) + if len(value) > 256 { + value = value[:256] + } + result[key] = value + } + return result +} + +func containsStringFoldV3(values []string, wanted string) bool { + for _, value := range values { + if strings.EqualFold(value, wanted) { + return true + } + } + return false +} + +// containsStringExactV3 compares attempt keys byte-for-byte: they are +// case-sensitive FNV hex digests, so case-folding would treat distinct keys +// as equal. +func containsStringExactV3(values []string, wanted string) bool { + wanted = strings.TrimSpace(wanted) + for _, value := range values { + if strings.TrimSpace(value) == wanted { + return true + } + } + return false +} + +func optionalIntEqualV3(left, right *int) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + return *left == *right +} + +func optionalFloatEqualV3(left, right *float64) bool { + if left == nil || right == nil { + return left == nil && right == nil + } + return *left == *right +} diff --git a/internal/api/handlers/playback_v3_shadow.go b/internal/api/handlers/playback_v3_shadow.go new file mode 100644 index 00000000..b27d9e84 --- /dev/null +++ b/internal/api/handlers/playback_v3_shadow.go @@ -0,0 +1,121 @@ +package handlers + +import ( + "context" + "log/slog" + "runtime/debug" + "time" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/playback" +) + +const playbackV3ShadowSetting = "playback.protocol_v3_shadow_enabled" + +func (h *PlaybackHandler) protocolV3ShadowEnabled(ctx context.Context) bool { + return h.settingFlagCachedV3(ctx, playbackV3ShadowSetting) +} + +// shadowLegacyPlaybackV3 compares the production legacy route with a v3 plan +// without advertising or executing that plan. Legacy capabilities lack the +// detailed v3 fields, so exact source facts are used only as an explicitly +// labelled validation inference; this is never a production compatibility +// claim and cannot enable v3 playback. +func (h *PlaybackHandler) shadowLegacyPlaybackV3(ctx context.Context, req startPlaybackRequest, requestedFile, effectiveFile *models.MediaFile, audioIndex int, productionMethod playback.PlayMethod, productionTranscodeAudio bool, sessionID string) { + // The shadow planner runs on a bare goroutine off the production start + // path; an escaped panic here would take down the whole process for a + // telemetry-only comparison, so it must fail closed instead. + defer func() { + if recovered := recover(); recovered != nil { + slog.ErrorContext(ctx, "playback v3 shadow planner panicked", + "component", "playback", "playback_session_id", sessionID, + "panic", recovered, "stack", string(debug.Stack())) + } + }() + if !h.protocolV3ShadowEnabled(ctx) || requestedFile == nil || effectiveFile == nil { + return + } + request := legacyShadowRequestV3(req, effectiveFile, audioIndex, sessionID) + result := playback.PlanPlaybackV3(playback.PlannerInputV3{ + Request: request, + RequestedFile: requestedFile, + EffectiveFile: effectiveFile, + AudioTrackIndex: audioIndex, + Settings: h.plannerSettingsV3(ctx), + Registry: h.transformationRegistryV3(ctx), + HLSRegistry: h.lazyHLSPlanningRegistryV3(ctx), + Now: time.Now(), + }) + attrs := []any{ + "component", "playback", + "playback_session_id", sessionID, + "requested_file_id", requestedFile.ID, + "effective_file_id", effectiveFile.ID, + "production_method", productionMethod, + "production_transcode_audio", productionTranscodeAudio, + "capability_basis", "legacy_source_exact_inference", + } + if result.Plan != nil { + attrs = append(attrs, + "shadow_plan_id", result.Plan.PlanID, + "shadow_delivery", result.Plan.Delivery, + "shadow_method", result.PlayMethod, + "shadow_transcode_audio", result.TranscodeAudio, + "shadow_reason", result.Plan.DecisionReason, + "route_match", result.PlayMethod == productionMethod && result.TranscodeAudio == productionTranscodeAudio, + ) + } else if result.Terminal != nil { + attrs = append(attrs, "shadow_terminal", result.Terminal.Reason, "route_match", false) + } + slog.InfoContext(ctx, "playback v3 shadow decision", attrs...) +} + +func legacyShadowRequestV3(req startPlaybackRequest, file *models.MediaFile, audioIndex int, sessionID string) playback.StartRequestV3 { + source := playback.SourceDescriptorFromFileV3(file, audioIndex) + hdr := legacyShadowHDRV3(req) + features := []string{playback.FeaturePlaybackPlanV3, playback.FeatureMedia3Only, playback.FeatureDetailedDecodeV3} + capabilities := playback.ClientCodecCapabilitiesV3{ + CodecsVideo: append([]string(nil), req.CodecsVideo...), + CodecsAudio: append([]string(nil), req.CodecsAudio...), + Containers: append([]string(nil), req.Containers...), + MaxResolution: req.MaxResolution, + HDR: req.HDR, + HDRDetails: hdr, + } + if containsStringFoldV3(req.CodecsVideo, source.VideoCodec) { + capabilities.VideoDecode = []playback.VideoDecodeCapabilityV3{{ + Codec: source.VideoCodec, Profiles: []string{source.VideoProfile}, Levels: []int{source.VideoLevel}, BitDepths: []int{source.BitDepth}, + MaxWidth: source.Width, MaxHeight: source.Height, MaxFrameRate: source.FrameRate, MaxBitrateKbps: source.BitrateKbps, Hardware: true, + }} + } + var passthrough *playback.AudioPassthroughV3 + if req.AudioPassthrough != nil { + passthrough = &playback.AudioPassthroughV3{PassthroughCodecs: append([]string(nil), req.AudioPassthrough.PassthroughCodecs...), SpatializerEnabled: req.AudioPassthrough.SpatializerEnabled, MaxChannels: req.AudioPassthrough.MaxChannels} + if containsStringFoldV3(req.AudioPassthrough.PassthroughCodecs, source.AudioCodec) && source.AudioChannels > 0 && source.AudioLayout != "" && source.AudioChannels <= req.AudioPassthrough.MaxChannels { + passthrough.Entries = []playback.AudioPassthroughEntryV3{{Codec: source.AudioCodec, ChannelCounts: []int{source.AudioChannels}, Layouts: []string{source.AudioLayout}}} + features = append(features, playback.FeatureLayoutPassthrough) + } + capabilities.AudioPassthrough = passthrough + } + engines := map[string]playback.EngineCapabilityV3{ + string(playback.EngineMedia3DirectV3): {Enabled: true, SupportedOnDevice: true, Subtitles: playback.EngineSubtitleCapabilitiesV3{EmbeddedText: true, SidecarText: true, EmbeddedBitmap: true, SidecarBitmap: true}}, + string(playback.EngineMedia3ProgressiveRemuxV3): {Enabled: true, SupportedOnDevice: true}, + string(playback.EngineMedia3HLSV3): {Enabled: true, SupportedOnDevice: true}, + } + return playback.StartRequestV3{ + ProtocolVersion: playback.ProtocolV3, ClientFeatures: features, FileID: file.ID, ProfileID: req.ProfileID, + PlaybackAttemptID: "shadow-" + sessionID, QualityPreference: "original", SubtitleFidelityPreference: playback.SubtitleFidelityCompatibleV3, + StartPosition: req.StartPosition, AudioTrackIndex: &audioIndex, Capabilities: capabilities, + ClientPlaybackContext: playback.ClientPlaybackContextV3{ProtocolVersion: playback.ProtocolV3, Features: features, Platform: "legacy-shadow", Output: playback.OutputContextV3{HDRDetails: hdr, AudioPassthrough: passthrough}, Engines: engines}, + } +} + +func legacyShadowHDRV3(req startPlaybackRequest) *playback.HDRCapabilitiesV3 { + if req.HdrDetails != nil { + return &playback.HDRCapabilitiesV3{HDR10: req.HdrDetails.HDR10, HDR10Plus: req.HdrDetails.HDR10Plus, HLG: req.HdrDetails.HLG, DolbyVisionProfiles: append([]int(nil), req.HdrDetails.DolbyVisionProfiles...)} + } + if req.HDR { + return &playback.HDRCapabilitiesV3{HDR10: true} + } + return nil +} diff --git a/internal/api/handlers/playback_v3_test.go b/internal/api/handlers/playback_v3_test.go new file mode 100644 index 00000000..25ef7c2a --- /dev/null +++ b/internal/api/handlers/playback_v3_test.go @@ -0,0 +1,1338 @@ +package handlers + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +type mutablePlaybackSettingsV3 struct { + mu sync.Mutex + values map[string]string +} + +type failingCompletePlanStoreV3 struct { + playback.PlanStoreV3 +} + +type staticNodePlannerV3 struct { + plan nodepool.Plan +} + +func (p staticNodePlannerV3) PlanSession(string, string, bool, int) nodepool.Plan { + return p.plan +} + +func (f *failingCompletePlanStoreV3) CompleteReplan(context.Context, string, string, string, json.RawMessage, playback.AttemptRecordV3) error { + return fmt.Errorf("injected complete replan failure") +} + +func TestShouldTryAlternateFileV3PinsOriginalQuality(t *testing.T) { + if shouldTryAlternateFileV3("original") || shouldTryAlternateFileV3(" ORIGINAL ") { + t.Fatal("original quality must pin the requested media file") + } + for _, quality := range []string{"auto", "2160p", "1080p", "480p"} { + if !shouldTryAlternateFileV3(quality) { + t.Fatalf("quality %q should permit alternate selection", quality) + } + } +} + +func TestReplanAllowsAlternateFileV3PinsSeekOperations(t *testing.T) { + tests := []struct { + name string + operation playback.ReplanOperationV3 + quality string + want bool + }{ + {name: "ordinary failure may use another version", operation: playback.ReplanOperationFailureRecoveryV3, quality: "auto", want: true}, + {name: "original quality remains pinned", operation: playback.ReplanOperationFailureRecoveryV3, quality: "original", want: false}, + {name: "exact seek reanchor pins current version", operation: playback.ReplanOperationSeekReanchorV3, quality: "auto", want: false}, + {name: "failed seek recovery pins current version", operation: playback.ReplanOperationSeekFailureRecoveryV3, quality: "auto", want: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := replanAllowsAlternateFileV3(test.operation, test.quality); got != test.want { + t.Fatalf("replanAllowsAlternateFileV3(%q, %q) = %v, want %v", test.operation, test.quality, got, test.want) + } + }) + } +} + +func (s *mutablePlaybackSettingsV3) Get(_ context.Context, key string) (string, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.values[key], nil +} + +func (s *mutablePlaybackSettingsV3) set(key, value string) { + s.mu.Lock() + defer s.mu.Unlock() + s.values[key] = value +} + +func TestHandleStartPlaybackV3DisabledDoesNotAllocateLegacySession(t *testing.T) { + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: v3HandlerFixtureFile(t)}) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "false"}} + + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, v3HandlerStartRequest()))) + req = req.WithContext(newAuthorizedPlaybackContext()) + rr := httptest.NewRecorder() + handler.HandleStartPlayback(rr, req) + + if rr.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + var response playback.DecisionResponseV3 + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.ProtocolVersion != playback.ProtocolV3 || len(response.ServerFeatures) != 0 || response.SessionID != "" { + t.Fatalf("response = %#v", response) + } + if got := len(manager.AllSessions()); got != 0 { + t.Fatalf("sessions = %d, want 0", got) + } +} + +func TestHandlePlaybackCapabilityV3ReadsFlagPerRequest(t *testing.T) { + settings := &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "false"}} + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.SettingsRepo = settings + + request := func() playback.CapabilityResponseV3 { + req := httptest.NewRequest(http.MethodGet, "/api/v1/playback/capability", nil).WithContext(newAuthorizedPlaybackContext()) + rr := httptest.NewRecorder() + handler.HandlePlaybackCapabilityV3(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + var response playback.CapabilityResponseV3 + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + return response + } + + if response := request(); response.Enabled || response.Reason != "disabled" { + t.Fatalf("disabled response = %#v", response) + } + settings.set("playback.protocol_v3_enabled", "true") + // The flag stays DB-backed (no restart needed for rollback) behind a + // short TTL cache; expiring the cache stands in for the TTL elapsing. + handler.v3FlagMu.Lock() + handler.v3Flags = nil + handler.v3FlagMu.Unlock() + if response := request(); !response.Enabled || len(response.Deliveries) != 4 || !playback.HasFeatureV3(response.Features, playback.FeatureSeekReanchorV3) { + t.Fatalf("enabled response = %#v", response) + } +} + +func TestHandleStartPlaybackV3ReturnsExecutableDirectPlan(t *testing.T) { + file := v3HandlerFixtureFile(t) + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "true", "allow_4k_transcode": "true"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, v3HandlerStartRequest()))) + req = req.WithContext(newAuthorizedPlaybackContext()) + rr := httptest.NewRecorder() + handler.HandleStartPlayback(rr, req) + + if rr.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + var response playback.DecisionResponseV3 + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.Outcome != playback.OutcomePlayableV3 || response.PlaybackPlan == nil { + t.Fatalf("response = %#v", response) + } + if response.PlaybackPlan.Delivery != playback.DeliveryOriginalHTTPV3 || response.PlaybackPlan.Engine != playback.EngineMedia3DirectV3 || response.PlaybackPlan.Stream.URL == "" { + t.Fatalf("plan = %#v", response.PlaybackPlan) + } + if response.PlaybackPlan.RequestedMediaFileID != file.ID || response.PlaybackPlan.EffectiveMediaFileID != file.ID || response.PlaybackPlan.Source.MediaFileID != file.ID { + t.Fatalf("source identity = %#v", response.PlaybackPlan) + } + if got := len(manager.AllSessions()); got != 1 { + t.Fatalf("sessions = %d, want 1", got) + } +} + +func TestHandleStartPlaybackV3DuplicateAttemptReturnsOriginalSession(t *testing.T) { + file := v3HandlerFixtureFile(t) + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "true", "allow_4k_transcode": "true"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + body := marshalV3StartRequest(t, v3HandlerStartRequest()) + + start := func() playback.DecisionResponseV3 { + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(body)).WithContext(newAuthorizedPlaybackContext()) + rr := httptest.NewRecorder() + handler.HandleStartPlayback(rr, req) + if rr.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + var response playback.DecisionResponseV3 + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + return response + } + first := start() + second := start() + if first.SessionID == "" || second.SessionID != first.SessionID { + t.Fatalf("first session %q, second %q", first.SessionID, second.SessionID) + } + if got := len(manager.AllSessions()); got != 1 { + t.Fatalf("sessions = %d, want 1", got) + } +} + +func TestHandleStartPlaybackV3RejectsProfileMismatch(t *testing.T) { + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: v3HandlerFixtureFile(t)}) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "true"}} + request := v3HandlerStartRequest() + request.ProfileID = "profile-other" + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, request))).WithContext(newAuthorizedPlaybackContext()) + rr := httptest.NewRecorder() + handler.HandleStartPlayback(rr, req) + if rr.Code != http.StatusBadRequest || len(manager.AllSessions()) != 0 { + t.Fatalf("status = %d, sessions = %d, body = %s", rr.Code, len(manager.AllSessions()), rr.Body.String()) + } +} + +func TestHandleReplanPlaybackV3UpdatesSelectedAudioAndReplaysIdempotently(t *testing.T) { + file := v3HandlerFixtureFile(t) + file.AudioTracks = append(file.AudioTracks, models.AudioTrack{Codec: "aac", Channels: 2, Layout: "stereo", Language: "spa"}) + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "true", "allow_4k_transcode": "true"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + startRequest := v3HandlerStartRequest() + startReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, startRequest))).WithContext(newAuthorizedPlaybackContext()) + startRR := httptest.NewRecorder() + handler.HandleStartPlayback(startRR, startReq) + if startRR.Code != http.StatusCreated { + t.Fatalf("start status = %d, body = %s", startRR.Code, startRR.Body.String()) + } + var started playback.DecisionResponseV3 + if err := json.Unmarshal(startRR.Body.Bytes(), &started); err != nil { + t.Fatal(err) + } + if started.PlaybackPlan == nil { + t.Fatal("start returned no plan") + } + audioIndex := 1 + bandwidthEstimate := 3_500 + bandwidthCap := 4_000 + failedKey := playback.PlanAttemptKeyV3(*started.PlaybackPlan, startRequest.OutputRouteGeneration, nil) + replan := playback.ReplanRequestV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: startRequest.PlaybackAttemptID, ReplanRequestID: "replan-0001", FailedPlanID: started.PlaybackPlan.PlanID, PlanAttemptID: "plan-attempt-0001", PlanAttemptKey: failedKey, AttemptedPlanKeys: []string{failedKey}, AttemptCount: 1, QualityPreference: "original", PositionSeconds: 12, OutputRouteGeneration: startRequest.OutputRouteGeneration, Metered: true, BandwidthEstimateKbps: &bandwidthEstimate, BandwidthCapKbps: &bandwidthCap, SelectedTracks: playback.SelectedTracksV3{Audio: &playback.TrackIdentityV3{ID: playback.TrackIDV3(file.ID, "audio", audioIndex), Index: &audioIndex}}, Failure: playback.FailureV3{Classification: "audio_renderer_error"}, Capabilities: startRequest.Capabilities, ClientPlaybackContext: startRequest.ClientPlaybackContext} + replanBody, err := json.Marshal(replan) + if err != nil { + t.Fatal(err) + } + + call := func() playback.DecisionResponseV3 { + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(replanBody))).WithContext(newAuthorizedPlaybackContext()) + req = withPlaybackRouteParam(req, "session_id", started.SessionID) + rr := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("replan status = %d, body = %s", rr.Code, rr.Body.String()) + } + var response playback.DecisionResponseV3 + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + return response + } + first := call() + second := call() + if first.PlaybackPlan == nil || second.PlaybackPlan == nil || first.PlaybackPlan.PlanID != second.PlaybackPlan.PlanID { + t.Fatalf("first=%#v second=%#v", first, second) + } + session, err := manager.GetSession(started.SessionID) + if err != nil { + t.Fatal(err) + } + if session.AudioTrackIndex != audioIndex { + t.Fatalf("audio index = %d, want %d", session.AudioTrackIndex, audioIndex) + } + record, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + if !record.NormalizedRequest.Metered || record.NormalizedRequest.BandwidthEstimateKbps == nil || *record.NormalizedRequest.BandwidthEstimateKbps != bandwidthEstimate || + record.NormalizedRequest.BandwidthCapKbps == nil || *record.NormalizedRequest.BandwidthCapKbps != bandwidthCap { + t.Fatalf("stored replan network evidence = %#v", record.NormalizedRequest) + } + replan.PositionSeconds++ + conflictBody, err := json.Marshal(replan) + if err != nil { + t.Fatal(err) + } + conflictReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(conflictBody))).WithContext(newAuthorizedPlaybackContext()) + conflictReq = withPlaybackRouteParam(conflictReq, "session_id", started.SessionID) + conflictRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(conflictRR, conflictReq) + if conflictRR.Code != http.StatusConflict || !strings.Contains(conflictRR.Body.String(), "idempotency_key_reused") { + t.Fatalf("conflict status = %d, body = %s", conflictRR.Code, conflictRR.Body.String()) + } +} + +func TestHandleReplanPlaybackV3RollsBackLiveSessionWhenPersistenceFails(t *testing.T) { + file := v3HandlerFixtureFile(t) + file.AudioTracks = append(file.AudioTracks, models.AudioTrack{Codec: "aac", Channels: 2, Layout: "stereo", Language: "spa"}) + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "true", "allow_4k_transcode": "true"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + startRequest := v3HandlerStartRequest() + startReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, startRequest))).WithContext(newAuthorizedPlaybackContext()) + startRR := httptest.NewRecorder() + handler.HandleStartPlayback(startRR, startReq) + if startRR.Code != http.StatusCreated { + t.Fatalf("start status = %d, body = %s", startRR.Code, startRR.Body.String()) + } + var started playback.DecisionResponseV3 + if err := json.Unmarshal(startRR.Body.Bytes(), &started); err != nil || started.PlaybackPlan == nil { + t.Fatalf("start response: err=%v response=%#v", err, started) + } + beforeSession, err := manager.GetSession(started.SessionID) + if err != nil { + t.Fatal(err) + } + beforeRecord, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + underlyingStore := handler.PlanStoreV3 + handler.PlanStoreV3 = &failingCompletePlanStoreV3{PlanStoreV3: underlyingStore} + + audioIndex := 1 + failedKey := playback.PlanAttemptKeyV3(*started.PlaybackPlan, startRequest.OutputRouteGeneration, nil) + replan := playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + PlaybackAttemptID: startRequest.PlaybackAttemptID, + ReplanRequestID: "replan-rollback-0001", + FailedPlanID: started.PlaybackPlan.PlanID, + PlanAttemptID: "plan-attempt-rollback-0001", + PlanAttemptKey: failedKey, + AttemptedPlanKeys: []string{failedKey}, + AttemptCount: 1, + QualityPreference: "original", + PositionSeconds: 12, + OutputRouteGeneration: startRequest.OutputRouteGeneration, + SelectedTracks: playback.SelectedTracksV3{Audio: &playback.TrackIdentityV3{ + ID: playback.TrackIDV3(file.ID, "audio", audioIndex), Index: &audioIndex, + }}, + Failure: playback.FailureV3{Classification: "audio_renderer_error"}, + Capabilities: startRequest.Capabilities, + ClientPlaybackContext: startRequest.ClientPlaybackContext, + } + body, err := json.Marshal(replan) + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(body))).WithContext(newAuthorizedPlaybackContext()) + req = withPlaybackRouteParam(req, "session_id", started.SessionID) + rr := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(rr, req) + if rr.Code != http.StatusInternalServerError { + t.Fatalf("replan status = %d, body = %s", rr.Code, rr.Body.String()) + } + + afterSession, err := manager.GetSession(started.SessionID) + if err != nil { + t.Fatal(err) + } + if afterSession.MediaFileID != beforeSession.MediaFileID || afterSession.AudioTrackIndex != beforeSession.AudioTrackIndex || + afterSession.PlayMethod != beforeSession.PlayMethod || afterSession.TranscodeNodeURL != beforeSession.TranscodeNodeURL { + t.Fatalf("live session was not rolled back: before=%#v after=%#v", beforeSession, afterSession) + } + afterRecord, err := underlyingStore.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + if afterRecord.CurrentPlanID != beforeRecord.CurrentPlanID || afterRecord.CurrentReplanRequestID != beforeRecord.CurrentReplanRequestID { + t.Fatalf("durable attempt changed after failed commit: before=%#v after=%#v", beforeRecord, afterRecord) + } +} + +func TestHandleReplanPlaybackV3SeekReanchorKeepsCurrentRecipeEligible(t *testing.T) { + file := v3HandlerFixtureFile(t) + file.SubtitleTracks = []models.SubtitleTrack{{Index: 0, Codec: "ass", Language: "eng"}} + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "true", "allow_4k_transcode": "false"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + startRequest := v3HandlerStartRequest() + subtitleIndex := 0 + startRequest.SubtitleTrackID = playback.TrackIDV3(file.ID, "subtitle", subtitleIndex) + startRequest.SubtitleTrackIndex = &subtitleIndex + startReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, startRequest))).WithContext(newAuthorizedPlaybackContext()) + startRR := httptest.NewRecorder() + handler.HandleStartPlayback(startRR, startReq) + if startRR.Code != http.StatusCreated { + t.Fatalf("start status = %d, body = %s", startRR.Code, startRR.Body.String()) + } + var started playback.DecisionResponseV3 + if err := json.Unmarshal(startRR.Body.Bytes(), &started); err != nil { + t.Fatal(err) + } + if started.PlaybackPlan == nil { + t.Fatal("start returned no plan") + } + if started.PlaybackPlan.Subtitle.Artifact == nil || started.PlaybackPlan.Subtitle.Artifact.Format != "ass" { + t.Fatalf("start returned no ASS artifact: %#v", started.PlaybackPlan.Subtitle) + } + if err := manager.UpdateProgress(started.SessionID, 12, true); err != nil { + t.Fatal(err) + } + currentKey := playback.PlanAttemptKeyV3(*started.PlaybackPlan, startRequest.OutputRouteGeneration, nil) + reanchor := playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, Operation: playback.ReplanOperationSeekReanchorV3, + PlaybackAttemptID: startRequest.PlaybackAttemptID, + ReplanRequestID: "seek-reanchor-0001", FailedPlanID: started.PlaybackPlan.PlanID, + PlanAttemptID: "plan-attempt-seek-0001", PlanAttemptKey: currentKey, + // Clients may include the current key defensively. A seek reanchor must + // ignore it because the recipe did not fail. + AttemptedPlanKeys: []string{currentKey}, AttemptCount: 1, + QualityPreference: "original", PositionSeconds: 321, + OutputRouteGeneration: startRequest.OutputRouteGeneration, + SelectedTracks: started.PlaybackPlan.SelectedTracks, + Failure: playback.FailureV3{}, + Capabilities: startRequest.Capabilities, ClientPlaybackContext: startRequest.ClientPlaybackContext, + } + body, err := json.Marshal(reanchor) + if err != nil { + t.Fatal(err) + } + call := func() playback.DecisionResponseV3 { + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(body))).WithContext(newAuthorizedPlaybackContext()) + req = withPlaybackRouteParam(req, "session_id", started.SessionID) + rr := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("reanchor status = %d, body = %s", rr.Code, rr.Body.String()) + } + var response playback.DecisionResponseV3 + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + return response + } + first := call() + second := call() + for index, response := range []playback.DecisionResponseV3{first, second} { + if response.PlaybackPlan == nil || response.SessionID != started.SessionID || + response.PlaybackPlan.SessionID != started.SessionID || + response.PlaybackPlan.PlanID != started.PlaybackPlan.PlanID || + response.PlaybackPlan.RequestedMediaFileID != started.PlaybackPlan.RequestedMediaFileID || + response.PlaybackPlan.EffectiveMediaFileID != started.PlaybackPlan.EffectiveMediaFileID || + !sameSelectedTracksV3(response.PlaybackPlan.SelectedTracks, started.PlaybackPlan.SelectedTracks) || + response.PlaybackPlan.Timeline.SourceStartSeconds != 321 || + playback.PlanAttemptKeyV3(*response.PlaybackPlan, startRequest.OutputRouteGeneration, nil) != currentKey { + t.Fatalf("reanchored response %d = %#v", index, response.PlaybackPlan) + } + if response.PlaybackPlan.Subtitle.Artifact == nil || + response.PlaybackPlan.Subtitle.Artifact.Format != started.PlaybackPlan.Subtitle.Artifact.Format || + response.PlaybackPlan.Subtitle.Artifact.MIMEType != started.PlaybackPlan.Subtitle.Artifact.MIMEType { + t.Fatalf("reanchored response %d changed the ASS artifact: %#v", index, response.PlaybackPlan.Subtitle) + } + if !playback.HasFeatureV3(response.ServerFeatures, playback.FeatureSeekReanchorV3) { + t.Fatalf("reanchored response %d omitted %q: %#v", index, playback.FeatureSeekReanchorV3, response.ServerFeatures) + } + } + liveSessionManager := handler.sessionMgr + handler.sessionMgr = playback.NewSessionManager(0, 0) + restartReplayReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(body))).WithContext(newAuthorizedPlaybackContext()) + restartReplayReq = withPlaybackRouteParam(restartReplayReq, "session_id", started.SessionID) + restartReplayRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(restartReplayRR, restartReplayReq) + handler.sessionMgr = liveSessionManager + if restartReplayRR.Code != http.StatusNotFound || !strings.Contains(restartReplayRR.Body.String(), playbackSessionNotFoundErrorCode) { + t.Fatalf("restart replay status = %d, body = %s", restartReplayRR.Code, restartReplayRR.Body.String()) + } + session, err := manager.GetSession(started.SessionID) + if err != nil { + t.Fatal(err) + } + if session.Position != 321 || !session.IsPaused { + t.Fatalf("session progress = (%v, paused=%v), want (321, paused=true)", session.Position, session.IsPaused) + } + record, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + if record.CurrentPlanID != started.PlaybackPlan.PlanID || record.EffectiveMediaFileID != file.ID || + record.NormalizedRequest.StartPosition == nil || *record.NormalizedRequest.StartPosition != 321 { + t.Fatalf("stored reanchor = %#v", record) + } + + mismatch := reanchor + mismatch.ReplanRequestID = "seek-reanchor-mismatch-0001" + mismatch.QualityPreference = "480p" + mismatchBody, err := json.Marshal(mismatch) + if err != nil { + t.Fatal(err) + } + mismatchReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(mismatchBody))).WithContext(newAuthorizedPlaybackContext()) + mismatchReq = withPlaybackRouteParam(mismatchReq, "session_id", started.SessionID) + mismatchRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(mismatchRR, mismatchReq) + if mismatchRR.Code != http.StatusOK { + t.Fatalf("mismatch status = %d, body = %s", mismatchRR.Code, mismatchRR.Body.String()) + } + var mismatchResponse playback.DecisionResponseV3 + if err := json.Unmarshal(mismatchRR.Body.Bytes(), &mismatchResponse); err != nil { + t.Fatal(err) + } + if mismatchResponse.Terminal == nil || mismatchResponse.Terminal.Reason != "seek_reanchor_intent_mismatch" { + t.Fatalf("mismatch response = %#v", mismatchResponse) + } + recordAfterMismatch, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + if recordAfterMismatch.CurrentPlanID != record.CurrentPlanID || recordAfterMismatch.NormalizedRequest.StartPosition == nil || *recordAfterMismatch.NormalizedRequest.StartPosition != 321 { + t.Fatalf("intent mismatch changed stored route: before=%#v after=%#v", record, recordAfterMismatch) + } + + newer := reanchor + newer.ReplanRequestID = "seek-reanchor-0002" + newer.PositionSeconds = 500 + newerBody, err := json.Marshal(newer) + if err != nil { + t.Fatal(err) + } + newerReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(newerBody))).WithContext(newAuthorizedPlaybackContext()) + newerReq = withPlaybackRouteParam(newerReq, "session_id", started.SessionID) + newerRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(newerRR, newerReq) + if newerRR.Code != http.StatusOK { + t.Fatalf("newer reanchor status = %d, body = %s", newerRR.Code, newerRR.Body.String()) + } + + staleRetryReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(body))).WithContext(newAuthorizedPlaybackContext()) + staleRetryReq = withPlaybackRouteParam(staleRetryReq, "session_id", started.SessionID) + staleRetryRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(staleRetryRR, staleRetryReq) + if staleRetryRR.Code != http.StatusConflict || !strings.Contains(staleRetryRR.Body.String(), "stale_playback_plan") { + t.Fatalf("stale reanchor replay status = %d, body = %s", staleRetryRR.Code, staleRetryRR.Body.String()) + } + latestRecord, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + if latestRecord.CurrentReplanRequestID != newer.ReplanRequestID || latestRecord.NormalizedRequest.StartPosition == nil || *latestRecord.NormalizedRequest.StartPosition != newer.PositionSeconds { + t.Fatalf("stale replay changed latest reanchor: %#v", latestRecord) + } + // Simulate a rolling-deploy writer which updated the current plan but did + // not know about CurrentReplanRequestID. The durable plan comparison must + // still reject A's cached response after B became active. + mixedWriterRecord := *latestRecord + mixedWriterRecord.CurrentReplanRequestID = reanchor.ReplanRequestID + handler.PlanStoreV3.(*playback.MemoryPlanStoreV3).ReplaceAttempt(context.Background(), mixedWriterRecord) + mixedWriterRetryReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(body))).WithContext(newAuthorizedPlaybackContext()) + mixedWriterRetryReq = withPlaybackRouteParam(mixedWriterRetryReq, "session_id", started.SessionID) + mixedWriterRetryRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(mixedWriterRetryRR, mixedWriterRetryReq) + if mixedWriterRetryRR.Code != http.StatusConflict || !strings.Contains(mixedWriterRetryRR.Body.String(), "stale_playback_plan") { + t.Fatalf("mixed-writer stale replay status = %d, body = %s", mixedWriterRetryRR.Code, mixedWriterRetryRR.Body.String()) + } + + beyondEnd := newer + beyondEnd.ReplanRequestID = "seek-reanchor-beyond-end-0001" + beyondEnd.PositionSeconds = float64(file.Duration) + 1 + beyondBody, err := json.Marshal(beyondEnd) + if err != nil { + t.Fatal(err) + } + beyondReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(beyondBody))).WithContext(newAuthorizedPlaybackContext()) + beyondReq = withPlaybackRouteParam(beyondReq, "session_id", started.SessionID) + beyondRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(beyondRR, beyondReq) + if beyondRR.Code != http.StatusOK || !strings.Contains(beyondRR.Body.String(), "invalid_seek_position") { + t.Fatalf("beyond-end reanchor status = %d, body = %s", beyondRR.Code, beyondRR.Body.String()) + } + + missingSince := time.Now() + file.MissingSince = &missingSince + missing := newer + missing.ReplanRequestID = "seek-reanchor-missing-source-0001" + missing.PositionSeconds = 600 + missingBody, err := json.Marshal(missing) + if err != nil { + t.Fatal(err) + } + missingReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(missingBody))).WithContext(newAuthorizedPlaybackContext()) + missingReq = withPlaybackRouteParam(missingReq, "session_id", started.SessionID) + missingRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(missingRR, missingReq) + if missingRR.Code != http.StatusOK || !strings.Contains(missingRR.Body.String(), "source_unavailable") { + t.Fatalf("missing-source reanchor status = %d, body = %s", missingRR.Code, missingRR.Body.String()) + } + unchanged, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + if unchanged.CurrentPlanID != latestRecord.CurrentPlanID || unchanged.EffectiveMediaFileID != latestRecord.EffectiveMediaFileID || + unchanged.NormalizedRequest.StartPosition == nil || *unchanged.NormalizedRequest.StartPosition != newer.PositionSeconds { + t.Fatalf("rejected seeks changed the active route: before=%#v after=%#v", latestRecord, unchanged) + } +} + +func TestHandleReplanPlaybackV3SeekFailureRecoveryNeverChangesMediaVersion(t *testing.T) { + source := v3HandlerFixtureFile(t) + source.Resolution = "2160p" + source.Bitrate = 32_000 + source.VideoTracks = append([]models.VideoTrack(nil), source.VideoTracks...) + source.VideoTracks[0].Level = 51 + source.VideoTracks[0].Width = 3840 + source.VideoTracks[0].Height = 2160 + source.VideoTracks[0].Bitrate = 32_000 + + alternateValue := *source + alternate := &alternateValue + alternate.ID = 84 + alternate.Resolution = "1080p" + alternate.Bitrate = 8_000 + alternate.VideoTracks = append([]models.VideoTrack(nil), source.VideoTracks...) + alternate.VideoTracks[0].Level = 41 + alternate.VideoTracks[0].Width = 1920 + alternate.VideoTracks[0].Height = 1080 + alternate.VideoTracks[0].Bitrate = 8_000 + + manager := playback.NewSessionManager(0, 0) + files := map[int]*models.MediaFile{ + source.ID: source, alternate.ID: alternate, + } + handler := NewPlaybackHandler(manager, mapPlaybackFileResolver{files: files}) + handler.FileVersionFetcher = testPlaybackFileVersionFetcher{byContent: map[string][]*models.MediaFile{ + source.ContentID: {source, alternate}, + }} + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "true", "allow_4k_transcode": "false"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + + startRequest := v3HandlerStartRequest() + startRequest.QualityPreference = "auto" + startRequest.Capabilities.MaxResolution = "2160p" + startRequest.Capabilities.VideoDecode[0].Levels = []int{51} + startRequest.Capabilities.VideoDecode[0].MaxWidth = 3840 + startRequest.Capabilities.VideoDecode[0].MaxHeight = 2160 + startRequest.Capabilities.VideoDecode[0].MaxBitrateKbps = 50_000 + startRequest.ClientPlaybackContext.Engines[string(playback.EngineMedia3HLSV3)] = playback.EngineCapabilityV3{Enabled: true, SupportedOnDevice: true} + + startReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, startRequest))).WithContext(newAuthorizedPlaybackContext()) + startRR := httptest.NewRecorder() + handler.HandleStartPlayback(startRR, startReq) + if startRR.Code != http.StatusCreated { + t.Fatalf("start status = %d, body = %s", startRR.Code, startRR.Body.String()) + } + var started playback.DecisionResponseV3 + if err := json.Unmarshal(startRR.Body.Bytes(), &started); err != nil { + t.Fatal(err) + } + if started.PlaybackPlan == nil || started.PlaybackPlan.EffectiveMediaFileID != source.ID { + t.Fatalf("initial 4K plan = %#v", started.PlaybackPlan) + } + if err := manager.UpdateProgress(started.SessionID, 15, true); err != nil { + t.Fatal(err) + } + + seekCapabilities := startRequest.Capabilities + seekCapabilities.VideoDecode = append([]playback.VideoDecodeCapabilityV3(nil), startRequest.Capabilities.VideoDecode...) + seekCapabilities.MaxResolution = "1080p" + seekCapabilities.VideoDecode[0].MaxWidth = 1920 + seekCapabilities.VideoDecode[0].MaxHeight = 1080 + seekCapabilities.VideoDecode[0].MaxBitrateKbps = 20_000 + seekContext := startRequest.ClientPlaybackContext + seekContext.Device.Model = "request-only-model" + currentKey := playback.PlanAttemptKeyV3(*started.PlaybackPlan, startRequest.OutputRouteGeneration, nil) + staleClientKey := "v3:0000000000000000" + replan := playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationSeekFailureRecoveryV3, + PlaybackAttemptID: startRequest.PlaybackAttemptID, + ReplanRequestID: "seek-failure-recovery-0001", + FailedPlanID: started.PlaybackPlan.PlanID, + PlanAttemptID: "plan-attempt-seek-failure-0001", + PlanAttemptKey: staleClientKey, + AttemptedPlanKeys: nil, + AttemptCount: 1, + QualityPreference: startRequest.QualityPreference, + PositionSeconds: 417, + OutputRouteGeneration: startRequest.OutputRouteGeneration, + SelectedTracks: started.PlaybackPlan.SelectedTracks, + Failure: playback.FailureV3{Classification: "decoder_failure", Message: "reanchored route failed"}, + Capabilities: seekCapabilities, + ClientPlaybackContext: seekContext, + } + body, err := json.Marshal(replan) + if err != nil { + t.Fatal(err) + } + replanReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(body))).WithContext(newAuthorizedPlaybackContext()) + replanReq = withPlaybackRouteParam(replanReq, "session_id", started.SessionID) + replanRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(replanRR, replanReq) + if replanRR.Code != http.StatusOK { + t.Fatalf("replan status = %d, body = %s", replanRR.Code, replanRR.Body.String()) + } + var response playback.DecisionResponseV3 + if err := json.Unmarshal(replanRR.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.PlaybackPlan == nil || response.Terminal != nil || + response.PlaybackPlan.RequestedMediaFileID != source.ID || response.PlaybackPlan.EffectiveMediaFileID != source.ID { + t.Fatalf("failed seek did not recover on the pinned media version: %#v", response) + } + if response.PlaybackPlan.Delivery == playback.DeliveryTranscodeHLSV3 || + response.PlaybackPlan.EffectiveRecipe.Width == nil || *response.PlaybackPlan.EffectiveRecipe.Width != 3840 || + response.PlaybackPlan.EffectiveRecipe.Height == nil || *response.PlaybackPlan.EffectiveRecipe.Height != 2160 { + t.Fatalf("failed seek downgraded or video-transcoded the pinned 4K source: %#v", response.PlaybackPlan) + } + if playback.PlanAttemptKeyV3(*response.PlaybackPlan, startRequest.OutputRouteGeneration, nil) == currentKey { + t.Fatalf("failed seek retried the failed route: %#v", response.PlaybackPlan) + } + record, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + if record.EffectiveMediaFileID != source.ID || record.CurrentPlan.EffectiveMediaFileID != source.ID { + t.Fatalf("failed seek changed the durable media version: %#v", record) + } + if record.NormalizedRequest.Capabilities.MaxResolution != startRequest.Capabilities.MaxResolution || + record.NormalizedRequest.ClientPlaybackContext.Device.Model != startRequest.ClientPlaybackContext.Device.Model { + t.Fatalf("failed seek accepted request-only capability evidence: %#v", record.NormalizedRequest) + } + session, err := manager.GetSession(started.SessionID) + if err != nil { + t.Fatal(err) + } + if session.Position != replan.PositionSeconds || !session.IsPaused { + t.Fatalf("session progress = (%v, paused=%v), want (%v, paused=true)", session.Position, session.IsPaused, replan.PositionSeconds) + } + + delete(files, source.ID) + for index, operation := range []playback.ReplanOperationV3{ + playback.ReplanOperationSeekReanchorV3, + playback.ReplanOperationSeekFailureRecoveryV3, + } { + missing := replan + missing.Operation = operation + missing.ReplanRequestID = fmt.Sprintf("seek-missing-current-%04d", index) + missing.FailedPlanID = response.PlaybackPlan.PlanID + missing.PlanAttemptKey = playback.PlanAttemptKeyV3(*response.PlaybackPlan, startRequest.OutputRouteGeneration, nil) + missing.AttemptedPlanKeys = []string{missing.PlanAttemptKey} + missing.SelectedTracks = response.PlaybackPlan.SelectedTracks + missing.PositionSeconds++ + if operation == playback.ReplanOperationSeekReanchorV3 { + missing.Failure = playback.FailureV3{} + } + missingBody, err := json.Marshal(missing) + if err != nil { + t.Fatal(err) + } + missingReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(missingBody))).WithContext(newAuthorizedPlaybackContext()) + missingReq = withPlaybackRouteParam(missingReq, "session_id", started.SessionID) + missingRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(missingRR, missingReq) + if missingRR.Code != http.StatusOK || !strings.Contains(missingRR.Body.String(), "source_unavailable") { + t.Fatalf("missing current %s status = %d, body = %s", operation, missingRR.Code, missingRR.Body.String()) + } + } + finalRecord, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + finalSession, err := manager.GetSession(started.SessionID) + if err != nil { + t.Fatal(err) + } + if finalRecord.EffectiveMediaFileID != source.ID || finalRecord.CurrentPlan.EffectiveMediaFileID != source.ID || finalSession.MediaFileID != source.ID { + t.Fatalf("missing 4K source fell through to alternate: record=%#v session=%#v", finalRecord, finalSession) + } +} + +func TestHandleReplanPlaybackV3SeekUsesEffectiveEditionWhenRequestedEditionIsGone(t *testing.T) { + effective := v3HandlerFixtureFile(t) + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, mapPlaybackFileResolver{files: map[int]*models.MediaFile{effective.ID: effective}}) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "true"}} + handler.ItemAccess = allowAllPlaybackItemAccess{} + + startRequest := v3HandlerStartRequest() + startReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(marshalV3StartRequest(t, startRequest))).WithContext(newAuthorizedPlaybackContext()) + startRR := httptest.NewRecorder() + handler.HandleStartPlayback(startRR, startReq) + if startRR.Code != http.StatusCreated { + t.Fatalf("start status = %d, body = %s", startRR.Code, startRR.Body.String()) + } + var started playback.DecisionResponseV3 + if err := json.Unmarshal(startRR.Body.Bytes(), &started); err != nil || started.PlaybackPlan == nil { + t.Fatalf("start response: err=%v response=%#v", err, started) + } + + const missingRequestedID = 84 + record, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + record.RequestedMediaFileID = missingRequestedID + record.NormalizedRequest.FileID = missingRequestedID + record.CurrentPlan.RequestedMediaFileID = missingRequestedID + record.CurrentPlan.PlanID = playback.DeterministicPlanIDV3( + record.PlaybackAttemptID, + missingRequestedID, + effective.ID, + record.CurrentPlan, + ) + record.CurrentPlanID = record.CurrentPlan.PlanID + handler.PlanStoreV3.(*playback.MemoryPlanStoreV3).ReplaceAttempt(context.Background(), *record) + + currentKey := playback.PlanAttemptKeyV3(record.CurrentPlan, record.NormalizedRequest.OutputRouteGeneration, nil) + reanchor := playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + Operation: playback.ReplanOperationSeekReanchorV3, + PlaybackAttemptID: record.PlaybackAttemptID, + ReplanRequestID: "seek-missing-requested-0001", + FailedPlanID: record.CurrentPlanID, + PlanAttemptID: "plan-attempt-missing-requested-0001", + PlanAttemptKey: currentKey, + AttemptedPlanKeys: []string{currentKey}, + AttemptCount: 1, + QualityPreference: record.NormalizedRequest.QualityPreference, + PositionSeconds: 300, + OutputRouteGeneration: record.NormalizedRequest.OutputRouteGeneration, + SelectedTracks: record.CurrentPlan.SelectedTracks, + Capabilities: record.NormalizedRequest.Capabilities, + ClientPlaybackContext: record.NormalizedRequest.ClientPlaybackContext, + } + body, err := json.Marshal(reanchor) + if err != nil { + t.Fatal(err) + } + reanchorReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(body))).WithContext(newAuthorizedPlaybackContext()) + reanchorReq = withPlaybackRouteParam(reanchorReq, "session_id", started.SessionID) + reanchorRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(reanchorRR, reanchorReq) + if reanchorRR.Code != http.StatusOK { + t.Fatalf("reanchor status = %d, body = %s", reanchorRR.Code, reanchorRR.Body.String()) + } + var response playback.DecisionResponseV3 + if err := json.Unmarshal(reanchorRR.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.PlaybackPlan == nil || response.PlaybackPlan.RequestedMediaFileID != missingRequestedID || + response.PlaybackPlan.EffectiveMediaFileID != effective.ID || response.PlaybackPlan.Timeline.SourceStartSeconds != reanchor.PositionSeconds { + t.Fatalf("reanchor did not retain the effective edition: %#v", response) + } + + current, err := handler.PlanStoreV3.GetAttempt(context.Background(), started.SessionID) + if err != nil { + t.Fatal(err) + } + outputContext := current.NormalizedRequest.ClientPlaybackContext + outputContext.Output.OutputRouteGeneration++ + currentKey = playback.PlanAttemptKeyV3(current.CurrentPlan, current.NormalizedRequest.OutputRouteGeneration, nil) + ordinary := playback.ReplanRequestV3{ + ProtocolVersion: playback.ProtocolV3, + PlaybackAttemptID: current.PlaybackAttemptID, + ReplanRequestID: "ordinary-missing-requested-0001", + FailedPlanID: current.CurrentPlanID, + PlanAttemptID: "plan-attempt-ordinary-missing-0001", + PlanAttemptKey: currentKey, + AttemptedPlanKeys: []string{currentKey}, + AttemptCount: 1, + QualityPreference: current.NormalizedRequest.QualityPreference, + PositionSeconds: 320, + OutputRouteGeneration: outputContext.Output.OutputRouteGeneration, + SelectedTracks: current.CurrentPlan.SelectedTracks, + Failure: playback.FailureV3{Classification: "output_route_changed"}, + Capabilities: current.NormalizedRequest.Capabilities, + ClientPlaybackContext: outputContext, + } + ordinaryBody, err := json.Marshal(ordinary) + if err != nil { + t.Fatal(err) + } + ordinaryReq := httptest.NewRequest(http.MethodPost, "/api/v1/playback/"+started.SessionID+"/replan", strings.NewReader(string(ordinaryBody))).WithContext(newAuthorizedPlaybackContext()) + ordinaryReq = withPlaybackRouteParam(ordinaryReq, "session_id", started.SessionID) + ordinaryRR := httptest.NewRecorder() + handler.HandleReplanPlaybackV3(ordinaryRR, ordinaryReq) + if ordinaryRR.Code != http.StatusOK { + t.Fatalf("ordinary replan status = %d, body = %s", ordinaryRR.Code, ordinaryRR.Body.String()) + } + var ordinaryResponse playback.DecisionResponseV3 + if err := json.Unmarshal(ordinaryRR.Body.Bytes(), &ordinaryResponse); err != nil { + t.Fatal(err) + } + if ordinaryResponse.PlaybackPlan == nil || ordinaryResponse.PlaybackPlan.RequestedMediaFileID != missingRequestedID || + ordinaryResponse.PlaybackPlan.EffectiveMediaFileID != effective.ID { + t.Fatalf("ordinary replan lost the requested/effective split: %#v", ordinaryResponse) + } +} + +func TestValidateSeekRecoveryRequestV3PinsCurrentIntent(t *testing.T) { + audioIndex := 0 + start := v3HandlerStartRequest() + start.AudioTrackID = playback.TrackIDV3(start.FileID, "audio", audioIndex) + start.AudioTrackIndex = &audioIndex + selected := playback.SelectedTracksV3{Audio: &playback.TrackIdentityV3{ID: start.AudioTrackID, Index: &audioIndex}} + plan := playback.PlanV3{ + PlanID: "plan:seek-current", + RequestedMediaFileID: start.FileID, + EffectiveMediaFileID: start.FileID, + SelectedTracks: selected, + } + record := &playback.AttemptRecordV3{ + RequestedMediaFileID: start.FileID, + EffectiveMediaFileID: start.FileID, + CurrentPlanID: plan.PlanID, + CurrentPlan: plan, + NormalizedRequest: start, + } + request := playback.ReplanRequestV3{ + Operation: playback.ReplanOperationSeekReanchorV3, + QualityPreference: start.QualityPreference, + OutputRouteGeneration: start.OutputRouteGeneration, + SelectedTracks: selected, + } + if err := validateSeekRecoveryRequestV3(record, request); err != nil { + t.Fatalf("valid seek reanchor guard: %v", err) + } + + // Capabilities and client context are structurally validated by the request + // decoder but are not route inputs for a same-recipe reanchor. + request.Capabilities.CodecsVideo = []string{"av1"} + request.ClientPlaybackContext.Device.Model = "changed-client-claim" + if err := validateSeekRecoveryRequestV3(record, request); err != nil { + t.Fatalf("ignored client evidence changed stored intent: %v", err) + } + + tests := []struct { + name string + mutate func(*playback.ReplanRequestV3) + }{ + {name: "quality", mutate: func(value *playback.ReplanRequestV3) { value.QualityPreference = "480p" }}, + {name: "output route", mutate: func(value *playback.ReplanRequestV3) { value.OutputRouteGeneration++ }}, + {name: "audio track", mutate: func(value *playback.ReplanRequestV3) { + index := 1 + value.SelectedTracks.Audio = &playback.TrackIdentityV3{ID: playback.TrackIDV3(start.FileID, "audio", index), Index: &index} + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + candidate := request + test.mutate(&candidate) + if err := validateSeekRecoveryRequestV3(record, candidate); err == nil { + t.Fatalf("%s change was accepted", test.name) + } + }) + } +} + +func TestValidateSeekReanchorPlanV3RejectsRouteDrift(t *testing.T) { + audioIndex := 0 + frameRate := 23.976 + audioChannels := 2 + current := playback.PlanV3{ + PlanID: "plan:seek-current", + Delivery: playback.DeliveryOriginalHTTPV3, + Engine: playback.EngineMedia3DirectV3, + Stream: playback.StreamV3{Protocol: playback.StreamHTTPProgressiveV3, Container: "mp4", MIMEType: "video/mp4", HeaderRefresh: playback.HeaderRefreshNoneV3}, + SelectedTracks: playback.SelectedTracksV3{Audio: &playback.TrackIdentityV3{ID: playback.TrackIDV3(42, "audio", audioIndex), Index: &audioIndex}}, + EffectiveRecipe: playback.EffectiveRecipeV3{VideoCodec: "h264", AudioCodec: "aac", FrameRate: &frameRate, AudioChannels: &audioChannels, AudioLayout: "stereo"}, + Claims: playback.ValidationClaimsV3{Video: playback.VideoClaimsV3{HDR10: true}}, + Subtitle: playback.SubtitleDecisionV3{Mode: playback.SubtitleRenderV3, TrackID: playback.TrackIDV3(42, "subtitle", 0), Artifact: &playback.SubtitleArtifactV3{ + MIMEType: "text/x-ssa", Format: "ass", + }}, + Transformations: []playback.TransformationV3{{Name: "subtitle-convert", Executor: "ffmpeg", RecipeVersion: "1", ValidatedClaims: []string{"ass"}}}, + AppliedQuirks: []playback.AppliedQuirkV3{{ID: "quirk-1", RegistryRevision: "1", Action: "force-remux"}}, + RuntimeCorrections: []string{"pcm_fallback"}, + RequestedMediaFileID: 42, + EffectiveMediaFileID: 42, + SubtitleFidelityPolicy: "preserve_styling", + } + record := &playback.AttemptRecordV3{ + RequestedMediaFileID: 42, + EffectiveMediaFileID: 42, + CurrentPlanID: current.PlanID, + CurrentPlan: current, + NormalizedRequest: playback.StartRequestV3{OutputRouteGeneration: 9}, + } + candidate := current + candidate.Timeline.SourceStartSeconds = 321 + if err := validateSeekReanchorPlanV3(record, &candidate); err != nil { + t.Fatalf("timeline-only change rejected: %v", err) + } + + tests := []struct { + name string + mutate func(*playback.PlanV3) + }{ + {name: "delivery recipe", mutate: func(value *playback.PlanV3) { value.Delivery = playback.DeliveryRemuxProgressiveV3 }}, + {name: "engine", mutate: func(value *playback.PlanV3) { value.Engine = playback.EngineMedia3HLSV3 }}, + {name: "stream MIME", mutate: func(value *playback.PlanV3) { value.Stream.MIMEType = "application/x-mpegURL" }}, + {name: "header refresh", mutate: func(value *playback.PlanV3) { value.Stream.HeaderRefresh = playback.HeaderRefreshSessionV3 }}, + {name: "frame rate", mutate: func(value *playback.PlanV3) { + changed := 24.0 + value.EffectiveRecipe.FrameRate = &changed + }}, + {name: "audio channels", mutate: func(value *playback.PlanV3) { + changed := 6 + value.EffectiveRecipe.AudioChannels = &changed + }}, + {name: "audio layout", mutate: func(value *playback.PlanV3) { value.EffectiveRecipe.AudioLayout = "5.1" }}, + {name: "claims", mutate: func(value *playback.PlanV3) { value.Claims.Video.HDR10 = false }}, + {name: "subtitle artifact MIME", mutate: func(value *playback.PlanV3) { + copy := *value.Subtitle.Artifact + copy.MIMEType = "text/vtt" + value.Subtitle.Artifact = © + }}, + {name: "subtitle artifact format", mutate: func(value *playback.PlanV3) { + copy := *value.Subtitle.Artifact + copy.Format = "vtt" + value.Subtitle.Artifact = © + }}, + {name: "subtitle fidelity", mutate: func(value *playback.PlanV3) { value.SubtitleFidelityPolicy = "compatibility" }}, + {name: "transformation claims", mutate: func(value *playback.PlanV3) { + value.Transformations = append([]playback.TransformationV3(nil), value.Transformations...) + value.Transformations[0].ValidatedClaims = []string{"vtt"} + }}, + {name: "quirk action", mutate: func(value *playback.PlanV3) { + value.AppliedQuirks = append([]playback.AppliedQuirkV3(nil), value.AppliedQuirks...) + value.AppliedQuirks[0].Action = "disable-passthrough" + }}, + {name: "effective version", mutate: func(value *playback.PlanV3) { value.EffectiveMediaFileID = 84 }}, + {name: "track", mutate: func(value *playback.PlanV3) { + index := 1 + value.SelectedTracks.Audio = &playback.TrackIdentityV3{ID: playback.TrackIDV3(42, "audio", index), Index: &index} + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + drifted := candidate + test.mutate(&drifted) + if err := validateSeekReanchorPlanV3(record, &drifted); err == nil { + t.Fatalf("%s drift was accepted", test.name) + } + }) + } +} + +func TestPrepareIdentityTransportV3ProgressiveRemuxNeverAdvertisesNativeSeek(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + session := &playback.Session{ID: "session-progressive", UserID: 7, ProfileID: "profile-1", MediaFileID: 42, PlayMethod: playback.PlayRemux, BasePlayMethod: playback.PlayRemux, AudioTrackIndex: 0} + for index, origin := range []float64{321.25, 654.5} { + plan := &playback.PlanV3{ + PlanID: "plan:progressive", + Delivery: playback.DeliveryRemuxProgressiveV3, + EffectiveMediaFileID: 42, + Timeline: playback.TimelineV3{SourceStartSeconds: origin, PlayerStartSeconds: origin, CanSeekAnywhere: true, SeekRestoration: "player_position"}, + } + transport := handler.prepareIdentityTransportV3(session, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayRemux}) + + parsed, err := url.Parse(transport.url) + if err != nil { + transport.rollback() + t.Fatal(err) + } + if parsed.Query().Get("st") == "" || parsed.Query().Get("seek") != strconv.FormatFloat(origin, 'f', -1, 64) { + transport.rollback() + t.Fatalf("progressive reanchor URL %d = %q", index, transport.url) + } + if plan.Timeline.PlayerStartSeconds != 0 || plan.Timeline.StreamOriginSeconds != origin || + plan.Timeline.TimelineOffsetSeconds != origin || plan.Timeline.CanSeekAnywhere || + plan.Timeline.SeekWindowStartSeconds == nil || *plan.Timeline.SeekWindowStartSeconds != origin || + plan.Timeline.SeekWindowEndSeconds != nil || plan.Timeline.SeekRestoration != "source_position" { + transport.rollback() + t.Fatalf("progressive reanchor timeline %d = %#v", index, plan.Timeline) + } + transport.rollback() + } +} + +func TestPrepareTransportV3RejectsNodeMissingRequiredTransformation(t *testing.T) { + startHits := 0 + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/hw-capabilities": + writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{{Name: "video_to_h264", Executor: "server", RecipeVersion: "2"}}}) + case "/transcode/start": + startHits++ + w.WriteHeader(http.StatusAccepted) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer remote.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.NodePlanner = staticNodePlannerV3{plan: nodepool.Plan{TranscodeNode: &nodepool.Node{URL: remote.URL}}} + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.local_transcode_fallback": "false"}} + plan := &playback.PlanV3{ + PlanID: "plan:remote-capability", + Delivery: playback.DeliveryTranscodeHLSV3, + Transformations: []playback.TransformationV3{ + {Name: "video_to_h264", Executor: "server", RecipeVersion: "1"}, + {Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}, + }, + } + request := httptest.NewRequest(http.MethodPost, "/", nil) + _, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}) + if transportErr == nil || transportErr.reason != "transcode_node_capability_unavailable" { + t.Fatalf("transport error = %#v", transportErr) + } + if startHits != 0 { + t.Fatalf("incompatible node received %d start requests", startHits) + } +} + +func TestPrepareTransportV3RequiresRemoteManifestReadiness(t *testing.T) { + var startRequest transcodenode.TranscodeStartRequest + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/hw-capabilities": + writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{ + {Name: "video_to_h264", Executor: "server", RecipeVersion: "1"}, + {Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}, + }}) + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + if err := json.NewDecoder(r.Body).Decode(&startRequest); err != nil { + t.Errorf("decode remote start: %v", err) + } + writeJSON(w, http.StatusAccepted, transcodenode.TranscodeStartResponse{SessionID: startRequest.SessionID, Status: "started"}) + case r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + defer remote.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + handler.NodePlanner = staticNodePlannerV3{plan: nodepool.Plan{TranscodeNode: &nodepool.Node{URL: remote.URL}}} + plan := &playback.PlanV3{ + PlanID: "plan:remote-ready", + Delivery: playback.DeliveryTranscodeHLSV3, + Transformations: []playback.TransformationV3{ + {Name: "video_to_h264", Executor: "server", RecipeVersion: "1"}, + {Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}, + }, + } + request := httptest.NewRequest(http.MethodPost, "/", nil) + transport, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-ready", UserID: 7, ProfileID: "profile-1"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}) + if transportErr != nil { + t.Fatalf("prepare remote transport: %v", transportErr) + } + defer transport.rollback() + if !startRequest.RequireReady { + t.Fatal("protocol-v3 remote start did not require manifest readiness") + } +} + +func TestHandleStartPlaybackUnknownProtocolUsesLegacyBranch(t *testing.T) { + file := v3HandlerFixtureFile(t) + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(`{"protocol_version":99,"file_id":42,"profile_id":"profile-1","play_method":"direct"}`)) + req = req.WithContext(newAuthorizedPlaybackContext()) + rr := httptest.NewRecorder() + handler.HandleStartPlayback(rr, req) + + if rr.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + var response playbackSessionResponse + if err := json.Unmarshal(rr.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if response.SessionID == "" || response.PlayMethod != string(playback.PlayDirect) { + t.Fatalf("legacy response = %#v", response) + } +} + +func TestHandleStartPlaybackLegacyBranchPreservesTrailingBodyBehavior(t *testing.T) { + file := v3HandlerFixtureFile(t) + manager := playback.NewSessionManager(0, 0) + handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", strings.NewReader(`{"file_id":42,"profile_id":"profile-1","play_method":"direct"} trailing`)).WithContext(newAuthorizedPlaybackContext()) + rr := httptest.NewRecorder() + handler.HandleStartPlayback(rr, req) + if rr.Code != http.StatusCreated { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } +} + +func TestConfigureHLSTimelineV3MatchesTransportSeekSemantics(t *testing.T) { + copyPlan := &playback.PlanV3{Timeline: playback.TimelineV3{SourceStartSeconds: 17.3}} + copySeek, copySegment := configureHLSTimelineV3(copyPlan, "copy", 2, 600) + if copySeek != 17.3 || copySegment != 8 || copyPlan.Timeline.StreamOriginSeconds != 17.3 || copyPlan.Timeline.TimelineOffsetSeconds != 17.3 || copyPlan.Timeline.PlayerStartSeconds != 0 || copyPlan.Timeline.CanSeekAnywhere || + copyPlan.Timeline.SeekWindowStartSeconds == nil || *copyPlan.Timeline.SeekWindowStartSeconds != 17.3 || + copyPlan.Timeline.SeekWindowEndSeconds == nil || *copyPlan.Timeline.SeekWindowEndSeconds != 600 || + copyPlan.Timeline.SeekRestoration != "source_position" { + t.Fatalf("copy timeline=%#v seek=%v segment=%d", copyPlan.Timeline, copySeek, copySegment) + } + + encodePlan := &playback.PlanV3{Timeline: playback.TimelineV3{SourceStartSeconds: 17.3}} + encodeSeek, encodeSegment := configureHLSTimelineV3(encodePlan, "h264", 2, 600) + if encodeSeek != 16 || encodeSegment != 8 || encodePlan.Timeline.StreamOriginSeconds != 0 || encodePlan.Timeline.TimelineOffsetSeconds != 0 || encodePlan.Timeline.PlayerStartSeconds != 17.3 || !encodePlan.Timeline.CanSeekAnywhere || + encodePlan.Timeline.SeekWindowStartSeconds != nil || encodePlan.Timeline.SeekWindowEndSeconds != nil || + encodePlan.Timeline.SeekRestoration != "player_position" { + t.Fatalf("encode timeline=%#v seek=%v segment=%d", encodePlan.Timeline, encodeSeek, encodeSegment) + } + unknownDurationPlan := &playback.PlanV3{Timeline: playback.TimelineV3{SourceStartSeconds: 17.3}} + configureHLSTimelineV3(unknownDurationPlan, "h264", 2, 0) + if unknownDurationPlan.Timeline.CanSeekAnywhere { + t.Fatalf("unknown-duration timeline = %#v", unknownDurationPlan.Timeline) + } +} + +func TestTransportGenerationV3IsUniqueAndSessionScoped(t *testing.T) { + first := transportGenerationV3("session-1", "plan:abcdef") + second := transportGenerationV3("session-1", "plan:abcdef") + if first == second || !strings.HasPrefix(first, "session-1-abcdef-") || !strings.HasPrefix(second, "session-1-abcdef-") { + t.Fatalf("generations = %q, %q", first, second) + } +} + +func TestRemuxDVModeForPlanV3ExecutesProfile8Strip(t *testing.T) { + plan := &playback.PlanV3{Source: playback.SourceDescriptorV3{DVProfile: 8}, Transformations: []playback.TransformationV3{{Name: "server_dv7_to_hdr10"}}} + if got := remuxDVModeForPlanV3(plan); got != playback.RemuxDVStripToHDR10V3 { + t.Fatalf("mode = %q", got) + } +} + +func TestHandlePlaybackRouteEventV3RejectsWhileDisabled(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{"playback.protocol_v3_enabled": "false"}} + req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/route-events", strings.NewReader(`{}`)).WithContext(newAuthorizedPlaybackContext()) + rr := httptest.NewRecorder() + handler.HandlePlaybackRouteEventV3(rr, req) + if rr.Code != http.StatusConflict { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } +} + +func TestSanitizeDiagnosticsV3PreservesPlayerFailureEvidence(t *testing.T) { + got := sanitizeDiagnosticsV3(map[string]string{ + "error_code": "2004", + "error_code_name": "ERROR_CODE_PARSING_CONTAINER_MALFORMED", + "error_cause": "ParserException", + "network_transport": "wifi", + "network_metered": "true", + "network_validated": "true", + "bandwidth_estimate_kbps": "3500", + "link_downstream_kbps": "5000", + "target_source_position_seconds": "321.5", + "reason": "seek_reanchor", + "message": "must not be persisted", + }) + if got["error_code"] != "2004" || got["error_code_name"] == "" || got["error_cause"] != "ParserException" { + t.Fatalf("failure diagnostics = %#v", got) + } + if _, ok := got["message"]; ok { + t.Fatalf("unapproved message persisted: %#v", got) + } + for _, key := range []string{"network_transport", "network_metered", "network_validated", "bandwidth_estimate_kbps", "link_downstream_kbps", "target_source_position_seconds", "reason"} { + if got[key] == "" { + t.Errorf("client diagnostic %q was stripped: %#v", key, got) + } + } +} + +func TestRouteEventV3AcceptsAndroidSeekEvents(t *testing.T) { + base := playback.RouteEventV3{ + ProtocolVersion: playback.ProtocolV3, + PlaybackAttemptID: "attempt-route-0001", + OutputRouteGeneration: 1, + } + for _, event := range []string{playback.RouteEventSeekReanchorRequestedV3, playback.RouteEventSeekReanchoredV3} { + candidate := base + candidate.Event = event + if !validRouteEventV3(candidate) { + t.Errorf("Android route event %q was rejected", event) + } + } +} + +func TestRemapSubtitleSelectionV3RejectsNegativeIndex(t *testing.T) { + index := -1 + request := playback.StartRequestV3{SubtitleTrackIndex: &index} + source := &models.MediaFile{ID: 1, ExternalSubtitles: []models.ExternalSubtitle{{Language: "eng", Format: "srt"}}} + target := &models.MediaFile{ID: 2, ExternalSubtitles: []models.ExternalSubtitle{{Language: "eng", Format: "srt"}}} + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + if err := handler.remapSubtitleSelectionV3(context.Background(), source, target, &request); err == nil { + t.Fatal("negative subtitle index was accepted") + } +} + +func TestRouteEventV3HasPerUserLimitAcrossAttemptIDs(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + for i := 0; i < 600; i++ { + attemptID := "attempt-" + strconv.Itoa(i/100) + if !handler.allowRouteEventV3(7, attemptID) { + t.Fatalf("event %d was rejected before the user limit", i) + } + } + if handler.allowRouteEventV3(7, "attempt-rotated") { + t.Fatal("rotating attempt IDs bypassed the per-user limit") + } +} + +func TestLegacyShadowRequestV3ProducesExplicitDetailedInference(t *testing.T) { + file := v3HandlerFixtureFile(t) + legacy := startPlaybackRequest{FileID: file.ID, ProfileID: "profile-1", CodecsVideo: []string{"h264"}, CodecsAudio: []string{"aac"}, Containers: []string{"mp4"}, MaxResolution: "1080p"} + request := legacyShadowRequestV3(legacy, file, 0, "session-1234") + if _, err := request.NormalizeAndValidate(); err != nil { + t.Fatalf("shadow request validation: %v", err) + } + if len(request.Capabilities.VideoDecode) != 1 || !request.Capabilities.VideoDecode[0].Hardware || !playback.HasFeatureV3(request.ClientFeatures, playback.FeatureDetailedDecodeV3) { + t.Fatalf("shadow request = %#v", request) + } +} + +func v3HandlerFixtureFile(t *testing.T) *models.MediaFile { + t.Helper() + return &models.MediaFile{ID: 42, ContentID: "movie-1", FilePath: writePlaybackTestMediaFile(t, "movie.mp4"), Container: "mp4", CodecVideo: "h264", CodecAudio: "aac", Resolution: "1080p", Bitrate: 8_000, AudioChannels: 2, Duration: 3600, VideoTracks: []models.VideoTrack{{Codec: "h264", Profile: "high", Level: 41, Width: 1920, Height: 1080, FrameRate: "24000/1001", Bitrate: 8_000, BitDepth: 8, VideoRange: "SDR", VideoRangeType: "SDR"}}, AudioTracks: []models.AudioTrack{{Codec: "aac", Channels: 2, Layout: "stereo"}}} +} + +func v3HandlerStartRequest() playback.StartRequestV3 { + return playback.StartRequestV3{ProtocolVersion: playback.ProtocolV3, ClientFeatures: []string{playback.FeaturePlaybackPlanV3, playback.FeatureMedia3Only, playback.FeatureDetailedDecodeV3}, FileID: 42, ProfileID: "profile-1", PlaybackAttemptID: "attempt-handler-0001", QualityPreference: "original", SubtitleFidelityPreference: playback.SubtitleFidelityCompatibleV3, OutputRouteGeneration: 1, Capabilities: playback.ClientCodecCapabilitiesV3{CodecsVideo: []string{"h264"}, CodecsVideoHardware: []string{"h264"}, CodecsAudio: []string{"aac"}, Containers: []string{"mp4"}, MaxResolution: "1080p", VideoDecode: []playback.VideoDecodeCapabilityV3{{Codec: "h264", Profiles: []string{"high"}, Levels: []int{41}, BitDepths: []int{8}, MaxWidth: 1920, MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 20_000, Hardware: true}}}, ClientPlaybackContext: playback.ClientPlaybackContextV3{ProtocolVersion: playback.ProtocolV3, Features: []string{playback.FeaturePlaybackPlanV3, playback.FeatureMedia3Only, playback.FeatureDetailedDecodeV3}, Platform: "android", FormFactor: "tv", AppVersion: "test", Output: playback.OutputContextV3{OutputRouteGeneration: 1}, Engines: map[string]playback.EngineCapabilityV3{string(playback.EngineMedia3DirectV3): {Enabled: true, SupportedOnDevice: true, Subtitles: playback.EngineSubtitleCapabilitiesV3{EmbeddedText: true, SidecarText: true}}}}} +} + +func marshalV3StartRequest(t *testing.T, request playback.StartRequestV3) string { + t.Helper() + body, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + return string(body) +} diff --git a/internal/api/handlers/playback_v3_union_test.go b/internal/api/handlers/playback_v3_union_test.go new file mode 100644 index 00000000..344cb0a1 --- /dev/null +++ b/internal/api/handlers/playback_v3_union_test.go @@ -0,0 +1,198 @@ +package handlers + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + + "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/playback" +) + +// enumeratingNodePlannerV3 is a SessionPlanner stub that also exposes pooled +// transcode node URLs, matching *nodepool.Planner's production shape. +type enumeratingNodePlannerV3 struct { + staticNodePlannerV3 + urls []string +} + +func (p enumeratingNodePlannerV3) TranscodeNodeURLs() []string { return p.urls } + +// presetLocalRegistryV3 pins the handler's local transformation registry so +// tests never probe the machine's real ffmpeg. +func presetLocalRegistryV3(h *PlaybackHandler, registry *playback.TransformationRegistryV3) { + h.v3RegistryOnce.Do(func() {}) + h.v3Registry = registry +} + +func TestHLSPlanningRegistryV3UnionsPooledNodeCapabilities(t *testing.T) { + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/hw-capabilities" { + w.WriteHeader(http.StatusNotFound) + return + } + writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{ + {Name: "video_to_h264", Executor: "server", RecipeVersion: "1"}, + {Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}, + }}) + })) + defer remote.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + presetLocalRegistryV3(handler, playback.NewTransformationRegistryV3([]playback.TransformationSpecV3{ + {Name: "video_to_h264", RecipeVersion: "1"}, + {Name: "audio_to_aac", RecipeVersion: "1"}, + {Name: "server_dv7_to_hdr10", RecipeVersion: "1"}, + })) + handler.NodePlanner = enumeratingNodePlannerV3{urls: []string{remote.URL}} + + registry := handler.hlsPlanningRegistryV3(context.Background()) + if !registry.Available("video_to_h264") || !registry.Available("audio_to_aac") { + t.Fatal("pooled node capabilities must widen the HLS planning registry") + } + if registry.Available("server_dv7_to_hdr10") { + t.Fatal("transformations no node advertises must stay unavailable") + } + if handler.transformationRegistryV3(context.Background()).Available("video_to_h264") { + t.Fatal("the local registry must not be widened by node capabilities") + } +} + +func TestHLSPlanningRegistryV3WithoutEnumeratorIsLocal(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + local := playback.NewTransformationRegistryV3([]playback.TransformationSpecV3{{Name: "audio_to_aac", RecipeVersion: "1", Available: true}}) + presetLocalRegistryV3(handler, local) + handler.NodePlanner = staticNodePlannerV3{plan: nodepool.Plan{}} + + if registry := handler.hlsPlanningRegistryV3(context.Background()); registry != local { + t.Fatal("a planner without node enumeration must plan from the local registry") + } +} + +func TestRemoteTransformationsV3FailureCacheSplit(t *testing.T) { + hits := 0 + fail := true + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits++ + if fail { + w.WriteHeader(http.StatusInternalServerError) + return + } + writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}}}) + })) + defer remote.Close() + + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + if _, err := handler.remoteTransformationsPlanningV3(context.Background(), remote.URL); err == nil { + t.Fatal("fetch against a failing node must error") + } + if _, err := handler.remoteTransformationsPlanningV3(context.Background(), remote.URL); err == nil { + t.Fatal("planning lookups must surface the memoized failure") + } + if hits != 1 { + t.Fatalf("failing node was fetched %d times; planning must memoize the failure", hits) + } + + // The transport path must fetch through the memoized failure: it may + // have been produced by a planning deadline far shorter than this + // path's budget, and rejecting the already-selected node on it would + // fail a start a fresh fetch could still validate. + fail = false + transformations, err := handler.remoteTransformationsV3(context.Background(), remote.URL) + if err != nil || len(transformations) != 1 { + t.Fatalf("transport lookup must refetch through a memoized failure: %v %#v", err, transformations) + } + if hits != 2 { + t.Fatalf("transport lookup fetched %d times, want 2", hits) + } + // The refetched success replaces the failure for planning too. + if _, err := handler.remoteTransformationsPlanningV3(context.Background(), remote.URL); err != nil { + t.Fatalf("planning lookup after transport success: %v", err) + } + if hits != 2 { + t.Fatalf("cached success was refetched (%d hits)", hits) + } +} + +// In a heterogeneous pool, a plan that needs server transformations must be +// placed on a node advertising them even when load balancing prefers an +// incapable node, while transformation-free plans keep load-based selection. +func TestPlanNodeSessionV3PrefersCapabilityMatchingNode(t *testing.T) { + capable := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, playback.HWAccelInfo{Transformations: []playback.TransformationV3{ + {Name: "video_to_h264", Executor: "server", RecipeVersion: "1"}, + {Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}, + }}) + })) + defer capable.Close() + incapable := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeJSON(w, http.StatusOK, playback.HWAccelInfo{}) + })) + defer incapable.Close() + + transcodes := nodepool.NewTranscodePool() + transcodes.SetNodes([]*nodepool.Node{ + {ID: 1, Name: "incapable", Type: nodepool.NodeTypeTranscode, URL: incapable.URL, Enabled: true, Healthy: true, ActiveJobs: 0}, + {ID: 2, Name: "capable", Type: nodepool.NodeTypeTranscode, URL: capable.URL, Enabled: true, Healthy: true, ActiveJobs: 5}, + }) + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + handler.JWTSecret = "test-secret" + handler.NodePlanner = nodepool.NewPlanner(nodepool.NewProxyPool(), transcodes) + + plan := &playback.PlanV3{ + PlanID: "plan:heterogeneous", + Delivery: playback.DeliveryTranscodeHLSV3, + Transformations: []playback.TransformationV3{ + {Name: "video_to_h264", Executor: "server", RecipeVersion: "1"}, + {Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}, + }, + } + selected := handler.planNodeSessionV3(context.Background(), &playback.Session{ID: "session-hetero"}, playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode}) + if selected.TranscodeNode == nil || selected.TranscodeNode.URL != capable.URL { + t.Fatalf("capability-requiring plan selected %+v, want the capable node", selected.TranscodeNode) + } + + free := &playback.PlanV3{PlanID: "plan:copy", Delivery: playback.DeliveryRemuxHLSV3, Transformations: []playback.TransformationV3{}} + loadBased := handler.planNodeSessionV3(context.Background(), &playback.Session{ID: "session-copy"}, playback.PlannerResultV3{Plan: free, PlayMethod: playback.PlayRemux}) + if loadBased.TranscodeNode == nil || loadBased.TranscodeNode.URL != incapable.URL { + t.Fatalf("transformation-free plan selected %+v, want load-based selection", loadBased.TranscodeNode) + } +} + +func TestPrepareTransportV3LocalFallbackRejectsUnavailableTransformations(t *testing.T) { + handler := NewPlaybackHandler(playback.NewSessionManager(0, 0)) + presetLocalRegistryV3(handler, playback.NewTransformationRegistryV3([]playback.TransformationSpecV3{ + {Name: "video_to_h264", RecipeVersion: "1"}, + {Name: "audio_to_aac", RecipeVersion: "1"}, + })) + plan := &playback.PlanV3{ + PlanID: "plan:local-capability", + Delivery: playback.DeliveryTranscodeHLSV3, + Transformations: []playback.TransformationV3{ + {Name: "video_to_h264", Executor: "server", RecipeVersion: "1"}, + {Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}, + }, + } + request := httptest.NewRequest(http.MethodPost, "/", nil) + _, transportErr := handler.prepareTransportV3(request, &playback.Session{ID: "session-local-capability"}, v3HandlerFixtureFile(t), playback.PlannerResultV3{Plan: plan, PlayMethod: playback.PlayTranscode, TargetVideoCodec: "h264", TargetAudioCodec: "aac"}) + if transportErr == nil || transportErr.reason != "transcode_node_capability_unavailable" || !transportErr.retryable { + t.Fatalf("transport error = %#v", transportErr) + } +} + +func TestPlanRequiresServerTransformationsV3(t *testing.T) { + if planRequiresServerTransformationsV3(nil) { + t.Fatal("nil plan must not require server transformations") + } + clientOnly := &playback.PlanV3{Transformations: []playback.TransformationV3{{Name: playback.ClientDV7ToDV81V3, Executor: "client", RecipeVersion: "1"}}} + if planRequiresServerTransformationsV3(clientOnly) { + t.Fatal("client-executed transformations must not require a server executor") + } + server := &playback.PlanV3{Transformations: []playback.TransformationV3{{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}}} + if !planRequiresServerTransformationsV3(server) { + t.Fatal("server-executed transformations must require executor validation") + } +} diff --git a/internal/api/handlers/plugins.go b/internal/api/handlers/plugins.go index 463af5c2..51916b48 100644 --- a/internal/api/handlers/plugins.go +++ b/internal/api/handlers/plugins.go @@ -157,6 +157,7 @@ type pluginInstallationResponse struct { Version string `json:"version"` InstallPath string `json:"install_path"` Enabled bool `json:"enabled"` + Kind string `json:"kind"` UpdatePolicy string `json:"update_policy"` AvailableVersion *string `json:"available_version,omitempty"` SourceKind string `json:"source_kind"` @@ -320,6 +321,11 @@ type pluginUserSettingsSummary struct { UserConfigSchema []pluginConfigSchemaJSON `json:"user_config_schema"` Routes []pluginRouteJSON `json:"routes"` Assets []pluginAssetJSON `json:"assets"` + // Category is the manifest's optional slash-delimited grouping path + // (e.g. "Tools/Utilities") used to group plugin entries in the + // user-facing Apps navigation. Empty (omitted) when the manifest + // declares no category. Additive-only per v1 API rules. + Category string `json:"category,omitempty"` } type pluginUserSettingsListResponse struct { @@ -877,6 +883,10 @@ func (h *PluginHandler) HandleUpdateInstallation(w http.ResponseWriter, r *http. writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load plugin installation") return } + if currentInstallation.IsBuiltin() { + writeError(w, http.StatusConflict, "builtin_installation", "Built-in host providers cannot be modified") + return + } if req.Enabled != nil && !*req.Enabled && currentInstallation.Enabled && h.service != nil { if err := h.service.Stop(id); err != nil && !errors.Is(err, pluginhost.ErrClientNotFound) { @@ -928,6 +938,9 @@ func (h *PluginHandler) HandleApplyUpdate(w http.ResponseWriter, r *http.Request writeError(w, http.StatusBadRequest, "bad_request", "Invalid installation ID") return } + if h.rejectBuiltinInstallation(w, r, id) { + return + } installation, err := h.service.UpdateToAvailableVersion(r.Context(), id) if err != nil { @@ -969,6 +982,9 @@ func (h *PluginHandler) HandlePutInstallationConfig(w http.ResponseWriter, r *ht writeError(w, http.StatusInternalServerError, "internal_error", "Plugin service not configured") return } + if h.rejectBuiltinInstallation(w, r, id) { + return + } if err := h.service.SetGlobalConfig(r.Context(), id, req.Key, req.Value); err != nil { var validationErr *plugins.ConfigValidationError @@ -1007,6 +1023,9 @@ func (h *PluginHandler) HandleTestInstallationConfig(w http.ResponseWriter, r *h writeError(w, http.StatusBadRequest, "bad_request", "key is required") return } + if h.rejectBuiltinInstallation(w, r, id) { + return + } if err := h.service.TestGlobalConfig(r.Context(), id, req.Key, req.Value); err != nil { if errors.Is(err, plugins.ErrInstallationNotFound) { @@ -1041,6 +1060,10 @@ func (h *PluginHandler) HandlePutAuthBinding(w http.ResponseWriter, r *http.Requ return } + if h.rejectBuiltinInstallation(w, r, id) { + return + } + var req pluginAuthBindingRequest if err := json.NewDecoder(r.Body).Decode(&req); err != nil { writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body") @@ -1074,6 +1097,10 @@ func (h *PluginHandler) HandlePutTaskBinding(w http.ResponseWriter, r *http.Requ return } + if h.rejectBuiltinInstallation(w, r, id) { + return + } + capabilityID := chi.URLParam(r, "capability_id") if strings.TrimSpace(capabilityID) == "" { writeError(w, http.StatusBadRequest, "bad_request", "capability_id is required") @@ -1101,6 +1128,21 @@ func (h *PluginHandler) HandlePutTaskBinding(w http.ResponseWriter, r *http.Requ writeJSON(w, http.StatusOK, pluginTaskBindingUpdateResponse{RestartRequired: true}) } +// rejectBuiltinInstallation writes a 409 and returns true when the target +// installation is the reserved builtin row, which no plugin-management +// endpoint may mutate. Lookup errors are left to the caller's own handling. +func (h *PluginHandler) rejectBuiltinInstallation(w http.ResponseWriter, r *http.Request, id int) bool { + installation, err := h.installations.GetByID(r.Context(), id) + if err != nil { + return false + } + if !installation.IsBuiltin() { + return false + } + writeError(w, http.StatusConflict, "builtin_installation", "Built-in host providers cannot be modified") + return true +} + func (h *PluginHandler) HandleDeleteInstallation(w http.ResponseWriter, r *http.Request) { id, err := parseNamedIDParam(r, "id") if err != nil { @@ -1113,6 +1155,10 @@ func (h *PluginHandler) HandleDeleteInstallation(w http.ResponseWriter, r *http. writeError(w, http.StatusNotFound, "not_found", "Plugin installation not found") return } + if errors.Is(err, plugins.ErrBuiltinInstallationImmutable) { + writeError(w, http.StatusConflict, "builtin_installation", "Built-in host providers cannot be uninstalled") + return + } slog.ErrorContext(r.Context(), "deleting plugin installation", "component", "api", "error", err) writeError(w, http.StatusInternalServerError, "internal_error", "Failed to delete plugin installation") return @@ -1151,6 +1197,11 @@ func (h *PluginHandler) HandleListUserPluginSettings(w http.ResponseWriter, r *h Installations: make([]pluginUserSettingsSummary, 0, len(installations)), } for _, installation := range installations { + // The reserved builtin row has no manifest on disk; without this skip + // the whole user-scoped settings list would 500. + if installation.IsBuiltin() { + continue + } manifest, err := plugins.LoadManifestFile(plugins.InstalledManifestPath(installation.InstallPath)) if err != nil { slog.ErrorContext(r.Context(), "loading plugin manifest", "component", "api", "installation_id", installation.ID, "error", err) @@ -1233,7 +1284,7 @@ func (h *PluginHandler) loadUserConfigInstallation( writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load plugin installation") return nil, nil, err } - if !installation.Enabled { + if !installation.Enabled || installation.IsBuiltin() { writeError(w, http.StatusNotFound, "not_found", "Plugin installation not found") return nil, nil, plugins.ErrInstallationNotFound } @@ -1277,6 +1328,12 @@ func (h *PluginHandler) buildInstallationResponses( } response := make([]pluginInstallationResponse, 0, len(installations)) for _, installation := range installations { + // The reserved builtin row is not a manageable plugin: old web builds + // would render a phantom entry with uninstall/upgrade buttons that + // error, and the chain editor does not need it in this list. + if installation.IsBuiltin() { + continue + } item, err := h.buildInstallationResponseWithBindings( ctx, installation, @@ -1390,6 +1447,7 @@ func (h *PluginHandler) buildInstallationResponseWithBindings( Version: installation.Version, InstallPath: installation.InstallPath, Enabled: installation.Enabled, + Kind: installation.Kind, UpdatePolicy: installation.UpdatePolicy, AvailableVersion: installation.AvailableVersion, SourceKind: sourceKind, @@ -1465,6 +1523,7 @@ func toUserPluginSettingsSummary( UserConfigSchema: configSchemasToJSON(manifest.GetUserConfigSchema()), Routes: routesToJSON(manifest.GetHttpRoutes()), Assets: assetsToJSON(manifest.GetAssets()), + Category: manifest.GetCategory(), } } diff --git a/internal/api/handlers/plugins_builtin_test.go b/internal/api/handlers/plugins_builtin_test.go new file mode 100644 index 00000000..0b2f3425 --- /dev/null +++ b/internal/api/handlers/plugins_builtin_test.go @@ -0,0 +1,170 @@ +package handlers + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strconv" + "strings" + "testing" + "time" + + "github.com/go-chi/chi/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/plugins" +) + +func pluginBuiltinTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func builtinTestHandler(pool *pgxpool.Pool) *PluginHandler { + return &PluginHandler{ + repositories: plugins.NewRepositoryStore(pool), + installations: plugins.NewInstallationStore(pool), + configs: plugins.NewRuntimeConfigStore(pool), + } +} + +func seedHandlerBuiltinInstallation(t *testing.T, pool *pgxpool.Pool) int { + t.Helper() + var id int + pluginID := fmt.Sprintf("test.builtin.handler-%d", time.Now().UnixNano()) + err := pool.QueryRow(context.Background(), + `INSERT INTO plugin_installations (plugin_id, version, install_path, enabled, update_policy, kind) + VALUES ($1, '0', '/nonexistent/silo-builtin-test', true, 'manual', 'builtin') + RETURNING id`, pluginID).Scan(&id) + if err != nil { + t.Fatalf("seed builtin installation: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM plugin_installations WHERE id = $1`, id) + }) + return id +} + +func requestWithIDParam(method, target, param string, id int) *http.Request { + return requestWithIDParamBody(method, target, param, id, nil) +} + +func requestWithIDParamBody(method, target, param string, id int, body io.Reader) *http.Request { + req := httptest.NewRequest(method, target, body) + rctx := chi.NewRouteContext() + rctx.URLParams.Add(param, strconv.Itoa(id)) + return req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)) +} + +// The manifest-less builtin row must not 500 the user-scoped plugin settings +// list the moment the migration lands (ship-blocker: the web sidebar fetches +// this for every logged-in user). +func TestHandleListUserPluginSettings_SkipsBuiltinRow(t *testing.T) { + pool := pluginBuiltinTestPool(t) + builtinID := seedHandlerBuiltinInstallation(t, pool) + h := builtinTestHandler(pool) + + rec := httptest.NewRecorder() + h.HandleListUserPluginSettings(rec, httptest.NewRequest(http.MethodGet, "/api/v1/settings/plugins", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var resp struct { + Installations []struct { + ID int `json:"id"` + } `json:"installations"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + for _, inst := range resp.Installations { + if inst.ID == builtinID { + t.Errorf("builtin installation %d leaked into user plugin settings", builtinID) + } + } +} + +// loadUserConfigInstallation must 404 builtin ids. +func TestHandleGetUserPluginSettings_BuiltinIDIs404(t *testing.T) { + pool := pluginBuiltinTestPool(t) + builtinID := seedHandlerBuiltinInstallation(t, pool) + h := builtinTestHandler(pool) + + rec := httptest.NewRecorder() + h.HandleGetUserPluginSettings(rec, requestWithIDParam(http.MethodGet, "/api/v1/settings/plugins/0", "installation_id", builtinID)) + if rec.Code != http.StatusNotFound { + t.Fatalf("status = %d, want 404; body=%s", rec.Code, rec.Body.String()) + } +} + +// GET /plugins/installations must filter builtin rows server-side, and the +// remaining rows carry the additive kind field. +func TestHandleListInstallations_FiltersBuiltinAndExposesKind(t *testing.T) { + pool := pluginBuiltinTestPool(t) + builtinID := seedHandlerBuiltinInstallation(t, pool) + h := builtinTestHandler(pool) + + rec := httptest.NewRecorder() + h.HandleListInstallations(rec, httptest.NewRequest(http.MethodGet, "/api/v1/plugins/installations", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + var resp []struct { + ID int `json:"id"` + Kind string `json:"kind"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + for _, inst := range resp { + if inst.ID == builtinID { + t.Errorf("builtin installation %d leaked into installations list", builtinID) + } + if inst.Kind == "" { + t.Errorf("installation %d missing additive kind field", inst.ID) + } + } +} + +// Mutating endpoints must reject the builtin row with a clear 4xx. +func TestBuiltinInstallationMutationsRejected(t *testing.T) { + pool := pluginBuiltinTestPool(t) + builtinID := seedHandlerBuiltinInstallation(t, pool) + h := builtinTestHandler(pool) + + cases := []struct { + name string + call func(rec *httptest.ResponseRecorder) + }{ + {"delete", func(rec *httptest.ResponseRecorder) { + h.HandleDeleteInstallation(rec, requestWithIDParam(http.MethodDelete, "/api/v1/plugins/installations/0", "id", builtinID)) + }}, + {"update", func(rec *httptest.ResponseRecorder) { + req := requestWithIDParamBody(http.MethodPatch, "/api/v1/plugins/installations/0", "id", builtinID, + strings.NewReader(`{"enabled": false}`)) + h.HandleUpdateInstallation(rec, req) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + rec := httptest.NewRecorder() + tc.call(rec) + if rec.Code < 400 || rec.Code >= 500 { + t.Fatalf("status = %d, want 4xx; body=%s", rec.Code, rec.Body.String()) + } + }) + } +} diff --git a/internal/api/handlers/plugins_user_settings_test.go b/internal/api/handlers/plugins_user_settings_test.go new file mode 100644 index 00000000..978dcfb0 --- /dev/null +++ b/internal/api/handlers/plugins_user_settings_test.go @@ -0,0 +1,60 @@ +package handlers + +import ( + "encoding/json" + "strings" + "testing" + + pluginv1 "github.com/Silo-Server/silo-plugin-sdk/pkg/pluginproto/silo/plugin/v1" + + "github.com/Silo-Server/silo-server/internal/plugins" +) + +func TestToUserPluginSettingsSummaryIncludesManifestCategory(t *testing.T) { + t.Parallel() + + installation := &plugins.Installation{ + ID: 42, + PluginID: "example-audiobooks", + Version: "1.2.3", + } + manifest := &pluginv1.PluginManifest{ + Category: "Tools/Utilities", + } + + got := toUserPluginSettingsSummary(installation, manifest) + + if got.Category != "Tools/Utilities" { + t.Fatalf("Category = %q, want %q", got.Category, "Tools/Utilities") + } + if got.ID != 42 || got.PluginID != "example-audiobooks" || got.Version != "1.2.3" { + t.Fatalf("identity fields = %#v", got) + } +} + +func TestToUserPluginSettingsSummaryOmitsEmptyCategory(t *testing.T) { + t.Parallel() + + installation := &plugins.Installation{ + ID: 7, + PluginID: "example-plain", + Version: "0.1.0", + } + + got := toUserPluginSettingsSummary(installation, &pluginv1.PluginManifest{}) + + if got.Category != "" { + t.Fatalf("Category = %q, want empty", got.Category) + } + + // The field is additive-only; ensure it stays absent from the JSON + // payload for manifests without a category so existing clients see an + // unchanged response shape. + encoded, err := json.Marshal(got) + if err != nil { + t.Fatalf("marshaling summary: %v", err) + } + if strings.Contains(string(encoded), "\"category\"") { + t.Fatalf("JSON payload should omit empty category, got %s", encoded) + } +} diff --git a/internal/api/handlers/stream.go b/internal/api/handlers/stream.go index 1e6b0e38..ef198fc3 100644 --- a/internal/api/handlers/stream.go +++ b/internal/api/handlers/stream.go @@ -159,7 +159,7 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) { seekSeconds = s } } - if err := playback.ServeRemux(w, r, file.FilePath, "mp4", seekSeconds, session.TranscodeAudio, session.AudioTrackIndex, file.PrimaryDVProfile()); err != nil { + if err := playback.ServeRemuxWithDVMode(w, r, file.FilePath, "mp4", seekSeconds, session.TranscodeAudio, session.AudioTrackIndex, file.PrimaryDVProfile(), session.RemuxDVMode, h.ffmpegPath()); err != nil { h.handleTransportStartFailure(r.Context(), session, file, err) } @@ -191,7 +191,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) { setPlaybackSessionLogContext(r, sessionID) trackParam := chi.URLParam(r, "track") - trackIndex, _, err := playback.ParseSubtitleTrackParam(trackParam) + trackIndex, requestedFormat, err := playback.ParseSubtitleTrackParam(trackParam) if err != nil { writeError(w, http.StatusBadRequest, "bad_request", "Invalid subtitle track index") return @@ -224,7 +224,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) { sub := file.ExternalSubtitles[trackIndex] // Serve ASS/SSA external subtitles as raw data for client-side rendering. - if playback.IsASS(sub.Format) { + if playback.IsASS(sub.Format) && requestedFormat != "vtt" { data, err := playback.LoadExternalSubtitleRaw(sub.Path) if err != nil { writeError(w, http.StatusInternalServerError, "internal_error", @@ -264,7 +264,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) { // demuxed, so the first byte lands within ~1s even on network // storage. Works identically for direct-play, remux, and // transcode because it doesn't depend on any other ffmpeg. - h.streamEmbeddedSubtitle(w, r, file, embeddedIndex, session) + h.streamEmbeddedSubtitle(w, r, file, embeddedIndex, session, requestedFormat) return } @@ -295,7 +295,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) { } // Serve ASS/SSA downloaded subtitles as raw data. - if playback.IsASS(string(dl.Format)) { + if playback.IsASS(string(dl.Format)) && requestedFormat != "vtt" { playback.ServeSubtitle(w, data, "ass") return } @@ -307,7 +307,7 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) { } // Convert to VTT using the playback conversion pipeline. - vttData, err := playback.ConvertToVTT(data, string(dl.Format)) + vttData, err := playback.ConvertToVTTWithFFmpeg(r.Context(), data, string(dl.Format), h.ffmpegPath()) if err != nil { writeError(w, http.StatusInternalServerError, "convert_error", "Failed to convert subtitle") return @@ -488,7 +488,7 @@ func (h *StreamHandler) handleTransportStartFailure(ctx context.Context, session // track, seeked to the best-known playback position, and pipes its // stdout directly to w. Because this ffmpeg is independent of the video // pipeline, it works the same for direct play, remux, and transcode. -func (h *StreamHandler) streamEmbeddedSubtitle(w http.ResponseWriter, r *http.Request, file *models.MediaFile, embeddedIndex int, session *playback.Session) { +func (h *StreamHandler) streamEmbeddedSubtitle(w http.ResponseWriter, r *http.Request, file *models.MediaFile, embeddedIndex int, session *playback.Session, requestedFormat ...string) { track := file.SubtitleTracks[embeddedIndex] outFormat := "vtt" switch { @@ -537,6 +537,20 @@ func (h *StreamHandler) streamEmbeddedSubtitle(w http.ResponseWriter, r *http.Re AllowWindow: allowWindow, FFmpegPath: h.ffmpegPath(), } + if len(requestedFormat) > 0 && requestedFormat[0] == "vtt" { + // Only text sources can be converted to WebVTT. A bitmap track (PGS + // reaches here because it is deliverable as .sup; DVD/DVB are rejected + // upstream) carries no text, so honoring the override would spawn an + // ffmpeg that always fails after the 200 and headers are committed. + // Reject before any spawn or header write. + if playback.NeedsBurnIn(track.Codec) { + writeError(w, http.StatusUnsupportedMediaType, "unsupported_media_type", + "Bitmap subtitle tracks cannot be converted to WebVTT") + return + } + opts.TargetFormat = "vtt" + outFormat = "vtt" + } w.Header().Set("Access-Control-Allow-Origin", "*") diff --git a/internal/api/handlers/stream_test.go b/internal/api/handlers/stream_test.go index 507be166..5aefc50c 100644 --- a/internal/api/handlers/stream_test.go +++ b/internal/api/handlers/stream_test.go @@ -196,6 +196,51 @@ func TestHandleSubtitle_NilMediaFileReturns404(t *testing.T) { } } +// A .vtt request for a bitmap (PGS) embedded track must be rejected up front: +// PGS passes the burn-in guard because it is deliverable as .sup, but it has +// no text to convert, so forcing WebVTT would spawn an ffmpeg that always +// fails after the 200 and headers are committed. +func TestHandleSubtitle_BitmapTrackVTTRequestReturns415(t *testing.T) { + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: "/tmp/movie.mkv", + Duration: 3600, + SubtitleTracks: []models.SubtitleTrack{ + {Index: 0, Language: "eng", Codec: "hdmv_pgs_subtitle"}, + }, + } + baseMgr := playback.NewSessionManager(0, 0) + session, err := baseMgr.StartSession(1, "profile-1", 42, playback.PlayDirect, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + handler := NewStreamHandler(baseMgr, testPlaybackFileResolver{file: file}) + req := httptest.NewRequest(http.MethodGet, "/api/v1/stream/"+session.ID+"/subtitles/0.vtt", nil) + req = req.WithContext(newAuthorizedPlaybackContext()) + routeCtx := chi.NewRouteContext() + routeCtx.URLParams.Add("session_id", session.ID) + routeCtx.URLParams.Add("track", "0.vtt") + req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx)) + + rr := httptest.NewRecorder() + handler.HandleSubtitle(rr, req) + + if rr.Code != http.StatusUnsupportedMediaType { + t.Fatalf("status = %d, body = %s; want 415", rr.Code, rr.Body.String()) + } + var body struct { + Error string `json:"error"` + } + if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil { + t.Fatalf("decode error body: %v (body = %s)", err, rr.Body.String()) + } + if body.Error != "unsupported_media_type" { + t.Fatalf("error code = %q, want %q", body.Error, "unsupported_media_type") + } +} + func TestSubtitleSourceFileIDPinsURLAcrossEffectiveFileSwitch(t *testing.T) { session := &playback.Session{MediaFileID: 200, RequestedMediaFileID: 100} diff --git a/internal/api/handlers/system.go b/internal/api/handlers/system.go index 175d8e8d..76944f0e 100644 --- a/internal/api/handlers/system.go +++ b/internal/api/handlers/system.go @@ -2,11 +2,8 @@ package handlers import ( "context" - "encoding/json" - "fmt" "log/slog" "net/http" - "time" "github.com/Silo-Server/silo-server/internal/buildinfo" "github.com/Silo-Server/silo-server/internal/nodepool" @@ -57,28 +54,5 @@ func (h *SystemHandler) HandleBuildInfo(w http.ResponseWriter, _ *http.Request) } func (h *SystemHandler) fetchRemoteHWAccel(ctx context.Context, node *nodepool.Node) (playback.HWAccelInfo, error) { - client := &http.Client{Timeout: 5 * time.Second} - req, err := http.NewRequestWithContext(ctx, http.MethodGet, node.URL+"/hw-capabilities", nil) - if err != nil { - return playback.HWAccelInfo{}, err - } - req.Header.Set("Authorization", "Bearer "+h.jwtSecret) - - resp, err := client.Do(req) - if err != nil { - return playback.HWAccelInfo{}, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return playback.HWAccelInfo{}, fmt.Errorf("node returned %d", resp.StatusCode) - } - - var info playback.HWAccelInfo - if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { - return playback.HWAccelInfo{}, err - } - info.Source = "transcode_node" - info.NodeURL = node.URL - return info, nil + return fetchRemoteTranscodeCapabilities(ctx, node.URL, h.jwtSecret) } diff --git a/internal/api/middleware/auth.go b/internal/api/middleware/auth.go index f081ea83..d6edc8c3 100644 --- a/internal/api/middleware/auth.go +++ b/internal/api/middleware/auth.go @@ -47,6 +47,8 @@ type AuthMiddleware struct { sessionValidator SessionValidator apiKeyValidator APIKeyValidator // nil if API keys not configured apiKeyUserLoader APIKeyUserLoader // nil if API keys not configured + + apiKeyLastUsed *auth.APIKeyLastUsedTracker } // NewAuthMiddleware creates a new AuthMiddleware with the given token validator @@ -57,6 +59,7 @@ func NewAuthMiddleware(tv TokenValidator, sv SessionValidator, akv APIKeyValidat sessionValidator: sv, apiKeyValidator: akv, apiKeyUserLoader: akul, + apiKeyLastUsed: auth.NewAPIKeyLastUsedTracker(akv, nil), } } @@ -98,10 +101,7 @@ func (am *AuthMiddleware) RequireAuth(next http.Handler) http.Handler { return } - // Update last_used_at asynchronously. - go func(id int64) { - _ = am.apiKeyValidator.UpdateLastUsed(context.Background(), id) - }(apiKey.ID) + am.apiKeyLastUsed.Touch(apiKey.ID) claims = &auth.Claims{ UserID: user.ID, diff --git a/internal/api/middleware/demo_guard.go b/internal/api/middleware/demo_guard.go index 9c2725f7..ba0400bf 100644 --- a/internal/api/middleware/demo_guard.go +++ b/internal/api/middleware/demo_guard.go @@ -17,7 +17,8 @@ type DemoSettingsReader interface { // Allowed: browsing, playback, favorites, watchlist, ratings, collections, // profiles, playback progress, watched state. // -// Blocked: API key management, downloads, history imports, subtitle downloads. +// Blocked: API key management, downloads, history imports, subtitle downloads, +// diagnostics report uploads/deletes. type DemoGuard struct { settings DemoSettingsReader } @@ -41,6 +42,9 @@ var demoBlockedRoutes = []blockedRoute{ {methods: []string{"POST"}, prefix: "/api/v1/subtitles/download"}, {methods: []string{"POST"}, prefix: "/api/v1/subtitles/upload"}, {methods: []string{"DELETE"}, prefix: "/api/v1/subtitles/"}, + // Diagnostics report uploads/deletes write DB rows and private-bucket blobs; + // GET /api/v1/diagnostics/status is unaffected (GETs always pass). + {methods: []string{"POST", "DELETE"}, prefix: "/api/v1/diagnostics/reports"}, } // Guard is an HTTP middleware that enforces demo mode restrictions. diff --git a/internal/api/router.go b/internal/api/router.go index b508072f..7165bec4 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -32,6 +32,7 @@ import ( "github.com/Silo-Server/silo-server/internal/catalogseed" "github.com/Silo-Server/silo-server/internal/clientip" "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/diagnostics" "github.com/Silo-Server/silo-server/internal/downloads" evt "github.com/Silo-Server/silo-server/internal/events" "github.com/Silo-Server/silo-server/internal/historyimport" @@ -50,6 +51,7 @@ import ( "github.com/Silo-Server/silo-server/internal/notifications" "github.com/Silo-Server/silo-server/internal/opslog" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/playback/planstore" "github.com/Silo-Server/silo-server/internal/plugins" "github.com/Silo-Server/silo-server/internal/policy" "github.com/Silo-Server/silo-server/internal/ratelimit" @@ -253,14 +255,22 @@ func NewRouter(deps Dependencies) chi.Router { // acting-admin profile policy, degrading admin routes to the plain // role check. var checkPrimaryProfile apimw.PrimaryProfileChecker + // lookupProfile resolves a profile for the given user, returning nil when it + // does not exist. Shared by the acting-admin primary check and the + // diagnostics profile-attribution validator so both read profile state + // (IsPrimary, IsChild) the same way. + var lookupProfile func(ctx context.Context, userID int, profileID string) (*userstore.Profile, error) if deps.UserStoreProvider != nil { userStores := deps.UserStoreProvider - checkPrimaryProfile = func(ctx context.Context, userID int, profileID string) (bool, bool, error) { + lookupProfile = func(ctx context.Context, userID int, profileID string) (*userstore.Profile, error) { store, err := userStores.ForUser(ctx, userID) if err != nil { - return false, false, err + return nil, err } - profile, err := store.GetProfile(ctx, profileID) + return store.GetProfile(ctx, profileID) + } + checkPrimaryProfile = func(ctx context.Context, userID int, profileID string) (bool, bool, error) { + profile, err := lookupProfile(ctx, userID, profileID) if err != nil { return false, false, err } @@ -307,6 +317,39 @@ func NewRouter(deps Dependencies) chi.Router { if deps.DB != nil { accessGroupStore = access.NewGroupStore(deps.DB) } + var diagnosticsStore diagnostics.ObjectStore + if deps.S3Private != nil { + diagnosticsStore = diagnostics.NewS3ObjectStore(deps.S3Private) + } + var diagnosticsHandler *handlers.DiagnosticsHandler + if deps.DB != nil { + diagnosticsService := diagnostics.NewService( + diagnostics.NewPostgresRepository(deps.DB), + settingsRepo, + diagnosticsStore, + slog.Default(), + ) + if lookupProfile != nil { + // Attribute reports only to a profile that belongs to the account and + // is not a child profile; child profiles must not perform diagnostics + // actions per the design, so reject their attribution here. + diagnosticsService.SetProfileAttributionValidator(diagnostics.NewProfileAttributionValidator( + func(ctx context.Context, userID int, profileID string) (bool, bool, error) { + profile, err := lookupProfile(ctx, userID, profileID) + if err != nil { + return false, false, err + } + if profile == nil { + return false, false, nil + } + return true, profile.IsChild, nil + }, + )) + } + diagnosticsHandler = handlers.NewDiagnosticsHandler( + diagnosticsService, + ) + } // Build auth handler and auth middleware if DB and config are available. var userRepo *auth.UserRepository @@ -805,6 +848,12 @@ func NewRouter(deps Dependencies) chi.Router { } else { playbackHandler = handlers.NewPlaybackHandler(deps.SessionMgr) } + if deps.DB != nil { + playbackHandler.PlanStoreV3 = planstore.NewPostgres(deps.DB) + } + // Maintenance also bounds the in-memory fallback store: without it a + // DB-less deployment accumulates attempts and replans forever. + playbackHandler.StartV3Maintenance(deps.AppContext) // Wire UserStoreProvider for progress/history persistence. if deps.UserStoreProvider != nil { @@ -850,11 +899,13 @@ func NewRouter(deps Dependencies) chi.Router { playbackHandler.PlaybackConfig = func() config.PlaybackConfig { return deps.CurrentConfig().Playback } - if cleaned, err := playbackHandler.CleanupOrphanedTranscodes(); err != nil { - slog.Warn("playback transcode cleanup failed", "dir", deps.Config.Playback.TranscodeDir, "error", err) - } else if cleaned > 0 { - slog.Info("playback transcode cleanup removed orphaned dirs", "dir", deps.Config.Playback.TranscodeDir, "count", cleaned) - } + // In integrated mode this and the jellycompat sweep both scan the same + // TranscodeDir but each snapshots only its own manager's live set, so a + // >24h idle dir owned by the other manager can be reaped. Bounded and + // safe: active dirs stay mtime-fresh (spared) and either side rebuilds + // from its token/recipe, so the worst case is a wasted rebuild. A shared + // active-set source across both managers would remove even that. + playback.StartPeriodicOrphanCleanup(deps.AppContext, "api", deps.Config.Playback.TranscodeDir, playbackHandler.CleanupOrphanedTranscodes, playback.OrphanCleanupInterval) } playbackHandler.ProbeEnsurer = deps.ProbeEnsurer playbackHandler.ChapterThumbnailQueuer = deps.ChapterThumbnailQueuer @@ -949,6 +1000,7 @@ func NewRouter(deps Dependencies) chi.Router { adminHandler.BootstrapSensitiveValues = deps.BootstrapSensitiveValues adminHandler.RestartStatus = restartStatus adminHandler.CatalogSearchStatus = catalogSearchService + adminHandler.DiagnosticsStore = diagnosticsStore if settingsRepo != nil { adminHandler.SettingsRepo = settingsRepo } @@ -1028,6 +1080,7 @@ func NewRouter(deps Dependencies) chi.Router { deps.PluginImageResolver, detailSvc, ) + adminImageHandler.EventsHub = deps.EventsHub } var adminIntroHandler *handlers.AdminIntroHandler @@ -1757,6 +1810,28 @@ func NewRouter(deps Dependencies) chi.Router { }) } + // Client diagnostics are account-scoped and must work before profile + // selection, so this route intentionally uses auth only plus the + // generic rate limiter, not viewer/profile middleware. + if diagnosticsHandler != nil && authMiddleware != nil { + r.Group(func(r chi.Router) { + r.Use(authMiddleware.RequireAuth) + // Demo mode blocks non-admin report uploads (a write to the + // private bucket and DB); the read-only status endpoint stays + // available because DemoGuard always lets GETs through. + if demoGuard != nil { + r.Use(demoGuard.Guard) + } + if deps.RateLimitMW != nil { + r.Use(deps.RateLimitMW.Handler) + } + r.Route("/diagnostics", func(r chi.Router) { + r.Get("/status", diagnosticsHandler.HandleStatus) + r.Post("/reports", diagnosticsHandler.HandleUpload) + }) + }) + } + // All remaining routes require auth. if authMiddleware != nil { r.Group(func(r chi.Router) { @@ -1817,6 +1892,10 @@ func NewRouter(deps Dependencies) chi.Router { r.Get("/preferences", notificationsHandler.HandleGetPreferences) r.Put("/preferences", notificationsHandler.HandleUpdatePreferences) r.Get("/push/apple/display/{delivery_id}", notificationsHandler.HandleApplePushDisplay) + // Platform-generic registration used by the Android + // client; Apple keeps its dedicated route above. + r.Post("/push/devices", notificationsHandler.HandleRegisterPushDevice) + r.Delete("/push/devices/{device_id}", notificationsHandler.HandleUnregisterPushDevice) r.Get("/email-preferences", notificationsHandler.HandleGetEmailPreferences) r.Put("/email-preferences", notificationsHandler.HandleUpdateEmailPreferences) r.Put("/email-preferences/address", notificationsHandler.HandleRequestEmailAddress) @@ -2279,6 +2358,7 @@ func NewRouter(deps Dependencies) chi.Router { playbackHandler.FFmpegLogSink = deps.FFmpegLogSink r.Route("/playback", func(r chi.Router) { + r.Get("/capability", playbackHandler.HandlePlaybackCapabilityV3) // HLS transcode delivery — no profile auth needed; // session ID (UUID) serves as the access token, same // pattern as /stream/{session_id}. @@ -2292,6 +2372,8 @@ func NewRouter(deps Dependencies) chi.Router { r.Group(func(r chi.Router) { r.Use(apimw.RequireProfile) r.Post("/start", playbackHandler.HandleStartPlayback) + r.Post("/{session_id}/replan", playbackHandler.HandleReplanPlaybackV3) + r.Post("/route-events", playbackHandler.HandlePlaybackRouteEventV3) r.Post("/{session_id}/progress", playbackHandler.HandleUpdateProgress) r.Patch("/{session_id}/audio", playbackHandler.HandleChangeAudioTrack) r.Delete("/{session_id}", playbackHandler.HandleStopPlayback) @@ -2475,6 +2557,7 @@ func NewRouter(deps Dependencies) chi.Router { } r.Get("/sessions", adminHandler.HandleListSessions) + r.Get("/sessions/capabilities", adminHandler.HandleGetSessionsCapabilities) r.Get("/playback-history", adminHandler.HandleListPlaybackHistory) r.Get("/unmatched", adminHandler.HandleListUnmatched) r.Get("/stats", adminHandler.HandleGetStats) @@ -2537,6 +2620,7 @@ func NewRouter(deps Dependencies) chi.Router { applePushHandler := handlers.NewAdminApplePushHandler(deps.Notifications, settingsRepo) if deps.Notifications != nil { r.Post("/notifications/push/apple/test", applePushHandler.HandleTest) + r.Post("/notifications/push/fcm/test", applePushHandler.HandleTestAndroid) } if settingsRepo != nil { r.Post("/notifications/push/relay/register", applePushHandler.HandleRegisterRelay) @@ -2864,6 +2948,9 @@ func NewRouter(deps Dependencies) chi.Router { r.Get("/logs/audit", adminLogsHandler.HandleListAuditLogs) r.Get("/logs/ws", adminLogsHandler.HandleLogStreamWebSocket) } + if diagnosticsHandler != nil { + handlers.RegisterAdminDiagnosticsRoutes(r, diagnosticsHandler) + } if adminPlaybackControlHandler != nil { r.Post("/sessions/{session_id}/pause", adminPlaybackControlHandler.HandlePauseSession) r.Post("/sessions/{session_id}/resume", adminPlaybackControlHandler.HandleResumeSession) diff --git a/internal/artworkkey/artworkkey.go b/internal/artworkkey/artworkkey.go new file mode 100644 index 00000000..01f4a32e --- /dev/null +++ b/internal/artworkkey/artworkkey.go @@ -0,0 +1,115 @@ +// Package artworkkey owns the object-key naming contract for cached artwork. +// Legacy names such as original.webp and revisioned names such as +// original.<revision>.webp are both supported. +package artworkkey + +import ( + "path" + "strconv" + "strings" +) + +const OriginalVariant = "original" + +// Build returns an object key for a variant under basePath. +func Build(basePath, variant, revision, ext string) string { + basePath = strings.TrimRight(strings.TrimSpace(basePath), "/") + variant = strings.TrimSpace(variant) + revision = strings.TrimSpace(revision) + if basePath == "" || variant == "" { + return "" + } + if ext == "" { + ext = ".webp" + } else if !strings.HasPrefix(ext, ".") { + ext = "." + ext + } + if revision == "" { + return basePath + "/" + variant + ext + } + return basePath + "/" + variant + "." + revision + ext +} + +// Original returns the original-variant key under basePath. +func Original(basePath, revision, ext string) string { + return Build(basePath, OriginalVariant, revision, ext) +} + +// Variant rewrites an original key to another variant while retaining any +// revision and extension. Unrecognized paths pass through unchanged. +func Variant(originalPath, variant string) string { + if originalPath == "" || variant == "" || variant == OriginalVariant { + return originalPath + } + dir := path.Dir(originalPath) + base := path.Base(originalPath) + if dir == "." || !strings.HasPrefix(base, OriginalVariant+".") { + return originalPath + } + return strings.TrimRight(dir, "/") + "/" + variant + strings.TrimPrefix(base, OriginalVariant) +} + +// Directory returns the image-type prefix containing every revision and +// variant for an artwork key, including a trailing slash. +func Directory(objectPath string) string { + objectPath = strings.TrimSpace(objectPath) + if objectPath == "" || strings.Contains(objectPath, "://") { + return "" + } + dir := path.Dir(objectPath) + if dir == "." || dir == "/" { + return "" + } + return strings.TrimRight(dir, "/") + "/" +} + +// Revision extracts the content revision from a revisioned key. Legacy keys +// return an empty string. +func Revision(objectPath string) string { + name := path.Base(strings.TrimSpace(objectPath)) + ext := path.Ext(name) + stem := strings.TrimSuffix(name, ext) + firstDot := strings.IndexByte(stem, '.') + if firstDot < 0 || firstDot == len(stem)-1 { + return "" + } + return stem[firstDot+1:] +} + +// VariantWidths returns the resize widths generated for an artwork type. This +// is the single source of truth for the variant ladder: image generation, +// object-key expansion, and garbage collection all derive from it. +func VariantWidths(imageType string) []int { + switch strings.ToLower(strings.TrimSpace(imageType)) { + case "backdrop": + return []int{1920, 1280, 300} + case "logo": + return []int{500} + default: // poster, still, profile + return []int{500, 300} + } +} + +// VariantNames returns the cached variants generated for an artwork type. +func VariantNames(imageType string) []string { + widths := VariantWidths(imageType) + names := make([]string, 0, len(widths)+1) + names = append(names, OriginalVariant) + for _, width := range widths { + names = append(names, "w"+strconv.Itoa(width)) + } + return names +} + +// ObjectKeys expands an original key to every expected key for its image type. +func ObjectKeys(originalPath, imageType string) []string { + if originalPath == "" || strings.Contains(originalPath, "://") { + return nil + } + names := VariantNames(imageType) + keys := make([]string, 0, len(names)) + for _, name := range names { + keys = append(keys, Variant(originalPath, name)) + } + return keys +} diff --git a/internal/artworkkey/artworkkey_test.go b/internal/artworkkey/artworkkey_test.go new file mode 100644 index 00000000..cc5bb024 --- /dev/null +++ b/internal/artworkkey/artworkkey_test.go @@ -0,0 +1,38 @@ +package artworkkey + +import "testing" + +func TestRevisionedArtworkKeys(t *testing.T) { + base := "tmdb/movies/550/poster" + original := Original(base, "abc123", ".webp") + if original != base+"/original.abc123.webp" { + t.Fatalf("Original() = %q", original) + } + if got := Variant(original, "w500"); got != base+"/w500.abc123.webp" { + t.Fatalf("Variant() = %q", got) + } + if got := Revision(original); got != "abc123" { + t.Fatalf("Revision() = %q", got) + } + if got := Directory(original); got != base+"/" { + t.Fatalf("Directory() = %q", got) + } +} + +func TestLegacyArtworkKeysRemainSupported(t *testing.T) { + original := "tmdb/movies/550/poster/original.webp" + if got := Variant(original, "w300"); got != "tmdb/movies/550/poster/w300.webp" { + t.Fatalf("Variant() = %q", got) + } + if got := Revision(original); got != "" { + t.Fatalf("Revision() = %q, want empty", got) + } +} + +func TestVariantOnlyRewritesOriginalFilename(t *testing.T) { + original := "tmdb/movies/original.segment/550/poster/original.abc123.webp" + want := "tmdb/movies/original.segment/550/poster/w500.abc123.webp" + if got := Variant(original, "w500"); got != want { + t.Fatalf("Variant() = %q, want %q", got, want) + } +} diff --git a/internal/audiobooks/enrichment.go b/internal/audiobooks/enrichment.go index 08da71e2..33a95297 100644 --- a/internal/audiobooks/enrichment.go +++ b/internal/audiobooks/enrichment.go @@ -37,7 +37,7 @@ import ( // method set by structural typing. Anything implementing CacheAudiobookCover can // be passed to scanner.ExtractAndUploadAudiobookCover via this interface. type audiobookCoverCacher interface { - CacheAudiobookCover(ctx context.Context, data []byte, contentID string) (basePath string, ext string, thumbhash string, err error) + CacheAudiobookCover(ctx context.Context, data []byte, contentID string) (storedPath string, thumbhash string, err error) } const ( @@ -570,7 +570,7 @@ func (e *Enricher) cacheRemotePoster(ctx context.Context, contentID string, resu return } - if storedPath := cachedOriginalImagePath(cached.BasePath, cached.Ext); storedPath != "" { + if storedPath := metadata.CachedImageOriginalPath(cached); storedPath != "" { result.PosterPath = storedPath } if cached.Thumbhash != "" { @@ -578,19 +578,6 @@ func (e *Enricher) cacheRemotePoster(ctx context.Context, contentID string, resu } } -func cachedOriginalImagePath(basePath, ext string) string { - if basePath == "" { - return "" - } - if strings.Contains(basePath, "/original.") { - return basePath - } - if ext == "" { - ext = ".jpg" - } - return strings.TrimRight(basePath, "/") + "/original" + ext -} - // persist writes the enriched metadata back to the database. func (e *Enricher) persist(ctx context.Context, contentID string, providerIDs map[string]string, result *metadata.MetadataResult) error { // Build the MetadataUpdate — only set fields that the provider returned. diff --git a/internal/audiobooks/enrichment_test.go b/internal/audiobooks/enrichment_test.go index 903847bc..e4189fd7 100644 --- a/internal/audiobooks/enrichment_test.go +++ b/internal/audiobooks/enrichment_test.go @@ -125,7 +125,7 @@ func TestCacheRemotePosterCachesProviderURL(t *testing.T) { if cacher.req.ContentType != "audiobooks" || cacher.req.ContentID != "content-1" { t.Fatalf("cache target = %q/%q", cacher.req.ContentType, cacher.req.ContentID) } - if result.PosterPath != "audiobook-metadata/audiobooks/content-1/poster/original.webp" { + if result.PosterPath != "audiobook-metadata/audiobooks/content-1/poster/original.test-revision.webp" { t.Fatalf("PosterPath = %q", result.PosterPath) } if result.PosterThumbhash != "thumb" { @@ -155,16 +155,17 @@ type fakeAudiobookImageCacher struct { req metadata.CacheImageRequest } -func (f *fakeAudiobookImageCacher) CacheAudiobookCover(context.Context, []byte, string) (string, string, string, error) { - return "", "", "", nil +func (f *fakeAudiobookImageCacher) CacheAudiobookCover(context.Context, []byte, string) (string, string, error) { + return "", "", nil } func (f *fakeAudiobookImageCacher) CacheImage(_ context.Context, req metadata.CacheImageRequest) (*metadata.CacheImageResult, error) { f.calls++ f.req = req return &metadata.CacheImageResult{ - BasePath: req.ProviderID + "/" + req.ContentType + "/" + req.ContentID + "/poster", - Thumbhash: "thumb", - Ext: ".webp", + BasePath: req.ProviderID + "/" + req.ContentType + "/" + req.ContentID + "/poster", + OriginalPath: req.ProviderID + "/" + req.ContentType + "/" + req.ContentID + "/poster/original.test-revision.webp", + Thumbhash: "thumb", + Ext: ".webp", }, nil } diff --git a/internal/auth/api_key_last_used.go b/internal/auth/api_key_last_used.go new file mode 100644 index 00000000..267e020a --- /dev/null +++ b/internal/auth/api_key_last_used.go @@ -0,0 +1,80 @@ +package auth + +import ( + "context" + "log/slog" + "sync" + "time" +) + +const ( + apiKeyLastUsedInterval = time.Minute + apiKeyLastUsedTimeout = 5 * time.Second +) + +// APIKeyLastUsedUpdater persists an API key's last-used timestamp. +type APIKeyLastUsedUpdater interface { + UpdateLastUsed(ctx context.Context, id int64) error +} + +// APIKeyLastUsedTracker coalesces last-used writes per key without blocking +// authentication requests. Stale entries are pruned inline once per interval, +// avoiding both unbounded historical-key retention and another goroutine. +type APIKeyLastUsedTracker struct { + updater APIKeyLastUsedUpdater + now func() time.Time + + mu sync.Mutex + lastUsedAt map[int64]time.Time + nextPrune time.Time +} + +// NewAPIKeyLastUsedTracker creates a tracker. A nil clock uses time.Now. +func NewAPIKeyLastUsedTracker(updater APIKeyLastUsedUpdater, now func() time.Time) *APIKeyLastUsedTracker { + if now == nil { + now = time.Now + } + return &APIKeyLastUsedTracker{ + updater: updater, + now: now, + lastUsedAt: make(map[int64]time.Time), + } +} + +// Touch records key usage asynchronously, at most once per interval per key. +func (t *APIKeyLastUsedTracker) Touch(id int64) { + if t == nil || t.updater == nil || !t.shouldUpdate(id) { + return + } + + go func() { + ctx, cancel := context.WithTimeout(context.Background(), apiKeyLastUsedTimeout) + defer cancel() + if err := t.updater.UpdateLastUsed(ctx, id); err != nil { + slog.DebugContext(ctx, "api key last-used update failed", "component", "auth", "id", id, "error", err) + } + }() +} + +func (t *APIKeyLastUsedTracker) shouldUpdate(id int64) bool { + now := t.now() + + t.mu.Lock() + defer t.mu.Unlock() + + if t.nextPrune.IsZero() || !now.Before(t.nextPrune) { + cutoff := now.Add(-apiKeyLastUsedInterval) + for keyID, lastUsed := range t.lastUsedAt { + if !lastUsed.After(cutoff) { + delete(t.lastUsedAt, keyID) + } + } + t.nextPrune = now.Add(apiKeyLastUsedInterval) + } + + if lastUsed, ok := t.lastUsedAt[id]; ok && now.Sub(lastUsed) < apiKeyLastUsedInterval { + return false + } + t.lastUsedAt[id] = now + return true +} diff --git a/internal/auth/api_key_last_used_test.go b/internal/auth/api_key_last_used_test.go new file mode 100644 index 00000000..3dfc6727 --- /dev/null +++ b/internal/auth/api_key_last_used_test.go @@ -0,0 +1,82 @@ +package auth + +import ( + "context" + "testing" + "time" +) + +type recordingLastUsedUpdater struct { + calls chan lastUsedCall +} + +type lastUsedCall struct { + id int64 + deadline time.Time +} + +func (u *recordingLastUsedUpdater) UpdateLastUsed(ctx context.Context, id int64) error { + deadline, _ := ctx.Deadline() + u.calls <- lastUsedCall{id: id, deadline: deadline} + return nil +} + +func TestAPIKeyLastUsedTrackerThrottlesPerKey(t *testing.T) { + now := time.Date(2026, time.July, 20, 12, 0, 0, 0, time.UTC) + tracker := NewAPIKeyLastUsedTracker(nil, func() time.Time { return now }) + + if !tracker.shouldUpdate(1) { + t.Fatal("first use should update") + } + if tracker.shouldUpdate(1) { + t.Fatal("repeat use inside the interval should be throttled") + } + if !tracker.shouldUpdate(2) { + t.Fatal("a different key should update independently") + } + + now = now.Add(apiKeyLastUsedInterval) + if !tracker.shouldUpdate(1) { + t.Fatal("use at the next interval should update") + } +} + +func TestAPIKeyLastUsedTrackerPrunesExpiredKeys(t *testing.T) { + now := time.Date(2026, time.July, 20, 12, 0, 0, 0, time.UTC) + tracker := NewAPIKeyLastUsedTracker(nil, func() time.Time { return now }) + + tracker.shouldUpdate(1) + tracker.shouldUpdate(2) + + now = now.Add(apiKeyLastUsedInterval) + tracker.shouldUpdate(3) + + if len(tracker.lastUsedAt) != 1 { + t.Fatalf("retained entries = %d, want 1", len(tracker.lastUsedAt)) + } + if _, ok := tracker.lastUsedAt[3]; !ok { + t.Fatal("current key was not retained") + } +} + +func TestAPIKeyLastUsedTrackerAddsWriteDeadline(t *testing.T) { + updater := &recordingLastUsedUpdater{calls: make(chan lastUsedCall, 1)} + tracker := NewAPIKeyLastUsedTracker(updater, nil) + tracker.Touch(42) + + select { + case call := <-updater.calls: + if call.id != 42 { + t.Fatalf("key id = %d, want 42", call.id) + } + if call.deadline.IsZero() { + t.Fatal("last-used update context has no deadline") + } + remaining := time.Until(call.deadline) + if remaining <= 0 || remaining > apiKeyLastUsedTimeout { + t.Fatalf("deadline after %s, want within (0, %s]", remaining, apiKeyLastUsedTimeout) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for last-used update") + } +} diff --git a/internal/branding/render.go b/internal/branding/render.go index e5a2d996..14792613 100644 --- a/internal/branding/render.go +++ b/internal/branding/render.go @@ -49,6 +49,23 @@ func (s Snapshot) ThemeColor() string { return DefaultThemeColor } +// RenderKey returns a stable identity covering every snapshot field the +// Render* functions read, so callers can cache rendered output and re-render +// only when the branding configuration actually changes. Keep in sync with +// RenderIndexHTML and RenderManifest. +func (s Snapshot) RenderKey() string { + return strings.Join([]string{ + s.ServerName, + s.LoginSubtitle, + s.AccentColor, + s.DefaultTheme, + s.assets[KindWordmark], + s.assets[KindMark], + s.assets[KindFavicon], + s.assets[KindLoginBg], + }, "\x00") +} + // Favicon link literal as it appears in web/index.html. Kept in sync with that // file; RenderIndexHTML rewrites it to a custom favicon when one is set. const indexFaviconLink = `<link rel="icon" href="/favicon.ico" sizes="any" />` diff --git a/internal/catalog/access_filter.go b/internal/catalog/access_filter.go index ec7b729e..fc83a268 100644 --- a/internal/catalog/access_filter.go +++ b/internal/catalog/access_filter.go @@ -137,10 +137,15 @@ func intInSlice(value int, values []int) bool { return false } -// FilterMediaFilesByAccess drops file versions that exceed the viewer's -// effective quality ceiling. +// FilterMediaFilesByAccess drops file versions the viewer cannot access: +// files in libraries outside their allowed set, in libraries they disabled, +// or above their effective quality ceiling — the same predicate as +// FileAllowedByAccess. func FilterMediaFilesByAccess(files []*models.MediaFile, filter AccessFilter) []*models.MediaFile { - if len(files) == 0 || strings.TrimSpace(filter.MaxPlaybackQuality) == "" { + unrestricted := filter.AllowedLibraryIDs == nil && + len(filter.DisabledLibraryIDs) == 0 && + strings.TrimSpace(filter.MaxPlaybackQuality) == "" + if len(files) == 0 || unrestricted { return files } diff --git a/internal/catalog/access_filter_test.go b/internal/catalog/access_filter_test.go new file mode 100644 index 00000000..2225e3cc --- /dev/null +++ b/internal/catalog/access_filter_test.go @@ -0,0 +1,51 @@ +package catalog + +import ( + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func TestFilterMediaFilesByAccess(t *testing.T) { + allowed := &models.MediaFile{ID: 1, MediaFolderID: 1, Resolution: "1080p"} + otherLibrary := &models.MediaFile{ID: 2, MediaFolderID: 2, Resolution: "2160p"} + files := []*models.MediaFile{allowed, otherLibrary} + + t.Run("no restrictions returns all files", func(t *testing.T) { + got := FilterMediaFilesByAccess(files, AccessFilter{}) + if len(got) != 2 { + t.Fatalf("expected 2 files, got %d", len(got)) + } + }) + + t.Run("allowed library ids filter without quality ceiling", func(t *testing.T) { + got := FilterMediaFilesByAccess(files, AccessFilter{AllowedLibraryIDs: []int{1}}) + if len(got) != 1 || got[0].ID != allowed.ID { + t.Fatalf("expected only file %d, got %v", allowed.ID, got) + } + }) + + t.Run("disabled library ids filter without quality ceiling", func(t *testing.T) { + got := FilterMediaFilesByAccess(files, AccessFilter{DisabledLibraryIDs: []int{2}}) + if len(got) != 1 || got[0].ID != allowed.ID { + t.Fatalf("expected only file %d, got %v", allowed.ID, got) + } + }) + + t.Run("quality ceiling filters", func(t *testing.T) { + got := FilterMediaFilesByAccess(files, AccessFilter{MaxPlaybackQuality: "1080p"}) + if len(got) != 1 || got[0].ID != allowed.ID { + t.Fatalf("expected only file %d, got %v", allowed.ID, got) + } + }) + + t.Run("matches FileAllowedByAccess predicate", func(t *testing.T) { + filter := AccessFilter{AllowedLibraryIDs: []int{1}, MaxPlaybackQuality: "1080p"} + got := FilterMediaFilesByAccess(files, filter) + for _, f := range got { + if !FileAllowedByAccess(f, filter) { + t.Fatalf("file %d returned despite failing FileAllowedByAccess", f.ID) + } + } + }) +} diff --git a/internal/catalog/artwork_selection.go b/internal/catalog/artwork_selection.go new file mode 100644 index 00000000..40d1729c --- /dev/null +++ b/internal/catalog/artwork_selection.go @@ -0,0 +1,322 @@ +package catalog + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + defaultArtworkGCGracePeriod = 24 * time.Hour + artworkTargetSeason = "season" + artworkTargetEpisode = "episode" + artworkImagePoster = "poster" + artworkImageBackdrop = "backdrop" + artworkImageLogo = "logo" + artworkImageStill = "still" + artworkPosterPathColumn = "poster_path" + artworkBackdropPathColumn = "backdrop_path" + artworkLogoPathColumn = "logo_path" + artworkPosterSourceColumn = "poster_source_path" + artworkBackdropSourceColumn = "backdrop_source_path" + artworkLogoSourceColumn = "logo_source_path" + artworkPosterThumbColumn = "poster_thumbhash" + artworkBackdropThumbColumn = "backdrop_thumbhash" +) + +// ErrUnsupportedArtworkSelection reports a target/image-type combination that +// has no artwork column. Handlers should reject these before caching anything. +var ErrUnsupportedArtworkSelection = errors.New("catalog: unsupported artwork selection") + +// ArtworkRevisionTracker registers every immutable artwork upload before its +// objects are written, allowing the collector to reclaim uploads that are +// never published. Confirmed live revisions remain dormant in the registry so +// database triggers can later reactivate them. +type ArtworkRevisionTracker struct { + pool *pgxpool.Pool + gracePeriod time.Duration +} + +func NewArtworkRevisionTracker(pool *pgxpool.Pool) *ArtworkRevisionTracker { + if pool == nil { + return nil + } + return &ArtworkRevisionTracker{pool: pool, gracePeriod: defaultArtworkGCGracePeriod} +} + +// TrackArtworkRevision records the exact object manifest for a revision before +// its upload starts. Image caching calls this before uploading, so it +// serializes with a collector that may already be deleting an older, +// currently-unreferenced copy. Revisions parked as referenced stay dormant: a +// re-cache of live artwork is not garbage, and displacement triggers re-arm +// the row if the reference later moves away. +func (t *ArtworkRevisionTracker) TrackArtworkRevision(ctx context.Context, originalPath, imageType string, objectKeys []string) error { + if t == nil || t.pool == nil { + return fmt.Errorf("catalog: artwork revision tracking is not configured") + } + originalPath = strings.TrimSpace(originalPath) + keys := compactArtworkObjectKeys(objectKeys) + if originalPath == "" || strings.Contains(originalPath, "://") || len(keys) == 0 { + return nil + } + notBefore := time.Now().Add(t.gracePeriod) + // deleted_at is cleared because this upsert precedes a re-upload of the + // exact manifest: the objects exist again once the cacher finishes. + _, err := t.pool.Exec(ctx, ` + INSERT INTO artwork_revision_gc_candidates ( + original_path, image_type, object_keys, not_before, next_attempt_at + ) VALUES ($1, $2, $3, $4, $4) + ON CONFLICT (original_path) DO UPDATE SET + object_keys = EXCLUDED.object_keys, + image_type = CASE + WHEN artwork_revision_gc_candidates.image_type = '' THEN EXCLUDED.image_type + ELSE artwork_revision_gc_candidates.image_type + END, + not_before = CASE + WHEN artwork_revision_gc_candidates.next_attempt_at IS NULL THEN artwork_revision_gc_candidates.not_before + ELSE EXCLUDED.not_before + END, + next_attempt_at = CASE + WHEN artwork_revision_gc_candidates.next_attempt_at IS NULL THEN NULL + ELSE EXCLUDED.next_attempt_at + END, + deleted_at = NULL, + attempt_count = 0, + locked_at = NULL, + locked_by = '', + last_error = '', + updated_at = NOW()`, originalPath, strings.ToLower(strings.TrimSpace(imageType)), keys, notBefore) + if err != nil { + return fmt.Errorf("catalog: track artwork revision: %w", err) + } + return nil +} + +// ArtworkSelection describes a manually selected, already-cached artwork +// revision. PublishArtworkSelection makes the database pointer and image lock +// visible atomically, then schedules the displaced revision for delayed cleanup. +type ArtworkSelection struct { + TargetType string + TargetContentID string + ParentContentID string + ImageType string + StoredPath string + SourcePath string + Thumbhash string + LockField int +} + +// ValidateArtworkSelectionTarget reports whether a target/image-type pair maps +// to a real artwork column, so handlers can reject unsupported requests before +// downloading or uploading anything. +func ValidateArtworkSelectionTarget(targetType, imageType string) error { + _, _, _, _, _, err := artworkTargetColumns( + strings.ToLower(strings.TrimSpace(targetType)), + strings.ToLower(strings.TrimSpace(imageType)), + ) + return err +} + +// PublishArtworkSelection atomically changes the selected artwork, records its +// source, locks automatic image refreshes, and queues the old immutable revision +// for reference-aware garbage collection after a grace period. +func (s *DetailService) PublishArtworkSelection(ctx context.Context, selection ArtworkSelection) error { + if s == nil || s.itemRepo == nil || s.itemRepo.pool == nil { + return fmt.Errorf("catalog: artwork persistence is not configured") + } + selection.TargetType = strings.ToLower(strings.TrimSpace(selection.TargetType)) + selection.ImageType = strings.ToLower(strings.TrimSpace(selection.ImageType)) + selection.TargetContentID = strings.TrimSpace(selection.TargetContentID) + selection.ParentContentID = strings.TrimSpace(selection.ParentContentID) + selection.StoredPath = strings.TrimSpace(selection.StoredPath) + if selection.TargetContentID == "" || selection.StoredPath == "" { + return fmt.Errorf("catalog: artwork target and stored path are required") + } + + table, pathColumn, sourceColumn, thumbhashColumn, notFound, err := artworkTargetColumns(selection.TargetType, selection.ImageType) + if err != nil { + return err + } + + tx, err := s.itemRepo.pool.Begin(ctx) + if err != nil { + return fmt.Errorf("catalog: begin artwork publication: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var previousPath string + selectSQL := fmt.Sprintf("SELECT COALESCE(%s, '') FROM %s WHERE content_id = $1 FOR UPDATE", pathColumn, table) + if err := tx.QueryRow(ctx, selectSQL, selection.TargetContentID).Scan(&previousPath); err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return notFound + } + return fmt.Errorf("catalog: lock artwork target: %w", err) + } + + setThumbhash := "" + args := []any{selection.StoredPath, selection.SourcePath, selection.TargetContentID} + if thumbhashColumn != "" { + setThumbhash = fmt.Sprintf(", %s = $4", thumbhashColumn) + args = append(args, selection.Thumbhash) + } + updateSQL := fmt.Sprintf(` + UPDATE %s + SET %s = $1, %s = $2%s, updated_at = NOW() + WHERE content_id = $3`, table, pathColumn, sourceColumn, setThumbhash) + if _, err := tx.Exec(ctx, updateSQL, args...); err != nil { + return fmt.Errorf("catalog: publish artwork pointer: %w", err) + } + + lockContentID := selection.TargetContentID + if selection.TargetType == artworkTargetSeason || selection.TargetType == artworkTargetEpisode { + lockContentID = selection.ParentContentID + } + if lockContentID == "" { + return fmt.Errorf("catalog: parent content ID is required for %s artwork", selection.TargetType) + } + tag, err := tx.Exec(ctx, ` + UPDATE media_items + SET locked_fields = CASE + WHEN $2 = ANY(COALESCE(locked_fields, '{}'::integer[])) THEN COALESCE(locked_fields, '{}'::integer[]) + ELSE array_append(COALESCE(locked_fields, '{}'::integer[]), $2) + END, + updated_at = NOW() + WHERE content_id = $1`, lockContentID, selection.LockField) + if err != nil { + return fmt.Errorf("catalog: lock manual artwork: %w", err) + } + if tag.RowsAffected() == 0 { + return ErrItemNotFound + } + // The selected revision is referenced by construction once this transaction + // commits, so park it dormant immediately: no first-pass verification cycle + // is needed, and database triggers re-arm it if a later writer displaces it. + if err := parkArtworkRevision(ctx, tx, selection.StoredPath, selection.ImageType, time.Now().Add(defaultArtworkGCGracePeriod)); err != nil { + return err + } + + if previousPath != "" && previousPath != selection.StoredPath { + if err := queueArtworkRevisionGC(ctx, tx, previousPath, selection.ImageType, time.Now().Add(defaultArtworkGCGracePeriod)); err != nil { + return err + } + } + + if err := tx.Commit(ctx); err != nil { + return fmt.Errorf("catalog: commit artwork publication: %w", err) + } + return nil +} + +// QueueArtworkRevisionGC schedules an unreferenced cached revision for cleanup. +// It is used to reclaim an upload when publication fails after object storage +// succeeded. Cleanup still verifies database references before deleting. +func (s *DetailService) QueueArtworkRevisionGC(ctx context.Context, originalPath, imageType string, notBefore time.Time) error { + if s == nil || s.itemRepo == nil || s.itemRepo.pool == nil { + return fmt.Errorf("catalog: artwork persistence is not configured") + } + return queueArtworkRevisionGC(ctx, s.itemRepo.pool, originalPath, imageType, notBefore) +} + +type artworkGCExecer interface { + Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) +} + +// queueArtworkRevisionGC arms a revision for verification after notBefore. The +// stored manifest (when one was tracked pre-upload) is kept; rows without one +// carry the image type so the collector can expand the object keys itself. +func queueArtworkRevisionGC(ctx context.Context, db artworkGCExecer, originalPath, imageType string, notBefore time.Time) error { + return upsertArtworkRevision(ctx, db, originalPath, imageType, notBefore, false) +} + +// parkArtworkRevision registers a revision as referenced and dormant. Only a +// displacement trigger or the collector's dormant sweep re-arms it. +func parkArtworkRevision(ctx context.Context, db artworkGCExecer, originalPath, imageType string, notBefore time.Time) error { + return upsertArtworkRevision(ctx, db, originalPath, imageType, notBefore, true) +} + +func upsertArtworkRevision( + ctx context.Context, + db artworkGCExecer, + originalPath string, + imageType string, + notBefore time.Time, + dormant bool, +) error { + originalPath = strings.TrimSpace(originalPath) + imageType = strings.ToLower(strings.TrimSpace(imageType)) + if originalPath == "" || strings.Contains(originalPath, "://") || imageType == "" { + return nil + } + _, err := db.Exec(ctx, ` + INSERT INTO artwork_revision_gc_candidates ( + original_path, image_type, object_keys, not_before, next_attempt_at + ) VALUES ($1, $2, '{}', $3::timestamptz, CASE WHEN $4 THEN NULL ELSE $3::timestamptz END) + ON CONFLICT (original_path) DO UPDATE SET + image_type = CASE + WHEN artwork_revision_gc_candidates.image_type = '' THEN EXCLUDED.image_type + ELSE artwork_revision_gc_candidates.image_type + END, + not_before = EXCLUDED.not_before, + next_attempt_at = EXCLUDED.next_attempt_at, + attempt_count = 0, + locked_at = NULL, + locked_by = '', + last_error = '', + updated_at = NOW()`, originalPath, imageType, notBefore, dormant) + if err != nil { + return fmt.Errorf("catalog: queue artwork revision cleanup: %w", err) + } + return nil +} + +func compactArtworkObjectKeys(keys []string) []string { + seen := make(map[string]struct{}, len(keys)) + result := make([]string, 0, len(keys)) + for _, key := range keys { + key = strings.TrimSpace(key) + if key == "" || strings.Contains(key, "://") { + continue + } + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + result = append(result, key) + } + return result +} + +// artworkTargetColumns maps a selection target to its artwork columns. Season +// and episode rows have dedicated tables; every other catalog item type +// (movie, series, audiobook, ebook, manga, ...) stores artwork on its +// media_items row. +func artworkTargetColumns(targetType, imageType string) (table, pathColumn, sourceColumn, thumbhashColumn string, notFound error, err error) { + switch targetType { + case artworkTargetSeason: + if imageType == artworkImagePoster { + return "seasons", artworkPosterPathColumn, artworkPosterSourceColumn, artworkPosterThumbColumn, ErrSeasonNotFound, nil + } + case artworkTargetEpisode: + if imageType == artworkImageStill { + return "episodes", "still_path", "still_source_path", "still_thumbhash", ErrEpisodeNotFound, nil + } + default: + table = "media_items" + notFound = ErrItemNotFound + switch imageType { + case artworkImagePoster: + return table, artworkPosterPathColumn, artworkPosterSourceColumn, artworkPosterThumbColumn, notFound, nil + case artworkImageBackdrop: + return table, artworkBackdropPathColumn, artworkBackdropSourceColumn, artworkBackdropThumbColumn, notFound, nil + case artworkImageLogo: + return table, artworkLogoPathColumn, artworkLogoSourceColumn, "", notFound, nil + } + } + return "", "", "", "", nil, fmt.Errorf("%w: %s does not accept %q images", ErrUnsupportedArtworkSelection, targetType, imageType) +} diff --git a/internal/catalog/artwork_selection_test.go b/internal/catalog/artwork_selection_test.go new file mode 100644 index 00000000..345fe322 --- /dev/null +++ b/internal/catalog/artwork_selection_test.go @@ -0,0 +1,101 @@ +package catalog + +import ( + "context" + "fmt" + "os" + "slices" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func newArtworkSelectionTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func TestQueueAndParkArtworkRevisionUpserts(t *testing.T) { + pool := newArtworkSelectionTestPool(t) + ctx := context.Background() + suffix := time.Now().UnixNano() + path := fmt.Sprintf("tmdb/movies/%d/poster/original.rev.webp", suffix) + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path = $1`, path) + }) + + // Arming registers the revision for verification after the grace period. + if err := queueArtworkRevisionGC(ctx, pool, path, "poster", time.Now().Add(time.Hour)); err != nil { + t.Fatalf("queueArtworkRevisionGC: %v", err) + } + var imageType string + var nextAttempt *time.Time + if err := pool.QueryRow(ctx, ` + SELECT image_type, next_attempt_at FROM artwork_revision_gc_candidates + WHERE original_path = $1`, path).Scan(&imageType, &nextAttempt); err != nil { + t.Fatalf("load armed candidate: %v", err) + } + if imageType != "poster" { + t.Fatalf("image_type = %q, want poster", imageType) + } + if nextAttempt == nil { + t.Fatal("armed candidate has NULL next_attempt_at") + } + + // Publication parks the selected revision: referenced by construction. + if err := parkArtworkRevision(ctx, pool, path, "poster", time.Now().Add(time.Hour)); err != nil { + t.Fatalf("parkArtworkRevision: %v", err) + } + if err := pool.QueryRow(ctx, ` + SELECT next_attempt_at FROM artwork_revision_gc_candidates + WHERE original_path = $1`, path).Scan(&nextAttempt); err != nil { + t.Fatalf("load parked candidate: %v", err) + } + if nextAttempt != nil { + t.Fatalf("parked candidate next_attempt_at = %v, want NULL", *nextAttempt) + } +} + +func TestTrackArtworkRevisionKeepsDormantRowsDormant(t *testing.T) { + pool := newArtworkSelectionTestPool(t) + ctx := context.Background() + suffix := time.Now().UnixNano() + path := fmt.Sprintf("tmdb/movies/%d/poster/original.live.webp", suffix) + keys := []string{path, fmt.Sprintf("tmdb/movies/%d/poster/w500.live.webp", suffix)} + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path = $1`, path) + }) + + if err := parkArtworkRevision(ctx, pool, path, "poster", time.Now().Add(time.Hour)); err != nil { + t.Fatalf("parkArtworkRevision: %v", err) + } + // A re-cache of live artwork must not re-arm the parked row. + tracker := NewArtworkRevisionTracker(pool) + if err := tracker.TrackArtworkRevision(ctx, path, "poster", keys); err != nil { + t.Fatalf("TrackArtworkRevision: %v", err) + } + + var nextAttempt *time.Time + var storedKeys []string + if err := pool.QueryRow(ctx, ` + SELECT next_attempt_at, object_keys FROM artwork_revision_gc_candidates + WHERE original_path = $1`, path).Scan(&nextAttempt, &storedKeys); err != nil { + t.Fatalf("load candidate: %v", err) + } + if nextAttempt != nil { + t.Fatalf("re-cache re-armed dormant row: next_attempt_at = %v", *nextAttempt) + } + if !slices.Equal(storedKeys, keys) { + t.Fatalf("stored manifest = %v, want exact tracked manifest %v", storedKeys, keys) + } +} diff --git a/internal/catalog/catalog_resolver.go b/internal/catalog/catalog_resolver.go index 1e01dd11..22994fc3 100644 --- a/internal/catalog/catalog_resolver.go +++ b/internal/catalog/catalog_resolver.go @@ -2253,6 +2253,16 @@ func filterCatalogSearchItems(items []*models.MediaItem, raw string) []*models.M filtered = append(filtered, item) } } + if len(filtered) == 0 && len(items) > 0 && eligibleForFuzzy(parsed) { + // The strict per-token substring filter erases the repo layer's + // typo-tolerant (trigram) matches: a misspelled token is never a + // substring of the titles it fuzzy-matched, so a typo query that + // found results via the fuzzy fallback would be filtered to zero here + // while the plain search box shows matches. When a fuzzy-eligible + // query would be emptied entirely, trust the repo's relevance + // ordering instead. + return items + } return filtered } diff --git a/internal/catalog/detail.go b/internal/catalog/detail.go index 44402469..ab97aef9 100644 --- a/internal/catalog/detail.go +++ b/internal/catalog/detail.go @@ -14,6 +14,7 @@ import ( "time" "github.com/Silo-Server/silo-server/internal/access" + "github.com/Silo-Server/silo-server/internal/artworkkey" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/overlays" "github.com/Silo-Server/silo-server/internal/playback" @@ -3620,10 +3621,7 @@ func cachedImageVariantPath(path, imageType, size string) string { if variant == "" { return path } - if strings.Contains(path, "/original.") { - return strings.Replace(path, "/original.", "/"+variant+".", 1) - } - return path + return artworkkey.Variant(path, variant) } // imageTypeFromCachedPath returns the image type segment ("poster", "backdrop", diff --git a/internal/catalog/folder_repo.go b/internal/catalog/folder_repo.go index a5780fcb..818a1c60 100644 --- a/internal/catalog/folder_repo.go +++ b/internal/catalog/folder_repo.go @@ -11,6 +11,7 @@ import ( "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" + "github.com/Silo-Server/silo-server/internal/artworkkey" "github.com/Silo-Server/silo-server/internal/models" ) @@ -894,14 +895,26 @@ func dirSetToSlice(set map[string]struct{}) []string { return dirs } +// imageDeletePrefix returns the S3 prefix to sweep when deleting the content +// that owns the cached image at path. Local artwork keys carry a content-hash +// segment (local/{contentType}/{contentID}/{hash8}/{imageType}/...), so local +// paths trim to local/{contentType}/{contentID}/ — deleting every stale hash +// prefix, not just the live one. Other keys keep the per-image-type dir. +func imageDeletePrefix(path string) string { + if !strings.HasPrefix(path, "local/") { + return pathDir(path) + } + segments := strings.SplitN(path, "/", 4) + if len(segments) < 4 { + return pathDir(path) + } + return segments[0] + "/" + segments[1] + "/" + segments[2] + "/" +} + // pathDir extracts the directory portion of an S3 image path. // e.g. "tmdb/movies/550/poster/original.webp" → "tmdb/movies/550/poster/" func pathDir(path string) string { - idx := strings.LastIndex(path, "/") - if idx <= 0 { - return "" - } - return path[:idx+1] + return artworkkey.Directory(path) } // rowQuerier is satisfied by both *pgxpool.Pool and pgx.Tx, letting read @@ -967,7 +980,7 @@ func collectRawImageDirs(ctx context.Context, q rowQuerier, contentIDs []string) } for _, p := range []string{p1, p2, p3} { if p != "" && !strings.Contains(p, "://") { - if dir := pathDir(p); dir != "" { + if dir := imageDeletePrefix(p); dir != "" { dirSet[dir] = struct{}{} } } @@ -1035,6 +1048,32 @@ func filterUnreferencedImageDirs(ctx context.Context, q rowQuerier, dirs, deleti } // UpdateLastScanned sets the last_scanned_at timestamp for the given folder. +// LibraryRootsForContent returns the media folder root paths the given +// content belongs to, resolved through media_item_libraries. The metadata +// image-cache processor uses this to confine local file:// artwork sources +// (metadata.LibraryRootResolver). +func (r *FolderRepository) LibraryRootsForContent(ctx context.Context, contentID string) ([]string, error) { + rows, err := r.pool.Query(ctx, ` + SELECT DISTINCT mfp.path + FROM media_folder_paths mfp + JOIN media_item_libraries mil ON mil.media_folder_id = mfp.media_folder_id + WHERE mil.content_id = $1 + `, contentID) + if err != nil { + return nil, fmt.Errorf("querying library roots for content %q: %w", contentID, err) + } + defer rows.Close() + var paths []string + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, fmt.Errorf("scanning library root path: %w", err) + } + paths = append(paths, path) + } + return paths, rows.Err() +} + func (r *FolderRepository) UpdateLastScanned(ctx context.Context, id int, scannedAt time.Time) error { tag, err := r.pool.Exec(ctx, "UPDATE media_folders SET last_scanned_at = $1 WHERE id = $2", diff --git a/internal/catalog/folder_repo_local_roots_test.go b/internal/catalog/folder_repo_local_roots_test.go new file mode 100644 index 00000000..09b13af4 --- /dev/null +++ b/internal/catalog/folder_repo_local_roots_test.go @@ -0,0 +1,77 @@ +package catalog + +import ( + "context" + "fmt" + "os" + "sort" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestLibraryRootsForContent(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + + suffix := time.Now().UnixNano() + contentID := fmt.Sprintf("roots-test-%d", suffix) + var folderID int + if err := pool.QueryRow(ctx, + `INSERT INTO media_folders (type, name, enabled) VALUES ('movies', $1, true) RETURNING id`, + fmt.Sprintf("roots-test-%d", suffix)).Scan(&folderID); err != nil { + t.Fatalf("seed folder: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_item_libraries WHERE content_id = $1`, contentID) + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) + _, _ = pool.Exec(ctx, `DELETE FROM media_folder_paths WHERE media_folder_id = $1`, folderID) + _, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, folderID) + }) + if _, err := pool.Exec(ctx, + `INSERT INTO media_items (content_id, type, title, status, genres) VALUES ($1, 'movie', 'Roots Test', 'matched', '{}'::text[])`, + contentID); err != nil { + t.Fatalf("seed item: %v", err) + } + for _, path := range []string{"/media/movies-a", "/media/movies-b"} { + if _, err := pool.Exec(ctx, + `INSERT INTO media_folder_paths (media_folder_id, path) VALUES ($1, $2)`, + folderID, fmt.Sprintf("%s-%d", path, suffix)); err != nil { + t.Fatalf("seed folder path: %v", err) + } + } + if _, err := pool.Exec(ctx, + `INSERT INTO media_item_libraries (content_id, media_folder_id) VALUES ($1, $2)`, + contentID, folderID); err != nil { + t.Fatalf("seed membership: %v", err) + } + + repo := NewFolderRepository(pool) + roots, err := repo.LibraryRootsForContent(ctx, contentID) + if err != nil { + t.Fatalf("LibraryRootsForContent: %v", err) + } + sort.Strings(roots) + want := []string{ + fmt.Sprintf("/media/movies-a-%d", suffix), + fmt.Sprintf("/media/movies-b-%d", suffix), + } + if len(roots) != 2 || roots[0] != want[0] || roots[1] != want[1] { + t.Fatalf("roots = %v, want %v", roots, want) + } + + // Unknown content has no roots (and no error). + none, err := repo.LibraryRootsForContent(ctx, contentID+"-missing") + if err != nil || len(none) != 0 { + t.Fatalf("missing content: roots=%v err=%v", none, err) + } +} diff --git a/internal/catalog/item_repo.go b/internal/catalog/item_repo.go index efa41285..1dee0fed 100644 --- a/internal/catalog/item_repo.go +++ b/internal/catalog/item_repo.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log/slog" "strconv" "strings" @@ -131,6 +132,40 @@ func (r *ItemRepository) GetPosterPath(ctx context.Context, contentID string) (s // incidental one-mention hits that flooded results before. const overviewMatchFloor = 0.15 +// fuzzyFallbackThreshold is the FTS result count below which SearchPage reaches +// for the trigram fuzzy fallback (buildFuzzySearchSQL). At or above it the FTS +// result set is considered rich enough that a "did you mean" pass would only add +// noise, so common searches stay on the unchanged FTS path and the fuzzy query +// fires only in the sparse/misspelling case. +const fuzzyFallbackThreshold = 5 + +// fuzzyMaxResults caps how many trigram fuzzy matches the fallback contributes. +// The combined result the fuzzy path serves is therefore bounded at +// (fuzzyFallbackThreshold-1) FTS rows + fuzzyMaxResults fuzzy rows, small enough +// to materialize once and slice per page. A "did you mean" fallback has no need +// to paginate hundreds of low-similarity typo matches; when the true match count +// exceeds this, the overflow is logged and the total is reported as inexact. +const fuzzyMaxResults = 50 + +// trgmWordSimilarityThreshold pins pg_trgm.strict_word_similarity_threshold +// for the fuzzy query via SET LOCAL. The <<% operator's selectivity is +// otherwise governed by a cluster-wide GUC a DBA (or another application) can +// change, silently altering both match quality and scan cost per deployment — +// and its server default (0.6) would reject ordinary one-edit typos +// ("avegners" vs "avengers" scores ~0.38), so pinning is load-bearing, not +// just hygiene. +const trgmWordSimilarityThreshold = 0.30 + +// fuzzyAugmentSimilarityFloor is the minimum WHOLE-TITLE similarity demanded of +// fuzzy rows when the sparse FTS block is non-empty. A correctly spelled query +// with a few genuine hits ("coraline") should only gain near-identical titles, +// not every title clearing the permissive base threshold — word similarity is +// deliberately not used here, since it scores embedded prefix words far too +// high ("coral" is 0.5 to "coraline"). A zero-hit query (a likely misspelling) +// skips the floor entirely so typo recall, including word-extent matches into +// long titles, stays high. +const fuzzyAugmentSimilarityFloor = 0.45 + // itemColumnNames lists, in scan order, every column selected by media_items // item queries. Shared by itemColumns, qualifiedItemColumns, and // qualifiedListItemColumns so the select lists can never drift from each @@ -194,6 +229,17 @@ func qualifiedItemColumns(alias string) string { return joinItemColumns(alias) } +// qualifiedItemColumnRefs renders plain alias-qualified column references +// without COALESCE or AS aliases, for contexts like GROUP BY where output +// aliases are invalid. +func qualifiedItemColumnRefs(alias string) string { + refs := make([]string, len(itemColumnNames)) + for i, col := range itemColumnNames { + refs[i] = alias + "." + col + } + return strings.Join(refs, ", ") +} + func qualifiedListItemColumns(alias string) string { exprs := make([]string, len(itemColumnNames)) for i, col := range itemColumnNames { @@ -823,7 +869,7 @@ func (r *ItemRepository) Delete(ctx context.Context, contentID string) ([]string } for _, p := range []string{p1, p2, p3} { if p != "" && !strings.Contains(p, "://") { - if dir := pathDir(p); dir != "" { + if dir := imageDeletePrefix(p); dir != "" { imageDirs = append(imageDirs, dir) } } @@ -873,26 +919,241 @@ func (r *ItemRepository) SearchPage( if filter.AllowedLibraryIDs != nil && len(filter.AllowedLibraryIDs) == 0 { return []*models.MediaItem{}, 0, false, includeTotal, nil } + + parsed := parseSearchQuery(query) + + // FTS block (the common path). queryLimit fetches one extra row in cursor + // mode so execSearchBlock can derive hasMore without a separate count. When + // the fuzzy fallback below could fire (first page of a fuzzy-eligible + // query) the probe is additionally floored at fuzzyFallbackThreshold rows: + // a tiny caller limit (e.g. an autocomplete asking for 2) with a few + // incidental exact hits would otherwise report hasMore and hide the FTS + // block's true size, so the sparse test below could never fire. Fetching up + // to the threshold reveals a genuinely sparse block regardless of limit; + // the returned page is still trimmed to limit. Where the fallback cannot + // fire, fetching past limit+1 would be pure waste, so the floor is skipped. queryLimit := limit if !includeTotal { queryLimit = limit + 1 + if offset == 0 && eligibleForFuzzy(parsed) && queryLimit < fuzzyFallbackThreshold { + queryLimit = fuzzyFallbackThreshold + } } - sql, countSQL, args := r.buildSearchSQLWithTotal(query, itemTypes, queryLimit, offset, filter, includeTotal) + sql, countSQL, args := r.buildSearchSQLFromParsed(parsed, itemTypes, queryLimit, offset, filter, includeTotal) if sql == "" { return []*models.MediaItem{}, 0, false, includeTotal, nil } - - rows, err := r.pool.Query(ctx, sql, args...) + ftsItems, ftsTotal, ftsHasMore, ftsUntrimmed, err := r.execSearchBlock(ctx, r.pool, sql, countSQL, args, limit, offset, includeTotal) if err != nil { - return nil, 0, false, includeTotal, fmt.Errorf("searching media items: %w", err) + return nil, 0, false, includeTotal, err + } + + // Fuzzy fallback trigger. The trigram (typo-tolerant) arm is deliberately NOT + // part of the FTS query: OR-ing the pg_trgm % operator into that WHERE forces + // a lossy bitmap heap recheck that rebuilds the three title tsvectors for + // every one of the thousands of near-miss rows the % operator surfaces, + // making *every* search take seconds. Instead we reach for the trigram index + // only when the FTS result set is fully known and sparse (a likely + // misspelling or word-boundary miss). + // + // Exact mode judges sparsity by the window count (page-independent), so every + // page of a sparse query agrees and the combined result paginates fully. + // + // Cursor mode has no window count, so it uses the pre-trim size of a probe + // floored at fuzzyFallbackThreshold rows: fewer than the threshold came + // back ⇒ the whole FTS block is smaller than the threshold ⇒ sparse, + // independent of the caller's page size. It only trusts this on the first + // page (offset 0), where searchWithFuzzyFallback serves fuzzy as a terminal + // augmentation (no phantom next page a bare offset cursor couldn't locate), + // and only when the whole FTS block fits inside the caller's page: a + // terminal page must never hide exact matches the plain cursor path would + // have surfaced via hasMore. Past offset 0 an empty FTS page can't be told + // from a rich one, so sparse stays false. + var sparse bool + if includeTotal { + sparse = ftsTotal < fuzzyFallbackThreshold + } else if offset == 0 { + sparse = len(ftsUntrimmed) < fuzzyFallbackThreshold && len(ftsUntrimmed) <= limit + } + if !sparse || !eligibleForFuzzy(parsed) { + return ftsItems, ftsTotal, ftsHasMore, includeTotal, nil + } + + // Hand the fallback the FTS block when the rows just fetched provably form + // the complete block, sparing it an identical second FTS query. Cursor mode + // always qualifies here (the floored probe returned fewer rows than it + // asked for); exact mode qualifies on the first page when the window count + // fit within the page. + ftsBlock, haveFTSBlock := []*models.MediaItem(nil), false + if !includeTotal { + ftsBlock, haveFTSBlock = ftsUntrimmed, true + } else if offset == 0 && ftsTotal <= len(ftsItems) { + ftsBlock, haveFTSBlock = ftsItems, true + } + return r.searchWithFuzzyFallback(ctx, parsed, itemTypes, limit, offset, filter, includeTotal, ftsBlock, haveFTSBlock) +} + +// searchWithFuzzyFallback serves the combined [FTS block][fuzzy block] result +// for a sparse, fuzzy-eligible query. Because fuzzy only fires when the FTS +// block is sparse (< fuzzyFallbackThreshold rows) and the fuzzy contribution is +// capped at fuzzyMaxResults, the ENTIRE combined result is small +// (< fuzzyFallbackThreshold + fuzzyMaxResults rows). We materialize it once and +// slice the requested page in memory. That keeps two-block pagination correct on +// every page — stable total, no cross-page duplicates, no offset arithmetic +// straddling the block boundary — for both exact and cursor callers. The +// FTS-only path in SearchPage is untouched and still streams strictly per page. +// +// When haveFTSBlock is true, ftsBlock is the complete sparse FTS block the +// caller already fetched and no second FTS query is issued; otherwise (exact +// mode past the first page, or a first page smaller than the block) the block +// is re-fetched here at offset 0 — its ids drive fuzzy dedup below. +func (r *ItemRepository) searchWithFuzzyFallback( + ctx context.Context, + parsed parsedSearchQuery, + itemTypes []string, + limit, offset int, + filter AccessFilter, + includeTotal bool, + ftsBlock []*models.MediaItem, + haveFTSBlock bool, +) ([]*models.MediaItem, int, bool, bool, error) { + if !haveFTSBlock { + ftsSQL, ftsCountSQL, ftsArgs := r.buildSearchSQLFromParsed(parsed, itemTypes, fuzzyFallbackThreshold, 0, filter, true) + var err error + ftsBlock, _, _, _, err = r.execSearchBlock(ctx, r.pool, ftsSQL, ftsCountSQL, ftsArgs, fuzzyFallbackThreshold, 0, true) + if err != nil { + return nil, 0, false, includeTotal, err + } + } + + // Fuzzy block, excluding every FTS-block id so no title appears in both + // blocks and the combined total is stable per page. Cursor mode serves the + // combined result as a single terminal page, so it never needs more fuzzy + // rows than the page has room for; exact mode materializes up to the cap. + // Either way one extra row is fetched so truncation is detectable without + // paying a COUNT(*) OVER () window count over the whole trigram match set. + // When the FTS block found real hits the fuzzy arm demands near-certain + // corrections (fuzzyAugmentSimilarityFloor) instead of base-threshold noise. + fuzzyLimit := fuzzyMaxResults + if !includeTotal && limit-len(ftsBlock) < fuzzyLimit { + fuzzyLimit = limit - len(ftsBlock) + } + minSimilarity := 0.0 + if len(ftsBlock) > 0 { + minSimilarity = fuzzyAugmentSimilarityFloor + } + var fuzzyBlock []*models.MediaItem + fuzzyTruncated := false + if fuzzyLimit > 0 { + fuzzySQL, _, fuzzyArgs := r.buildFuzzySearchFromParsed(parsed, itemTypes, fuzzyLimit+1, 0, filter, false, contentIDsFromMediaItems(ftsBlock), minSimilarity) + if fuzzySQL != "" { + var err error + fuzzyBlock, fuzzyTruncated, err = r.execFuzzyBlock(ctx, fuzzySQL, fuzzyArgs, fuzzyLimit) + if err != nil { + return nil, 0, false, includeTotal, err + } + if fuzzyTruncated { + slog.DebugContext(ctx, "fuzzy search fallback truncated to cap", + "query", parsed.Text, "cap", fuzzyLimit) + } + } + } + + combined := make([]*models.MediaItem, 0, len(ftsBlock)+len(fuzzyBlock)) + combined = append(combined, ftsBlock...) + combined = append(combined, fuzzyBlock...) + total := len(combined) + + // Slice the requested page out of the materialized list, clamped so a + // caller passing a negative offset or an overflowing limit (the exported + // SearchPage is reachable outside the HTTP parser, which otherwise + // guarantees offset>=0 and limit>0) gets a sane page instead of a + // reversed-slice panic. + lo := min(max(offset, 0), total) + hi := lo + max(limit, 0) + if hi < lo || hi > total { // hi < lo: lo+limit overflowed; serve the rest + hi = total + } + page := combined[lo:hi] + + if !includeTotal { + // Cursor mode reaches here only at offset 0 with the whole FTS block + // inside the page (see SearchPage gate), so no exact match is hidden; + // only fuzzy augmentation past the page is cut. Serve as a terminal + // page: report only what we return and no next page, since a bare + // offset cursor can't locate the FTS/fuzzy boundary on a follow-up + // request. + return page, len(page), false, false, nil + } + // When the fuzzy arm hit its cap the true match count is larger than the + // materialized list, so surface the total as inexact rather than presenting + // the cap as an exact count. + return page, total, hi < total, !fuzzyTruncated, nil +} + +// execFuzzyBlock runs the fuzzy data query inside a transaction that pins +// pg_trgm.strict_word_similarity_threshold, so the <<% operator's selectivity +// cannot drift with cluster configuration (its 0.6 server default would reject +// ordinary typos outright). The query was built with LIMIT fuzzyLimit+1; the +// extra row only signals truncation and is trimmed from the returned block. +func (r *ItemRepository) execFuzzyBlock(ctx context.Context, dataSQL string, args []any, fuzzyLimit int) ([]*models.MediaItem, bool, error) { + tx, err := r.pool.Begin(ctx) + if err != nil { + return nil, false, fmt.Errorf("beginning fuzzy search tx: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + if _, err := tx.Exec(ctx, fmt.Sprintf("SET LOCAL pg_trgm.strict_word_similarity_threshold = %g", trgmWordSimilarityThreshold)); err != nil { + return nil, false, fmt.Errorf("pinning trigram word-similarity threshold: %w", err) + } + block, _, truncated, _, err := r.execSearchBlock(ctx, tx, dataSQL, "", args, fuzzyLimit, 0, false) + if err != nil { + return nil, false, err + } + if err := tx.Commit(ctx); err != nil { + return nil, false, fmt.Errorf("committing fuzzy search tx: %w", err) + } + return block, truncated, nil +} + +// searchQuerier is the subset of pgxpool.Pool / pgx.Tx that execSearchBlock +// needs, so a block can run either directly on the pool or inside a +// transaction (the fuzzy block pins a GUC with SET LOCAL). +type searchQuerier interface { + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +// execSearchBlock runs one search data query (FTS or fuzzy fallback) and its +// count-fallback sibling, returning that block's page. It is the shared engine +// behind SearchPage's two blocks so both derive hasMore/total identically. +// +// In cursor mode (includeTotal == false) the caller fetches limit+1 rows so +// hasMore is len(items) > limit; the extra row is trimmed here. In exact mode +// the data query carries COUNT(*) OVER () for the total, which emits no rows on +// an empty page (e.g. OFFSET past the last row); only then, and only past +// offset 0, is the count sibling re-run to recover the real total. Both queries +// must end with the trailing limit/offset args so the count sibling can drop +// them. +// +// The fourth return value is the untrimmed row set the data query actually +// returned (before the cursor-mode +1 row is trimmed; in exact mode it equals +// the page). SearchPage uses it in cursor mode to judge FTS-block sparsity +// independently of the caller's page size and to hand the fuzzy fallback the +// complete block without a second FTS query. +func (r *ItemRepository) execSearchBlock(ctx context.Context, q searchQuerier, dataSQL, countSQL string, args []any, limit, offset int, includeTotal bool) ([]*models.MediaItem, int, bool, []*models.MediaItem, error) { + rows, err := q.Query(ctx, dataSQL, args...) + if err != nil { + return nil, 0, false, nil, fmt.Errorf("searching media items: %w", err) } defer rows.Close() if !includeTotal { items, err := scanItems(rows) if err != nil { - return nil, 0, false, false, err + return nil, 0, false, nil, err } + untrimmed := items hasMore := len(items) > limit if hasMore { items = items[:limit] @@ -901,28 +1162,23 @@ func (r *ItemRepository) SearchPage( if hasMore { total++ } - return items, total, hasMore, false, nil + return items, total, hasMore, untrimmed, nil } items, total, err := scanItemsWithTotal(rows) if err != nil { - return nil, 0, false, true, err + return nil, 0, false, nil, err } hasMore := total > offset+len(items) - // COUNT(*) OVER () emits no rows when the data SELECT is empty, so total - // stays 0 even when the broader result set has matching rows (e.g. OFFSET - // past the last page). Re-query the count to give callers the real total. - // Skip when offset == 0 because in that case an empty page genuinely means - // total = 0. if len(items) == 0 && offset > 0 { // Drop the trailing limit/offset args from the data query. countArgs := args[:len(args)-2] - if err := r.pool.QueryRow(ctx, countSQL, countArgs...).Scan(&total); err != nil { - return nil, 0, false, true, fmt.Errorf("count fallback for empty search page: %w", err) + if err := q.QueryRow(ctx, countSQL, countArgs...).Scan(&total); err != nil { + return nil, 0, false, nil, fmt.Errorf("count fallback for empty search page: %w", err) } hasMore = total > offset+len(items) } - return items, total, hasMore, true, nil + return items, total, hasMore, items, nil } // buildSearchSQL assembles the unified search query, returning the SQL string @@ -949,12 +1205,181 @@ func (r *ItemRepository) SearchPage( // parsed.Year (or NULL) // parsed.Phrase // limit, offset +// +// searchTextFromParsed derives the effective search text shared by the FTS and +// fuzzy builders: the parsed Text, or (when the query was only quotes/whitespace +// that parsed to empty Text) a whitespace-collapsed, quote-stripped fallback off +// the raw query. Centralized so the two builders can never search different text. +func searchTextFromParsed(parsed parsedSearchQuery) string { + if parsed.Text != "" { + return parsed.Text + } + return collapseSearchWhitespace(strings.ReplaceAll(strings.TrimSpace(parsed.Raw), "\"", " ")) +} + func (r *ItemRepository) buildSearchSQL(query string, itemTypes []string, limit, offset int, filter AccessFilter) (dataSQL, countSQL string, args []any) { return r.buildSearchSQLWithTotal(query, itemTypes, limit, offset, filter, true) } func (r *ItemRepository) buildSearchSQLWithTotal(query string, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool) (dataSQL, countSQL string, args []any) { - return r.buildMixedSearchSQLWithTotal(query, itemTypes, limit, offset, filter, includeTotal) + return r.buildSearchSQLFromParsed(parseSearchQuery(query), itemTypes, limit, offset, filter, includeTotal) +} + +// buildSearchSQLFromParsed is the FTS query builder proper; the string-taking +// wrappers above parse first. SearchPage parses once and calls this directly so +// the same parsedSearchQuery feeds the FTS builder, the fuzzy builder, and the +// eligibility gate without re-parsing. The mixed builder unions media_items and +// episode candidates into one ranked page. +func (r *ItemRepository) buildSearchSQLFromParsed(parsed parsedSearchQuery, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool) (dataSQL, countSQL string, args []any) { + return r.buildMixedSearchSQLFromParsed(parsed, itemTypes, limit, offset, filter, includeTotal) +} + +// appendSearchScopeFilters appends the scope predicates shared by the FTS search +// (buildSearchSQLWithTotal) and the trigram fuzzy fallback (buildFuzzySearchSQL): +// item type, allowed/disabled libraries, the access filter, and the +// manga-chapter exclusion. It mutates conditions/args/argIdx in place and +// returns the FROM clause (always "media_items mi"; library scoping is enforced +// with independent EXISTS/NOT EXISTS subqueries rather than a JOIN). Both callers +// append these in the same order so a single helper keeps the two queries' +// filtering provably identical. +func appendSearchScopeFilters(itemTypes []string, filter AccessFilter, conditions *[]string, args *[]any, argIdx *int) (fromClause string) { + fromClause = "media_items mi" + + if len(itemTypes) > 0 { + placeholders := make([]string, 0, len(itemTypes)) + for _, itemType := range itemTypes { + if strings.TrimSpace(itemType) == "" { + continue + } + placeholders = append(placeholders, fmt.Sprintf("$%d", *argIdx)) + *args = append(*args, strings.ToLower(strings.TrimSpace(itemType))) + *argIdx++ + } + if len(placeholders) > 0 { + *conditions = append(*conditions, fmt.Sprintf("mi.type IN (%s)", strings.Join(placeholders, ", "))) + } + } + + // Library allow/deny via the shared leak-safe helper: an item linked to both a + // passing and a disabled library must not slip through. The old JOIN + + // NOT(mil.media_folder_id = ANY(...)) form let the passing membership row + // satisfy the deny check (audit 2026-05-01 §3.3), so both the FTS search and + // the fuzzy fallback that share this helper used it. appendLibraryAccessConditions + // emits independent EXISTS/NOT EXISTS subqueries keyed on mi.content_id and + // needs no JOIN. + appendLibraryAccessConditions("mi.content_id", filter, conditions, args, argIdx) + + applyAccessFilter("mi", AccessFilter{MaxContentRating: filter.MaxContentRating, ExcludedMediaTypes: filter.ExcludedMediaTypes}, conditions, args, argIdx) + + // Manga chapters (type='ebook' rows linked into a manga series) are internal + // sub-units and must never surface as standalone search results. + *conditions = append(*conditions, MangaChapterExclusionWhere("mi")) + return fromClause +} + +// buildFuzzySearchSQL assembles the trigram fuzzy-fallback query invoked by +// SearchPage only when the FTS query is sparse. Unlike buildSearchSQLWithTotal, +// the sole match predicate is the pg_trgm strict-word-similarity operator +// (<<%) against the indexed title_normalized generated column (migration 105's +// gin_trgm_ops index serves it), and the ranking signals are +// strict_word_similarity()/similarity() on that same column. Crucially it +// never references the title tsvectors, so the low-similarity rows the +// operator can surface never pay a per-row tsvector rebuild — that rebuild +// over the fuzzy candidate set was the entire cause of the multi-second search +// regression. The operator's base selectivity is pinned by the caller +// (execFuzzyBlock sets pg_trgm.strict_word_similarity_threshold via SET LOCAL) +// so it cannot drift with cluster configuration. +// +// Scope note: only title_normalized — a generated column over `title` — is +// fuzzy-matched, because it is the only column with a gin_trgm_ops index +// (migration 105). Typos of original_title or sort_title, which the FTS arm +// covers exactly, are deliberately out of the fuzzy arm's scope; extending it +// would need matching normalized columns + trigram indexes (follow-up). +// +// includeTotal toggles the COUNT(*) OVER () window total the same way +// buildSearchSQLWithTotal does, so the fuzzy block honors the caller's cursor +// vs. exact-count mode. excludeContentIDs drops rows already returned by the FTS +// block so a title matching both is not shown twice. minSimilarity > 0 adds an +// explicit similarity floor above the base threshold — used when the FTS block +// had real hits so augmentation only admits near-certain corrections. Argument +// order: $1 searchText, then the similarity floor (if any), then the shared +// scope placeholders, then the exclusion array, then limit/offset. +func (r *ItemRepository) buildFuzzySearchSQL(query string, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool, excludeContentIDs []string, minSimilarity float64) (dataSQL, countSQL string, args []any) { + return r.buildFuzzySearchFromParsed(parseSearchQuery(query), itemTypes, limit, offset, filter, includeTotal, excludeContentIDs, minSimilarity) +} + +// buildFuzzySearchFromParsed is the fuzzy query builder proper; the string-taking +// wrapper above parses first. SearchPage parses once and calls this directly. +func (r *ItemRepository) buildFuzzySearchFromParsed(parsed parsedSearchQuery, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool, excludeContentIDs []string, minSimilarity float64) (dataSQL, countSQL string, args []any) { + searchText := searchTextFromParsed(parsed) + if searchText == "" { + return "", "", nil + } + + args = []any{searchText} + argIdx := 2 + // Strict word similarity (<<%) rather than full-string similarity (%): a + // typo of one word must still reach long titles ("avegners" → "Avengers: + // Endgame"), where full-string similarity is diluted by every extra trigram + // the rest of the title contributes. <<% scores the query against the best + // word-boundary extent of the title instead, and at equal thresholds it is + // a strict superset of % (the whole string is itself a valid extent). The + // same gin_trgm_ops index serves both operators. + conditions := []string{"public.normalize_search_text($1) <<% mi.title_normalized"} + if minSimilarity > 0 { + // The floor is whole-title similarity, NOT word similarity: augmenting + // a query that already has hits must only admit near-identical titles, + // and word similarity rates embedded prefix words far too high (see + // fuzzyAugmentSimilarityFloor). + conditions = append(conditions, fmt.Sprintf("similarity(public.normalize_search_text($1), mi.title_normalized) >= $%d", argIdx)) + args = append(args, minSimilarity) + argIdx++ + } + + fromClause := appendSearchScopeFilters(itemTypes, filter, &conditions, &args, &argIdx) + + if len(excludeContentIDs) > 0 { + conditions = append(conditions, fmt.Sprintf("NOT (mi.content_id = ANY($%d))", argIdx)) + args = append(args, excludeContentIDs) + argIdx++ + } + + whereClause := "WHERE " + strings.Join(conditions, " AND ") + + // GROUP BY is required so the MAX(similarity(...)) ranking aggregate is legal, + // mirroring buildSearchSQLWithTotal; COUNT(*) OVER () then counts distinct + // content_ids. qualifiedItemColumns aliases the coalesced columns so the outer + // SELECT can re-reference them; GROUP BY uses the raw refs. + qualifiedCols := qualifiedItemColumns("mi") + groupByCols := qualifiedItemColumnRefs("mi") + // fuzzy_rank orders by how well the query matches SOME word extent of the + // title; fuzzy_full_rank tie-breaks by whole-title closeness so "The + // Avengers" sorts above "Avengers: Endgame" for the typo "avegners". Both + // are computed only over the matched candidate set. + scoredCTE := fmt.Sprintf(` + WITH scored AS ( + SELECT + %s, + MAX(strict_word_similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_rank, + MAX(similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_full_rank + FROM %s + %s + GROUP BY %s + ) + `, qualifiedCols, fromClause, whereClause, groupByCols) + + totalColumn := "" + if includeTotal { + totalColumn = ", COUNT(*) OVER () AS total_count" + } + dataSQL = scoredCTE + fmt.Sprintf(` + SELECT %s%s + FROM scored + ORDER BY fuzzy_rank DESC, fuzzy_full_rank DESC, LOWER(title) ASC, content_id ASC + LIMIT $%d OFFSET $%d`, itemColumns, totalColumn, argIdx, argIdx+1) + countSQL = scoredCTE + `SELECT COUNT(*) FROM scored` + args = append(args, limit, offset) + return dataSQL, countSQL, args } // ListUnmatchedByFolderAndPathPrefix returns content IDs for unmatched-style diff --git a/internal/catalog/item_repo_test.go b/internal/catalog/item_repo_test.go index 4922761e..9e6e6dbe 100644 --- a/internal/catalog/item_repo_test.go +++ b/internal/catalog/item_repo_test.go @@ -382,6 +382,223 @@ func TestItemRepo_Search_EmptyQueryReturnsEmpty(t *testing.T) { } } +// TestEligibleForFuzzy pins the min-token gate that guards the fuzzy title +// fallback: the longest normalized token must clear fuzzyMinTokenLen, so short, +// non-selective queries stay on the exact FTS/prefix path. +func TestEligibleForFuzzy(t *testing.T) { + cases := []struct { + query string + want bool + }{ + {"avegners", true}, // 8-char typo + {"sponge bob", true}, // longest token "sponge" (6) + {"a vengers", true}, // judged on "vengers" (7), not "a" + {"dune", true}, // exactly at the floor + {"the", false}, // 3 chars + {"a b c", false}, // all short + {"and the", false}, // "and" normalized away, "the" is 3 + {" ", false}, // empty + } + for _, tc := range cases { + if got := eligibleForFuzzy(parseSearchQuery(tc.query)); got != tc.want { + t.Errorf("eligibleForFuzzy(%q) = %v, want %v", tc.query, got, tc.want) + } + } +} + +// TestSearchTextFromParsed pins the shared searchText derivation used by both +// the FTS and fuzzy builders: parsed Text when present, else a quote-stripped +// fallback off the raw query. Both builders must agree so they search the same +// text. +func TestSearchTextFromParsed(t *testing.T) { + cases := []struct { + raw string + want string + }{ + {"interstellar", "interstellar"}, + {`"the matrix"`, "the matrix"}, + {" spaced out ", "spaced out"}, + {`"`, ""}, + {" ", ""}, + } + for _, tc := range cases { + if got := searchTextFromParsed(parseSearchQuery(tc.raw)); got != tc.want { + t.Errorf("searchTextFromParsed(%q) = %q, want %q", tc.raw, got, tc.want) + } + } +} + +// TestParseSearchQuery_NormalizedText asserts parseSearchQuery precomputes the +// full normalized text that eligibleForFuzzy's token gate reads (so the gate +// does not re-normalize on every sparse search). +func TestParseSearchQuery_NormalizedText(t *testing.T) { + if got := parseSearchQuery("The Avengers").NormalizedText; got != "the avengers" { + t.Fatalf("NormalizedText = %q, want %q", got, "the avengers") + } + // "and" is dropped and folded, matching normalizeTitleForComparison. + if got := parseSearchQuery("law and order").NormalizedText; got != "law order" { + t.Fatalf("NormalizedText = %q, want %q", got, "law order") + } +} + +// TestItemRepo_Search_FTSQueryHasNoFuzzyArm asserts that the FTS query never +// carries the trigram % arm or its similarity ranking. Fusing them forced a +// lossy bitmap recheck that rebuilt the title tsvectors for thousands of +// near-miss rows on every search; the fuzzy path is now a separate query +// (buildFuzzySearchSQL) invoked only as a fallback by SearchPage. +func TestItemRepo_Search_FTSQueryHasNoFuzzyArm(t *testing.T) { + repo := &ItemRepository{} + dataSQL, countSQL, _ := repo.buildSearchSQL("avegners", []string{"movie"}, 20, 0, AccessFilter{}) + for _, sql := range []string{dataSQL, countSQL} { + if strings.Contains(sql, "% mi.title_normalized") { + t.Fatalf("FTS query must not include the trigram %% arm; got:\n%s", sql) + } + if strings.Contains(sql, "fuzzy_rank") { + t.Fatalf("FTS query must not include fuzzy_rank; got:\n%s", sql) + } + if strings.Contains(sql, "similarity(") { + t.Fatalf("FTS query must not compute similarity(); got:\n%s", sql) + } + } +} + +// TestItemRepo_BuildFuzzySearchSQL asserts that the fuzzy fallback query scores +// only on the indexed title_normalized column (no title tsvector rebuild), +// matches via strict word similarity so long titles stay reachable, ranks by +// descending word similarity with whole-title closeness as tie-break, excludes +// already-seen content_ids, and applies the same scope filters (type, manga +// exclusion) as the FTS query. +func TestItemRepo_BuildFuzzySearchSQL(t *testing.T) { + repo := &ItemRepository{} + dataSQL, countSQL, args := repo.buildFuzzySearchSQL("avegners", []string{"movie"}, 20, 0, AccessFilter{}, true, []string{"abc", "def"}, 0) + + // <<%, not %: full-string similarity is diluted by long titles ("avegners" + // must reach "Avengers: Endgame"); the same gin_trgm_ops index serves both. + if !strings.Contains(dataSQL, "public.normalize_search_text($1) <<% mi.title_normalized") { + t.Fatalf("expected strict-word-similarity arm against title_normalized; got:\n%s", dataSQL) + } + if !strings.Contains(dataSQL, "MAX(strict_word_similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_rank") { + t.Fatalf("expected strict word similarity ranking on title_normalized; got:\n%s", dataSQL) + } + if !strings.Contains(dataSQL, "MAX(similarity(public.normalize_search_text($1), mi.title_normalized)) AS fuzzy_full_rank") { + t.Fatalf("expected whole-title similarity tie-break rank; got:\n%s", dataSQL) + } + // The whole point of the separate query: it must never rebuild the title + // tsvectors that made the fused query slow. + if strings.Contains(dataSQL, "to_tsvector") || strings.Contains(dataSQL, "ts_rank_cd") { + t.Fatalf("fuzzy query must not rebuild title tsvectors; got:\n%s", dataSQL) + } + if !strings.Contains(dataSQL, "ORDER BY fuzzy_rank DESC, fuzzy_full_rank DESC, LOWER(title) ASC, content_id ASC") { + t.Fatalf("expected similarity-ordered results; got:\n%s", dataSQL) + } + if !strings.Contains(dataSQL, "NOT (mi.content_id = ANY($") { + t.Fatalf("expected exclusion of already-seen content_ids; got:\n%s", dataSQL) + } + if !strings.Contains(dataSQL, "mi.type IN ($") { + t.Fatalf("expected shared type scope filter; got:\n%s", dataSQL) + } + if !strings.Contains(countSQL, "SELECT COUNT(*) FROM scored") { + t.Fatalf("expected count sibling over the scored CTE; got:\n%s", countSQL) + } + // Arg order: $1 searchText, $2 type, $3 exclusion array, $4 limit, $5 offset. + if len(args) != 5 { + t.Fatalf("expected 5 args, got %d: %#v", len(args), args) + } + if args[0] != "avegners" { + t.Fatalf("expected $1=searchText; got %#v", args[0]) + } + if args[len(args)-2] != 20 || args[len(args)-1] != 0 { + t.Fatalf("expected trailing limit/offset args; got %#v", args[len(args)-2:]) + } +} + +// TestItemRepo_BuildFuzzySearchSQL_CursorModeOmitsWindowCount asserts that the +// fuzzy fallback honors cursor mode: with includeTotal=false it must not carry +// COUNT(*) OVER (), matching buildSearchSQLWithTotal's skip-total contract so +// SearchPage's cursor path stays a pure keyset page. +func TestItemRepo_BuildFuzzySearchSQL_CursorModeOmitsWindowCount(t *testing.T) { + repo := &ItemRepository{} + dataSQL, _, _ := repo.buildFuzzySearchSQL("avegners", []string{"movie"}, 21, 0, AccessFilter{}, false, nil, 0) + if strings.Contains(dataSQL, "COUNT(*) OVER") { + t.Fatalf("cursor-mode fuzzy query must omit window count; got:\n%s", dataSQL) + } + if strings.Contains(dataSQL, "total_count") { + t.Fatalf("cursor-mode fuzzy query must not select total_count; got:\n%s", dataSQL) + } +} + +// TestItemRepo_BuildFuzzySearchSQL_SimilarityFloor asserts the augment floor +// contract: minSimilarity > 0 adds an explicit similarity() >= $n predicate +// (used when the FTS block had real hits, so augmentation only admits +// near-certain corrections), and minSimilarity == 0 omits it so zero-hit typo +// queries keep the base pinned threshold's recall. +func TestItemRepo_BuildFuzzySearchSQL_SimilarityFloor(t *testing.T) { + repo := &ItemRepository{} + + floorSQL, _, floorArgs := repo.buildFuzzySearchSQL("avegners", []string{"movie"}, 20, 0, AccessFilter{}, true, nil, fuzzyAugmentSimilarityFloor) + // Whole-title similarity, not word similarity: the augment floor must not + // admit embedded prefix words ("coral" scores 0.5 word-similarity to + // "coraline"). + if !strings.Contains(floorSQL, "AND similarity(public.normalize_search_text($1), mi.title_normalized) >= $2") { + t.Fatalf("expected explicit whole-title similarity floor predicate as $2; got:\n%s", floorSQL) + } + if len(floorArgs) < 2 || floorArgs[1] != fuzzyAugmentSimilarityFloor { + t.Fatalf("expected $2 = fuzzyAugmentSimilarityFloor; got args %#v", floorArgs) + } + // Trailing limit/offset must still close the arg list for the count sibling. + if floorArgs[len(floorArgs)-2] != 20 || floorArgs[len(floorArgs)-1] != 0 { + t.Fatalf("expected trailing limit/offset args; got %#v", floorArgs[len(floorArgs)-2:]) + } + + baseSQL, _, baseArgs := repo.buildFuzzySearchSQL("avegners", []string{"movie"}, 20, 0, AccessFilter{}, true, nil, 0) + if strings.Contains(baseSQL, "AND similarity(public.normalize_search_text($1), mi.title_normalized) >= $2") { + t.Fatalf("zero floor must not add a similarity predicate; got:\n%s", baseSQL) + } + if len(baseArgs) != len(floorArgs)-1 { + t.Fatalf("zero floor should bind one fewer arg: base %d vs floor %d", len(baseArgs), len(floorArgs)) + } +} + +// TestItemRepo_BuildFuzzySearchSQL_EmptyQueryReturnsEmpty guards the same +// empty-input contract as buildSearchSQL. +func TestItemRepo_BuildFuzzySearchSQL_EmptyQueryReturnsEmpty(t *testing.T) { + repo := &ItemRepository{} + dataSQL, countSQL, args := repo.buildFuzzySearchSQL(" ", []string{"movie"}, 20, 0, AccessFilter{}, true, nil, 0) + if dataSQL != "" || countSQL != "" || args != nil { + t.Fatalf("expected empty result for blank query; got dataSQL=%q countSQL=%q args=%#v", dataSQL, countSQL, args) + } +} + +// TestItemRepo_Search_LibraryScopeUsesIndependentExistsPredicates pins the +// leak-safe library scoping shared by the FTS search and the fuzzy fallback via +// appendSearchScopeFilters. The prior JOIN + NOT(mil.media_folder_id = ANY(...)) +// form let an item linked to BOTH a disabled and a non-disabled library survive +// the deny check on the passing membership row, so a typo (fuzzy) or exact +// search could surface items from a disabled library. Both builders must instead +// emit independent EXISTS/NOT EXISTS subqueries and no membership JOIN. +func TestItemRepo_Search_LibraryScopeUsesIndependentExistsPredicates(t *testing.T) { + repo := &ItemRepository{} + filter := AccessFilter{DisabledLibraryIDs: []int{9}} + + ftsSQL, _, _ := repo.buildSearchSQL("avatar", []string{"movie"}, 20, 0, filter) + fuzzySQL, _, _ := repo.buildFuzzySearchSQL("avatar", []string{"movie"}, 20, 0, filter, true, nil, 0) + + for name, sql := range map[string]string{"fts": ftsSQL, "fuzzy": fuzzySQL} { + if strings.Contains(sql, "JOIN media_item_libraries") { + t.Fatalf("%s query must not JOIN media_item_libraries for scoping; got:\n%s", name, sql) + } + // Disabled-only path requires positive membership (argument-free EXISTS) + // so orphan items don't slip through the vacuous NOT EXISTS... + if !strings.Contains(sql, "EXISTS (SELECT 1 FROM media_item_libraries mil WHERE mil.content_id = mi.content_id)") { + t.Fatalf("%s query must require library membership via an argument-free EXISTS; got:\n%s", name, sql) + } + // ...plus the disabled NOT EXISTS. + if !strings.Contains(sql, "NOT EXISTS (SELECT 1 FROM media_item_libraries mil WHERE mil.content_id = mi.content_id AND mil.media_folder_id = ANY($") { + t.Fatalf("%s query must exclude disabled libraries via NOT EXISTS; got:\n%s", name, sql) + } + } +} + // TestItemRepo_Search_UsesTitleNormalizedColumn asserts that buildSearchSQL // reads the mi.title_normalized stored generated column for the title rank // arms (exact_title_match and contiguous_title_match), so the LIKE diff --git a/internal/catalog/library_repo.go b/internal/catalog/library_repo.go index 6eb1cc98..d0e8e49d 100644 --- a/internal/catalog/library_repo.go +++ b/internal/catalog/library_repo.go @@ -12,6 +12,7 @@ import ( "github.com/Silo-Server/silo-server/internal/access" "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/pathscope" ) // LibraryItemRepository provides CRUD operations for the media_item_libraries @@ -421,7 +422,19 @@ func (r *LibraryItemRepository) Delete(ctx context.Context, contentID string, fo // longer has any non-missing files in the given folder. It also deletes orphaned // media items once they no longer belong to any library. Returns removed // membership count, deleted item count, orphaned S3 image dirs, and any error. -func (r *LibraryItemRepository) ReconcileFolderMembership(ctx context.Context, folderID int) (int, int, []string, error) { +// +// protectedPathPrefixes lists library roots that are currently unreachable +// (dead drive, lost mount). Membership removal proceeds regardless — browse +// and home queries hide items via media_item_libraries, so removal is what +// keeps a title with no playable files out of the catalog — but items whose +// files in this folder sit under a protected prefix are exempt from the +// orphan delete. Deleting them is not losslessly recoverable (user +// collections cascade via library_collection_items, manual metadata edits and +// cached artwork are lost), whereas a hidden membership-less item restores +// automatically: when the root returns, the scanner clears missing_since on +// its surviving media_files rows and syncPresentLibraryState re-inserts the +// membership from those rows. +func (r *LibraryItemRepository) ReconcileFolderMembership(ctx context.Context, folderID int, protectedPathPrefixes []string) (int, int, []string, error) { tx, err := r.pool.Begin(ctx) if err != nil { return 0, 0, nil, fmt.Errorf("beginning membership reconciliation transaction: %w", err) @@ -470,11 +483,28 @@ func (r *LibraryItemRepository) ReconcileFolderMembership(ctx context.Context, f deletedItems := 0 var orphanedImageDirs []string - if len(removedContentIDs) > 0 { - // Find items that will become orphaned (no remaining library memberships). - orphanIDs, err := collectOrphanIDs(ctx, tx, removedContentIDs) - if err != nil { - return 0, 0, nil, err + // Find both newly orphaned items and items preserved by an earlier + // protected-root pass. The latter no longer have a membership to return from + // the DELETE above, but their surviving media_files row still ties them to + // this folder so they can be reconsidered after the root recovers. + orphanIDs, err := collectOrphanIDs(ctx, tx, removedContentIDs) + if err != nil { + return 0, 0, nil, err + } + previouslyProtected, err := collectFolderFileOrphanIDs(ctx, tx, folderID) + if err != nil { + return 0, 0, nil, err + } + orphanIDs = appendUniqueStrings(orphanIDs, previouslyProtected...) + if len(orphanIDs) > 0 { + + // Exempt orphans whose files sit under an unreachable root: the files + // still exist, the root is just offline. See the doc comment above. + if len(orphanIDs) > 0 && len(protectedPathPrefixes) > 0 { + orphanIDs, err = excludeOrphansUnderProtectedPrefixes(ctx, tx, orphanIDs, folderID, protectedPathPrefixes) + if err != nil { + return 0, 0, nil, err + } } // Collect image paths before deletion. @@ -516,3 +546,85 @@ func (r *LibraryItemRepository) ReconcileFolderMembership(ctx context.Context, f return len(removedContentIDs), deletedItems, orphanedImageDirs, nil } + +func collectFolderFileOrphanIDs(ctx context.Context, tx pgx.Tx, folderID int) ([]string, error) { + rows, err := tx.Query(ctx, ` + SELECT DISTINCT mf.content_id + FROM media_files mf + WHERE mf.media_folder_id = $1 + AND mf.content_id IS NOT NULL + AND mf.content_id <> '' + AND NOT EXISTS ( + SELECT 1 FROM media_item_libraries mil WHERE mil.content_id = mf.content_id + ) + `, folderID) + if err != nil { + return nil, fmt.Errorf("finding previously protected folder orphans: %w", err) + } + defer rows.Close() + ids, err := pgx.CollectRows(rows, pgx.RowTo[string]) + if err != nil { + return nil, fmt.Errorf("collecting previously protected folder orphans: %w", err) + } + return ids, nil +} + +func appendUniqueStrings(values []string, additions ...string) []string { + seen := make(map[string]struct{}, len(values)+len(additions)) + for _, value := range values { + seen[value] = struct{}{} + } + for _, value := range additions { + if _, ok := seen[value]; ok { + continue + } + seen[value] = struct{}{} + values = append(values, value) + } + return values +} + +// excludeOrphansUnderProtectedPrefixes returns the subset of orphanIDs that +// have no media_files row in the folder at or under any protected prefix. +// Matching uses the exact-path + escaped prefix-LIKE shape shared with the +// scanner's root matching, so a sibling root that merely shares a string +// prefix (/mnt/movies2 vs /mnt/movies) is never protected by accident. +// +// Known limitation: the check is scoped to the reconciled folder's own files +// (mf.media_folder_id = folderID). An item shared across two libraries whose +// only surviving files sit under ANOTHER folder's currently-unreachable root +// is not exempted here — if its membership in this folder is genuinely +// removed while the other folder's root is offline, the item is purged even +// though its files under the dead root would have resurrected. Closing this +// would require probing every enabled folder's roots (or persisting per- +// folder unreachable state) on each reconcile; accepted for now given how +// narrow the window is. +func excludeOrphansUnderProtectedPrefixes(ctx context.Context, tx pgx.Tx, orphanIDs []string, folderID int, prefixes []string) ([]string, error) { + conds, condArgs := pathscope.CoverageClauses("mf.file_path", prefixes, 3) + args := append([]any{orphanIDs, folderID}, condArgs...) + + rows, err := tx.Query(ctx, fmt.Sprintf(` + SELECT cid FROM unnest($1::text[]) AS cid + WHERE NOT EXISTS ( + SELECT 1 + FROM media_files mf + WHERE mf.media_folder_id = $2 + AND mf.content_id = cid + AND (%s) + ) + `, strings.Join(conds, " OR ")), args...) + if err != nil { + return nil, fmt.Errorf("filtering orphans under protected prefixes: %w", err) + } + defer rows.Close() + + ids := make([]string, 0, len(orphanIDs)) + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, fmt.Errorf("scanning unprotected orphan id: %w", err) + } + ids = append(ids, id) + } + return ids, rows.Err() +} diff --git a/internal/catalog/library_repo_dead_root_test.go b/internal/catalog/library_repo_dead_root_test.go new file mode 100644 index 00000000..53245bdd --- /dev/null +++ b/internal/catalog/library_repo_dead_root_test.go @@ -0,0 +1,133 @@ +package catalog + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +// TestReconcileFolderMembershipProtectsUnreachableRoots verifies the +// dead-root exemption in the orphan purge: memberships are still removed for +// content with no non-missing files (that is what hides the item from +// browse/home, which gate on media_item_libraries), but the media_items row +// of an item whose files sit under a protected (unreachable) root must +// survive so user collections, metadata edits, and artwork are not destroyed +// by a temporary outage. An orphan with no protected files is still purged. +func TestReconcileFolderMembershipProtectsUnreachableRoots(t *testing.T) { + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + + suffix := time.Now().UnixNano() + deadRoot := fmt.Sprintf("/drp-dead-%d", suffix) + goneItem := fmt.Sprintf("drp-gone-%d", suffix) + protectedItem := fmt.Sprintf("drp-protected-%d", suffix) + + var folderID int + if err := pool.QueryRow(ctx, + `INSERT INTO media_folders (type, name, enabled) VALUES ('movies', 'DRP Test', true) RETURNING id`, + ).Scan(&folderID); err != nil { + t.Fatalf("seed folder: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_files WHERE media_folder_id = $1`, folderID) + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = ANY($1)`, []string{goneItem, protectedItem}) + _, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, folderID) + }) + + for _, contentID := range []string{goneItem, protectedItem} { + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres) + VALUES ($1, 'movie', 'DRP Item', 'matched', '{}'::text[]) + `, contentID); err != nil { + t.Fatalf("seed item %s: %v", contentID, err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO media_item_libraries (content_id, media_folder_id, first_seen_at) + VALUES ($1, $2, NOW()) + `, contentID, folderID); err != nil { + t.Fatalf("seed membership %s: %v", contentID, err) + } + } + missingSince := time.Now().UTC().Add(-72 * time.Hour) + seedFile := func(contentID, path string) { + if _, err := pool.Exec(ctx, ` + INSERT INTO media_files (content_id, media_folder_id, file_path, file_size, missing_since) + VALUES ($1, $2, $3, 1024, $4) + `, contentID, folderID, path, missingSince); err != nil { + t.Fatalf("seed file %s: %v", path, err) + } + } + // The protected item's only file lives under the unreachable root; the + // other item's file was under a root that is reachable (its absence is a + // genuine deletion). + seedFile(protectedItem, deadRoot+"/Alpha (2020)/Alpha (2020).mkv") + seedFile(goneItem, fmt.Sprintf("/drp-live-%d/Beta (2021)/Beta (2021).mkv", suffix)) + + repo := NewLibraryItemRepository(pool) + removed, deletedItems, _, err := repo.ReconcileFolderMembership(ctx, folderID, []string{deadRoot}) + if err != nil { + t.Fatalf("ReconcileFolderMembership: %v", err) + } + if removed != 2 { + t.Fatalf("removed memberships = %d, want 2 (hiding still requires removal)", removed) + } + if deletedItems != 1 { + t.Fatalf("deleted items = %d, want 1 (only the unprotected orphan)", deletedItems) + } + + var exists bool + if err := pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM media_items WHERE content_id = $1)`, protectedItem, + ).Scan(&exists); err != nil { + t.Fatalf("check protected item: %v", err) + } + if !exists { + t.Fatal("protected item was purged despite its files sitting under an unreachable root") + } + if err := pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM media_items WHERE content_id = $1)`, goneItem, + ).Scan(&exists); err != nil { + t.Fatalf("check gone item: %v", err) + } + if exists { + t.Fatal("unprotected orphan item survived; purge regressed") + } + if err := pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM media_item_libraries WHERE media_folder_id = $1)`, folderID, + ).Scan(&exists); err != nil { + t.Fatalf("check memberships: %v", err) + } + if exists { + t.Fatal("memberships remain; items with only missing files must be hidden") + } + + // Once the root is reachable again, reconciliation must revisit the item + // even though its membership was removed during the protected pass. + removed, deletedItems, _, err = repo.ReconcileFolderMembership(ctx, folderID, nil) + if err != nil { + t.Fatalf("ReconcileFolderMembership after recovery: %v", err) + } + if removed != 0 || deletedItems != 1 { + t.Fatalf("recovery reconciliation removed/deleted = %d/%d, want 0/1", removed, deletedItems) + } + if err := pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM media_items WHERE content_id = $1)`, protectedItem, + ).Scan(&exists); err != nil { + t.Fatalf("check recovered orphan: %v", err) + } + if exists { + t.Fatal("previously protected orphan survived after its root recovered") + } +} diff --git a/internal/catalog/local_image_paths_test.go b/internal/catalog/local_image_paths_test.go new file mode 100644 index 00000000..938646a8 --- /dev/null +++ b/internal/catalog/local_image_paths_test.go @@ -0,0 +1,37 @@ +package catalog + +import "testing" + +func TestImageTypeFromCachedPathLocalHashedLayout(t *testing.T) { + // Local keys interpose a content-hash segment BEFORE the image type + // (local/{contentType}/{contentID}/{hash8}/{imageType}/{variant}.{ext}) + // so the variant's parent directory stays the image type. + cases := map[string]string{ + "local/movies/movie-1/deadbeef/poster/original.webp": "poster", + "local/movies/movie-1/deadbeef/backdrop/original.webp": "backdrop", + "local/series/show-1/cafef00d/logo/w500.webp": "logo", + } + for path, want := range cases { + if got := imageTypeFromCachedPath(path); got != want { + t.Errorf("imageTypeFromCachedPath(%q) = %q, want %q", path, got, want) + } + } +} + +func TestImageDeletePrefixTrimsLocalPathsToContentRoot(t *testing.T) { + // On item deletion, local paths trim to local/{contentType}/{contentID} + // so every stale hash prefix is swept, not just the live one. + cases := map[string]string{ + "local/movies/movie-1/deadbeef/poster/original.webp": "local/movies/movie-1/", + "local/series/show-1/cafef00d/logo/w500.webp": "local/series/show-1/", + // Remote cached keys keep the existing per-image-type prefix. + "tmdb/movies/550/poster/original.webp": "tmdb/movies/550/poster/", + // Legacy scanner keys without a hash segment stay whole-path dirs. + "local/ebooks/book-1/poster/original.webp": "local/ebooks/book-1/", + } + for path, want := range cases { + if got := imageDeletePrefix(path); got != want { + t.Errorf("imageDeletePrefix(%q) = %q, want %q", path, got, want) + } + } +} diff --git a/internal/catalog/search_postgres_mixed.go b/internal/catalog/search_postgres_mixed.go index 5d7624d3..871b924d 100644 --- a/internal/catalog/search_postgres_mixed.go +++ b/internal/catalog/search_postgres_mixed.go @@ -69,22 +69,18 @@ const mixedSearchOrder = `exact_title_match DESC, contiguous_title_match DESC, y phrase_rank DESC, title_rank DESC, title_prefix_rank DESC, overview_rank DESC, LOWER(title) ASC, content_id ASC` -// buildMixedSearchSQLWithTotal builds one ranked candidate set from the two +// buildMixedSearchSQLFromParsed builds one ranked candidate set from the two // physical catalog sources. The scored CTE deliberately carries only ranking // fields; the wide MediaItem projection is hydrated after LIMIT/OFFSET so a // broad match never sorts posters, arrays, or metadata blobs for every hit. -func (r *ItemRepository) buildMixedSearchSQLWithTotal( - query string, +func (r *ItemRepository) buildMixedSearchSQLFromParsed( + parsed parsedSearchQuery, itemTypes []string, limit, offset int, filter AccessFilter, includeTotal bool, ) (dataSQL, countSQL string, args []any) { - parsed := parseSearchQuery(query) - searchText := parsed.Text - if searchText == "" { - searchText = collapseSearchWhitespace(strings.ReplaceAll(strings.TrimSpace(query), "\"", " ")) - } + searchText := searchTextFromParsed(parsed) if searchText == "" { return "", "", nil } diff --git a/internal/catalog/search_query.go b/internal/catalog/search_query.go index 788ab333..d487d7e0 100644 --- a/internal/catalog/search_query.go +++ b/internal/catalog/search_query.go @@ -11,6 +11,11 @@ type parsedSearchQuery struct { Text string Phrase string ExactTitleHint string + // NormalizedText is normalizeTitleForComparison(Text) computed once at parse + // time. Text already folds phrase + remainder together, so this is the full + // normalized query used by eligibleForFuzzy's token gate (which would + // otherwise re-normalize on every sparse search). + NormalizedText string Year *int } @@ -37,10 +42,31 @@ func parseSearchQuery(raw string) parsedSearchQuery { Text: text, Phrase: phrase, ExactTitleHint: normalizeTitleForComparison(firstNonEmptySearchValue(phrase, text)), + NormalizedText: normalizeTitleForComparison(text), Year: year, } } +// fuzzyMinTokenLen is the shortest normalized token that may enable the trigram +// fuzzy title fallback. A token shorter than this forms too few trigrams to use +// the gin_trgm_ops index selectively (and a 1-2 char token can't use it at +// all), so for short queries the fuzzy fallback is skipped and search stays on +// the exact FTS/prefix path. The gate is applied to the longest token so a stray +// short token ("a vengers") is judged on "vengers", not "a". +const fuzzyMinTokenLen = 4 + +// eligibleForFuzzy reports whether a parsed query clears the min-token gate for +// the trigram fuzzy title fallback. +func eligibleForFuzzy(parsed parsedSearchQuery) bool { + longest := 0 + for _, tok := range strings.Fields(parsed.NormalizedText) { + if n := len([]rune(tok)); n > longest { + longest = n + } + } + return longest >= fuzzyMinTokenLen +} + func extractBalancedPhrase(input string) (string, string) { start := strings.Index(input, "\"") if start == -1 { diff --git a/internal/catalogseed/service.go b/internal/catalogseed/service.go index 83451734..46468abd 100644 --- a/internal/catalogseed/service.go +++ b/internal/catalogseed/service.go @@ -2580,6 +2580,7 @@ func toVideoTrackRecords(tracks []models.VideoTrack) []VideoTrackRecord { Bitrate: track.Bitrate, VideoRange: track.VideoRange, VideoRangeType: track.VideoRangeType, + ColorRange: track.ColorRange, ColorPrimaries: track.ColorPrimaries, ColorSpace: track.ColorSpace, ColorTransfer: track.ColorTransfer, diff --git a/internal/catalogseed/service_test.go b/internal/catalogseed/service_test.go index 9cd576ae..421eefcd 100644 --- a/internal/catalogseed/service_test.go +++ b/internal/catalogseed/service_test.go @@ -3,8 +3,30 @@ package catalogseed import ( "reflect" "testing" + + "github.com/Silo-Server/silo-server/internal/models" ) +func TestToVideoTrackRecordsPreservesColorRange(t *testing.T) { + got := toVideoTrackRecords([]models.VideoTrack{ + {ColorRange: "tv"}, + {ColorRange: "pc"}, + {ColorRange: "unknown"}, + }) + + if len(got) != 3 { + t.Fatalf("records length = %d, want 3", len(got)) + } + if got[0].ColorRange != "tv" || got[1].ColorRange != "pc" || got[2].ColorRange != "unknown" { + t.Fatalf( + "ColorRange values = [%q, %q, %q], want [tv, pc, unknown]", + got[0].ColorRange, + got[1].ColorRange, + got[2].ColorRange, + ) + } +} + func TestCatalogSeedSearchUpsertIDsIncludesChangedItemsAndEmbeddings(t *testing.T) { itemStates := map[string]bool{ "movie-1": true, diff --git a/internal/catalogseed/types.go b/internal/catalogseed/types.go index 39e2d435..d4037a58 100644 --- a/internal/catalogseed/types.go +++ b/internal/catalogseed/types.go @@ -156,6 +156,7 @@ type VideoTrackRecord struct { Bitrate int `json:"bitrate,omitempty"` VideoRange string `json:"video_range,omitempty"` VideoRangeType string `json:"video_range_type,omitempty"` + ColorRange string `json:"color_range,omitempty"` ColorPrimaries string `json:"color_primaries,omitempty"` ColorSpace string `json:"color_space,omitempty"` ColorTransfer string `json:"color_transfer,omitempty"` diff --git a/internal/diagnostics/admin.go b/internal/diagnostics/admin.go new file mode 100644 index 00000000..957c32de --- /dev/null +++ b/internal/diagnostics/admin.go @@ -0,0 +1,121 @@ +package diagnostics + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "time" +) + +const ReportDownloadContentType = BundleContentType + +var ErrReportNotReady = errors.New("diagnostic report is not ready") + +func (s *Service) ListForAdmin(ctx context.Context, filters ListFilters) (ListResult, error) { + if s.reports == nil { + return ListResult{}, ErrReportStoreUnavailable + } + return s.reports.ListForAdmin(ctx, filters) +} + +func (s *Service) GetReport(ctx context.Context, id string) (*Report, error) { + if s.reports == nil { + return nil, ErrReportStoreUnavailable + } + return s.reports.GetByID(ctx, id) +} + +func (s *Service) PresignReportDownload(ctx context.Context, report *Report, expiry time.Duration) (string, error) { + bucket, key, err := s.readyReportBlobLocation(report) + if err != nil { + return "", err + } + if expiry <= 0 { + expiry = 15 * time.Minute + } + return s.store.PresignGetURL(ctx, bucket, key, expiry) +} + +func (s *Service) EffectiveReportDownloadTTL(requested time.Duration) time.Duration { + if requested <= 0 { + return requested + } + effectiveStore, ok := s.store.(interface { + EffectivePresignTTL(time.Duration) time.Duration + }) + if !ok { + return requested + } + effective := effectiveStore.EffectivePresignTTL(requested) + if effective <= 0 { + return requested + } + return effective +} + +func (s *Service) OpenReportDownload(ctx context.Context, report *Report) (io.ReadCloser, error) { + bucket, key, err := s.readyReportBlobLocation(report) + if err != nil { + return nil, err + } + return s.store.GetObject(ctx, bucket, key) +} + +func (s *Service) DeleteReport(ctx context.Context, id string) (*Report, error) { + if s.reports == nil { + return nil, ErrReportStoreUnavailable + } + // Delete the row before the blob: a DB failure after the object is gone + // would otherwise leave a visible report whose bundle can no longer be + // downloaded. DeleteByID returns the deleted row (and ErrNotFound when + // absent), so the blob location is captured from it. If the blob delete + // fails the row is already gone, so log the bucket/key for an operator (and + // the orphan reconciler) to reap instead of failing the delete. + deleted, err := s.reports.DeleteByID(ctx, id) + if err != nil { + return nil, err + } + if err := deleteReportObjects(ctx, s.store, deleted, s.logger); err != nil { + logDeferredBlobDeletion(ctx, s.logger, s.store, deleted, err) + } + return deleted, nil +} + +func reportBlobBucket(report *Report, store ObjectStore) string { + bucket := stringValue(report.BlobBucket) + if bucket == "" && store != nil { + bucket = store.Bucket() + } + return bucket +} + +func (s *Service) readyReportBlobLocation(report *Report) (string, string, error) { + if report == nil { + return "", "", ErrNotFound + } + if report.State != StateReady { + return "", "", ErrReportNotReady + } + if s.store == nil || strings.TrimSpace(s.store.Bucket()) == "" { + return "", "", ErrStorageUnavailable + } + + bucket := stringValue(report.BlobBucket) + if bucket == "" { + bucket = s.store.Bucket() + } + key := stringValue(report.BlobKey) + if key == "" { + return "", "", fmt.Errorf("%w: diagnostic report has no blob key", ErrStorageUnavailable) + } + return bucket, key, nil +} + +func stringValue(value *string) string { + if value == nil { + return "" + } + return strings.TrimSpace(*value) +} diff --git a/internal/diagnostics/bundle.go b/internal/diagnostics/bundle.go new file mode 100644 index 00000000..76aee2c5 --- /dev/null +++ b/internal/diagnostics/bundle.go @@ -0,0 +1,581 @@ +package diagnostics + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "io" + "math" + "path" + "strings" + + "github.com/Silo-Server/silo-server/internal/diagnostics/contract" +) + +const ( + DefaultMaxBundleEntries = 16 + DefaultMaxCompressionRatio = int64(200) + defaultBundleReadBufferSize = 32 * 1024 + // Standard tar writers (GNU tar, Python tarfile, Apache Commons Compress) + // pad the archive with zero blocks to a record boundary, 10240 bytes by + // default; 64 KiB also covers writers configured with larger blocking + // factors. + maxTarTrailingPaddingBytes = 64 * 1024 +) + +var ( + ErrCompressedTooLarge = errors.New("diagnostics bundle compressed size exceeds limit") + ErrUncompressedTooLarge = errors.New("diagnostics bundle uncompressed size exceeds limit") + ErrEntryTooLarge = errors.New("diagnostics bundle entry size exceeds limit") + ErrTooManyEntries = errors.New("diagnostics bundle has too many entries") + ErrInvalidBundle = errors.New("invalid diagnostics bundle") + ErrCompressionRatio = errors.New("diagnostics bundle compression ratio exceeds limit") +) + +type BundleLimits struct { + MaxCompressedBytes int64 + MaxUncompressedBytes int64 + MaxEntryBytes int64 + MaxEntries int + MaxCompressionRatio int64 +} + +type BundleInfo struct { + CompressedBytes int64 + // UncompressedBytes is the total size of the decompressed tar stream — + // headers, entry payloads, end-of-archive marker, and padding — matching + // what a client counts on the way into its gzip writer (and `gzip -l`). + UncompressedBytes int64 + SHA256 string + Entries []string + // EmbeddedManifest is the raw bytes of the archive's first entry + // (manifest.json). Per the bundle contract this is the part-1 manifest with + // the `archive` object removed; Ingest compares the two so a stored archive + // can never carry a manifest that disagrees with the accepted report. + EmbeddedManifest []byte +} + +var allowedBundleEntries = allowlistMap(contract.ArchiveEntryAllowlist) + +func ValidateBundle(r io.Reader, limits BundleLimits) (BundleInfo, error) { + limits = normalizeBundleLimits(limits) + metered := newCompressedMeter(r, limits.MaxCompressedBytes) + + gz, err := gzip.NewReader(metered) + if err != nil { + if isBundleUploadAbortError(err) { + return BundleInfo{}, err + } + if errors.Is(err, ErrCompressedTooLarge) { + return BundleInfo{}, ErrCompressedTooLarge + } + return BundleInfo{}, fmt.Errorf("%w: open gzip: %v", ErrInvalidBundle, err) + } + gz.Multistream(false) + gzipClosed := false + defer func() { + if !gzipClosed { + _ = gz.Close() + } + }() + + uncompressed := &uncompressedCounter{r: gz} + tr := tar.NewReader(uncompressed) + info := BundleInfo{ + Entries: make([]string, 0, len(allowedBundleEntries)), + } + buffer := make([]byte, defaultBundleReadBufferSize) + var payloadBytes int64 + + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return BundleInfo{}, classifyBundleReadError(err) + } + + if len(info.Entries) >= limits.MaxEntries { + return BundleInfo{}, ErrTooManyEntries + } + + name, err := validateEntryHeader(hdr) + if err != nil { + return BundleInfo{}, err + } + isManifestEntry := len(info.Entries) == 0 + if isManifestEntry && name != "manifest.json" { + return BundleInfo{}, fmt.Errorf("%w: first entry must be manifest.json", ErrInvalidBundle) + } + info.Entries = append(info.Entries, name) + + if hdr.Size > limits.MaxEntryBytes { + return BundleInfo{}, ErrEntryTooLarge + } + if isManifestEntry && hdr.Size > MaxManifestBytes { + return BundleInfo{}, fmt.Errorf("%w: embedded manifest exceeds %d bytes", ErrInvalidBundle, MaxManifestBytes) + } + if payloadBytes+hdr.Size > limits.MaxUncompressedBytes { + return BundleInfo{}, ErrUncompressedTooLarge + } + + entry := &tarEntryReader{tr: tr, limits: limits, metered: metered, total: &payloadBytes} + var manifestCapture *bytes.Buffer + var consumeErr error + switch { + case isManifestEntry: + // Capture the embedded manifest so Ingest can compare it against the + // part-1 manifest; it is the only entry buffered whole. + manifestCapture = &bytes.Buffer{} + manifestCapture.Grow(int(hdr.Size)) + _, consumeErr = io.Copy(manifestCapture, entry) + case isNDJSONBundleEntry(name): + // logs.jsonl / breadcrumbs.jsonl: newline-delimited JSON objects, + // validated line-by-line so memory stays bounded to one capped line. + consumeErr = validateNDJSONObjectEntry(entry) + case isJSONObjectBundleEntry(name): + // device.json / crash/*.json: a single JSON object, validated with a + // streaming token decoder (no whole-entry buffering). + consumeErr = validateJSONObjectEntry(entry) + default: + // Opaque members (crash/stack.txt, crash/tombstone.pb) carry no JSON + // contract; drain them so their bytes still count toward the size, + // ratio, and count limits. + _, consumeErr = io.Copy(io.Discard, entry) + } + if consumeErr != nil { + return BundleInfo{}, consumeErr + } + if entry.entryBytes != hdr.Size { + return BundleInfo{}, fmt.Errorf("%w: entry size mismatch for %s", ErrInvalidBundle, name) + } + if manifestCapture != nil { + info.EmbeddedManifest = manifestCapture.Bytes() + } + } + + if len(info.Entries) == 0 { + return BundleInfo{}, fmt.Errorf("%w: empty archive", ErrInvalidBundle) + } + if err := rejectPostTarData(uncompressed, buffer); err != nil { + return BundleInfo{}, err + } + if err := gz.Close(); err != nil { + return BundleInfo{}, classifyBundleReadError(err) + } + gzipClosed = true + if err := rejectTrailingCompressedData(metered); err != nil { + return BundleInfo{}, err + } + if uncompressed.count > limits.MaxUncompressedBytes { + return BundleInfo{}, ErrUncompressedTooLarge + } + if ratioExceeded(uncompressed.count, metered.Count(), limits.MaxCompressionRatio) { + return BundleInfo{}, ErrCompressionRatio + } + + info.UncompressedBytes = uncompressed.count + info.CompressedBytes = metered.Count() + info.SHA256 = hex.EncodeToString(metered.Sum()) + return info, nil +} + +func normalizeBundleLimits(limits BundleLimits) BundleLimits { + if limits.MaxCompressedBytes <= 0 { + limits.MaxCompressedBytes = DefaultMaxBundleBytes + } + if limits.MaxUncompressedBytes <= 0 { + limits.MaxUncompressedBytes = DefaultMaxUncompressed + } + if limits.MaxEntryBytes <= 0 || limits.MaxEntryBytes > limits.MaxUncompressedBytes { + limits.MaxEntryBytes = limits.MaxUncompressedBytes + } + if limits.MaxEntries <= 0 { + limits.MaxEntries = DefaultMaxBundleEntries + } + if limits.MaxCompressionRatio <= 0 { + limits.MaxCompressionRatio = DefaultMaxCompressionRatio + } + return limits +} + +func validateEntryHeader(hdr *tar.Header) (string, error) { + if hdr == nil { + return "", fmt.Errorf("%w: missing tar header", ErrInvalidBundle) + } + // PAX/GNU extension records are consumed by archive/tar before the next + // header surfaces, so bytes carried in them never reach the allowlist and + // count checks. The contract fixtures are plain USTAR; reject anything else + // rather than let extension records smuggle data past validation. + if hdr.Format == tar.FormatPAX || hdr.Format == tar.FormatGNU { + return "", fmt.Errorf("%w: unsupported tar format", ErrInvalidBundle) + } + if len(hdr.PAXRecords) > 0 { + return "", fmt.Errorf("%w: unsupported tar extension records", ErrInvalidBundle) + } + if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { + return "", fmt.Errorf("%w: unsupported tar entry type", ErrInvalidBundle) + } + if hdr.Size < 0 { + return "", fmt.Errorf("%w: negative tar entry size", ErrInvalidBundle) + } + // Reject names that are not already trimmed instead of normalizing them: + // otherwise a padded name like "manifest.json " would be recorded as the + // allowlisted entry while the archive stores a different literal member. + name := hdr.Name + if name != strings.TrimSpace(name) { + return "", fmt.Errorf("%w: unsafe entry name", ErrInvalidBundle) + } + if name == "" || strings.Contains(name, "\\") || strings.HasPrefix(name, "/") || path.IsAbs(name) { + return "", fmt.Errorf("%w: unsafe entry name", ErrInvalidBundle) + } + clean := path.Clean(name) + if clean == "." || clean != name || strings.HasPrefix(clean, "../") || strings.Contains(clean, "/../") { + return "", fmt.Errorf("%w: unsafe entry name", ErrInvalidBundle) + } + if _, ok := allowedBundleEntries[name]; !ok { + return "", fmt.Errorf("%w: disallowed entry %q", ErrInvalidBundle, name) + } + return name, nil +} + +// tarEntryReader streams one tar entry's payload while charging every read +// against the bundle's entry-size, total-size, and compression-ratio limits — +// the same accounting the opaque drain path applied — so structural validators +// can pull the payload without letting a member slip past the guards. It also +// tracks entryBytes so the caller can compare it against the declared header +// size. +type tarEntryReader struct { + tr *tar.Reader + limits BundleLimits + metered *compressedMeter + total *int64 + entryBytes int64 +} + +func (er *tarEntryReader) Read(p []byte) (int, error) { + n, err := er.tr.Read(p) + if n > 0 { + er.entryBytes += int64(n) + if er.entryBytes > er.limits.MaxEntryBytes { + return n, ErrEntryTooLarge + } + *er.total += int64(n) + if *er.total > er.limits.MaxUncompressedBytes { + return n, ErrUncompressedTooLarge + } + if ratioExceeded(*er.total, er.metered.Count(), er.limits.MaxCompressionRatio) { + return n, ErrCompressionRatio + } + } + if err != nil && !errors.Is(err, io.EOF) { + return n, classifyBundleReadError(err) + } + return n, err +} + +func isNDJSONBundleEntry(name string) bool { + return name == "logs.jsonl" || name == "breadcrumbs.jsonl" +} + +func isJSONObjectBundleEntry(name string) bool { + return name == "device.json" || + (strings.HasPrefix(name, "crash/") && strings.HasSuffix(name, ".json")) +} + +// validateJSONObjectEntry streams a single JSON object from r, rejecting +// anything that is not exactly one top-level object. It uses the token decoder +// so memory stays bounded to the nesting depth rather than the entry size. +func validateJSONObjectEntry(r io.Reader) error { + dec := json.NewDecoder(r) + tok, err := dec.Token() + if err != nil { + return classifyEntryContentError(err) + } + if delim, ok := tok.(json.Delim); !ok || delim != '{' { + return fmt.Errorf("%w: entry is not a JSON object", ErrInvalidBundle) + } + depth := 1 + for depth > 0 { + tok, err := dec.Token() + if err != nil { + return classifyEntryContentError(err) + } + if delim, ok := tok.(json.Delim); ok { + switch delim { + case '{', '[': + depth++ + case '}', ']': + depth-- + } + } + } + // A well-formed entry ends here; anything after the top-level object is + // smuggled content. + if _, err := dec.Token(); !errors.Is(err, io.EOF) { + if err == nil { + return fmt.Errorf("%w: trailing data after JSON object", ErrInvalidBundle) + } + return classifyEntryContentError(err) + } + return nil +} + +// validateNDJSONObjectEntry streams newline-delimited JSON, requiring each +// non-blank line to be a single JSON object. Lines are validated one at a time +// with a byte cap so a pathological entry can never buffer unbounded memory. +func validateNDJSONObjectEntry(r io.Reader) error { + buffer := make([]byte, defaultBundleReadBufferSize) + line := make([]byte, 0, 512) + for { + n, err := r.Read(buffer) + if err != nil && !errors.Is(err, io.EOF) { + return classifyEntryContentError(err) + } + chunk := buffer[:n] + for len(chunk) > 0 { + idx := bytes.IndexByte(chunk, '\n') + if idx < 0 { + line = append(line, chunk...) + if len(line) > contract.MaxLogLineBytes { + return errLogLineTooLarge() + } + break + } + line = append(line, chunk[:idx]...) + if len(line) > contract.MaxLogLineBytes { + return errLogLineTooLarge() + } + if verr := validateJSONObjectLine(line); verr != nil { + return verr + } + line = line[:0] + chunk = chunk[idx+1:] + } + if errors.Is(err, io.EOF) { + break + } + } + // Handle a trailing line without a closing newline. + if len(bytes.TrimSpace(line)) > 0 { + if len(line) > contract.MaxLogLineBytes { + return errLogLineTooLarge() + } + return validateJSONObjectLine(line) + } + return nil +} + +func validateJSONObjectLine(line []byte) error { + trimmed := bytes.TrimSpace(line) + if len(trimmed) == 0 { + return nil + } + if trimmed[0] != '{' || !json.Valid(trimmed) { + return fmt.Errorf("%w: entry contains a line that is not a JSON object", ErrInvalidBundle) + } + return nil +} + +func errLogLineTooLarge() error { + return fmt.Errorf("%w: entry line exceeds %d bytes", ErrInvalidBundle, contract.MaxLogLineBytes) +} + +// classifyEntryContentError maps a decode-time error to the right bundle +// sentinel: the streaming limit sentinels and existing bundle errors pass +// through unchanged; a truncated/empty stream or malformed JSON becomes +// ErrInvalidBundle. +func classifyEntryContentError(err error) error { + switch { + case errors.Is(err, ErrEntryTooLarge), + errors.Is(err, ErrUncompressedTooLarge), + errors.Is(err, ErrCompressionRatio), + errors.Is(err, ErrCompressedTooLarge), + errors.Is(err, ErrInvalidBundle): + return err + case isBundleUploadAbortError(err): + return err + case errors.Is(err, io.EOF), errors.Is(err, io.ErrUnexpectedEOF): + return fmt.Errorf("%w: truncated or empty JSON entry", ErrInvalidBundle) + default: + return fmt.Errorf("%w: %v", ErrInvalidBundle, err) + } +} + +func rejectPostTarData(r io.Reader, buffer []byte) error { + var padding int64 + for { + n, err := r.Read(buffer) + if errors.Is(err, ErrCompressedTooLarge) { + return ErrCompressedTooLarge + } + if n > 0 { + for _, b := range buffer[:n] { + if b != 0 { + return fmt.Errorf("%w: trailing data after tar archive", ErrInvalidBundle) + } + } + padding += int64(n) + if padding > maxTarTrailingPaddingBytes { + return fmt.Errorf("%w: excessive padding after tar archive", ErrInvalidBundle) + } + } + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return classifyBundleReadError(err) + } + } +} + +func rejectTrailingCompressedData(metered *compressedMeter) error { + var one [1]byte + n, err := metered.Read(one[:]) + if errors.Is(err, ErrCompressedTooLarge) { + return ErrCompressedTooLarge + } + if n > 0 { + return fmt.Errorf("%w: trailing data after gzip stream", ErrInvalidBundle) + } + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return classifyBundleReadError(err) + } + return nil +} + +func ratioExceeded(uncompressed, compressed, maxRatio int64) bool { + if compressed <= 0 || maxRatio <= 0 { + return false + } + if compressed > math.MaxInt64/maxRatio { + return false + } + return uncompressed > compressed*maxRatio +} + +func classifyBundleReadError(err error) error { + if isBundleUploadAbortError(err) { + return err + } + if errors.Is(err, ErrCompressedTooLarge) { + return ErrCompressedTooLarge + } + if errors.Is(err, io.ErrUnexpectedEOF) { + return fmt.Errorf("%w: truncated archive", ErrInvalidBundle) + } + return fmt.Errorf("%w: %v", ErrInvalidBundle, err) +} + +type bundleUploadAbortError struct { + err error +} + +func (e *bundleUploadAbortError) Error() string { + if e == nil || e.err == nil { + return "diagnostics bundle upload aborted" + } + return e.err.Error() +} + +func (e *bundleUploadAbortError) Unwrap() error { + if e == nil { + return nil + } + return e.err +} + +func isBundleUploadAbortError(err error) bool { + var abortErr *bundleUploadAbortError + return errors.As(err, &abortErr) +} + +func allowlistMap(entries []string) map[string]struct{} { + allowed := make(map[string]struct{}, len(entries)) + for _, entry := range entries { + allowed[entry] = struct{}{} + } + return allowed +} + +// uncompressedCounter tracks the total number of decompressed bytes read out +// of the gzip stream, including tar headers, end-of-archive blocks, and +// record padding. +type uncompressedCounter struct { + r io.Reader + count int64 +} + +func (c *uncompressedCounter) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.count += int64(n) + return n, err +} + +type compressedMeter struct { + r io.Reader + max int64 + n int64 + hash hash.Hash + limit bool +} + +func newCompressedMeter(r io.Reader, max int64) *compressedMeter { + return &compressedMeter{ + r: r, + max: max, + hash: sha256.New(), + limit: max > 0, + } +} + +func (m *compressedMeter) Read(p []byte) (int, error) { + if m.limit { + remaining := m.max - m.n + if remaining < 0 { + return 0, ErrCompressedTooLarge + } + if int64(len(p)) > remaining+1 { + p = p[:int(remaining+1)] + } + } + n, err := m.r.Read(p) + if n > 0 { + m.n += int64(n) + _, _ = m.hash.Write(p[:n]) + if m.limit && m.n > m.max { + return n, ErrCompressedTooLarge + } + } + return n, err +} + +func (m *compressedMeter) ReadByte() (byte, error) { + var one [1]byte + n, err := m.Read(one[:]) + if n == 1 { + if errors.Is(err, io.EOF) { + err = nil + } + return one[0], err + } + return 0, err +} + +func (m *compressedMeter) Count() int64 { + return m.n +} + +func (m *compressedMeter) Sum() []byte { + return m.hash.Sum(nil) +} diff --git a/internal/diagnostics/bundle_test.go b/internal/diagnostics/bundle_test.go new file mode 100644 index 00000000..bd6da681 --- /dev/null +++ b/internal/diagnostics/bundle_test.go @@ -0,0 +1,402 @@ +package diagnostics + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "errors" + "io" + "strings" + "testing" +) + +type testArchiveEntry struct { + name string + body []byte + typeflag byte +} + +func TestValidateBundle(t *testing.T) { + validEntries := []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{"schema_version":1}`)}, + {name: "logs.jsonl", body: []byte("{}\n")}, + } + + tests := []struct { + name string + entries []testArchiveEntry + limits BundleLimits + mutate func([]byte) []byte + wantErr error + wantEntries string + }{ + { + name: "happy path", + entries: validEntries, + }, + { + name: "wrong first entry", + entries: []testArchiveEntry{ + {name: "device.json", body: []byte(`{}`)}, + {name: "manifest.json", body: []byte(`{}`)}, + }, + wantErr: ErrInvalidBundle, + }, + { + name: "disallowed name", + entries: []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + {name: "extra.txt", body: []byte("nope")}, + }, + wantErr: ErrInvalidBundle, + }, + { + name: "traversal name", + entries: []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + {name: "../logs.jsonl", body: []byte("nope")}, + }, + wantErr: ErrInvalidBundle, + }, + { + name: "ratio bomb", + entries: []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + // Valid newline-delimited JSON objects that still compress far + // beyond the ratio limit, so the ratio guard trips before the + // content validator can pass judgement on every line. + {name: "logs.jsonl", body: bytes.Repeat([]byte("{}\n"), 400*1024)}, + }, + limits: BundleLimits{ + MaxUncompressedBytes: 4 * 1024 * 1024, + }, + wantErr: ErrCompressionRatio, + }, + { + name: "rejects malformed device.json", + entries: []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + {name: "device.json", body: []byte(`{"identity": }`)}, + }, + wantErr: ErrInvalidBundle, + }, + { + name: "rejects non-object device.json", + entries: []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + {name: "device.json", body: []byte(`[1,2,3]`)}, + }, + wantErr: ErrInvalidBundle, + }, + { + name: "rejects trailing data in device.json", + entries: []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + {name: "device.json", body: []byte(`{}{}`)}, + }, + wantErr: ErrInvalidBundle, + }, + { + name: "rejects malformed logs.jsonl line", + entries: []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + {name: "logs.jsonl", body: []byte("{}\nnot json\n")}, + }, + wantErr: ErrInvalidBundle, + }, + { + name: "rejects non-object logs.jsonl line", + entries: []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + {name: "logs.jsonl", body: []byte("{}\n[1,2]\n")}, + }, + wantErr: ErrInvalidBundle, + }, + { + name: "accepts well-formed device.json and crash json", + entries: []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + {name: "device.json", body: []byte(`{"identity":{"model":"Pixel"}}` + "\n")}, + {name: "logs.jsonl", body: []byte(`{"lvl":"I","msg":"hi"}` + "\n")}, + {name: "crash/summary.json", body: []byte(`{"summary":"boom"}`)}, + {name: "crash/tombstone.pb", body: []byte{0x00, 0x01, 0x02, 0x03}}, + }, + wantEntries: "manifest.json,device.json,logs.jsonl,crash/summary.json,crash/tombstone.pb", + }, + { + name: "over count", + entries: append([]testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + }, repeatedEntries("logs.jsonl", 16)...), + wantErr: ErrTooManyEntries, + }, + { + name: "oversize", + entries: validEntries, + limits: BundleLimits{MaxCompressedBytes: 64}, + wantErr: ErrCompressedTooLarge, + }, + { + name: "truncated gzip", + entries: validEntries, + mutate: func(raw []byte) []byte { + return raw[:len(raw)-8] + }, + wantErr: ErrInvalidBundle, + }, + { + name: "rejects trailing tar data inside gzip stream", + entries: validEntries, + mutate: func([]byte) []byte { + return buildTestArchiveWithPostTarPayload(t, validEntries, []byte("hidden")) + }, + wantErr: ErrInvalidBundle, + }, + { + name: "accepts tar record padding", + entries: validEntries, + mutate: func([]byte) []byte { + return buildTestArchiveWithPostTarPayload(t, validEntries, make([]byte, 8*1024)) + }, + }, + { + name: "rejects nonzero byte inside trailing padding", + entries: validEntries, + mutate: func([]byte) []byte { + padding := make([]byte, 8*1024) + padding[len(padding)-1] = 1 + return buildTestArchiveWithPostTarPayload(t, validEntries, padding) + }, + wantErr: ErrInvalidBundle, + }, + { + name: "rejects excessive trailing padding", + entries: validEntries, + mutate: func([]byte) []byte { + return buildTestArchiveWithPostTarPayload(t, validEntries, make([]byte, maxTarTrailingPaddingBytes+1)) + }, + wantErr: ErrInvalidBundle, + }, + { + name: "rejects concatenated gzip member", + entries: validEntries, + mutate: func(raw []byte) []byte { + return append(raw, buildTestGzipMember(t, bytes.Repeat([]byte{0}, 1024))...) + }, + wantErr: ErrInvalidBundle, + }, + { + name: "rejects symlinks", + entries: []testArchiveEntry{ + {name: "manifest.json", body: []byte(`{}`)}, + {name: "logs.jsonl", typeflag: tar.TypeSymlink}, + }, + wantErr: ErrInvalidBundle, + }, + { + name: "rejects padded entry name", + entries: []testArchiveEntry{ + {name: "manifest.json ", body: []byte(`{}`)}, + }, + wantErr: ErrInvalidBundle, + }, + { + name: "rejects pax extension records", + entries: validEntries, + mutate: func([]byte) []byte { + return buildTestArchivePAX(t) + }, + wantErr: ErrInvalidBundle, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + raw := buildTestArchive(t, tt.entries) + if tt.mutate != nil { + raw = tt.mutate(raw) + } + info, err := ValidateBundle(bytes.NewReader(raw), tt.limits) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("ValidateBundle error = %v, want %v", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("ValidateBundle error = %v", err) + } + if info.CompressedBytes != int64(len(raw)) { + t.Fatalf("compressed bytes = %d, want %d", info.CompressedBytes, len(raw)) + } + sum := sha256.Sum256(raw) + if info.SHA256 != fmtHex(sum[:]) { + t.Fatalf("sha256 = %s, want %s", info.SHA256, fmtHex(sum[:])) + } + wantEntries := tt.wantEntries + if wantEntries == "" { + wantEntries = "manifest.json,logs.jsonl" + } + if got := strings.Join(info.Entries, ","); got != wantEntries { + t.Fatalf("entries = %s, want %s", got, wantEntries) + } + gzr, err := gzip.NewReader(bytes.NewReader(raw)) + if err != nil { + t.Fatalf("open gzip for uncompressed size: %v", err) + } + wantUncompressed, err := io.Copy(io.Discard, gzr) + if err != nil { + t.Fatalf("measure uncompressed size: %v", err) + } + if info.UncompressedBytes != wantUncompressed { + t.Fatalf("uncompressed bytes = %d, want %d", info.UncompressedBytes, wantUncompressed) + } + }) + } +} + +func repeatedEntries(name string, count int) []testArchiveEntry { + entries := make([]testArchiveEntry, 0, count) + for i := 0; i < count; i++ { + entries = append(entries, testArchiveEntry{name: name, body: []byte("{}\n")}) + } + return entries +} + +func buildTestArchive(t *testing.T, entries []testArchiveEntry) []byte { + t.Helper() + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, entry := range entries { + typeflag := entry.typeflag + if typeflag == 0 { + typeflag = tar.TypeReg + } + hdr := &tar.Header{ + Name: entry.name, + Mode: 0o600, + Size: int64(len(entry.body)), + Typeflag: typeflag, + } + if typeflag != tar.TypeReg && typeflag != tar.TypeRegA { + hdr.Size = 0 + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("write header: %v", err) + } + if hdr.Size > 0 { + if _, err := tw.Write(entry.body); err != nil { + t.Fatalf("write body: %v", err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatalf("close tar: %v", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("close gzip: %v", err) + } + return buf.Bytes() +} + +func buildTestArchiveWithPostTarPayload(t *testing.T, entries []testArchiveEntry, trailing []byte) []byte { + t.Helper() + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + for _, entry := range entries { + typeflag := entry.typeflag + if typeflag == 0 { + typeflag = tar.TypeReg + } + hdr := &tar.Header{ + Name: entry.name, + Mode: 0o600, + Size: int64(len(entry.body)), + Typeflag: typeflag, + } + if typeflag != tar.TypeReg && typeflag != tar.TypeRegA { + hdr.Size = 0 + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("write header: %v", err) + } + if hdr.Size > 0 { + if _, err := tw.Write(entry.body); err != nil { + t.Fatalf("write body: %v", err) + } + } + } + if err := tw.Close(); err != nil { + t.Fatalf("close tar: %v", err) + } + if _, err := gz.Write(trailing); err != nil { + t.Fatalf("write trailing gzip payload: %v", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("close gzip: %v", err) + } + return buf.Bytes() +} + +// buildTestArchivePAX writes an archive whose first entry uses PAX extended +// headers (forced by a custom PAX record). archive/tar consumes those records +// before the entry header surfaces, so validation must reject the format even +// though the visible name is allowlisted. +func buildTestArchivePAX(t *testing.T) []byte { + t.Helper() + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + body := []byte(`{"schema_version":1}`) + hdr := &tar.Header{ + Name: "manifest.json", + Mode: 0o600, + Size: int64(len(body)), + Typeflag: tar.TypeReg, + Format: tar.FormatPAX, + PAXRecords: map[string]string{"SILO.smuggled": "payload"}, + } + if err := tw.WriteHeader(hdr); err != nil { + t.Fatalf("write pax header: %v", err) + } + if _, err := tw.Write(body); err != nil { + t.Fatalf("write pax body: %v", err) + } + if err := tw.Close(); err != nil { + t.Fatalf("close tar: %v", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("close gzip: %v", err) + } + return buf.Bytes() +} + +func buildTestGzipMember(t *testing.T, payload []byte) []byte { + t.Helper() + + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + if _, err := gz.Write(payload); err != nil { + t.Fatalf("write gzip payload: %v", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("close gzip: %v", err) + } + return buf.Bytes() +} + +func fmtHex(raw []byte) string { + const table = "0123456789abcdef" + out := make([]byte, len(raw)*2) + for i, b := range raw { + out[i*2] = table[b>>4] + out[i*2+1] = table[b&0x0f] + } + return string(out) +} diff --git a/internal/diagnostics/cleanup.go b/internal/diagnostics/cleanup.go new file mode 100644 index 00000000..ff591366 --- /dev/null +++ b/internal/diagnostics/cleanup.go @@ -0,0 +1,257 @@ +package diagnostics + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "time" +) + +const ( + ObjectPrefix = diagnosticObjectPrefix + "/" + DefaultReceivingGrace = time.Hour +) + +type CleanupRepository interface { + DeleteByID(ctx context.Context, id string) (*Report, error) + MarkFailed(ctx context.Context, id string) error + RetentionCandidates(ctx context.Context, olderThan time.Time, perUserByteCap int64) ([]Report, error) + StaleReceiving(ctx context.Context, grace time.Duration) ([]Report, error) + LiveBlobKeys(ctx context.Context, keys []string) (map[string]ReportState, error) +} + +type CleanupOptions struct { + Now func() time.Time + StaleReceivingGrace time.Duration + Logger *slog.Logger +} + +type CleanupResult struct { + RetentionReportsDeleted int `json:"retention_reports_deleted"` + StaleReportsDeleted int `json:"stale_reports_deleted"` + OrphanObjectsDeleted int `json:"orphan_objects_deleted"` +} + +func (r CleanupResult) ReportsDeleted() int { + return r.RetentionReportsDeleted + r.StaleReportsDeleted +} + +func CleanupOnce( + ctx context.Context, + repo CleanupRepository, + settingsStore SettingsStore, + store ObjectStore, + logger *slog.Logger, +) (CleanupResult, error) { + settings, err := LoadSettings(ctx, settingsStore) + if err != nil { + return CleanupResult{}, fmt.Errorf("load diagnostics cleanup settings: %w", err) + } + return CleanupReports(ctx, repo, store, settings, CleanupOptions{Logger: logger}) +} + +func CleanupReports( + ctx context.Context, + repo CleanupRepository, + store ObjectStore, + settings Settings, + opts CleanupOptions, +) (CleanupResult, error) { + if repo == nil { + return CleanupResult{}, nil + } + now := func() time.Time { return time.Now().UTC() } + if opts.Now != nil { + now = opts.Now + } + logger := opts.Logger + if logger == nil { + logger = slog.Default() + } + grace := opts.StaleReceivingGrace + if grace <= 0 { + grace = DefaultReceivingGrace + } + + cutoff := time.Time{} + if settings.RetentionDays > 0 { + cutoff = now().Add(-time.Duration(settings.RetentionDays) * 24 * time.Hour) + } + + // A single poisoned report (e.g. a permission error on one S3 object) must + // not abort the whole run, or every scheduled pass re-hits it first and + // blocks retention, stale reconciliation, and orphan cleanup indefinitely. + // Log-and-continue per report; aggregate per-item failures and still surface + // genuine query/iteration errors. + var result CleanupResult + var errs []error + + candidates, err := repo.RetentionCandidates(ctx, cutoff, settings.MaxBytesPerUser) + if err != nil { + errs = append(errs, fmt.Errorf("load diagnostic retention candidates: %w", err)) + return result, errors.Join(errs...) + } + for _, report := range candidates { + // Row first: a DB failure after the object is gone would leave a visible + // ready report whose bundle 404s on download. If the blob delete then + // fails the row is already gone, so log it for the orphan pass below to + // reap rather than aborting the run (see the admin DeleteReport path). + if _, err := repo.DeleteByID(ctx, report.ID); err != nil && !IsReportNotFound(err) { + errs = append(errs, fmt.Errorf("delete diagnostic retention row %s: %w", report.ID, err)) + continue + } + if err := deleteReportObjects(ctx, store, &report, logger); err != nil { + logDeferredBlobDeletion(ctx, logger, store, &report, err) + } + result.RetentionReportsDeleted++ + } + + stale, err := repo.StaleReceiving(ctx, grace) + if err != nil { + errs = append(errs, fmt.Errorf("load stale diagnostic reports: %w", err)) + return result, errors.Join(errs...) + } + for _, report := range stale { + // Same rows-first ordering as retention: mark/delete the row before the + // blob so a mid-cleanup DB failure never leaves a row pointing at a + // missing object, and let orphan cleanup reap a blob that fails to delete. + if report.State == StateReceiving { + if err := repo.MarkFailed(ctx, report.ID); err != nil && !IsReportNotFound(err) { + errs = append(errs, fmt.Errorf("mark stale diagnostic report failed %s: %w", report.ID, err)) + continue + } + } + if _, err := repo.DeleteByID(ctx, report.ID); err != nil && !IsReportNotFound(err) { + errs = append(errs, fmt.Errorf("delete stale diagnostic row %s: %w", report.ID, err)) + continue + } + if err := deleteReportObjects(ctx, store, &report, logger); err != nil { + logDeferredBlobDeletion(ctx, logger, store, &report, err) + } + result.StaleReportsDeleted++ + } + + if store == nil || strings.TrimSpace(store.Bucket()) == "" { + return result, errors.Join(errs...) + } + keys, err := store.ListObjects(ctx, ObjectPrefix) + if err != nil { + errs = append(errs, fmt.Errorf("list diagnostic objects: %w", err)) + return result, errors.Join(errs...) + } + if len(keys) == 0 { + return result, errors.Join(errs...) + } + liveKeys, err := repo.LiveBlobKeys(ctx, keys) + if err != nil { + errs = append(errs, fmt.Errorf("load live diagnostic blob keys: %w", err)) + return result, errors.Join(errs...) + } + for _, key := range keys { + if _, ok := liveKeys[key]; ok { + continue + } + if err := deleteObjectIfPresent(ctx, store, store.Bucket(), key); err != nil { + errs = append(errs, fmt.Errorf("delete orphan diagnostic object %s: %w", key, err)) + continue + } + logger.InfoContext(ctx, "diagnostic orphan object deleted", + "component", "diagnostics", + "key", key, + ) + result.OrphanObjectsDeleted++ + } + return result, errors.Join(errs...) +} + +func deleteReportObjects(ctx context.Context, store ObjectStore, report *Report, logger *slog.Logger) error { + if report == nil { + return nil + } + keys := reportObjectKeys(*report) + if store == nil || strings.TrimSpace(store.Bucket()) == "" { + if len(keys) == 0 { + return nil + } + logSkippedObjectDeletion(ctx, logger, report.ID, keys) + return nil + } + bucket := stringValue(report.BlobBucket) + if bucket == "" { + bucket = store.Bucket() + } + for _, key := range keys { + if err := deleteObjectIfPresent(ctx, store, bucket, key); err != nil { + return err + } + } + return nil +} + +func logSkippedObjectDeletion(ctx context.Context, logger *slog.Logger, reportID string, keys []string) { + if logger == nil { + logger = slog.Default() + } + logger.WarnContext(ctx, "diagnostic report blob deletion skipped because storage is unavailable", + "component", "diagnostics", + "report_id", reportID, + "keys", keys, + ) +} + +// logDeferredBlobDeletion records a report blob that could not be deleted after +// its DB row was already removed. The row is the source of truth, so the object +// is now an orphan for the orphan-cleanup pass (this run or a later one) to +// reap; we log the bucket and keys rather than fail so one unreachable object +// can't block row deletion. Shared by the retention/stale loops and the admin +// DeleteReport path. +func logDeferredBlobDeletion(ctx context.Context, logger *slog.Logger, store ObjectStore, report *Report, err error) { + if logger == nil { + logger = slog.Default() + } + logger.ErrorContext(ctx, "diagnostic report blob deletion failed after row deletion", + "component", "diagnostics", + "report_id", report.ID, + "bucket", reportBlobBucket(report, store), + "keys", reportObjectKeys(*report), + "error", err, + ) +} + +func deleteObjectIfPresent(ctx context.Context, store ObjectStore, bucket, key string) error { + key = strings.TrimSpace(key) + if key == "" { + return nil + } + if err := store.DeleteObject(ctx, bucket, key); err != nil && !IsObjectNotFound(err) { + return err + } + return nil +} + +func reportObjectKeys(report Report) []string { + keys := make([]string, 0, 2) + if key := stringValue(report.BlobKey); key != "" { + keys = append(keys, key) + } + if report.ID != "" && report.UserID > 0 { + deterministic := reportObjectKey(report.UserID, report.ID) + seen := false + for _, key := range keys { + if key == deterministic { + seen = true + break + } + } + if !seen { + keys = append(keys, deterministic) + } + } + return keys +} + +func IsReportNotFound(err error) bool { + return errors.Is(err, ErrNotFound) +} diff --git a/internal/diagnostics/cleanup_test.go b/internal/diagnostics/cleanup_test.go new file mode 100644 index 00000000..c42a441e --- /dev/null +++ b/internal/diagnostics/cleanup_test.go @@ -0,0 +1,296 @@ +package diagnostics + +import ( + "context" + "errors" + "io" + "log/slog" + "testing" + "time" +) + +func TestCleanupReportsDeletesRowBeforeBlobAndToleratesMissingObject(t *testing.T) { + ops := []string{} + repo := &fakeCleanupRepo{ + retention: []Report{ + testCleanupReport("r1", 7, StateReady, "diagnostics/7/r1.tar.gz"), + testCleanupReport("r2", 7, StateReady, "diagnostics/7/r2.tar.gz"), + }, + ops: &ops, + } + store := &fakeCleanupStore{ + bucket: "private", + missingKeys: map[string]bool{"diagnostics/7/r1.tar.gz": true}, + ops: &ops, + } + + result, err := CleanupReports(context.Background(), repo, store, Settings{ + RetentionDays: 30, + MaxBytesPerUser: DefaultMaxBytesPerUser, + }, CleanupOptions{ + Now: func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) }, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + if err != nil { + t.Fatalf("CleanupReports: %v", err) + } + if result.RetentionReportsDeleted != 2 { + t.Fatalf("RetentionReportsDeleted = %d, want 2", result.RetentionReportsDeleted) + } + wantOps := []string{ + "delete-row:r1", + "delete-object:diagnostics/7/r1.tar.gz", + "delete-row:r2", + "delete-object:diagnostics/7/r2.tar.gz", + } + assertStrings(t, ops, wantOps) +} + +func TestCleanupReportsTreatsBlobFailureAsNonFatal(t *testing.T) { + ops := []string{} + repo := &fakeCleanupRepo{ + retention: []Report{ + testCleanupReport("r1", 7, StateReady, "diagnostics/7/r1.tar.gz"), + testCleanupReport("r2", 7, StateReady, "diagnostics/7/r2.tar.gz"), + }, + ops: &ops, + } + store := &fakeCleanupStore{ + bucket: "private", + deleteErrs: map[string]error{"diagnostics/7/r1.tar.gz": errors.New("s3 access denied")}, + ops: &ops, + } + + result, err := CleanupReports(context.Background(), repo, store, Settings{ + RetentionDays: 30, + MaxBytesPerUser: DefaultMaxBytesPerUser, + }, CleanupOptions{ + Now: func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) }, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + // r1's row is deleted first, so its blob delete failing does not abort the + // run or roll back the row: it is logged for orphan cleanup to reap, both + // rows count as deleted, and no error is surfaced. + if err != nil { + t.Fatalf("CleanupReports: %v", err) + } + if result.RetentionReportsDeleted != 2 { + t.Fatalf("RetentionReportsDeleted = %d, want 2", result.RetentionReportsDeleted) + } + assertStrings(t, ops, []string{ + "delete-row:r1", + "delete-object:diagnostics/7/r1.tar.gz", + "delete-row:r2", + "delete-object:diagnostics/7/r2.tar.gz", + }) +} + +func TestCleanupReportsCleansStaleReceiving(t *testing.T) { + ops := []string{} + repo := &fakeCleanupRepo{ + stale: []Report{ + testCleanupReport("r1", 7, StateReceiving, ""), + }, + ops: &ops, + } + store := &fakeCleanupStore{bucket: "private", ops: &ops} + + result, err := CleanupReports(context.Background(), repo, store, Settings{ + RetentionDays: 30, + MaxBytesPerUser: DefaultMaxBytesPerUser, + }, CleanupOptions{Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}) + if err != nil { + t.Fatalf("CleanupReports: %v", err) + } + if result.StaleReportsDeleted != 1 { + t.Fatalf("StaleReportsDeleted = %d, want 1", result.StaleReportsDeleted) + } + assertStrings(t, ops, []string{ + "mark-failed:r1", + "delete-row:r1", + "delete-object:diagnostics/7/r1.tar.gz", + }) +} + +func TestCleanupReportsDeletesRowsWhenStorageUnavailable(t *testing.T) { + ops := []string{} + repo := &fakeCleanupRepo{ + retention: []Report{ + testCleanupReport("r1", 7, StateReady, "diagnostics/7/r1.tar.gz"), + }, + stale: []Report{ + testCleanupReport("r2", 7, StateReceiving, "diagnostics/7/r2.tar.gz"), + }, + ops: &ops, + } + + result, err := CleanupReports(context.Background(), repo, nil, Settings{ + RetentionDays: 30, + MaxBytesPerUser: DefaultMaxBytesPerUser, + }, CleanupOptions{ + Now: func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) }, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + }) + if err != nil { + t.Fatalf("CleanupReports: %v", err) + } + if result.RetentionReportsDeleted != 1 || result.StaleReportsDeleted != 1 { + t.Fatalf("result = %#v, want one retention and one stale deletion", result) + } + assertStrings(t, ops, []string{ + "delete-row:r1", + "mark-failed:r2", + "delete-row:r2", + }) +} + +func TestCleanupReportsDeletesUnmatchedObjects(t *testing.T) { + ops := []string{} + repo := &fakeCleanupRepo{ + live: map[string]ReportState{ + "diagnostics/7/ready.tar.gz": StateReady, + "diagnostics/7/receiving.tar.gz": StateReceiving, + }, + ops: &ops, + } + store := &fakeCleanupStore{ + bucket: "private", + list: []string{ + "diagnostics/7/ready.tar.gz", + "diagnostics/7/receiving.tar.gz", + "diagnostics/7/orphan.tar.gz", + "diagnostics/7/failed.tar.gz", + }, + ops: &ops, + } + + result, err := CleanupReports(context.Background(), repo, store, Settings{ + RetentionDays: 30, + MaxBytesPerUser: DefaultMaxBytesPerUser, + }, CleanupOptions{Logger: slog.New(slog.NewTextHandler(io.Discard, nil))}) + if err != nil { + t.Fatalf("CleanupReports: %v", err) + } + if result.OrphanObjectsDeleted != 2 { + t.Fatalf("OrphanObjectsDeleted = %d, want 2", result.OrphanObjectsDeleted) + } + wantDeleted := []string{"diagnostics/7/orphan.tar.gz", "diagnostics/7/failed.tar.gz"} + if !sameStrings(store.deleted, wantDeleted) { + t.Fatalf("deleted objects = %v, want %v", store.deleted, wantDeleted) + } +} + +func testCleanupReport(id string, userID int, state ReportState, blobKey string) Report { + report := Report{ID: id, UserID: userID, State: state} + if blobKey != "" { + report.BlobKey = &blobKey + } + return report +} + +type fakeCleanupRepo struct { + retention []Report + stale []Report + live map[string]ReportState + ops *[]string +} + +func (f *fakeCleanupRepo) DeleteByID(_ context.Context, id string) (*Report, error) { + f.record("delete-row:" + id) + return &Report{ID: id}, nil +} + +func (f *fakeCleanupRepo) MarkFailed(_ context.Context, id string) error { + f.record("mark-failed:" + id) + return nil +} + +func (f *fakeCleanupRepo) RetentionCandidates(context.Context, time.Time, int64) ([]Report, error) { + return append([]Report(nil), f.retention...), nil +} + +func (f *fakeCleanupRepo) StaleReceiving(context.Context, time.Duration) ([]Report, error) { + return append([]Report(nil), f.stale...), nil +} + +func (f *fakeCleanupRepo) LiveBlobKeys(_ context.Context, keys []string) (map[string]ReportState, error) { + live := make(map[string]ReportState) + for _, key := range keys { + if state, ok := f.live[key]; ok { + live[key] = state + } + } + return live, nil +} + +func (f *fakeCleanupRepo) record(op string) { + if f.ops != nil { + *f.ops = append(*f.ops, op) + } +} + +type fakeCleanupStore struct { + bucket string + list []string + missingKeys map[string]bool + deleteErrs map[string]error + deleted []string + ops *[]string +} + +func (f *fakeCleanupStore) PutStream(context.Context, string, string, io.Reader, string) error { + return nil +} + +func (f *fakeCleanupStore) GetObject(context.Context, string, string) (io.ReadCloser, error) { + return nil, errors.New("not implemented") +} + +func (f *fakeCleanupStore) DeleteObject(_ context.Context, _ string, key string) error { + f.record("delete-object:" + key) + f.deleted = append(f.deleted, key) + if f.missingKeys[key] { + return ErrObjectNotFound + } + if err := f.deleteErrs[key]; err != nil { + return err + } + return nil +} + +func (f *fakeCleanupStore) ListObjects(context.Context, string) ([]string, error) { + return append([]string(nil), f.list...), nil +} + +func (f *fakeCleanupStore) PresignGetURL(context.Context, string, string, time.Duration) (string, error) { + return "", errors.New("not implemented") +} + +func (f *fakeCleanupStore) Bucket() string { + return f.bucket +} + +func (f *fakeCleanupStore) record(op string) { + if f.ops != nil { + *f.ops = append(*f.ops, op) + } +} + +func assertStrings(t *testing.T, got, want []string) { + t.Helper() + if !sameStrings(got, want) { + t.Fatalf("ops = %v, want %v", got, want) + } +} + +func sameStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/diagnostics/contract/contract.go b/internal/diagnostics/contract/contract.go new file mode 100644 index 00000000..6709f049 --- /dev/null +++ b/internal/diagnostics/contract/contract.go @@ -0,0 +1,1069 @@ +package contract + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "regexp" + "slices" + "strconv" + "strings" + "time" +) + +const ( + SchemaVersion = 1 + MaxStackExcerptBytes = 8192 + // MaxLogLineBytes bounds a single newline-delimited entry line + // (logs.jsonl / breadcrumbs.jsonl) during streaming bundle validation. It + // is well above the per-field caps a log line can legitimately reach (msg + // 2048, tag/run 128, plus registered attrs) so it never rejects a valid + // line, while keeping per-line memory bounded when validating. + MaxLogLineBytes = 64 * 1024 +) + +var ( + manifestRequiredFields = []string{ + "schema_version", + "report", + "destination", + "consent", + "device_summary", + "playback_session_ids", + "log_summary", + "archive", + } + manifestReportRequiredFields = []string{ + "type", + "captured_at", + "capture_session_id", + "app_version", + "app_build", + "platform", + "os_version", + } + manifestDestinationRequiredFields = []string{"server_instance_id"} + manifestConsentRequiredFields = []string{"mode", "notice_version"} + manifestCrashRequiredFields = []string{"summary", "source", "provenance", "occurred_at"} + manifestDeviceSummaryFields = []string{"manufacturer", "model", "os", "form_factor"} + manifestLogSummaryRequiredFields = []string{"lines", "bytes_gz", "dropped_lines", "categories", "debug_logging"} + manifestArchiveRequiredFields = []string{"entries", "bytes", "uncompressed_bytes", "sha256"} + + reportTypes = []string{"crash", "anr", "native_crash", "hang", "abnormal_exit", "manual"} + platforms = []string{"android", "android-tv", "ios", "tvos"} + crashSources = []string{"ueh", "exit_info", "metrickit", "exit_sentinel"} + crashProvenances = []string{"pre_failure", "post_restart", "metric_reporting_period"} + consentModes = []string{"prompt", "always", "manual"} + logLevels = []string{"V", "D", "I", "W", "E"} + logCategories = []string{"playback", "focus", "network", "lifecycle", "browse", "cast", "download", "crash", "other"} + deviceProvenances = []string{ + "pre_failure", + "post_restart", + } + deviceRequiredFields = []string{"captured_at", "provenance", "identity", "display", "audio", "video_codecs", "network"} + + ArchiveEntryAllowlist = []string{ + "manifest.json", + "device.json", + "logs.jsonl", + "crash/summary.json", + "crash/stack.txt", + "crash/tombstone.pb", + "crash/metrickit.json", + "breadcrumbs.jsonl", + } + + sha256Pattern = regexp.MustCompile(`^[A-Fa-f0-9]{64}$`) +) + +type Manifest struct { + SchemaVersion int + Report Report + Destination Destination + Consent Consent + Crash *Crash + DeviceSummary DeviceSummary + PlaybackSessionIDs []string + LogSummary LogSummary + Archive Archive +} + +type Report struct { + Type string + CapturedAt time.Time + CaptureSessionID string + AppVersion string + AppBuild string + Platform string + OSVersion string + ProfileID string +} + +type Destination struct { + ServerInstanceID string +} + +type Consent struct { + Mode string + NoticeVersion int +} + +type Crash struct { + Summary string + StackExcerpt string + Thread string + Foreground *bool + Source string + Provenance string + OccurredAt time.Time +} + +type DeviceSummary struct { + Manufacturer string + Model string + OS string + FormFactor string +} + +type LogSummary struct { + Lines int64 + BytesGz int64 + DroppedLines int64 + Categories []string + DebugLogging bool +} + +type Archive struct { + Entries []string + // Bytes is the size of the bundle part as transmitted (the gzip stream). + Bytes int64 + // UncompressedBytes is the total size of the decompressed tar stream, + // including headers, end-of-archive blocks, and record padding — the + // byte count a client observes between its tar writer and gzip writer. + UncompressedBytes int64 + // SHA256 is the hex digest of the bundle part as transmitted. + SHA256 string +} + +type DeviceSnapshot struct { + CapturedAt time.Time + Provenance string + Identity json.RawMessage + Display json.RawMessage + Audio json.RawMessage + VideoCodecs json.RawMessage + Network json.RawMessage + OriginalJSON json.RawMessage +} + +type LogLine struct { + Timestamp time.Time + Run string + Level string + Category string + Tag string + Message string + Attrs map[string]any +} + +type manifestWire struct { + SchemaVersion *int `json:"schema_version"` + Report *reportWire `json:"report"` + Destination *destinationWire `json:"destination"` + Consent *consentWire `json:"consent"` + Crash *crashWire `json:"crash"` + DeviceSummary *deviceSummaryWire `json:"device_summary"` + PlaybackSessionIDs *[]string `json:"playback_session_ids"` + LogSummary *logSummaryWire `json:"log_summary"` + Archive *archiveWire `json:"archive"` +} + +type reportWire struct { + Type *string `json:"type"` + CapturedAt *string `json:"captured_at"` + CaptureSessionID *string `json:"capture_session_id"` + AppVersion *string `json:"app_version"` + AppBuild *string `json:"app_build"` + Platform *string `json:"platform"` + OSVersion *string `json:"os_version"` + ProfileID *string `json:"profile_id"` +} + +type destinationWire struct { + ServerInstanceID *string `json:"server_instance_id"` +} + +type consentWire struct { + Mode *string `json:"mode"` + NoticeVersion *int `json:"notice_version"` +} + +type crashWire struct { + Summary *string `json:"summary"` + StackExcerpt *string `json:"stack_excerpt"` + Thread *string `json:"thread"` + Foreground *bool `json:"foreground"` + Source *string `json:"source"` + Provenance *string `json:"provenance"` + OccurredAt *string `json:"occurred_at"` +} + +type deviceSummaryWire struct { + Manufacturer *string `json:"manufacturer"` + Model *string `json:"model"` + OS *string `json:"os"` + FormFactor *string `json:"form_factor"` +} + +type logSummaryWire struct { + Lines *int64 `json:"lines"` + BytesGz *int64 `json:"bytes_gz"` + DroppedLines *int64 `json:"dropped_lines"` + Categories *[]string `json:"categories"` + DebugLogging *bool `json:"debug_logging"` +} + +type archiveWire struct { + Entries *[]string `json:"entries"` + Bytes *int64 `json:"bytes"` + UncompressedBytes *int64 `json:"uncompressed_bytes"` + SHA256 *string `json:"sha256"` +} + +type logLineWire struct { + Timestamp *string `json:"ts"` + Run *string `json:"run"` + Level *string `json:"lvl"` + Category *string `json:"cat"` + Tag *string `json:"tag"` + Message *string `json:"msg"` + Attrs json.RawMessage `json:"attrs"` +} + +type attrValueType string + +const ( + attrString attrValueType = "string" + attrInteger attrValueType = "integer" +) + +var attrRegistry = map[string]map[string]attrValueType{ + "playback": { + "sink": attrString, + "fmt": attrString, + "decoder": attrString, + "width": attrInteger, + "height": attrInteger, + "hdr_mode": attrString, + "bitrate_kbps": attrInteger, + "dropped_frames": attrInteger, + "audio_underruns": attrInteger, + }, + "focus": { + "target": attrString, + "action": attrString, + }, + "network": { + "method": attrString, + "path": attrString, + "status": attrInteger, + "duration_ms": attrInteger, + }, + "lifecycle": { + "state": attrString, + }, + "crash": { + "fingerprint": attrString, + "source": attrString, + }, +} + +func ValidateManifest(data []byte) (Manifest, error) { + var wire manifestWire + if err := decodeJSON(data, &wire); err != nil { + return Manifest{}, err + } + + if wire.SchemaVersion == nil { + return Manifest{}, requiredError("manifest.schema_version") + } + if *wire.SchemaVersion != SchemaVersion { + return Manifest{}, fieldError("manifest.schema_version", "unsupported value %d", *wire.SchemaVersion) + } + + if wire.Report == nil { + return Manifest{}, requiredError("manifest.report") + } + report, err := validateReport(*wire.Report) + if err != nil { + return Manifest{}, err + } + + if wire.Destination == nil { + return Manifest{}, requiredError("manifest.destination") + } + destination, err := validateDestination(*wire.Destination) + if err != nil { + return Manifest{}, err + } + + if wire.Consent == nil { + return Manifest{}, requiredError("manifest.consent") + } + consent, err := validateConsent(*wire.Consent) + if err != nil { + return Manifest{}, err + } + + // Per the spec the crash object is present for every event report type and + // absent for manual captures. Enforce both directions so an event report + // can't be accepted with no crash summary/source/occurred_at to diagnose. + var crash *Crash + switch { + case report.Type == "manual": + if wire.Crash != nil { + return Manifest{}, fieldError("manifest.crash", "must be absent for manual reports") + } + case wire.Crash == nil: + return Manifest{}, requiredError("manifest.crash") + default: + validated, err := validateCrash(*wire.Crash) + if err != nil { + return Manifest{}, err + } + crash = &validated + } + + if wire.DeviceSummary == nil { + return Manifest{}, requiredError("manifest.device_summary") + } + deviceSummary, err := validateDeviceSummary(*wire.DeviceSummary) + if err != nil { + return Manifest{}, err + } + + if wire.PlaybackSessionIDs == nil { + return Manifest{}, requiredError("manifest.playback_session_ids") + } + playbackSessionIDs, err := validateStringSlice("manifest.playback_session_ids", *wire.PlaybackSessionIDs, 20, 1, 128, false) + if err != nil { + return Manifest{}, err + } + + if wire.LogSummary == nil { + return Manifest{}, requiredError("manifest.log_summary") + } + logSummary, err := validateLogSummary(*wire.LogSummary) + if err != nil { + return Manifest{}, err + } + + if wire.Archive == nil { + return Manifest{}, requiredError("manifest.archive") + } + archive, err := validateArchive(*wire.Archive) + if err != nil { + return Manifest{}, err + } + + return Manifest{ + SchemaVersion: *wire.SchemaVersion, + Report: report, + Destination: destination, + Consent: consent, + Crash: crash, + DeviceSummary: deviceSummary, + PlaybackSessionIDs: playbackSessionIDs, + LogSummary: logSummary, + Archive: archive, + }, nil +} + +func ValidateDevice(data []byte) (DeviceSnapshot, error) { + var raw map[string]json.RawMessage + if err := decodeJSON(data, &raw); err != nil { + return DeviceSnapshot{}, err + } + + capturedAt, err := requiredRawDateTime(raw, "captured_at", "device.captured_at") + if err != nil { + return DeviceSnapshot{}, err + } + provenance, err := requiredRawString(raw, "provenance", "device.provenance", 1, 32) + if err != nil { + return DeviceSnapshot{}, err + } + if err := validateEnum("device.provenance", provenance, deviceProvenances); err != nil { + return DeviceSnapshot{}, err + } + + identity, err := requiredRawSection(raw, "identity", "device.identity", sectionObject) + if err != nil { + return DeviceSnapshot{}, err + } + display, err := requiredRawSection(raw, "display", "device.display", sectionObject) + if err != nil { + return DeviceSnapshot{}, err + } + audio, err := requiredRawSection(raw, "audio", "device.audio", sectionObject) + if err != nil { + return DeviceSnapshot{}, err + } + videoCodecs, err := requiredRawSection(raw, "video_codecs", "device.video_codecs", sectionArray) + if err != nil { + return DeviceSnapshot{}, err + } + network, err := requiredRawSection(raw, "network", "device.network", sectionObject) + if err != nil { + return DeviceSnapshot{}, err + } + + return DeviceSnapshot{ + CapturedAt: capturedAt, + Provenance: provenance, + Identity: append(json.RawMessage(nil), identity...), + Display: append(json.RawMessage(nil), display...), + Audio: append(json.RawMessage(nil), audio...), + VideoCodecs: append(json.RawMessage(nil), videoCodecs...), + Network: append(json.RawMessage(nil), network...), + OriginalJSON: append(json.RawMessage(nil), bytes.TrimSpace(data)...), + }, nil +} + +func ValidateLogLine(data []byte) (LogLine, error) { + var wire logLineWire + if err := decodeJSON(data, &wire); err != nil { + return LogLine{}, err + } + + timestamp, err := validateDateTime("log.ts", wire.Timestamp) + if err != nil { + return LogLine{}, err + } + run, err := validateRequiredString("log.run", wire.Run, 1, 128) + if err != nil { + return LogLine{}, err + } + level, err := validateRequiredString("log.lvl", wire.Level, 1, 1) + if err != nil { + return LogLine{}, err + } + if err := validateEnum("log.lvl", level, logLevels); err != nil { + return LogLine{}, err + } + category, err := validateRequiredString("log.cat", wire.Category, 1, 32) + if err != nil { + return LogLine{}, err + } + if err := validateEnum("log.cat", category, logCategories); err != nil { + return LogLine{}, err + } + tag, err := validateRequiredString("log.tag", wire.Tag, 1, 128) + if err != nil { + return LogLine{}, err + } + msg, err := validateRequiredString("log.msg", wire.Message, 1, 2048) + if err != nil { + return LogLine{}, err + } + + attrs, err := validateAttrs(category, wire.Attrs) + if err != nil { + return LogLine{}, err + } + + return LogLine{ + Timestamp: timestamp, + Run: run, + Level: level, + Category: category, + Tag: tag, + Message: msg, + Attrs: attrs, + }, nil +} + +func ArchiveEntryAllowed(name string) bool { + return slices.Contains(ArchiveEntryAllowlist, name) +} + +func validateReport(w reportWire) (Report, error) { + reportType, err := validateRequiredString("manifest.report.type", w.Type, 1, 32) + if err != nil { + return Report{}, err + } + if err := validateEnum("manifest.report.type", reportType, reportTypes); err != nil { + return Report{}, err + } + capturedAt, err := validateDateTime("manifest.report.captured_at", w.CapturedAt) + if err != nil { + return Report{}, err + } + captureSessionID, err := validateRequiredString("manifest.report.capture_session_id", w.CaptureSessionID, 1, 128) + if err != nil { + return Report{}, err + } + appVersion, err := validateRequiredString("manifest.report.app_version", w.AppVersion, 1, 64) + if err != nil { + return Report{}, err + } + appBuild, err := validateRequiredString("manifest.report.app_build", w.AppBuild, 1, 64) + if err != nil { + return Report{}, err + } + platform, err := validateRequiredString("manifest.report.platform", w.Platform, 1, 32) + if err != nil { + return Report{}, err + } + if err := validateEnum("manifest.report.platform", platform, platforms); err != nil { + return Report{}, err + } + osVersion, err := validateRequiredString("manifest.report.os_version", w.OSVersion, 1, 128) + if err != nil { + return Report{}, err + } + profileID, err := validateOptionalString("manifest.report.profile_id", w.ProfileID, 128) + if err != nil { + return Report{}, err + } + + return Report{ + Type: reportType, + CapturedAt: capturedAt, + CaptureSessionID: captureSessionID, + AppVersion: appVersion, + AppBuild: appBuild, + Platform: platform, + OSVersion: osVersion, + ProfileID: profileID, + }, nil +} + +func validateDestination(w destinationWire) (Destination, error) { + serverInstanceID, err := validateRequiredString("manifest.destination.server_instance_id", w.ServerInstanceID, 1, 128) + if err != nil { + return Destination{}, err + } + return Destination{ServerInstanceID: serverInstanceID}, nil +} + +func validateConsent(w consentWire) (Consent, error) { + mode, err := validateRequiredString("manifest.consent.mode", w.Mode, 1, 32) + if err != nil { + return Consent{}, err + } + if err := validateEnum("manifest.consent.mode", mode, consentModes); err != nil { + return Consent{}, err + } + if w.NoticeVersion == nil { + return Consent{}, requiredError("manifest.consent.notice_version") + } + if *w.NoticeVersion < 1 { + return Consent{}, fieldError("manifest.consent.notice_version", "must be >= 1") + } + return Consent{Mode: mode, NoticeVersion: *w.NoticeVersion}, nil +} + +func validateCrash(w crashWire) (Crash, error) { + summary, err := validateRequiredString("manifest.crash.summary", w.Summary, 1, MaxStackExcerptBytes) + if err != nil { + return Crash{}, err + } + stackExcerpt, err := validateOptionalString("manifest.crash.stack_excerpt", w.StackExcerpt, MaxStackExcerptBytes) + if err != nil { + return Crash{}, err + } + thread, err := validateOptionalString("manifest.crash.thread", w.Thread, 128) + if err != nil { + return Crash{}, err + } + source, err := validateRequiredString("manifest.crash.source", w.Source, 1, 32) + if err != nil { + return Crash{}, err + } + if err := validateEnum("manifest.crash.source", source, crashSources); err != nil { + return Crash{}, err + } + provenance, err := validateRequiredString("manifest.crash.provenance", w.Provenance, 1, 32) + if err != nil { + return Crash{}, err + } + if err := validateEnum("manifest.crash.provenance", provenance, crashProvenances); err != nil { + return Crash{}, err + } + occurredAt, err := validateDateTime("manifest.crash.occurred_at", w.OccurredAt) + if err != nil { + return Crash{}, err + } + + return Crash{ + Summary: summary, + StackExcerpt: stackExcerpt, + Thread: thread, + Foreground: w.Foreground, + Source: source, + Provenance: provenance, + OccurredAt: occurredAt, + }, nil +} + +func validateDeviceSummary(w deviceSummaryWire) (DeviceSummary, error) { + manufacturer, err := validateRequiredString("manifest.device_summary.manufacturer", w.Manufacturer, 1, 128) + if err != nil { + return DeviceSummary{}, err + } + model, err := validateRequiredString("manifest.device_summary.model", w.Model, 1, 128) + if err != nil { + return DeviceSummary{}, err + } + osVersion, err := validateRequiredString("manifest.device_summary.os", w.OS, 1, 128) + if err != nil { + return DeviceSummary{}, err + } + formFactor, err := validateRequiredString("manifest.device_summary.form_factor", w.FormFactor, 1, 64) + if err != nil { + return DeviceSummary{}, err + } + return DeviceSummary{ + Manufacturer: manufacturer, + Model: model, + OS: osVersion, + FormFactor: formFactor, + }, nil +} + +func validateLogSummary(w logSummaryWire) (LogSummary, error) { + lines, err := validateNonNegativeInt64("manifest.log_summary.lines", w.Lines) + if err != nil { + return LogSummary{}, err + } + bytesGz, err := validateNonNegativeInt64("manifest.log_summary.bytes_gz", w.BytesGz) + if err != nil { + return LogSummary{}, err + } + droppedLines, err := validateNonNegativeInt64("manifest.log_summary.dropped_lines", w.DroppedLines) + if err != nil { + return LogSummary{}, err + } + if w.Categories == nil { + return LogSummary{}, requiredError("manifest.log_summary.categories") + } + categories, err := validateStringSlice("manifest.log_summary.categories", *w.Categories, 9, 1, 32, true) + if err != nil { + return LogSummary{}, err + } + for i, category := range categories { + if err := validateEnum(fmt.Sprintf("manifest.log_summary.categories[%d]", i), category, logCategories); err != nil { + return LogSummary{}, err + } + } + if w.DebugLogging == nil { + return LogSummary{}, requiredError("manifest.log_summary.debug_logging") + } + return LogSummary{ + Lines: lines, + BytesGz: bytesGz, + DroppedLines: droppedLines, + Categories: categories, + DebugLogging: *w.DebugLogging, + }, nil +} + +func validateArchive(w archiveWire) (Archive, error) { + if w.Entries == nil { + return Archive{}, requiredError("manifest.archive.entries") + } + entries, err := validateStringSlice("manifest.archive.entries", *w.Entries, len(ArchiveEntryAllowlist), 1, 64, true) + if err != nil { + return Archive{}, err + } + hasManifest := false + for i, entry := range entries { + if !ArchiveEntryAllowed(entry) { + return Archive{}, fieldError(fmt.Sprintf("manifest.archive.entries[%d]", i), "unsupported value %q", entry) + } + if entry == "manifest.json" { + hasManifest = true + } + } + if !hasManifest { + return Archive{}, fieldError("manifest.archive.entries", "must include manifest.json") + } + bytesCount, err := validateNonNegativeInt64("manifest.archive.bytes", w.Bytes) + if err != nil { + return Archive{}, err + } + uncompressedBytes, err := validateNonNegativeInt64("manifest.archive.uncompressed_bytes", w.UncompressedBytes) + if err != nil { + return Archive{}, err + } + sha256Value, err := validateRequiredString("manifest.archive.sha256", w.SHA256, 64, 64) + if err != nil { + return Archive{}, err + } + if !sha256Pattern.MatchString(sha256Value) { + return Archive{}, fieldError("manifest.archive.sha256", "must be 64 hexadecimal characters") + } + return Archive{ + Entries: entries, + Bytes: bytesCount, + UncompressedBytes: uncompressedBytes, + SHA256: strings.ToLower(sha256Value), + }, nil +} + +type sectionKind int + +const ( + sectionObject sectionKind = iota + sectionArray +) + +func requiredRawSection(raw map[string]json.RawMessage, key, path string, kind sectionKind) (json.RawMessage, error) { + value, ok := raw[key] + if !ok { + return nil, requiredError(path) + } + if isRawNull(value) { + return nil, fieldError(path, "must not be null") + } + + if sentinel, ok, err := rawSentinel(value); err != nil { + return nil, err + } else if ok { + if sentinel != "unknown" && sentinel != "not_collected" { + return nil, fieldError(path, "unsupported sentinel %q", sentinel) + } + return append(json.RawMessage(nil), value...), nil + } + + switch kind { + case sectionObject: + if err := validateDeviceObject(path, value, 0); err != nil { + return nil, err + } + case sectionArray: + if err := validateVideoCodecs(path, value); err != nil { + return nil, err + } + default: + return nil, fieldError(path, "unsupported section kind") + } + return append(json.RawMessage(nil), value...), nil +} + +func validateVideoCodecs(path string, raw json.RawMessage) error { + var items []json.RawMessage + if err := decodeJSON(raw, &items); err != nil { + return fieldError(path, "must be an array or unknown/not_collected") + } + if len(items) > 128 { + return fieldError(path, "must contain <= 128 items") + } + for i, item := range items { + if err := validateDeviceObject(fmt.Sprintf("%s[%d]", path, i), item, 0); err != nil { + return err + } + } + return nil +} + +func validateDeviceObject(path string, raw json.RawMessage, depth int) error { + if depth > 8 { + return fieldError(path, "is nested too deeply") + } + var obj map[string]json.RawMessage + if err := decodeJSON(raw, &obj); err != nil { + return fieldError(path, "must be an object or unknown/not_collected") + } + if len(obj) > 64 { + return fieldError(path, "must contain <= 64 properties") + } + for key, value := range obj { + if err := validateStringBytes(path+" key", key, 1, 128); err != nil { + return err + } + if err := validateDeviceValue(path+"."+key, value, depth+1); err != nil { + return err + } + } + return nil +} + +func validateDeviceValue(path string, raw json.RawMessage, depth int) error { + if depth > 8 { + return fieldError(path, "is nested too deeply") + } + if isRawNull(raw) { + return nil + } + if _, ok, err := rawSentinel(raw); err != nil { + return err + } else if ok { + var s string + if err := json.Unmarshal(raw, &s); err != nil { + return fieldError(path, "must be a valid string") + } + return validateStringBytes(path, s, 0, 512) + } + + switch firstRawByte(raw) { + case '{': + return validateDeviceObject(path, raw, depth+1) + case '[': + var items []json.RawMessage + if err := decodeJSON(raw, &items); err != nil { + return fieldError(path, "must be an array") + } + if len(items) > 128 { + return fieldError(path, "must contain <= 128 items") + } + for i, item := range items { + if err := validateDeviceValue(fmt.Sprintf("%s[%d]", path, i), item, depth+1); err != nil { + return err + } + } + return nil + case '"': + var s string + if err := json.Unmarshal(raw, &s); err != nil { + return fieldError(path, "must be a valid string") + } + return validateStringBytes(path, s, 0, 512) + case 't', 'f': + var b bool + if err := json.Unmarshal(raw, &b); err != nil { + return fieldError(path, "must be a boolean") + } + return nil + default: + var n json.Number + if err := decodeJSON(raw, &n); err != nil { + return fieldError(path, "must be a string, number, boolean, null, array, or object") + } + if _, err := strconv.ParseFloat(n.String(), 64); err != nil { + return fieldError(path, "must be a valid number") + } + return nil + } +} + +func validateAttrs(category string, raw json.RawMessage) (map[string]any, error) { + if len(bytes.TrimSpace(raw)) == 0 { + return nil, nil + } + if isRawNull(raw) { + return nil, fieldError("log.attrs", "must be an object") + } + + var attrs map[string]json.RawMessage + if err := decodeJSON(raw, &attrs); err != nil { + return nil, fieldError("log.attrs", "must be an object") + } + + registered := attrRegistry[category] + if len(registered) == 0 { + return nil, nil + } + + filtered := make(map[string]any) + for key, value := range attrs { + valueType, ok := registered[key] + if !ok { + continue + } + decoded, err := validateAttrValue("log.attrs."+key, value, valueType) + if err != nil { + return nil, err + } + filtered[key] = decoded + } + if len(filtered) == 0 { + return nil, nil + } + return filtered, nil +} + +func validateAttrValue(path string, raw json.RawMessage, valueType attrValueType) (any, error) { + switch valueType { + case attrString: + var s string + if err := json.Unmarshal(raw, &s); err != nil { + return nil, fieldError(path, "must be a string") + } + if err := validateStringBytes(path, s, 0, 512); err != nil { + return nil, err + } + return s, nil + case attrInteger: + if b := firstRawByte(raw); b != '-' && (b < '0' || b > '9') { + return nil, fieldError(path, "must be an integer") + } + var n json.Number + if err := decodeJSON(raw, &n); err != nil { + return nil, fieldError(path, "must be an integer") + } + i, err := strconv.ParseInt(n.String(), 10, 64) + if err != nil { + return nil, fieldError(path, "must be an integer") + } + return i, nil + default: + return nil, fieldError(path, "has unsupported registered type %q", valueType) + } +} + +func validateDateTime(path string, value *string) (time.Time, error) { + s, err := validateRequiredString(path, value, 1, 64) + if err != nil { + return time.Time{}, err + } + parsed, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return time.Time{}, fieldError(path, "must be RFC3339 date-time") + } + return parsed, nil +} + +func requiredRawDateTime(raw map[string]json.RawMessage, key, path string) (time.Time, error) { + s, err := requiredRawString(raw, key, path, 1, 64) + if err != nil { + return time.Time{}, err + } + parsed, err := time.Parse(time.RFC3339Nano, s) + if err != nil { + return time.Time{}, fieldError(path, "must be RFC3339 date-time") + } + return parsed, nil +} + +func requiredRawString(raw map[string]json.RawMessage, key, path string, minBytes, maxBytes int) (string, error) { + value, ok := raw[key] + if !ok { + return "", requiredError(path) + } + var s string + if err := json.Unmarshal(value, &s); err != nil { + return "", fieldError(path, "must be a string") + } + if err := validateStringBytes(path, s, minBytes, maxBytes); err != nil { + return "", err + } + return s, nil +} + +func validateRequiredString(path string, value *string, minBytes, maxBytes int) (string, error) { + if value == nil { + return "", requiredError(path) + } + if err := validateStringBytes(path, *value, minBytes, maxBytes); err != nil { + return "", err + } + return *value, nil +} + +func validateOptionalString(path string, value *string, maxBytes int) (string, error) { + if value == nil { + return "", nil + } + if err := validateStringBytes(path, *value, 0, maxBytes); err != nil { + return "", err + } + return *value, nil +} + +func validateStringBytes(path, value string, minBytes, maxBytes int) error { + if len(value) < minBytes { + return fieldError(path, "must be at least %d bytes", minBytes) + } + if maxBytes > 0 && len(value) > maxBytes { + return fieldError(path, "exceeds %d bytes", maxBytes) + } + return nil +} + +func validateNonNegativeInt64(path string, value *int64) (int64, error) { + if value == nil { + return 0, requiredError(path) + } + if *value < 0 { + return 0, fieldError(path, "must be >= 0") + } + return *value, nil +} + +func validateStringSlice(path string, values []string, maxItems, minStringBytes, maxStringBytes int, unique bool) ([]string, error) { + if maxItems > 0 && len(values) > maxItems { + return nil, fieldError(path, "must contain <= %d items", maxItems) + } + if unique { + seen := make(map[string]struct{}, len(values)) + for i, value := range values { + if _, ok := seen[value]; ok { + return nil, fieldError(fmt.Sprintf("%s[%d]", path, i), "must be unique") + } + seen[value] = struct{}{} + } + } + for i, value := range values { + if err := validateStringBytes(fmt.Sprintf("%s[%d]", path, i), value, minStringBytes, maxStringBytes); err != nil { + return nil, err + } + } + return append([]string(nil), values...), nil +} + +func validateEnum(path, value string, allowed []string) error { + if !slices.Contains(allowed, value) { + return fieldError(path, "unsupported value %q", value) + } + return nil +} + +func decodeJSON(data []byte, out any) error { + dec := json.NewDecoder(bytes.NewReader(data)) + dec.UseNumber() + if err := dec.Decode(out); err != nil { + return fmt.Errorf("invalid JSON: %w", err) + } + var trailing any + if err := dec.Decode(&trailing); err != io.EOF { + return fmt.Errorf("invalid JSON: trailing data") + } + return nil +} + +func rawSentinel(raw json.RawMessage) (string, bool, error) { + if firstRawByte(raw) != '"' { + return "", false, nil + } + var s string + if err := json.Unmarshal(raw, &s); err != nil { + return "", false, fieldError("device", "contains invalid string") + } + if s == "unknown" || s == "not_collected" { + return s, true, nil + } + return s, true, nil +} + +func firstRawByte(raw json.RawMessage) byte { + trimmed := bytes.TrimSpace(raw) + if len(trimmed) == 0 { + return 0 + } + return trimmed[0] +} + +func isRawNull(raw json.RawMessage) bool { + return bytes.Equal(bytes.TrimSpace(raw), []byte("null")) +} + +func requiredError(path string) error { + return fieldError(path, "required") +} + +func fieldError(path, format string, args ...any) error { + return fmt.Errorf("%s: %s", path, fmt.Sprintf(format, args...)) +} diff --git a/internal/diagnostics/contract/contract_test.go b/internal/diagnostics/contract/contract_test.go new file mode 100644 index 00000000..e3119562 --- /dev/null +++ b/internal/diagnostics/contract/contract_test.go @@ -0,0 +1,323 @@ +package contract + +import ( + "bufio" + "bytes" + "encoding/json" + "os" + "path/filepath" + "slices" + "sort" + "strings" + "testing" +) + +const contractSchemaRoot = "../../../docs/design/schemas/client-diagnostics" + +func TestValidFixtures(t *testing.T) { + fixtures := mustGlob(t, "v1/fixtures/valid/*") + if len(fixtures) == 0 { + t.Fatal("valid fixtures missing") + } + + for _, fixture := range fixtures { + fixture := fixture + t.Run(filepath.Base(fixture), func(t *testing.T) { + data := mustReadFixture(t, fixture) + switch { + case strings.HasSuffix(fixture, "device.json"): + if _, err := ValidateDevice(data); err != nil { + t.Fatalf("ValidateDevice() error = %v", err) + } + case strings.HasSuffix(fixture, ".jsonl"): + validateLogLinesFixture(t, data) + case strings.HasSuffix(fixture, ".json"): + if _, err := ValidateManifest(data); err != nil { + t.Fatalf("ValidateManifest() error = %v", err) + } + default: + t.Fatalf("unexpected fixture extension: %s", fixture) + } + }) + } +} + +func TestInvalidManifestFixtures(t *testing.T) { + fixtures := mustGlob(t, "v1/fixtures/invalid/*.json") + if len(fixtures) < 6 { + t.Fatalf("invalid fixtures = %d, want at least 6", len(fixtures)) + } + + seenErrors := map[string]string{} + for _, fixture := range fixtures { + fixture := fixture + t.Run(filepath.Base(fixture), func(t *testing.T) { + _, err := ValidateManifest(mustReadFixture(t, fixture)) + if err == nil { + t.Fatal("ValidateManifest() error = nil, want failure") + } + msg := err.Error() + if prior := seenErrors[msg]; prior != "" { + t.Fatalf("error %q also used by %s", msg, prior) + } + seenErrors[msg] = filepath.Base(fixture) + }) + } +} + +func TestValidateManifestRequiresCrashForEventReports(t *testing.T) { + // A non-manual report with the crash object stripped must be rejected so + // event reports always carry crash summary/source/occurred_at. + var m map[string]json.RawMessage + if err := json.Unmarshal(mustReadFixture(t, "v1/fixtures/valid/android-tv-crash-ueh.json"), &m); err != nil { + t.Fatalf("parse crash fixture: %v", err) + } + delete(m, "crash") + stripped, err := json.Marshal(m) + if err != nil { + t.Fatalf("marshal stripped manifest: %v", err) + } + _, err = ValidateManifest(stripped) + if err == nil || !strings.Contains(err.Error(), "crash") { + t.Fatalf("ValidateManifest() error = %v, want crash required for event report", err) + } +} + +func TestValidateManifestRejectsCrashOnManualReports(t *testing.T) { + var m map[string]json.RawMessage + if err := json.Unmarshal(mustReadFixture(t, "v1/fixtures/valid/android-tv-crash-ueh.json"), &m); err != nil { + t.Fatalf("parse crash fixture: %v", err) + } + var report map[string]json.RawMessage + if err := json.Unmarshal(m["report"], &report); err != nil { + t.Fatalf("parse report: %v", err) + } + report["type"] = json.RawMessage(`"manual"`) + reportJSON, err := json.Marshal(report) + if err != nil { + t.Fatalf("marshal report: %v", err) + } + m["report"] = reportJSON + withCrash, err := json.Marshal(m) + if err != nil { + t.Fatalf("marshal manifest: %v", err) + } + _, err = ValidateManifest(withCrash) + if err == nil || !strings.Contains(err.Error(), "crash") { + t.Fatalf("ValidateManifest() error = %v, want crash rejected for manual report", err) + } +} + +func TestSchemaEnumsAndRequiredFieldsStayInSync(t *testing.T) { + manifest := mustReadObject(t, "v1/manifest.schema.json") + assertStringsEqual(t, "manifest.required", schemaStrings(t, manifest, "required"), manifestRequiredFields) + assertConstInt(t, "manifest.schema_version.const", schemaValue(t, manifest, "properties", "schema_version", "const"), SchemaVersion) + assertStringsEqual(t, "manifest.report.required", schemaStrings(t, manifest, "properties", "report", "required"), manifestReportRequiredFields) + assertStringsEqual(t, "manifest.destination.required", schemaStrings(t, manifest, "properties", "destination", "required"), manifestDestinationRequiredFields) + assertStringsEqual(t, "manifest.consent.required", schemaStrings(t, manifest, "properties", "consent", "required"), manifestConsentRequiredFields) + assertStringsEqual(t, "manifest.crash.required", schemaStrings(t, manifest, "properties", "crash", "required"), manifestCrashRequiredFields) + assertStringsEqual(t, "manifest.device_summary.required", schemaStrings(t, manifest, "properties", "device_summary", "required"), manifestDeviceSummaryFields) + assertStringsEqual(t, "manifest.log_summary.required", schemaStrings(t, manifest, "properties", "log_summary", "required"), manifestLogSummaryRequiredFields) + assertStringsEqual(t, "manifest.archive.required", schemaStrings(t, manifest, "properties", "archive", "required"), manifestArchiveRequiredFields) + assertStringsEqual(t, "manifest.report.type.enum", schemaStrings(t, manifest, "properties", "report", "properties", "type", "enum"), reportTypes) + assertStringsEqual(t, "manifest.report.platform.enum", schemaStrings(t, manifest, "properties", "report", "properties", "platform", "enum"), platforms) + assertStringsEqual(t, "manifest.consent.mode.enum", schemaStrings(t, manifest, "properties", "consent", "properties", "mode", "enum"), consentModes) + assertStringsEqual(t, "manifest.crash.source.enum", schemaStrings(t, manifest, "properties", "crash", "properties", "source", "enum"), crashSources) + assertStringsEqual(t, "manifest.crash.provenance.enum", schemaStrings(t, manifest, "properties", "crash", "properties", "provenance", "enum"), crashProvenances) + assertStringsEqual(t, "manifest.log_summary.categories.enum", schemaStrings(t, manifest, "properties", "log_summary", "properties", "categories", "items", "enum"), logCategories) + assertStringsEqual(t, "manifest.archive.entries.enum", schemaStrings(t, manifest, "properties", "archive", "properties", "entries", "items", "enum"), ArchiveEntryAllowlist) + + logline := mustReadObject(t, "v1/logline.schema.json") + assertStringsEqual(t, "logline.required", schemaStrings(t, logline, "required"), []string{"ts", "run", "lvl", "cat", "tag", "msg"}) + assertStringsEqual(t, "logline.lvl.enum", schemaStrings(t, logline, "properties", "lvl", "enum"), logLevels) + assertStringsEqual(t, "logline.cat.enum", schemaStrings(t, logline, "properties", "cat", "enum"), logCategories) + + device := mustReadObject(t, "v1/device.schema.json") + assertStringsEqual(t, "device.required", schemaStrings(t, device, "required"), deviceRequiredFields) + assertStringsEqual(t, "device.provenance.enum", schemaStrings(t, device, "properties", "provenance", "enum"), deviceProvenances) +} + +func TestAttrRegistryStaysInSync(t *testing.T) { + var registry struct { + Categories map[string]map[string]struct { + Type string `json:"type"` + } `json:"categories"` + } + if err := json.Unmarshal(mustReadFixture(t, "v1/attr-registry.json"), ®istry); err != nil { + t.Fatalf("parse attr-registry.json: %v", err) + } + + got := make(map[string]map[string]string, len(registry.Categories)) + for category, keys := range registry.Categories { + got[category] = make(map[string]string, len(keys)) + for key, spec := range keys { + got[category][key] = spec.Type + } + } + want := make(map[string]map[string]string, len(attrRegistry)) + for category, keys := range attrRegistry { + want[category] = make(map[string]string, len(keys)) + for key, valueType := range keys { + want[category][key] = string(valueType) + } + } + + if !mapsEqual(got, want) { + t.Fatalf("attr registry mismatch\ngot: %#v\nwant: %#v", got, want) + } +} + +func TestValidateLogLineDropsUnregisteredAttrs(t *testing.T) { + line := []byte(`{"ts":"2026-07-19T18:22:29Z","run":"run_1","lvl":"I","cat":"playback","tag":"Player","msg":"started","attrs":{"sink":"HDMI","unregistered":"drop-me"}}`) + got, err := ValidateLogLine(line) + if err != nil { + t.Fatalf("ValidateLogLine() error = %v", err) + } + if _, ok := got.Attrs["sink"]; !ok { + t.Fatalf("registered attr missing: %#v", got.Attrs) + } + if _, ok := got.Attrs["unregistered"]; ok { + t.Fatalf("unregistered attr was not dropped: %#v", got.Attrs) + } +} + +func TestValidateLogLineRejectsRegisteredAttrTypeMismatch(t *testing.T) { + line := []byte(`{"ts":"2026-07-19T18:22:29Z","run":"run_1","lvl":"I","cat":"network","tag":"HTTP","msg":"done","attrs":{"status":"200"}}`) + if _, err := ValidateLogLine(line); err == nil { + t.Fatal("ValidateLogLine() error = nil, want type mismatch") + } +} + +func validateLogLinesFixture(t *testing.T, data []byte) { + t.Helper() + + scanner := bufio.NewScanner(bytes.NewReader(data)) + scanner.Buffer(make([]byte, 0, 1024), 1024*1024) + lineCount := 0 + for scanner.Scan() { + line := bytes.TrimSpace(scanner.Bytes()) + if len(line) == 0 { + continue + } + lineCount++ + if _, err := ValidateLogLine(line); err != nil { + t.Fatalf("line %d: ValidateLogLine() error = %v", lineCount, err) + } + } + if err := scanner.Err(); err != nil { + t.Fatalf("scan loglines fixture: %v", err) + } + if lineCount == 0 { + t.Fatal("loglines fixture has no log lines") + } +} + +func mustGlob(t *testing.T, pattern string) []string { + t.Helper() + matches, err := filepath.Glob(filepath.Join(contractSchemaRoot, filepath.FromSlash(pattern))) + if err != nil { + t.Fatalf("glob %s: %v", pattern, err) + } + for i, match := range matches { + rel, err := filepath.Rel(contractSchemaRoot, match) + if err != nil { + t.Fatalf("rel %s: %v", match, err) + } + matches[i] = filepath.ToSlash(rel) + } + sort.Strings(matches) + return matches +} + +func mustReadFixture(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(filepath.Join(contractSchemaRoot, filepath.FromSlash(path))) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return data +} + +func mustReadObject(t *testing.T, path string) map[string]any { + t.Helper() + var obj map[string]any + if err := json.Unmarshal(mustReadFixture(t, path), &obj); err != nil { + t.Fatalf("parse %s: %v", path, err) + } + return obj +} + +func schemaStrings(t *testing.T, root any, path ...string) []string { + t.Helper() + value := schemaValue(t, root, path...) + items, ok := value.([]any) + if !ok { + t.Fatalf("%s is %T, want array", strings.Join(path, "."), value) + } + out := make([]string, 0, len(items)) + for _, item := range items { + s, ok := item.(string) + if !ok { + t.Fatalf("%s item is %T, want string", strings.Join(path, "."), item) + } + out = append(out, s) + } + return out +} + +func schemaValue(t *testing.T, root any, path ...string) any { + t.Helper() + current := root + for _, key := range path { + obj, ok := current.(map[string]any) + if !ok { + t.Fatalf("%s parent is %T, want object", key, current) + } + next, ok := obj[key] + if !ok { + t.Fatalf("missing schema path %s", strings.Join(path, ".")) + } + current = next + } + return current +} + +func assertConstInt(t *testing.T, label string, got any, want int) { + t.Helper() + gotFloat, ok := got.(float64) + if !ok { + t.Fatalf("%s = %T, want JSON number", label, got) + } + if int(gotFloat) != want || gotFloat != float64(want) { + t.Fatalf("%s = %v, want %d", label, got, want) + } +} + +func assertStringsEqual(t *testing.T, label string, got, want []string) { + t.Helper() + got = append([]string(nil), got...) + want = append([]string(nil), want...) + sort.Strings(got) + sort.Strings(want) + if !slices.Equal(got, want) { + t.Fatalf("%s mismatch\ngot: %v\nwant: %v", label, got, want) + } +} + +func mapsEqual(got, want map[string]map[string]string) bool { + if len(got) != len(want) { + return false + } + for category, gotKeys := range got { + wantKeys, ok := want[category] + if !ok || len(gotKeys) != len(wantKeys) { + return false + } + for key, gotType := range gotKeys { + if wantKeys[key] != gotType { + return false + } + } + } + return true +} diff --git a/internal/diagnostics/repo.go b/internal/diagnostics/repo.go new file mode 100644 index 00000000..91cb077a --- /dev/null +++ b/internal/diagnostics/repo.go @@ -0,0 +1,672 @@ +package diagnostics + +import ( + "context" + "database/sql" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +const ( + diagnosticQuotaLockNamespace = 173 + defaultShortIDRetries = 8 + defaultAdminListLimit = 50 + maxAdminListLimit = 200 +) + +type PostgresRepository struct { + pool *pgxpool.Pool +} + +func NewPostgresRepository(pool *pgxpool.Pool) *PostgresRepository { + return &PostgresRepository{pool: pool} +} + +func (r *PostgresRepository) InsertReceiving(ctx context.Context, input InsertReceivingInput) (InsertReceivingResult, error) { + if err := validateInsertReceivingInput(input); err != nil { + return InsertReceivingResult{}, err + } + + tx, err := r.pool.Begin(ctx) + if err != nil { + return InsertReceivingResult{}, fmt.Errorf("begin diagnostic report insert: %w", err) + } + defer tx.Rollback(ctx) + + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1::int4, $2::int4)`, + diagnosticQuotaLockNamespace, input.UserID); err != nil { + return InsertReceivingResult{}, fmt.Errorf("acquire diagnostic quota lock: %w", err) + } + + now := input.Now + if now.IsZero() { + now = time.Now().UTC() + } + if err := r.checkQuotas(ctx, tx, input, now); err != nil { + return InsertReceivingResult{}, err + } + + shortIDGenerator := input.ShortIDGenerator + if shortIDGenerator == nil { + shortIDGenerator = NewShortID + } + reportIDGenerator := input.ReportIDGenerator + if reportIDGenerator == nil { + reportIDGenerator = uuid.NewString + } + retries := input.ShortIDCollisionRetries + if retries <= 0 { + retries = defaultShortIDRetries + } + + crashSummary := truncateCrashSummary(input.CrashSummary) + sessionIDs := input.PlaybackSessionIDs + if sessionIDs == nil { + // pgx binds a nil slice as SQL NULL, which bypasses the column's + // '{}' default and violates its NOT NULL constraint. + sessionIDs = []string{} + } + for attempt := 0; attempt < retries; attempt++ { + id := reportIDGenerator() + if _, err := uuid.Parse(id); err != nil { + return InsertReceivingResult{}, fmt.Errorf("%w: report id must be a UUID", ErrInvalidReportInput) + } + shortID, err := shortIDGenerator() + if err != nil { + return InsertReceivingResult{}, fmt.Errorf("generate diagnostic short id: %w", err) + } + shortID, err = ParseShortID(shortID) + if err != nil { + return InsertReceivingResult{}, fmt.Errorf("%w: generated short id is invalid", err) + } + + var insertedID, insertedShortID string + err = tx.QueryRow(ctx, ` + INSERT INTO client_diagnostic_reports ( + id, short_id, user_id, profile_id, state, captured_at, report_type, + platform, app_version, crash_summary, manifest, playback_session_ids, + blob_bytes + ) + VALUES ($1::uuid, $2, $3, $4, 'receiving', $5, $6, $7, $8, $9, $10::jsonb, $11, $12) + ON CONFLICT DO NOTHING + RETURNING id::text, short_id + `, id, shortID, input.UserID, nullableString(input.ProfileID), input.CapturedAt, + strings.TrimSpace(input.ReportType), strings.TrimSpace(input.Platform), + strings.TrimSpace(input.AppVersion), nullableString(crashSummary), + string(input.Manifest), sessionIDs, input.ExpectedBlobBytes).Scan(&insertedID, &insertedShortID) + if errors.Is(err, pgx.ErrNoRows) { + continue + } + if err != nil { + return InsertReceivingResult{}, fmt.Errorf("insert diagnostic report: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return InsertReceivingResult{}, fmt.Errorf("commit diagnostic report insert: %w", err) + } + return InsertReceivingResult{ID: insertedID, ShortID: insertedShortID}, nil + } + + return InsertReceivingResult{}, ErrShortIDExhausted +} + +func (r *PostgresRepository) checkQuotas(ctx context.Context, tx pgx.Tx, input InsertReceivingInput, now time.Time) error { + if input.MaxReportsPerUserDay > 0 { + var count int + if err := tx.QueryRow(ctx, ` + SELECT COUNT(*) + FROM client_diagnostic_reports + WHERE user_id = $1 + AND state IN ('receiving', 'ready') + AND received_at >= date_trunc('day', $2::timestamptz) + `, input.UserID, now).Scan(&count); err != nil { + return fmt.Errorf("count diagnostic reports for quota: %w", err) + } + if count >= input.MaxReportsPerUserDay { + return &QuotaError{Kind: QuotaKindReportsPerDay, Limit: int64(input.MaxReportsPerUserDay)} + } + } + + if input.MaxBytesPerUser > 0 { + // Count 'receiving' rows too: a new upload reserves its expected size in + // blob_bytes before the bundle streams in, so concurrent or multi-node + // uploads can't each pass a near-limit check and both mark ready. Ready + // rows overwrite the reservation with their actual size on completion. + var used int64 + if err := tx.QueryRow(ctx, ` + SELECT COALESCE(SUM(blob_bytes), 0) + FROM client_diagnostic_reports + WHERE user_id = $1 + AND state IN ('receiving', 'ready') + `, input.UserID).Scan(&used); err != nil { + return fmt.Errorf("sum diagnostic bytes for quota: %w", err) + } + if input.ExpectedBlobBytes > input.MaxBytesPerUser || used+input.ExpectedBlobBytes > input.MaxBytesPerUser { + return &QuotaError{Kind: QuotaKindBytesPerUser, Limit: input.MaxBytesPerUser} + } + } + return nil +} + +func (r *PostgresRepository) MarkReady(ctx context.Context, id string, blob BlobInfo) error { + tag, err := r.pool.Exec(ctx, ` + UPDATE client_diagnostic_reports + SET state = 'ready', + blob_bucket = $2, + blob_key = $3, + blob_bytes = $4, + uncompressed_bytes = $5, + blob_sha256 = $6 + WHERE id = $1::uuid + `, id, strings.TrimSpace(blob.Bucket), strings.TrimSpace(blob.Key), blob.Bytes, blob.UncompressedBytes, strings.TrimSpace(blob.SHA256)) + if err != nil { + return fmt.Errorf("mark diagnostic report ready: %w", err) + } + if tag.RowsAffected() == 0 { + return ErrNotFound + } + return nil +} + +func (r *PostgresRepository) MarkFailed(ctx context.Context, id string) error { + tag, err := r.pool.Exec(ctx, ` + UPDATE client_diagnostic_reports + SET state = 'failed' + WHERE id = $1::uuid + `, id) + if err != nil { + return fmt.Errorf("mark diagnostic report failed: %w", err) + } + if tag.RowsAffected() == 0 { + return ErrNotFound + } + return nil +} + +func (r *PostgresRepository) GetByID(ctx context.Context, id string) (*Report, error) { + row := r.pool.QueryRow(ctx, reportSelectSQL()+` + WHERE id = $1::uuid + `, id) + report, err := scanReport(row) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, err + } + return report, nil +} + +func (r *PostgresRepository) ListForAdmin(ctx context.Context, filters ListFilters) (ListResult, error) { + limit := filters.Limit + if limit <= 0 || limit > maxAdminListLimit { + limit = defaultAdminListLimit + } + + conditions := []string{"1=1"} + args := make([]any, 0, 10) + argIdx := 1 + + if filters.UserID != nil { + conditions = append(conditions, fmt.Sprintf("user_id = $%d", argIdx)) + args = append(args, *filters.UserID) + argIdx++ + } + if strings.TrimSpace(filters.Platform) != "" { + conditions = append(conditions, fmt.Sprintf("platform = $%d", argIdx)) + args = append(args, strings.TrimSpace(filters.Platform)) + argIdx++ + } + if strings.TrimSpace(filters.ReportType) != "" { + conditions = append(conditions, fmt.Sprintf("report_type = $%d", argIdx)) + args = append(args, strings.TrimSpace(filters.ReportType)) + argIdx++ + } + if filters.From != nil { + conditions = append(conditions, fmt.Sprintf("received_at >= $%d", argIdx)) + args = append(args, *filters.From) + argIdx++ + } + if filters.To != nil { + conditions = append(conditions, fmt.Sprintf("received_at <= $%d", argIdx)) + args = append(args, *filters.To) + argIdx++ + } + if strings.TrimSpace(filters.ShortID) != "" { + shortID, err := ParseShortID(filters.ShortID) + if err != nil { + return ListResult{}, err + } + conditions = append(conditions, fmt.Sprintf("lower(short_id) = lower($%d)", argIdx)) + args = append(args, shortID) + argIdx++ + } + if strings.TrimSpace(filters.Cursor) != "" { + cursorReceivedAt, cursorID, err := decodeReportCursor(filters.Cursor) + if err != nil { + return ListResult{}, err + } + conditions = append(conditions, fmt.Sprintf("(received_at, id) < ($%d, $%d::uuid)", argIdx, argIdx+1)) + args = append(args, cursorReceivedAt, cursorID) + argIdx += 2 + } + + query := fmt.Sprintf("%s\n\t\tWHERE %s\n\t\tORDER BY received_at DESC, id DESC\n\t\tLIMIT $%d", + reportListSelectSQL(), strings.Join(conditions, " AND "), argIdx) + args = append(args, limit+1) + + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return ListResult{}, fmt.Errorf("list diagnostic reports: %w", err) + } + defer rows.Close() + + reports := make([]Report, 0, limit+1) + for rows.Next() { + report, err := scanReportSummary(rows) + if err != nil { + return ListResult{}, err + } + reports = append(reports, *report) + } + if err := rows.Err(); err != nil { + return ListResult{}, fmt.Errorf("iterate diagnostic reports: %w", err) + } + + result := ListResult{} + if len(reports) > limit { + last := reports[limit-1] + result.NextCursor = encodeReportCursor(last.ReceivedAt, last.ID) + reports = reports[:limit] + } + result.Reports = reports + return result, nil +} + +func (r *PostgresRepository) DeleteByID(ctx context.Context, id string) (*Report, error) { + row := r.pool.QueryRow(ctx, ` + DELETE FROM client_diagnostic_reports + WHERE id = $1::uuid + RETURNING id::text, short_id, user_id, profile_id, state, captured_at, received_at, + report_type, platform, app_version, crash_summary, manifest, playback_session_ids, + blob_bucket, blob_key, blob_bytes, uncompressed_bytes, blob_sha256, + COALESCE(manifest->'report'->>'app_build', '') + `, id) + report, err := scanReport(row) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + return nil, err + } + return report, nil +} + +func (r *PostgresRepository) RetentionCandidates(ctx context.Context, olderThan time.Time, perUserByteCap int64) ([]Report, error) { + if olderThan.IsZero() && perUserByteCap <= 0 { + return nil, nil + } + + query := reportCleanupSelectSQL() + ` + WHERE received_at < $1 + ORDER BY received_at ASC, id ASC + ` + args := []any{olderThan} + if olderThan.IsZero() { + query = reportCleanupSelectSQL() + ` + WHERE false + ` + args = nil + } + + if perUserByteCap > 0 { + byteCapQuery := ` + WITH quota_candidates AS ( + SELECT id + FROM ( + SELECT id, + SUM(COALESCE(blob_bytes, 0)) OVER ( + PARTITION BY user_id + ORDER BY received_at DESC, id DESC + ) AS retained_bytes + FROM client_diagnostic_reports + WHERE state = 'ready' + ) ranked + WHERE retained_bytes > $1 + ) + ` + reportCleanupSelectSQL() + ` + WHERE id IN (SELECT id FROM quota_candidates) + ORDER BY received_at ASC, id ASC + ` + if olderThan.IsZero() { + return r.queryReportCleanup(ctx, byteCapQuery, perUserByteCap) + } + + query = ` + WITH quota_candidates AS ( + SELECT id + FROM ( + SELECT id, + SUM(COALESCE(blob_bytes, 0)) OVER ( + PARTITION BY user_id + ORDER BY received_at DESC, id DESC + ) AS retained_bytes + FROM client_diagnostic_reports + WHERE state = 'ready' + ) ranked + WHERE retained_bytes > $2 + ) + ` + reportCleanupSelectSQL() + ` + WHERE received_at < $1 OR id IN (SELECT id FROM quota_candidates) + ORDER BY received_at ASC, id ASC + ` + args = []any{olderThan, perUserByteCap} + } + + return r.queryReportCleanup(ctx, query, args...) +} + +func (r *PostgresRepository) StaleReceiving(ctx context.Context, grace time.Duration) ([]Report, error) { + if grace <= 0 { + grace = time.Hour + } + cutoff := time.Now().UTC().Add(-grace) + return r.queryReportCleanup(ctx, reportCleanupSelectSQL()+` + WHERE state IN ('receiving', 'failed') + AND received_at < $1 + ORDER BY received_at ASC, id ASC + `, cutoff) +} + +func (r *PostgresRepository) LiveBlobKeys(ctx context.Context, keys []string) (map[string]ReportState, error) { + live := make(map[string]ReportState, len(keys)) + if len(keys) == 0 { + return live, nil + } + rows, err := r.pool.Query(ctx, ` + SELECT candidate.key, report.state + FROM unnest($1::text[]) AS candidate(key) + JOIN client_diagnostic_reports report + ON report.blob_key = candidate.key + OR ($2 || report.user_id::text || '/' || report.id::text || '.tar.gz') = candidate.key + WHERE report.state IN ('ready', 'receiving') + `, keys, ObjectPrefix) + if err != nil { + return nil, fmt.Errorf("load live diagnostic blob keys: %w", err) + } + defer rows.Close() + + for rows.Next() { + var key string + var state ReportState + if err := rows.Scan(&key, &state); err != nil { + return nil, fmt.Errorf("scan live diagnostic blob key: %w", err) + } + live[key] = state + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate live diagnostic blob keys: %w", err) + } + return live, nil +} + +// queryReportCleanup runs a reportCleanupSelectSQL-shaped query; the returned +// reports carry no manifest and no app_build. Only the retention and +// stale-cleanup batches use it. +func (r *PostgresRepository) queryReportCleanup(ctx context.Context, query string, args ...any) ([]Report, error) { + rows, err := r.pool.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("query diagnostic reports: %w", err) + } + defer rows.Close() + + reports := []Report{} + for rows.Next() { + report, err := scanReportCleanup(rows) + if err != nil { + return nil, err + } + reports = append(reports, *report) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate diagnostic reports: %w", err) + } + return reports, nil +} + +func validateInsertReceivingInput(input InsertReceivingInput) error { + if input.UserID <= 0 { + return fmt.Errorf("%w: user id is required", ErrInvalidReportInput) + } + if input.CapturedAt.IsZero() { + return fmt.Errorf("%w: captured_at is required", ErrInvalidReportInput) + } + if strings.TrimSpace(input.ReportType) == "" { + return fmt.Errorf("%w: report type is required", ErrInvalidReportInput) + } + if strings.TrimSpace(input.Platform) == "" { + return fmt.Errorf("%w: platform is required", ErrInvalidReportInput) + } + if strings.TrimSpace(input.AppVersion) == "" { + return fmt.Errorf("%w: app version is required", ErrInvalidReportInput) + } + if len(input.Manifest) == 0 || !json.Valid(input.Manifest) { + return fmt.Errorf("%w: manifest must be valid JSON", ErrInvalidReportInput) + } + if input.ExpectedBlobBytes < 0 { + return fmt.Errorf("%w: expected blob bytes cannot be negative", ErrInvalidReportInput) + } + return nil +} + +// reportSelectSQL is the full projection including the manifest JSONB (up to +// MaxManifestBytes per row). Reserve it for single-report detail/download paths; +// the admin list uses reportListSelectSQL and retention/stale batches use +// reportCleanupSelectSQL so browsing a page or running a cleanup pass doesn't +// drag every report's manifest through the DB. +func reportSelectSQL() string { + return `SELECT id::text, short_id, user_id, profile_id, state, captured_at, received_at, + report_type, platform, app_version, crash_summary, manifest, playback_session_ids, + blob_bucket, blob_key, blob_bytes, uncompressed_bytes, blob_sha256, + COALESCE(manifest->'report'->>'app_build', '') + FROM client_diagnostic_reports` +} + +// reportListSelectSQL is the admin-list projection: it omits the full manifest +// column but still extracts app_build from the manifest JSONB (a cheap text +// extraction) so list rows can show the build number without shipping the whole +// manifest. Reports scanned with scanReportSummary therefore have a nil +// Manifest. Cleanup batches must not pay for the JSONB extraction, so they use +// reportCleanupSelectSQL instead. +func reportListSelectSQL() string { + return `SELECT id::text, short_id, user_id, profile_id, state, captured_at, received_at, + report_type, platform, app_version, crash_summary, playback_session_ids, + blob_bucket, blob_key, blob_bytes, uncompressed_bytes, blob_sha256, + COALESCE(manifest->'report'->>'app_build', '') + FROM client_diagnostic_reports` +} + +// reportCleanupSelectSQL is the retention/stale-cleanup projection. It omits +// both the manifest column and the app_build extraction because cleanup only +// needs id/state/blob fields to delete a row and its object; touching the JSONB +// per candidate would turn a scheduled pass over many large manifests into +// needless manifest I/O. Reports scanned with scanReportCleanup therefore have +// a nil Manifest and an empty AppBuild. +func reportCleanupSelectSQL() string { + return `SELECT id::text, short_id, user_id, profile_id, state, captured_at, received_at, + report_type, platform, app_version, crash_summary, playback_session_ids, + blob_bucket, blob_key, blob_bytes, uncompressed_bytes, blob_sha256 + FROM client_diagnostic_reports` +} + +type reportScanner interface { + Scan(dest ...any) error +} + +// reportNulls holds the nullable columns shared by the full and summary +// projections so both scanners apply them the same way. +type reportNulls struct { + profileID sql.NullString + crashSummary sql.NullString + blobBucket sql.NullString + blobKey sql.NullString + blobSHA256 sql.NullString + blobBytes sql.NullInt64 + uncompressedBytes sql.NullInt64 +} + +func (n *reportNulls) apply(report *Report) { + if n.profileID.Valid { + report.ProfileID = &n.profileID.String + } + if n.crashSummary.Valid { + report.CrashSummary = &n.crashSummary.String + } + if report.PlaybackSessionIDs == nil { + report.PlaybackSessionIDs = []string{} + } + if n.blobBucket.Valid { + report.BlobBucket = &n.blobBucket.String + } + if n.blobKey.Valid { + report.BlobKey = &n.blobKey.String + } + if n.blobBytes.Valid { + report.BlobBytes = &n.blobBytes.Int64 + } + if n.uncompressedBytes.Valid { + report.UncompressedBytes = &n.uncompressedBytes.Int64 + } + if n.blobSHA256.Valid { + report.BlobSHA256 = &n.blobSHA256.String + } +} + +// scanReport scans the full reportSelectSQL projection, including the manifest. +func scanReport(row reportScanner) (*Report, error) { + var report Report + var nulls reportNulls + var manifest []byte + if err := row.Scan( + &report.ID, + &report.ShortID, + &report.UserID, + &nulls.profileID, + &report.State, + &report.CapturedAt, + &report.ReceivedAt, + &report.ReportType, + &report.Platform, + &report.AppVersion, + &nulls.crashSummary, + &manifest, + &report.PlaybackSessionIDs, + &nulls.blobBucket, + &nulls.blobKey, + &nulls.blobBytes, + &nulls.uncompressedBytes, + &nulls.blobSHA256, + &report.AppBuild, + ); err != nil { + return nil, fmt.Errorf("scan diagnostic report: %w", err) + } + if len(manifest) > 0 { + report.Manifest = append(json.RawMessage(nil), manifest...) + } + nulls.apply(&report) + return &report, nil +} + +// summaryScanTargets returns the scan destinations shared by the admin-list and +// cleanup projections, in column order. The two scanners differ only in whether +// app_build is appended, so keeping the common prefix in one place stops the +// long Scan lists from drifting apart. +func summaryScanTargets(report *Report, nulls *reportNulls) []any { + return []any{ + &report.ID, + &report.ShortID, + &report.UserID, + &nulls.profileID, + &report.State, + &report.CapturedAt, + &report.ReceivedAt, + &report.ReportType, + &report.Platform, + &report.AppVersion, + &nulls.crashSummary, + &report.PlaybackSessionIDs, + &nulls.blobBucket, + &nulls.blobKey, + &nulls.blobBytes, + &nulls.uncompressedBytes, + &nulls.blobSHA256, + } +} + +// scanReportSummary scans the reportListSelectSQL projection, which omits the +// manifest but includes app_build; the returned report's Manifest is left nil. +func scanReportSummary(row reportScanner) (*Report, error) { + var report Report + var nulls reportNulls + targets := append(summaryScanTargets(&report, &nulls), &report.AppBuild) + if err := row.Scan(targets...); err != nil { + return nil, fmt.Errorf("scan diagnostic report: %w", err) + } + nulls.apply(&report) + return &report, nil +} + +// scanReportCleanup scans the reportCleanupSelectSQL projection, which omits +// both the manifest and app_build; the returned report's Manifest is nil and +// AppBuild is empty. Cleanup only reads id/state/blob fields. +func scanReportCleanup(row reportScanner) (*Report, error) { + var report Report + var nulls reportNulls + if err := row.Scan(summaryScanTargets(&report, &nulls)...); err != nil { + return nil, fmt.Errorf("scan diagnostic report: %w", err) + } + nulls.apply(&report) + return &report, nil +} + +func nullableString(s *string) any { + if s == nil { + return nil + } + if strings.TrimSpace(*s) == "" { + return nil + } + return *s +} + +func encodeReportCursor(receivedAt time.Time, id string) string { + return base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%d|%s", receivedAt.UnixNano(), id))) +} + +func decodeReportCursor(cursor string) (time.Time, string, error) { + raw, err := base64.RawURLEncoding.DecodeString(cursor) + if err != nil { + return time.Time{}, "", fmt.Errorf("%w: decode: %v", ErrInvalidCursor, err) + } + parts := strings.Split(string(raw), "|") + if len(parts) != 2 { + return time.Time{}, "", ErrInvalidCursor + } + nanos, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return time.Time{}, "", fmt.Errorf("%w: timestamp: %v", ErrInvalidCursor, err) + } + id := parts[1] + if _, err := uuid.Parse(id); err != nil { + return time.Time{}, "", fmt.Errorf("%w: id: %v", ErrInvalidCursor, err) + } + return time.Unix(0, nanos).UTC(), id, nil +} diff --git a/internal/diagnostics/service.go b/internal/diagnostics/service.go new file mode 100644 index 00000000..60040d91 --- /dev/null +++ b/internal/diagnostics/service.go @@ -0,0 +1,572 @@ +package diagnostics + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "reflect" + "slices" + "strconv" + "strings" + "time" + + "github.com/Silo-Server/silo-server/internal/diagnostics/contract" +) + +const ( + MaxManifestBytes = int64(64 * 1024) + BundleContentType = "application/gzip" + diagnosticObjectPrefix = "diagnostics" + + failedUploadCompensationTimeout = 10 * time.Second +) + +var ( + ErrDisabled = errors.New("diagnostics disabled") + ErrStorageUnavailable = errors.New("diagnostics storage unavailable") + ErrTooLarge = errors.New("diagnostics upload too large") + ErrUnsupportedSchema = errors.New("diagnostics schema version unsupported") + ErrDestinationMismatch = errors.New("diagnostics destination mismatch") + ErrStaleConsent = errors.New("diagnostics consent notice is stale") + ErrArchiveMismatch = errors.New("diagnostics archive metadata mismatch") + ErrReportStoreUnavailable = errors.New("diagnostics report store unavailable") + ErrProfileMismatch = errors.New("diagnostics profile id mismatch") + ErrChildProfileForbidden = errors.New("diagnostics child profile attribution forbidden") +) + +type AvailabilityStatus string + +const ( + StatusAvailable AvailabilityStatus = "available" + StatusDisabled AvailabilityStatus = "disabled" + StatusStorageUnavailable AvailabilityStatus = "storage_unavailable" +) + +type Status struct { + Status AvailabilityStatus `json:"status"` + ServerInstanceID string `json:"server_instance_id"` + AcceptedSchemaVersions []int `json:"accepted_schema_versions"` + MaxBundleBytes int64 `json:"max_bundle_bytes"` + MaxManifestBytes int64 `json:"max_manifest_bytes"` + RetentionDays int `json:"retention_days"` + ConsentNoticeVersion int `json:"consent_notice_version"` +} + +type IngestResult struct { + ReportID string `json:"report_id"` + ShortID string `json:"short_id"` +} + +type Service struct { + reports ReportStore + settings SettingsStore + store ObjectStore + profileValidator ProfileAttributionValidator + logger *slog.Logger + now func() time.Time +} + +func NewService(reports ReportStore, settings SettingsStore, store ObjectStore, logger *slog.Logger) *Service { + if logger == nil { + logger = slog.Default() + } + return &Service{ + reports: reports, + settings: settings, + store: store, + logger: logger, + now: func() time.Time { return time.Now().UTC() }, + } +} + +type ProfileAttributionValidator interface { + ProfileBelongsToUser(ctx context.Context, userID int, profileID string) (bool, error) +} + +type ProfileAttributionValidatorFunc func(ctx context.Context, userID int, profileID string) (bool, error) + +func (f ProfileAttributionValidatorFunc) ProfileBelongsToUser(ctx context.Context, userID int, profileID string) (bool, error) { + return f(ctx, userID, profileID) +} + +// ProfileLookup resolves whether profileID exists for userID and whether it is +// a child profile. It backs NewProfileAttributionValidator. +type ProfileLookup func(ctx context.Context, userID int, profileID string) (found bool, isChild bool, err error) + +// NewProfileAttributionValidator builds a ProfileAttributionValidator that +// attributes a report to a profile only when it belongs to the user and is not +// a child profile. The client diagnostics design forbids child profiles from +// performing diagnostics actions, so a nonconforming client that sends a child +// profile's ID (via X-Profile-Id or a manifest profile_id) has the upload +// rejected with ErrChildProfileForbidden rather than silently losing the +// attribution. A profile that is simply not the user's (found == false) is not +// an error: attribution is dropped and the upload proceeds unattributed. +func NewProfileAttributionValidator(lookup ProfileLookup) ProfileAttributionValidator { + return ProfileAttributionValidatorFunc(func(ctx context.Context, userID int, profileID string) (bool, error) { + found, isChild, err := lookup(ctx, userID, profileID) + if err != nil { + return false, err + } + if found && isChild { + return false, ErrChildProfileForbidden + } + return found, nil + }) +} + +func (s *Service) SetProfileAttributionValidator(validator ProfileAttributionValidator) { + s.profileValidator = validator +} + +func (s *Service) Status(ctx context.Context, userID int) (Status, error) { + settings, err := s.currentSettings(ctx) + if err != nil { + return Status{}, err + } + return s.statusFromSettings(settings), nil +} + +func (s *Service) Ingest(ctx context.Context, userID int, profileID *string, manifestJSON []byte, bundle io.Reader) (IngestResult, error) { + settings, err := s.currentSettings(ctx) + if err != nil { + s.logRejected(ctx, "", userID, "", "", 0, "settings_error") + return IngestResult{}, err + } + status := s.statusFromSettings(settings).Status + switch status { + case StatusDisabled: + s.logRejected(ctx, "", userID, "", "", 0, "disabled") + return IngestResult{}, ErrDisabled + case StatusStorageUnavailable: + s.logRejected(ctx, "", userID, "", "", 0, "storage_unavailable") + return IngestResult{}, ErrStorageUnavailable + } + if s.reports == nil { + s.logRejected(ctx, "", userID, "", "", 0, "report_store_unavailable") + return IngestResult{}, ErrReportStoreUnavailable + } + if len(manifestJSON) > int(MaxManifestBytes) { + s.logRejected(ctx, "", userID, "", "", int64(len(manifestJSON)), "too_large") + return IngestResult{}, ErrTooLarge + } + if manifestHasUnsupportedSchema(manifestJSON) { + s.logRejected(ctx, "", userID, "", "", int64(len(manifestJSON)), "unsupported_schema") + return IngestResult{}, ErrUnsupportedSchema + } + + manifest, err := contract.ValidateManifest(manifestJSON) + if err != nil { + s.logRejected(ctx, "", userID, "", "", int64(len(manifestJSON)), "invalid_manifest") + return IngestResult{}, fmt.Errorf("%w: invalid manifest: %v", ErrInvalidBundle, err) + } + if manifest.Archive.Bytes > settings.MaxBundleBytes { + s.logRejected(ctx, "", userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, "too_large") + return IngestResult{}, ErrTooLarge + } + if manifest.Destination.ServerInstanceID != settings.ServerInstanceID { + s.logRejected(ctx, "", userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, "destination_mismatch") + return IngestResult{}, ErrDestinationMismatch + } + if manifest.Consent.NoticeVersion != settings.ConsentNoticeVersion { + s.logRejected(ctx, "", userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, "stale_consent") + return IngestResult{}, ErrStaleConsent + } + + reportProfileID, err := s.validatedProfileID(ctx, userID, profileID, manifest.Report.ProfileID) + if err != nil { + s.logRejected(ctx, "", userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, "profile_validation_error") + return IngestResult{}, err + } + crashSummary := crashSummaryFromManifest(manifest) + // ExpectedBlobBytes reserves the client-claimed archive size against the + // per-user byte quota before the bundle streams in. Trusting the claim here is + // safe: a report only reaches MarkReady (which rewrites blob_bytes to the + // measured size) after archiveMatches confirms the claimed bytes/sha/entries + // exactly equal the streamed bundle, so no stored report can ever occupy more + // bytes than it reserved. A client that under-claims to under-reserve then + // uploads a larger bundle fails archiveMatches and is compensated (blob + // deleted, row marked failed) before it counts, gaining no storage. The + // residual cost of such rejected churn is bandwidth, not storage — bounded by + // the per-account single in-flight upload cap, not by max_bytes_per_user, + // which is a storage cap enforced oldest-first by retention. + reserved, err := s.reports.InsertReceiving(ctx, InsertReceivingInput{ + UserID: userID, + ProfileID: reportProfileID, + CapturedAt: manifest.Report.CapturedAt, + ReportType: manifest.Report.Type, + Platform: manifest.Report.Platform, + AppVersion: manifest.Report.AppVersion, + CrashSummary: crashSummary, + Manifest: append(json.RawMessage(nil), manifestJSON...), + PlaybackSessionIDs: manifest.PlaybackSessionIDs, + ExpectedBlobBytes: manifest.Archive.Bytes, + MaxReportsPerUserDay: settings.MaxReportsPerUserDay, + MaxBytesPerUser: settings.MaxBytesPerUser, + Now: s.now(), + }) + if err != nil { + reason := "insert_failed" + if errors.Is(err, ErrQuotaExceeded) { + reason = "quota_exceeded" + } else { + s.logger.ErrorContext(ctx, "diagnostic report insert failed", + "component", "diagnostics", + "user_id", userID, + "error", err, + ) + } + s.logRejected(ctx, "", userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, reason) + return IngestResult{}, err + } + + key := reportObjectKey(userID, reserved.ID) + info, err := s.putValidatedBundle(ctx, key, bundle, settings) + if err != nil { + s.compensateFailedUpload(ctx, reserved.ID, key) + rejectErr := classifyBundleUploadError(err) + s.logger.ErrorContext(ctx, "diagnostic bundle validation failed", + "component", "diagnostics", + "report_id", reserved.ID, + "user_id", userID, + "error", err, + ) + s.logRejected(ctx, reserved.ID, userID, manifest.Report.Platform, manifest.Report.Type, manifest.Archive.Bytes, rejectReason(rejectErr)) + return IngestResult{}, rejectErr + } + if !archiveMatches(manifest.Archive, info) { + s.compensateFailedUpload(ctx, reserved.ID, key) + s.logRejected(ctx, reserved.ID, userID, manifest.Report.Platform, manifest.Report.Type, info.CompressedBytes, "archive_mismatch") + return IngestResult{}, ErrArchiveMismatch + } + if !embeddedManifestMatches(manifestJSON, info.EmbeddedManifest) { + s.compensateFailedUpload(ctx, reserved.ID, key) + s.logRejected(ctx, reserved.ID, userID, manifest.Report.Platform, manifest.Report.Type, info.CompressedBytes, "manifest_mismatch") + return IngestResult{}, ErrArchiveMismatch + } + + if err := s.reports.MarkReady(ctx, reserved.ID, BlobInfo{ + Bucket: s.store.Bucket(), + Key: key, + Bytes: info.CompressedBytes, + UncompressedBytes: info.UncompressedBytes, + SHA256: info.SHA256, + }); err != nil { + s.compensateFailedUpload(ctx, reserved.ID, key) + s.logRejected(ctx, reserved.ID, userID, manifest.Report.Platform, manifest.Report.Type, info.CompressedBytes, "mark_ready_failed") + return IngestResult{}, err + } + + s.logger.InfoContext(ctx, "diagnostic report accepted", + "component", "diagnostics", + "report_id", reserved.ID, + "user_id", userID, + "platform", manifest.Report.Platform, + "type", manifest.Report.Type, + "bytes", info.CompressedBytes, + "result", "accepted", + ) + return IngestResult{ReportID: reserved.ID, ShortID: reserved.ShortID}, nil +} + +func (s *Service) currentSettings(ctx context.Context) (Settings, error) { + settings, err := LoadSettings(ctx, s.settings) + if err != nil { + return Settings{}, err + } + if strings.TrimSpace(settings.ServerInstanceID) != "" || s.settings == nil { + return settings, nil + } + + instanceID, err := ensureServerInstanceID(ctx, s.settings) + if err != nil { + return Settings{}, err + } + settings.ServerInstanceID = instanceID + return settings, nil +} + +func (s *Service) statusFromSettings(settings Settings) Status { + availability := StatusDisabled + if settings.UploadsEnabled { + availability = StatusAvailable + if s.store == nil || strings.TrimSpace(s.store.Bucket()) == "" { + availability = StatusStorageUnavailable + } + } + + return Status{ + Status: availability, + ServerInstanceID: settings.ServerInstanceID, + AcceptedSchemaVersions: []int{contract.SchemaVersion}, + MaxBundleBytes: settings.MaxBundleBytes, + MaxManifestBytes: MaxManifestBytes, + RetentionDays: settings.RetentionDays, + ConsentNoticeVersion: settings.ConsentNoticeVersion, + } +} + +func (s *Service) putValidatedBundle(ctx context.Context, key string, bundle io.Reader, settings Settings) (BundleInfo, error) { + pr, pw := io.Pipe() + resultCh := make(chan bundleValidationResult, 1) + go func() { + info, err := ValidateBundle(pr, BundleLimits{ + MaxCompressedBytes: settings.MaxBundleBytes, + MaxUncompressedBytes: settings.MaxUncompressedBytes, + }) + if err != nil { + _ = pr.CloseWithError(err) + } + resultCh <- bundleValidationResult{info: info, err: err} + }() + + reader := &validatingUploadReader{src: bundle, pipe: pw} + putErr := s.store.PutStream(ctx, s.store.Bucket(), key, reader, BundleContentType) + if putErr != nil { + reader.abort(&bundleUploadAbortError{err: putErr}) + } + result := <-resultCh + if putErr != nil { + if result.err != nil && !isBundleUploadAbortError(result.err) { + return result.info, result.err + } + return BundleInfo{}, fmt.Errorf("%w: store diagnostic bundle: %w", ErrStorageUnavailable, putErr) + } + if result.err != nil { + return result.info, result.err + } + return result.info, nil +} + +func (s *Service) compensateFailedUpload(ctx context.Context, reportID, key string) { + compensationCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), failedUploadCompensationTimeout) + defer cancel() + + if s.store != nil && key != "" { + if err := s.store.DeleteObject(compensationCtx, s.store.Bucket(), key); err != nil { + s.logger.WarnContext(compensationCtx, "diagnostic report blob compensation failed", + "component", "diagnostics", + "report_id", reportID, + "key", key, + "error", err, + ) + } + } + if s.reports != nil && reportID != "" { + if err := s.reports.MarkFailed(compensationCtx, reportID); err != nil { + s.logger.WarnContext(compensationCtx, "diagnostic report failed-state compensation failed", + "component", "diagnostics", + "report_id", reportID, + "error", err, + ) + } + } +} + +func (s *Service) logRejected(ctx context.Context, reportID string, userID int, platform, reportType string, bytes int64, reason string) { + s.logger.InfoContext(ctx, "diagnostic report rejected", + "component", "diagnostics", + "report_id", reportID, + "user_id", userID, + "platform", platform, + "type", reportType, + "bytes", bytes, + "result", "rejected", + "reason", reason, + ) +} + +type bundleValidationResult struct { + info BundleInfo + err error +} + +type validatingUploadReader struct { + src io.Reader + pipe *io.PipeWriter + closed bool +} + +func (r *validatingUploadReader) Read(p []byte) (int, error) { + n, readErr := r.src.Read(p) + if n > 0 { + if _, err := r.pipe.Write(p[:n]); err != nil { + return n, err + } + } + if readErr != nil { + r.close(readErr) + } + return n, readErr +} + +func (r *validatingUploadReader) abort(err error) { + r.close(err) +} + +func (r *validatingUploadReader) close(err error) { + if r.closed { + return + } + r.closed = true + if errors.Is(err, io.EOF) { + _ = r.pipe.Close() + return + } + _ = r.pipe.CloseWithError(err) +} + +func classifyBundleUploadError(err error) error { + switch { + case errors.Is(err, ErrCompressedTooLarge), + errors.Is(err, ErrUncompressedTooLarge), + errors.Is(err, ErrEntryTooLarge): + return ErrTooLarge + case errors.Is(err, ErrInvalidBundle), + errors.Is(err, ErrTooManyEntries), + errors.Is(err, ErrCompressionRatio): + return fmt.Errorf("%w: %v", ErrInvalidBundle, err) + default: + return err + } +} + +func rejectReason(err error) string { + switch { + case errors.Is(err, ErrTooLarge): + return "too_large" + case errors.Is(err, ErrUnsupportedSchema): + return "unsupported_schema" + case errors.Is(err, ErrInvalidBundle): + return "invalid_bundle" + default: + return "storage_error" + } +} + +func manifestHasUnsupportedSchema(data []byte) bool { + var header struct { + SchemaVersion *int `json:"schema_version"` + } + if err := json.Unmarshal(data, &header); err != nil { + return false + } + return header.SchemaVersion != nil && *header.SchemaVersion != contract.SchemaVersion +} + +// resolveProfileID chooses the profile a report is attributed to from the +// X-Profile-Id header and the manifest's report.profile_id fallback. When both +// are present they must agree: a header captured under a different profile than +// the manifest means the client switched profiles between capture and upload, +// and silently preferring either source would attribute the DB row to one +// profile while the stored manifest/download still names the other. Reject that +// as ErrProfileMismatch. Single-source cases (only one present, or both equal) +// resolve to that value; when neither is present the report is unattributed. +func resolveProfileID(profileID *string, fallback string) (*string, error) { + header := "" + if profileID != nil { + header = strings.TrimSpace(*profileID) + } + manifestID := strings.TrimSpace(fallback) + + switch { + case header != "" && manifestID != "" && header != manifestID: + return nil, fmt.Errorf("%w: header %q, manifest %q", ErrProfileMismatch, header, manifestID) + case header != "": + return &header, nil + case manifestID != "": + return &manifestID, nil + default: + return nil, nil + } +} + +func (s *Service) validatedProfileID(ctx context.Context, userID int, profileID *string, fallback string) (*string, error) { + candidate, err := resolveProfileID(profileID, fallback) + if err != nil { + return nil, err + } + if candidate == nil { + return nil, nil + } + if s.profileValidator == nil { + return nil, nil + } + found, err := s.profileValidator.ProfileBelongsToUser(ctx, userID, *candidate) + if err != nil { + // A child-profile attribution is a client-facing rejection, not an + // internal failure: surface it verbatim so the handler can return a + // clear error instead of a generic 500. + if errors.Is(err, ErrChildProfileForbidden) { + return nil, err + } + return nil, fmt.Errorf("validate diagnostic profile attribution: %w", err) + } + if !found { + return nil, nil + } + return candidate, nil +} + +func crashSummaryFromManifest(manifest contract.Manifest) *string { + if manifest.Crash == nil { + return nil + } + summary := manifest.Crash.Summary + return &summary +} + +func reportObjectKey(userID int, reportID string) string { + return diagnosticObjectPrefix + "/" + strconv.Itoa(userID) + "/" + reportID + ".tar.gz" +} + +func archiveMatches(archive contract.Archive, info BundleInfo) bool { + return archive.Bytes == info.CompressedBytes && + archive.UncompressedBytes == info.UncompressedBytes && + strings.EqualFold(archive.SHA256, info.SHA256) && + slices.Equal(archive.Entries, info.Entries) +} + +// embeddedManifestMatches reports whether the manifest.json embedded as the +// archive's first entry equals the received part-1 manifest with its `archive` +// object removed, per the bundle contract. Both are decoded to JSON before +// comparison so formatting or key-ordering differences don't matter. +func embeddedManifestMatches(received, embedded []byte) bool { + receivedMap, err := decodeJSONObject(received) + if err != nil { + return false + } + delete(receivedMap, "archive") + embeddedMap, err := decodeJSONObject(embedded) + if err != nil { + return false + } + return reflect.DeepEqual(receivedMap, embeddedMap) +} + +func decodeJSONObject(data []byte) (map[string]any, error) { + dec := json.NewDecoder(bytes.NewReader(data)) + // UseNumber keeps every JSON number as its exact decimal text (json.Number) + // instead of float64. Without it, two manifests that differ only in a large + // integer above 2^53 (e.g. log_summary.lines) both round to the same float + // and DeepEqual reports a false match, letting a tampered bundle through. + dec.UseNumber() + var obj map[string]any + if err := dec.Decode(&obj); err != nil { + return nil, err + } + // json.Unmarshal rejects trailing bytes after the value; Decoder.Decode does + // not, and Decoder.More() misses a stray closing delimiter (e.g. `{...}}`) + // because it only reports iteration inside an array/object. Re-assert that + // strictness by requiring the stream to be exhausted: a second Decode must + // hit io.EOF, so a manifest with any trailing token — junk or an extra + // `}`/`]` — is rejected and cannot compare equal. Trailing whitespace is + // fine; the decoder skips it and still returns io.EOF. + if err := dec.Decode(new(json.RawMessage)); err != io.EOF { + return nil, errors.New("unexpected trailing data after JSON object") + } + return obj, nil +} diff --git a/internal/diagnostics/service_test.go b/internal/diagnostics/service_test.go new file mode 100644 index 00000000..c4b13ccf --- /dev/null +++ b/internal/diagnostics/service_test.go @@ -0,0 +1,782 @@ +package diagnostics + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "log/slog" + "strconv" + "strings" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/diagnostics/contract" +) + +func TestServiceIngestStoresReadyReport(t *testing.T) { + bundle, manifest, info := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "prof_1") + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + + result, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)) + if err != nil { + t.Fatalf("Ingest: %v", err) + } + if result.ReportID != "11111111-1111-1111-1111-111111111111" { + t.Fatalf("ReportID = %q, want reserved report ID", result.ReportID) + } + if result.ShortID != "SILO-ABCDEF123456" { + t.Fatalf("ShortID = %q, want SILO-ABCDEF123456", result.ShortID) + } + if len(repo.ready) != 1 || repo.ready[0] != result.ReportID { + t.Fatalf("ready reports = %v, want [%s]", repo.ready, result.ReportID) + } + if len(repo.readyBlobs) != 1 { + t.Fatalf("ready blobs = %v, want one", repo.readyBlobs) + } + wantKey := "diagnostics/42/11111111-1111-1111-1111-111111111111.tar.gz" + readyBlob := repo.readyBlobs[0] + if readyBlob.Bucket != "private" || readyBlob.Key != wantKey { + t.Fatalf("ready blob location = %s/%s, want private/%s", readyBlob.Bucket, readyBlob.Key, wantKey) + } + if readyBlob.SHA256 != info.SHA256 { + t.Fatalf("ready blob sha256 = %q, want %q", readyBlob.SHA256, info.SHA256) + } + if len(store.puts) != 1 || store.puts[0] != wantKey { + t.Fatalf("stored keys = %v, want [%s]", store.puts, wantKey) + } + if !bytes.Equal(store.data, bundle) { + t.Fatal("stored bundle does not match uploaded bundle") + } + if len(repo.failed) != 0 || len(store.deleted) != 0 { + t.Fatalf("unexpected compensation: failed=%v deleted=%v", repo.failed, store.deleted) + } +} + +func TestServiceIngestRejectsDestinationMismatch(t *testing.T) { + bundle, info := testDiagnosticsBundle(t) + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + + manifest := testManifestJSON(t, "other-server", DefaultConsentNoticeVer, info) + _, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)) + if !errors.Is(err, ErrDestinationMismatch) { + t.Fatalf("Ingest error = %v, want ErrDestinationMismatch", err) + } + if repo.insertCalls != 0 { + t.Fatalf("InsertReceiving calls = %d, want 0", repo.insertCalls) + } + if len(store.puts) != 0 { + t.Fatalf("PutStream calls = %d, want 0", len(store.puts)) + } +} + +func TestServiceIngestRejectsStaleConsent(t *testing.T) { + bundle, info := testDiagnosticsBundle(t) + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + + manifest := testManifestJSON(t, "server-1", DefaultConsentNoticeVer+1, info) + _, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)) + if !errors.Is(err, ErrStaleConsent) { + t.Fatalf("Ingest error = %v, want ErrStaleConsent", err) + } + if repo.insertCalls != 0 { + t.Fatalf("InsertReceiving calls = %d, want 0", repo.insertCalls) + } + if len(store.puts) != 0 { + t.Fatalf("PutStream calls = %d, want 0", len(store.puts)) + } +} + +func TestServiceIngestArchiveMismatchCompensates(t *testing.T) { + bundle, info := testDiagnosticsBundle(t) + info.SHA256 = strings.Repeat("0", 64) + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + + manifest := testManifestJSON(t, "server-1", DefaultConsentNoticeVer, info) + _, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)) + if !errors.Is(err, ErrArchiveMismatch) { + t.Fatalf("Ingest error = %v, want ErrArchiveMismatch", err) + } + if repo.insertCalls != 1 { + t.Fatalf("InsertReceiving calls = %d, want 1", repo.insertCalls) + } + if len(store.puts) != 1 { + t.Fatalf("PutStream calls = %d, want 1", len(store.puts)) + } + wantKey := "diagnostics/42/11111111-1111-1111-1111-111111111111.tar.gz" + if len(store.deleted) != 1 || store.deleted[0] != wantKey { + t.Fatalf("deleted keys = %v, want [%s]", store.deleted, wantKey) + } + if len(repo.failed) != 1 || repo.failed[0] != "11111111-1111-1111-1111-111111111111" { + t.Fatalf("failed reports = %v, want reserved report id", repo.failed) + } + if len(repo.ready) != 0 { + t.Fatalf("ready reports = %v, want none", repo.ready) + } +} + +func TestServiceIngestRejectsEmbeddedManifestMismatch(t *testing.T) { + // The tar embeds a manifest.json that disagrees with the part-1 manifest + // (different os_version) while the archive size/sha fields still match the + // stored bundle, so this is caught only by the embedded-manifest check. + embedded := diagnosticsManifestBody("server-1", DefaultConsentNoticeVer, "prof_1", "15", nil) + bundle := buildManifestBundle(t, embedded) + info, err := ValidateBundle(bytes.NewReader(bundle), BundleLimits{}) + if err != nil { + t.Fatalf("ValidateBundle: %v", err) + } + manifest := diagnosticsManifestBody("server-1", DefaultConsentNoticeVer, "prof_1", "99", &info) + if _, err := contract.ValidateManifest(manifest); err != nil { + t.Fatalf("test manifest invalid: %v\n%s", err, manifest) + } + + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + + _, err = svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)) + if !errors.Is(err, ErrArchiveMismatch) { + t.Fatalf("Ingest error = %v, want ErrArchiveMismatch", err) + } + wantReportID := "11111111-1111-1111-1111-111111111111" + if len(repo.failed) != 1 || repo.failed[0] != wantReportID { + t.Fatalf("failed reports = %v, want [%s]", repo.failed, wantReportID) + } + if len(store.deleted) != 1 { + t.Fatalf("deleted keys = %v, want compensation delete", store.deleted) + } + if len(repo.ready) != 0 { + t.Fatalf("ready reports = %v, want none", repo.ready) + } +} + +func TestEmbeddedManifestMatchesPreservesLargeIntegers(t *testing.T) { + // The received (part-1) and embedded manifests differ only in + // log_summary.lines by 1, at a magnitude above 2^53 where float64 cannot + // represent adjacent integers. Decoding with UseNumber must keep them distinct + // so a bundle whose embedded manifest was tampered is rejected, not accepted. + received := []byte(`{"log_summary":{"lines":9007199254740993},"archive":{"bytes":1}}`) + embedded := []byte(`{"log_summary":{"lines":9007199254740992}}`) + if embeddedManifestMatches(received, embedded) { + t.Fatal("embeddedManifestMatches = true for manifests differing by a large integer, want false") + } + + // Sanity: with the archive object stripped from the received manifest, an + // otherwise identical large integer still compares equal. + sameReceived := []byte(`{"log_summary":{"lines":9007199254740993},"archive":{"bytes":1}}`) + sameEmbedded := []byte(`{"log_summary":{"lines":9007199254740993}}`) + if !embeddedManifestMatches(sameReceived, sameEmbedded) { + t.Fatal("embeddedManifestMatches = false for identical manifests, want true") + } +} + +func TestEmbeddedManifestMatchesRejectsTrailingData(t *testing.T) { + // A stray closing delimiter after an otherwise-matching object must be + // rejected. Decoder.More() returns false here (it only tracks array/object + // iteration), so the decode-to-EOF check is what catches it; without it a + // bundle whose embedded manifest is not a single clean JSON object would + // still compare equal to the accepted part-1 manifest. + received := []byte(`{"log_summary":{"lines":1},"archive":{"bytes":1}}`) + for _, trailer := range []string{"}", "]", "garbage", `{"extra":1}`} { + embedded := []byte(`{"log_summary":{"lines":1}}` + trailer) + if embeddedManifestMatches(received, embedded) { + t.Fatalf("embeddedManifestMatches = true for embedded manifest with trailing %q, want false", trailer) + } + } + + // Trailing whitespace after the object is not trailing data; it must still + // compare equal. + embeddedWithWhitespace := []byte("{\"log_summary\":{\"lines\":1}}\n ") + if !embeddedManifestMatches(received, embeddedWithWhitespace) { + t.Fatal("embeddedManifestMatches = false for embedded manifest with trailing whitespace, want true") + } +} + +func TestServiceIngestReturnsStorageErrorForMidStreamPutFailure(t *testing.T) { + bundle, info := testDiagnosticsBundle(t) + putErr := errors.New("s3: connection reset by peer") + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{ + bucket: "private", + putErr: putErr, + putReadBytes: 8, + } + svc := newTestDiagnosticsService(repo, store) + + manifest := testManifestJSON(t, "server-1", DefaultConsentNoticeVer, info) + _, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)) + if !errors.Is(err, ErrStorageUnavailable) || !errors.Is(err, putErr) { + t.Fatalf("Ingest error = %v, want retryable storage error wrapping put failure", err) + } + if errors.Is(err, ErrInvalidBundle) { + t.Fatalf("Ingest error = %v, must not be classified as ErrInvalidBundle", err) + } + wantReportID := "11111111-1111-1111-1111-111111111111" + if len(repo.failed) != 1 || repo.failed[0] != wantReportID { + t.Fatalf("failed reports = %v, want [%s]", repo.failed, wantReportID) + } + wantKey := "diagnostics/42/" + wantReportID + ".tar.gz" + if len(store.deleted) != 1 || store.deleted[0] != wantKey { + t.Fatalf("deleted keys = %v, want [%s]", store.deleted, wantKey) + } + if len(repo.ready) != 0 { + t.Fatalf("ready reports = %v, want none", repo.ready) + } +} + +func TestServiceIngestCompensatesWithDetachedContext(t *testing.T) { + bundle, info := testDiagnosticsBundle(t) + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{ + bucket: "private", + putErr: errors.New("s3: connection reset by peer"), + putReadBytes: 8, + } + svc := newTestDiagnosticsService(repo, store) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + manifest := testManifestJSON(t, "server-1", DefaultConsentNoticeVer, info) + _, err := svc.Ingest(ctx, 42, nil, manifest, bytes.NewReader(bundle)) + if err == nil { + t.Fatal("Ingest error = nil, want storage error") + } + if len(store.deleteCtxErrs) != 1 || store.deleteCtxErrs[0] != nil { + t.Fatalf("delete ctx errors = %v, want [nil]", store.deleteCtxErrs) + } + if len(repo.markFailedCtxErrs) != 1 || repo.markFailedCtxErrs[0] != nil { + t.Fatalf("mark failed ctx errors = %v, want [nil]", repo.markFailedCtxErrs) + } +} + +func TestServiceIngestStoresOnlyValidatedProfileAttribution(t *testing.T) { + // Header-only attribution (manifest carries no profile_id): the header is + // the single source, so it is validated and stored. + bundle, manifest, _ := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "") + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + svc.SetProfileAttributionValidator(ProfileAttributionValidatorFunc(func(_ context.Context, userID int, profileID string) (bool, error) { + return userID == 42 && profileID == "prof_header", nil + })) + + headerProfileID := "prof_header" + if _, err := svc.Ingest(context.Background(), 42, &headerProfileID, manifest, bytes.NewReader(bundle)); err != nil { + t.Fatalf("Ingest: %v", err) + } + if repo.insertInput.ProfileID == nil || *repo.insertInput.ProfileID != "prof_header" { + t.Fatalf("ProfileID = %v, want prof_header", repo.insertInput.ProfileID) + } + + // Manifest-only attribution that fails validation is dropped, not stored. + bundle, manifest, _ = testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "prof_1") + repo = &fakeDiagnosticReportStore{} + store = &fakeDiagnosticObjectStore{bucket: "private"} + svc = newTestDiagnosticsService(repo, store) + svc.SetProfileAttributionValidator(ProfileAttributionValidatorFunc(func(context.Context, int, string) (bool, error) { + return false, nil + })) + if _, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)); err != nil { + t.Fatalf("Ingest with unvalidated profile: %v", err) + } + if repo.insertInput.ProfileID != nil { + t.Fatalf("ProfileID = %v, want nil for unvalidated attribution", *repo.insertInput.ProfileID) + } +} + +func TestServiceIngestRejectsMismatchedProfileID(t *testing.T) { + // The manifest was captured under prof_manifest but the request carries a + // different X-Profile-Id (prof_header) because the client switched profiles + // between capture and upload. The two sources must not be silently + // reconciled; the upload is rejected and no report row is created. + bundle, manifest, _ := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "prof_manifest") + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + // Even a validator that would accept any profile must not paper over the + // mismatch: resolution fails before attribution validation runs. + svc.SetProfileAttributionValidator(ProfileAttributionValidatorFunc(func(context.Context, int, string) (bool, error) { + return true, nil + })) + + headerProfileID := "prof_header" + _, err := svc.Ingest(context.Background(), 42, &headerProfileID, manifest, bytes.NewReader(bundle)) + if !errors.Is(err, ErrProfileMismatch) { + t.Fatalf("Ingest error = %v, want ErrProfileMismatch", err) + } + if repo.insertInput.ProfileID != nil { + t.Fatalf("ProfileID = %v, want no insert on profile mismatch", *repo.insertInput.ProfileID) + } + if len(repo.ready) != 0 || len(store.puts) != 0 { + t.Fatalf("no report should be stored on mismatch: ready=%v puts=%v", repo.ready, store.puts) + } +} + +func TestServiceIngestAcceptsMatchingProfileID(t *testing.T) { + // Header and manifest agree: attribution resolves to that single value. + bundle, manifest, _ := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "prof_1") + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + svc.SetProfileAttributionValidator(ProfileAttributionValidatorFunc(func(_ context.Context, userID int, profileID string) (bool, error) { + return userID == 42 && profileID == "prof_1", nil + })) + + headerProfileID := "prof_1" + if _, err := svc.Ingest(context.Background(), 42, &headerProfileID, manifest, bytes.NewReader(bundle)); err != nil { + t.Fatalf("Ingest: %v", err) + } + if repo.insertInput.ProfileID == nil || *repo.insertInput.ProfileID != "prof_1" { + t.Fatalf("ProfileID = %v, want prof_1", repo.insertInput.ProfileID) + } +} + +func TestNewProfileAttributionValidatorRejectsChildProfiles(t *testing.T) { + tests := []struct { + name string + found bool + isChild bool + lookErr error + want bool + wantErr bool + wantChild bool + }{ + {name: "non-child profile attributed", found: true, isChild: false, want: true}, + {name: "child profile forbidden", found: true, isChild: true, wantErr: true, wantChild: true}, + {name: "missing profile dropped", found: false, isChild: false, want: false}, + {name: "lookup error propagated", lookErr: errors.New("boom"), wantErr: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + validator := NewProfileAttributionValidator(func(_ context.Context, userID int, profileID string) (bool, bool, error) { + if userID != 42 || profileID != "prof_1" { + t.Fatalf("lookup called with userID=%d profileID=%q", userID, profileID) + } + return tc.found, tc.isChild, tc.lookErr + }) + got, err := validator.ProfileBelongsToUser(context.Background(), 42, "prof_1") + if tc.wantErr { + if err == nil { + t.Fatal("ProfileBelongsToUser error = nil, want error") + } + if tc.wantChild && !errors.Is(err, ErrChildProfileForbidden) { + t.Fatalf("ProfileBelongsToUser error = %v, want ErrChildProfileForbidden", err) + } + return + } + if err != nil { + t.Fatalf("ProfileBelongsToUser: %v", err) + } + if got != tc.want { + t.Fatalf("ProfileBelongsToUser = %v, want %v", got, tc.want) + } + }) + } +} + +func TestServiceIngestRejectsChildProfileAttribution(t *testing.T) { + // A nonconforming client attributes the report to a child profile. The + // validator rejects it with ErrChildProfileForbidden, so the upload is + // refused outright rather than silently stored without attribution. + bundle, manifest, _ := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "prof_child") + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + svc.SetProfileAttributionValidator(ProfileAttributionValidatorFunc(func(context.Context, int, string) (bool, error) { + return false, ErrChildProfileForbidden + })) + + _, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)) + if !errors.Is(err, ErrChildProfileForbidden) { + t.Fatalf("Ingest error = %v, want ErrChildProfileForbidden", err) + } + if repo.insertInput.ProfileID != nil { + t.Fatalf("ProfileID = %v, want no insert on child profile", *repo.insertInput.ProfileID) + } + if len(repo.ready) != 0 || len(store.puts) != 0 { + t.Fatalf("no report should be stored on child rejection: ready=%v puts=%v", repo.ready, store.puts) + } +} + +func TestServiceIngestAcceptsManifestWithoutProfileID(t *testing.T) { + bundle, manifest, _ := testDiagnosticsUpload(t, "server-1", DefaultConsentNoticeVer, "") + repo := &fakeDiagnosticReportStore{} + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + + if _, err := svc.Ingest(context.Background(), 42, nil, manifest, bytes.NewReader(bundle)); err != nil { + t.Fatalf("Ingest: %v", err) + } + if repo.insertInput.ProfileID != nil { + t.Fatalf("ProfileID = %v, want nil", *repo.insertInput.ProfileID) + } +} + +func TestServiceDeleteReportDeletesRowWhenStorageUnavailable(t *testing.T) { + report := testReadyDiagnosticReport("22222222-2222-2222-2222-222222222222", 42) + repo := &fakeDiagnosticReportStore{ + getReport: &report, + deleteReport: &report, + } + svc := newTestDiagnosticsService(repo, nil) + + deleted, err := svc.DeleteReport(context.Background(), report.ID) + if err != nil { + t.Fatalf("DeleteReport: %v", err) + } + if deleted == nil || deleted.ID != report.ID { + t.Fatalf("deleted report = %#v, want %s", deleted, report.ID) + } + if len(repo.deleted) != 1 || repo.deleted[0] != report.ID { + t.Fatalf("deleted rows = %v, want [%s]", repo.deleted, report.ID) + } +} + +func TestServiceDeleteReportKeepsBlobWhenRowDeletionFails(t *testing.T) { + report := testReadyDiagnosticReport("22222222-2222-2222-2222-222222222222", 42) + repo := &fakeDiagnosticReportStore{ + getReport: &report, + deleteErr: errors.New("db down"), + } + store := &fakeDiagnosticObjectStore{bucket: "private"} + svc := newTestDiagnosticsService(repo, store) + + if _, err := svc.DeleteReport(context.Background(), report.ID); err == nil { + t.Fatal("DeleteReport error = nil, want row deletion failure") + } + // The blob must survive a failed row deletion so a retry can still recover + // the original artifact. + if len(store.deleted) != 0 { + t.Fatalf("blob deleted despite row deletion failure: %v", store.deleted) + } +} + +func TestServiceDeleteReportSucceedsWhenBlobDeletionFails(t *testing.T) { + report := testReadyDiagnosticReport("22222222-2222-2222-2222-222222222222", 42) + repo := &fakeDiagnosticReportStore{ + getReport: &report, + deleteReport: &report, + } + store := &fakeDiagnosticObjectStore{bucket: "private", deleteErr: errors.New("s3 down")} + svc := newTestDiagnosticsService(repo, store) + + deleted, err := svc.DeleteReport(context.Background(), report.ID) + if err != nil { + t.Fatalf("DeleteReport error = %v, want success after row deletion", err) + } + if deleted == nil || deleted.ID != report.ID { + t.Fatalf("deleted report = %#v, want %s", deleted, report.ID) + } + if len(repo.deleted) != 1 || repo.deleted[0] != report.ID { + t.Fatalf("deleted rows = %v, want [%s]", repo.deleted, report.ID) + } + // The row is gone; the orphaned blob delete was attempted (and failed), and + // the reconciler will reap it later. + if len(store.deleted) == 0 { + t.Fatal("blob deletion was not attempted after row removal") + } +} + +func newTestDiagnosticsService(repo ReportStore, store ObjectStore) *Service { + settings := newMemorySettingsStore(map[string]string{ + KeyUploadsEnabled: "true", + KeyMaxBundleBytes: "10485760", + KeyMaxUncompressedBytes: "67108864", + KeyMaxReportsPerUserDay: "20", + KeyRetentionDays: "30", + KeyMaxBytesPerUser: "209715200", + KeyConsentNoticeVersion: "1", + KeyServerInstanceID: "server-1", + }) + svc := NewService(repo, settings, store, slog.New(slog.NewTextHandler(io.Discard, nil))) + svc.now = func() time.Time { return time.Date(2026, 7, 20, 12, 0, 0, 0, time.UTC) } + return svc +} + +func testDiagnosticsBundle(t *testing.T) ([]byte, BundleInfo) { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + payload := []byte(`{}`) + if err := tw.WriteHeader(&tar.Header{ + Name: "manifest.json", + Mode: 0o600, + Size: int64(len(payload)), + }); err != nil { + t.Fatalf("write tar header: %v", err) + } + if _, err := tw.Write(payload); err != nil { + t.Fatalf("write tar payload: %v", err) + } + if err := tw.Close(); err != nil { + t.Fatalf("close tar: %v", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("close gzip: %v", err) + } + data := buf.Bytes() + info, err := ValidateBundle(bytes.NewReader(data), BundleLimits{}) + if err != nil { + t.Fatalf("ValidateBundle: %v", err) + } + return data, info +} + +func testManifestJSON(t *testing.T, serverID string, noticeVersion int, archive BundleInfo) []byte { + t.Helper() + return testManifestJSONWithProfile(t, serverID, noticeVersion, archive, "prof_1") +} + +func testManifestJSONWithProfile(t *testing.T, serverID string, noticeVersion int, archive BundleInfo, profileID string) []byte { + t.Helper() + data := diagnosticsManifestBody(serverID, noticeVersion, profileID, "15", &archive) + if _, err := contract.ValidateManifest(data); err != nil { + t.Fatalf("test manifest invalid: %v\n%s", err, string(data)) + } + return data +} + +// diagnosticsManifestBody builds a manifest JSON body. When info is nil the +// `archive` object is omitted, producing the embedded manifest.json contract +// form (part-1 manifest minus `archive`). +func diagnosticsManifestBody(serverID string, noticeVersion int, profileID, osVersion string, info *BundleInfo) []byte { + profileField := "" + if profileID != "" { + profileField = `, + "profile_id": "` + profileID + `"` + } + archiveField := "" + if info != nil { + archiveField = `, + "archive": { + "entries": ["manifest.json"], + "bytes": ` + strconv.FormatInt(info.CompressedBytes, 10) + `, + "uncompressed_bytes": ` + strconv.FormatInt(info.UncompressedBytes, 10) + `, + "sha256": "` + info.SHA256 + `" + }` + } + return []byte(`{ + "schema_version": 1, + "report": { + "type": "manual", + "captured_at": "2026-07-20T12:00:00Z", + "capture_session_id": "run_test", + "app_version": "1.0.0", + "app_build": "100", + "platform": "android", + "os_version": "` + osVersion + `"` + profileField + ` + }, + "destination": { "server_instance_id": "` + serverID + `" }, + "consent": { "mode": "manual", "notice_version": ` + strconv.Itoa(noticeVersion) + ` }, + "device_summary": { + "manufacturer": "Google", + "model": "Pixel", + "os": "15", + "form_factor": "phone" + }, + "playback_session_ids": ["ps_1"], + "log_summary": { + "lines": 1, + "bytes_gz": 1, + "dropped_lines": 0, + "categories": ["crash"], + "debug_logging": true + }` + archiveField + ` + }`) +} + +// testDiagnosticsUpload builds a coordinated (bundle, manifest) pair: the tar's +// embedded manifest.json is the part-1 manifest minus `archive`, and the part-1 +// manifest's archive fields match the built bundle — the shape Ingest accepts. +func testDiagnosticsUpload(t *testing.T, serverID string, noticeVersion int, profileID string) ([]byte, []byte, BundleInfo) { + t.Helper() + embedded := diagnosticsManifestBody(serverID, noticeVersion, profileID, "15", nil) + bundle := buildManifestBundle(t, embedded) + info, err := ValidateBundle(bytes.NewReader(bundle), BundleLimits{}) + if err != nil { + t.Fatalf("ValidateBundle: %v", err) + } + manifest := diagnosticsManifestBody(serverID, noticeVersion, profileID, "15", &info) + if _, err := contract.ValidateManifest(manifest); err != nil { + t.Fatalf("test manifest invalid: %v\n%s", err, manifest) + } + return bundle, manifest, info +} + +func buildManifestBundle(t *testing.T, manifestBody []byte) []byte { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + if err := tw.WriteHeader(&tar.Header{ + Name: "manifest.json", + Mode: 0o600, + Size: int64(len(manifestBody)), + }); err != nil { + t.Fatalf("write tar header: %v", err) + } + if _, err := tw.Write(manifestBody); err != nil { + t.Fatalf("write tar payload: %v", err) + } + if err := tw.Close(); err != nil { + t.Fatalf("close tar: %v", err) + } + if err := gz.Close(); err != nil { + t.Fatalf("close gzip: %v", err) + } + return buf.Bytes() +} + +type fakeDiagnosticReportStore struct { + insertCalls int + insertInput InsertReceivingInput + insertErr error + ready []string + readyBlobs []BlobInfo + failed []string + markFailedCtxErrs []error + getReport *Report + getErr error + deleteReport *Report + deleteErr error + deleted []string +} + +func (f *fakeDiagnosticReportStore) InsertReceiving(_ context.Context, input InsertReceivingInput) (InsertReceivingResult, error) { + f.insertCalls++ + f.insertInput = input + if f.insertErr != nil { + return InsertReceivingResult{}, f.insertErr + } + return InsertReceivingResult{ + ID: "11111111-1111-1111-1111-111111111111", + ShortID: "SILO-ABCDEF123456", + }, nil +} + +func (f *fakeDiagnosticReportStore) MarkReady(_ context.Context, id string, blob BlobInfo) error { + f.ready = append(f.ready, id) + f.readyBlobs = append(f.readyBlobs, blob) + return nil +} + +func (f *fakeDiagnosticReportStore) MarkFailed(ctx context.Context, id string) error { + f.failed = append(f.failed, id) + f.markFailedCtxErrs = append(f.markFailedCtxErrs, ctx.Err()) + return nil +} + +func (f *fakeDiagnosticReportStore) GetByID(context.Context, string) (*Report, error) { + if f.getErr != nil { + return nil, f.getErr + } + if f.getReport != nil { + report := *f.getReport + return &report, nil + } + return nil, ErrNotFound +} + +func (f *fakeDiagnosticReportStore) ListForAdmin(context.Context, ListFilters) (ListResult, error) { + return ListResult{}, nil +} + +func (f *fakeDiagnosticReportStore) DeleteByID(_ context.Context, id string) (*Report, error) { + f.deleted = append(f.deleted, id) + if f.deleteErr != nil { + return nil, f.deleteErr + } + if f.deleteReport != nil { + report := *f.deleteReport + return &report, nil + } + return nil, ErrNotFound +} + +func (f *fakeDiagnosticReportStore) RetentionCandidates(context.Context, time.Time, int64) ([]Report, error) { + return nil, nil +} + +func (f *fakeDiagnosticReportStore) StaleReceiving(context.Context, time.Duration) ([]Report, error) { + return nil, nil +} + +type fakeDiagnosticObjectStore struct { + bucket string + puts []string + deleted []string + deleteCtxErrs []error + deleteErr error + data []byte + putErr error + putReadBytes int64 +} + +func (f *fakeDiagnosticObjectStore) PutStream(_ context.Context, _ string, key string, r io.Reader, _ string) error { + if f.putErr != nil { + if f.putReadBytes > 0 { + _, _ = io.CopyN(io.Discard, r, f.putReadBytes) + } + f.puts = append(f.puts, key) + return f.putErr + } + data, err := io.ReadAll(r) + if err != nil { + return err + } + f.puts = append(f.puts, key) + f.data = append(f.data[:0], data...) + return nil +} + +func (f *fakeDiagnosticObjectStore) GetObject(context.Context, string, string) (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(f.data)), nil +} + +func (f *fakeDiagnosticObjectStore) DeleteObject(ctx context.Context, _ string, key string) error { + f.deleted = append(f.deleted, key) + f.deleteCtxErrs = append(f.deleteCtxErrs, ctx.Err()) + return f.deleteErr +} + +func (f *fakeDiagnosticObjectStore) ListObjects(context.Context, string) ([]string, error) { + return nil, nil +} + +func (f *fakeDiagnosticObjectStore) PresignGetURL(context.Context, string, string, time.Duration) (string, error) { + return "", nil +} + +func (f *fakeDiagnosticObjectStore) Bucket() string { + return f.bucket +} + +func testReadyDiagnosticReport(id string, userID int) Report { + bucket := "private" + key := reportObjectKey(userID, id) + bytes := int64(128) + uncompressed := int64(256) + sha := strings.Repeat("a", 64) + return Report{ + ID: id, + ShortID: "SILO-ABCDEF123456", + UserID: userID, + State: StateReady, + BlobBucket: &bucket, + BlobKey: &key, + BlobBytes: &bytes, + UncompressedBytes: &uncompressed, + BlobSHA256: &sha, + } +} diff --git a/internal/diagnostics/settings.go b/internal/diagnostics/settings.go new file mode 100644 index 00000000..592cdeb4 --- /dev/null +++ b/internal/diagnostics/settings.go @@ -0,0 +1,256 @@ +package diagnostics + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/google/uuid" +) + +const ( + KeyUploadsEnabled = "diagnostics.uploads_enabled" + KeyMaxBundleBytes = "diagnostics.max_bundle_bytes" + KeyMaxUncompressedBytes = "diagnostics.max_uncompressed_bytes" + KeyMaxReportsPerUserDay = "diagnostics.max_reports_per_user_per_day" + KeyRetentionDays = "diagnostics.retention_days" + KeyMaxBytesPerUser = "diagnostics.max_bytes_per_user" + KeyConsentNoticeVersion = "diagnostics.consent_notice_version" + KeyCleanupIntervalMinutes = "diagnostics.cleanup_interval_minutes" + KeyServerInstanceID = "diagnostics.server_instance_id" + DefaultUploadsEnabled = false + DefaultMaxBundleBytes = int64(10 * 1024 * 1024) + DefaultMaxUncompressed = int64(64 * 1024 * 1024) + DefaultMaxReportsPerDay = 20 + DefaultRetentionDays = 30 + DefaultMaxBytesPerUser = int64(200 * 1024 * 1024) + DefaultConsentNoticeVer = 1 + // DefaultCleanupIntervalMinutes matches the prior opslog-shared cadence so + // splitting diagnostics onto its own key preserves current behavior. + DefaultCleanupIntervalMinutes = 15 + // maxCleanupIntervalMinutes caps the parsed interval well below the point + // where minutes * time.Minute overflows int64 nanoseconds (~1.5e8 minutes, + // which would wrap a huge configured value into a tiny — or negative — + // duration). 7 days is far longer than any sane cleanup cadence. + maxCleanupIntervalMinutes = 7 * 24 * 60 + defaultUploadsEnabledStr = "false" + defaultMaxBundleStr = "10485760" + defaultMaxUncompressed = "67108864" + defaultMaxReportsStr = "20" + defaultRetentionDaysStr = "30" + defaultMaxBytesUserStr = "209715200" + defaultConsentNoticeStr = "1" + defaultCleanupIntervalStr = "15" +) + +// SettingsStore is the read/write surface over server_settings used by the +// diagnostics feature gate. catalog.ServerSettingsRepo satisfies it. +type SettingsStore interface { + Get(ctx context.Context, key string) (string, error) + Set(ctx context.Context, key, value string) error +} + +// conditionalSettingsStore is the optional insert-if-absent surface used to seed +// generated singletons (the server instance ID) without racing concurrent +// nodes. catalog.ServerSettingsRepo satisfies it. +type conditionalSettingsStore interface { + SetIfAbsent(ctx context.Context, key, value string) (bool, error) +} + +type Settings struct { + UploadsEnabled bool + MaxBundleBytes int64 + MaxUncompressedBytes int64 + MaxReportsPerUserDay int + RetentionDays int + MaxBytesPerUser int64 + ConsentNoticeVersion int + ServerInstanceID string +} + +func SeedDefaults(ctx context.Context, store SettingsStore) error { + defaults := map[string]string{ + KeyUploadsEnabled: defaultUploadsEnabledStr, + KeyMaxBundleBytes: defaultMaxBundleStr, + KeyMaxUncompressedBytes: defaultMaxUncompressed, + KeyMaxReportsPerUserDay: defaultMaxReportsStr, + KeyRetentionDays: defaultRetentionDaysStr, + KeyMaxBytesPerUser: defaultMaxBytesUserStr, + KeyConsentNoticeVersion: defaultConsentNoticeStr, + KeyCleanupIntervalMinutes: defaultCleanupIntervalStr, + } + for key, value := range defaults { + existing, err := store.Get(ctx, key) + if err != nil { + return fmt.Errorf("seed diagnostics defaults for %s: %w", key, err) + } + if existing != "" { + continue + } + if err := store.Set(ctx, key, value); err != nil { + return fmt.Errorf("seed diagnostics default %s: %w", key, err) + } + } + + if _, err := ensureServerInstanceID(ctx, store); err != nil { + return err + } + return nil +} + +// ensureServerInstanceID returns the diagnostics server instance ID, generating +// and persisting one when absent. Seeding is atomic when the store supports +// insert-if-absent: concurrent nodes converge on the single winning value +// instead of overwriting each other (which would reject already-issued +// destination IDs as destination_mismatch). +func ensureServerInstanceID(ctx context.Context, store SettingsStore) (string, error) { + existing, err := store.Get(ctx, KeyServerInstanceID) + if err != nil { + return "", fmt.Errorf("seed diagnostics server instance id: %w", err) + } + if id := strings.TrimSpace(existing); id != "" { + return id, nil + } + + instanceID, err := newServerInstanceID() + if err != nil { + return "", err + } + + conditional, ok := store.(conditionalSettingsStore) + if !ok { + if err := store.Set(ctx, KeyServerInstanceID, instanceID); err != nil { + return "", fmt.Errorf("seed diagnostics server instance id: %w", err) + } + return instanceID, nil + } + + if _, err := conditional.SetIfAbsent(ctx, KeyServerInstanceID, instanceID); err != nil { + return "", fmt.Errorf("seed diagnostics server instance id: %w", err) + } + // Re-read so a node that lost the insert race adopts the winning value. + winner, err := store.Get(ctx, KeyServerInstanceID) + if err != nil { + return "", fmt.Errorf("seed diagnostics server instance id: %w", err) + } + if id := strings.TrimSpace(winner); id != "" { + return id, nil + } + return instanceID, nil +} + +// LoadCleanupInterval returns the configured diagnostics cleanup interval. It is +// intentionally independent of the opslog cleanup cadence so tuning one does not +// silently move the other. +func LoadCleanupInterval(ctx context.Context, store SettingsStore) time.Duration { + minutes := DefaultCleanupIntervalMinutes + if store == nil { + return time.Duration(minutes) * time.Minute + } + if raw, err := store.Get(ctx, KeyCleanupIntervalMinutes); err == nil && strings.TrimSpace(raw) != "" { + if parsed := parseInt(raw); parsed > 0 { + minutes = parsed + if minutes > maxCleanupIntervalMinutes { + minutes = maxCleanupIntervalMinutes + } + } + } + return time.Duration(minutes) * time.Minute +} + +func DefaultSettings() Settings { + return Settings{ + UploadsEnabled: DefaultUploadsEnabled, + MaxBundleBytes: DefaultMaxBundleBytes, + MaxUncompressedBytes: DefaultMaxUncompressed, + MaxReportsPerUserDay: DefaultMaxReportsPerDay, + RetentionDays: DefaultRetentionDays, + MaxBytesPerUser: DefaultMaxBytesPerUser, + ConsentNoticeVersion: DefaultConsentNoticeVer, + } +} + +func LoadSettings(ctx context.Context, store SettingsStore) (Settings, error) { + settings := DefaultSettings() + if store == nil { + return settings, nil + } + // get distinguishes a genuine read failure — propagated so callers can + // surface a retryable error — from a missing or empty value, which falls + // back to the default. Silently defaulting on a transient DB error could + // report uploads disabled or the wrong quota limits to clients making + // decisions from /diagnostics/status. + get := func(key string) (string, error) { + raw, err := store.Get(ctx, key) + if err != nil { + return "", fmt.Errorf("load diagnostics setting %s: %w", key, err) + } + return strings.TrimSpace(raw), nil + } + + if raw, err := get(KeyUploadsEnabled); err != nil { + return Settings{}, err + } else if raw != "" { + if parsed, parseErr := strconv.ParseBool(raw); parseErr == nil { + settings.UploadsEnabled = parsed + } + } + if raw, err := get(KeyMaxBundleBytes); err != nil { + return Settings{}, err + } else if parsed := parseInt64(raw); parsed > 0 { + settings.MaxBundleBytes = parsed + } + if raw, err := get(KeyMaxUncompressedBytes); err != nil { + return Settings{}, err + } else if parsed := parseInt64(raw); parsed > 0 { + settings.MaxUncompressedBytes = parsed + } + if raw, err := get(KeyMaxReportsPerUserDay); err != nil { + return Settings{}, err + } else if parsed := parseInt(raw); parsed > 0 { + settings.MaxReportsPerUserDay = parsed + } + if raw, err := get(KeyRetentionDays); err != nil { + return Settings{}, err + } else if parsed := parseInt(raw); parsed > 0 { + settings.RetentionDays = parsed + } + if raw, err := get(KeyMaxBytesPerUser); err != nil { + return Settings{}, err + } else if parsed := parseInt64(raw); parsed > 0 { + settings.MaxBytesPerUser = parsed + } + if raw, err := get(KeyConsentNoticeVersion); err != nil { + return Settings{}, err + } else if parsed := parseInt(raw); parsed > 0 { + settings.ConsentNoticeVersion = parsed + } + if raw, err := get(KeyServerInstanceID); err != nil { + return Settings{}, err + } else { + settings.ServerInstanceID = raw + } + return settings, nil +} + +func newServerInstanceID() (string, error) { + return uuid.NewString(), nil +} + +func parseInt(s string) int { + v, err := strconv.Atoi(strings.TrimSpace(s)) + if err != nil { + return 0 + } + return v +} + +func parseInt64(s string) int64 { + v, err := strconv.ParseInt(strings.TrimSpace(s), 10, 64) + if err != nil { + return 0 + } + return v +} diff --git a/internal/diagnostics/settings_test.go b/internal/diagnostics/settings_test.go new file mode 100644 index 00000000..9bdabd14 --- /dev/null +++ b/internal/diagnostics/settings_test.go @@ -0,0 +1,234 @@ +package diagnostics + +import ( + "context" + "errors" + "strconv" + "testing" + "time" +) + +func TestSeedDefaultsPreservesExistingServerInstanceID(t *testing.T) { + store := newMemorySettingsStore(map[string]string{ + KeyServerInstanceID: "existing-instance", + }) + if err := SeedDefaults(context.Background(), store); err != nil { + t.Fatalf("SeedDefaults: %v", err) + } + if got := store.values[KeyServerInstanceID]; got != "existing-instance" { + t.Fatalf("server instance id = %q, want existing-instance", got) + } + if got := store.values[KeyUploadsEnabled]; got != defaultUploadsEnabledStr { + t.Fatalf("uploads enabled default = %q, want %q", got, defaultUploadsEnabledStr) + } +} + +func TestSeedDefaultsGeneratesDedicatedServerInstanceIDWhenMissing(t *testing.T) { + const jellyfinServerIDKey = "jellyfin_compat.server_id" + store := newMemorySettingsStore(map[string]string{ + jellyfinServerIDKey: "constant-jellyfin-id", + }) + if err := SeedDefaults(context.Background(), store); err != nil { + t.Fatalf("SeedDefaults: %v", err) + } + got := store.values[KeyServerInstanceID] + if len(got) != 36 { + t.Fatalf("server instance id length = %d, want UUID string", len(got)) + } + if got == store.values[jellyfinServerIDKey] { + t.Fatal("diagnostics server instance id reused jellyfin_compat.server_id") + } + if store.values[jellyfinServerIDKey] != "constant-jellyfin-id" { + t.Fatalf("jellyfin server id changed to %q", store.values[jellyfinServerIDKey]) + } +} + +func TestSeedDefaultsAdoptsConcurrentInstanceIDWinner(t *testing.T) { + // A store that supports insert-if-absent seeds the instance ID atomically: + // when another node wins the race, this node adopts the winning value + // instead of overwriting it. + store := &racingSettingsStore{ + memorySettingsStore: memorySettingsStore{values: map[string]string{}}, + concurrentWinner: "winner-instance", + } + if err := SeedDefaults(context.Background(), store); err != nil { + t.Fatalf("SeedDefaults: %v", err) + } + if got := store.values[KeyServerInstanceID]; got != "winner-instance" { + t.Fatalf("server instance id = %q, want winner-instance", got) + } + if store.setIfAbsentCalls != 1 { + t.Fatalf("SetIfAbsent calls = %d, want 1", store.setIfAbsentCalls) + } +} + +func TestLoadCleanupIntervalUsesDiagnosticsKey(t *testing.T) { + store := newMemorySettingsStore(map[string]string{ + KeyCleanupIntervalMinutes: "45", + }) + if got := LoadCleanupInterval(context.Background(), store); got != 45*time.Minute { + t.Fatalf("LoadCleanupInterval = %v, want 45m", got) + } + if got := LoadCleanupInterval(context.Background(), newMemorySettingsStore(nil)); got != DefaultCleanupIntervalMinutes*time.Minute { + t.Fatalf("LoadCleanupInterval default = %v, want %d minutes", got, DefaultCleanupIntervalMinutes) + } +} + +func TestLoadCleanupIntervalCapsHugeValues(t *testing.T) { + // The boundary: the largest value that still fits under the cap is accepted + // verbatim, while anything larger (including values that would overflow + // int64 nanoseconds when multiplied by time.Minute) clamps to the cap + // instead of wrapping into a tiny or negative duration. + atCap := newMemorySettingsStore(map[string]string{ + KeyCleanupIntervalMinutes: strconv.Itoa(maxCleanupIntervalMinutes), + }) + if got := LoadCleanupInterval(context.Background(), atCap); got != maxCleanupIntervalMinutes*time.Minute { + t.Fatalf("LoadCleanupInterval at cap = %v, want %v", got, maxCleanupIntervalMinutes*time.Minute) + } + + for _, raw := range []string{ + strconv.Itoa(maxCleanupIntervalMinutes + 1), + "153722868", // wraps int64 nanoseconds without the cap + "9223372036854775807", // math.MaxInt64 minutes + } { + store := newMemorySettingsStore(map[string]string{KeyCleanupIntervalMinutes: raw}) + got := LoadCleanupInterval(context.Background(), store) + if got != maxCleanupIntervalMinutes*time.Minute { + t.Fatalf("LoadCleanupInterval(%s) = %v, want capped %v", raw, got, maxCleanupIntervalMinutes*time.Minute) + } + if got <= 0 { + t.Fatalf("LoadCleanupInterval(%s) = %v, want a positive duration", raw, got) + } + } +} + +func TestLoadSettingsPropagatesReadErrors(t *testing.T) { + // A genuine read failure must surface as an error so callers can return a + // retryable failure instead of silently reporting defaults (e.g. uploads + // disabled) to clients. + store := &failingSettingsStore{ + memorySettingsStore: memorySettingsStore{values: map[string]string{KeyUploadsEnabled: "true"}}, + failKey: KeyUploadsEnabled, + } + if _, err := LoadSettings(context.Background(), store); err == nil { + t.Fatal("LoadSettings did not propagate a store read error") + } + + // A later key failing must also propagate, not be swallowed as a default. + store = &failingSettingsStore{ + memorySettingsStore: memorySettingsStore{values: map[string]string{KeyUploadsEnabled: "true"}}, + failKey: KeyMaxBytesPerUser, + } + if _, err := LoadSettings(context.Background(), store); err == nil { + t.Fatal("LoadSettings did not propagate a late store read error") + } +} + +func TestLoadSettingsParsesTypedValues(t *testing.T) { + store := newMemorySettingsStore(map[string]string{ + KeyUploadsEnabled: "true", + KeyMaxBundleBytes: "123", + KeyMaxUncompressedBytes: "456", + KeyMaxReportsPerUserDay: "7", + KeyRetentionDays: "8", + KeyMaxBytesPerUser: "900", + KeyConsentNoticeVersion: "2", + KeyServerInstanceID: "server-1", + }) + + got, err := LoadSettings(context.Background(), store) + if err != nil { + t.Fatalf("LoadSettings: %v", err) + } + if !got.UploadsEnabled || + got.MaxBundleBytes != 123 || + got.MaxUncompressedBytes != 456 || + got.MaxReportsPerUserDay != 7 || + got.RetentionDays != 8 || + got.MaxBytesPerUser != 900 || + got.ConsentNoticeVersion != 2 || + got.ServerInstanceID != "server-1" { + t.Fatalf("LoadSettings parsed unexpected values: %+v", got) + } +} + +func TestLoadSettingsFallsBackForInvalidNumbers(t *testing.T) { + store := newMemorySettingsStore(map[string]string{ + KeyMaxBundleBytes: "-1", + KeyMaxUncompressedBytes: "not-a-number", + KeyMaxReportsPerUserDay: "0", + KeyRetentionDays: "0", + KeyMaxBytesPerUser: "-2", + KeyConsentNoticeVersion: "0", + }) + + got, err := LoadSettings(context.Background(), store) + if err != nil { + t.Fatalf("LoadSettings: %v", err) + } + want := DefaultSettings() + if got.MaxBundleBytes != want.MaxBundleBytes || + got.MaxUncompressedBytes != want.MaxUncompressedBytes || + got.MaxReportsPerUserDay != want.MaxReportsPerUserDay || + got.RetentionDays != want.RetentionDays || + got.MaxBytesPerUser != want.MaxBytesPerUser || + got.ConsentNoticeVersion != want.ConsentNoticeVersion { + t.Fatalf("LoadSettings = %+v, want defaults %+v", got, want) + } +} + +type memorySettingsStore struct { + values map[string]string +} + +func newMemorySettingsStore(values map[string]string) *memorySettingsStore { + store := &memorySettingsStore{values: map[string]string{}} + for k, v := range values { + store.values[k] = v + } + return store +} + +func (s *memorySettingsStore) Get(_ context.Context, key string) (string, error) { + return s.values[key], nil +} + +func (s *memorySettingsStore) Set(_ context.Context, key, value string) error { + s.values[key] = value + return nil +} + +// failingSettingsStore returns an error when Get is called for failKey, +// simulating a transient database read failure for one setting. +type failingSettingsStore struct { + memorySettingsStore + failKey string +} + +func (s *failingSettingsStore) Get(ctx context.Context, key string) (string, error) { + if key == s.failKey { + return "", errors.New("transient read failure") + } + return s.memorySettingsStore.Get(ctx, key) +} + +// racingSettingsStore adds insert-if-absent semantics and simulates another node +// winning the instance-ID seed race via concurrentWinner. +type racingSettingsStore struct { + memorySettingsStore + concurrentWinner string + setIfAbsentCalls int +} + +func (s *racingSettingsStore) SetIfAbsent(_ context.Context, key, value string) (bool, error) { + s.setIfAbsentCalls++ + if existing := s.values[key]; existing != "" { + return false, nil + } + if s.concurrentWinner != "" { + s.values[key] = s.concurrentWinner + return false, nil + } + s.values[key] = value + return true, nil +} diff --git a/internal/diagnostics/shortid.go b/internal/diagnostics/shortid.go new file mode 100644 index 00000000..a98baf5c --- /dev/null +++ b/internal/diagnostics/shortid.go @@ -0,0 +1,50 @@ +package diagnostics + +import ( + "crypto/rand" + "errors" + "strings" +) + +const ( + shortIDPrefix = "SILO-" + shortIDPayloadLength = 12 + shortIDChars = "0123456789ABCDEFGHJKMNPQRSTVWXYZ" +) + +var ErrInvalidShortID = errors.New("invalid diagnostics short id") + +func NewShortID() (string, error) { + var random [shortIDPayloadLength]byte + if _, err := rand.Read(random[:]); err != nil { + return "", err + } + var out [shortIDPayloadLength]byte + for i, b := range random { + out[i] = shortIDChars[int(b)&31] + } + return shortIDPrefix + string(out[:]), nil +} + +func ParseShortID(raw string) (string, error) { + id := strings.ToUpper(strings.TrimSpace(raw)) + payload := strings.TrimPrefix(id, shortIDPrefix) + if len(payload) != shortIDPayloadLength { + return "", ErrInvalidShortID + } + for i := 0; i < len(payload); i++ { + if !isShortIDChar(payload[i]) { + return "", ErrInvalidShortID + } + } + return shortIDPrefix + payload, nil +} + +func isShortIDChar(ch byte) bool { + for i := 0; i < len(shortIDChars); i++ { + if shortIDChars[i] == ch { + return true + } + } + return false +} diff --git a/internal/diagnostics/shortid_test.go b/internal/diagnostics/shortid_test.go new file mode 100644 index 00000000..5ca71bbe --- /dev/null +++ b/internal/diagnostics/shortid_test.go @@ -0,0 +1,43 @@ +package diagnostics + +import ( + "errors" + "strings" + "testing" +) + +func TestNewShortIDIsParseable(t *testing.T) { + id, err := NewShortID() + if err != nil { + t.Fatalf("NewShortID: %v", err) + } + if len(id) != len(shortIDPrefix)+shortIDPayloadLength { + t.Fatalf("length = %d, want %d", len(id), len(shortIDPrefix)+shortIDPayloadLength) + } + if !strings.HasPrefix(id, shortIDPrefix) { + t.Fatalf("id = %q, want %s prefix", id, shortIDPrefix) + } + if _, err := ParseShortID(id); err != nil { + t.Fatalf("ParseShortID(%q): %v", id, err) + } +} + +func TestParseShortIDNormalizesCaseAndOptionalPrefix(t *testing.T) { + for _, raw := range []string{"abcdef123456", "silo-abcdef123456", " SILO-ABCDEF123456 "} { + got, err := ParseShortID(raw) + if err != nil { + t.Fatalf("ParseShortID(%q): %v", raw, err) + } + if got != "SILO-ABCDEF123456" { + t.Fatalf("ParseShortID(%q) = %q, want SILO-ABCDEF123456", raw, got) + } + } +} + +func TestParseShortIDRejectsAmbiguousCharacters(t *testing.T) { + for _, raw := range []string{"ABCDEF12345I", "ABCDEF12345L", "ABCDEF12345O", "ABCDEF12345U"} { + if _, err := ParseShortID(raw); !errors.Is(err, ErrInvalidShortID) { + t.Fatalf("ParseShortID(%q) error = %v, want ErrInvalidShortID", raw, err) + } + } +} diff --git a/internal/diagnostics/store.go b/internal/diagnostics/store.go new file mode 100644 index 00000000..e84f5df6 --- /dev/null +++ b/internal/diagnostics/store.go @@ -0,0 +1,91 @@ +package diagnostics + +import ( + "context" + "errors" + "io" + "time" + + "github.com/Silo-Server/silo-server/internal/s3client" +) + +var ErrObjectNotFound = errors.New("diagnostics object not found") + +// ObjectStore is the diagnostics-owned object storage surface. It is narrow +// enough for fake-backed ingest/admin tests and wraps the private S3 bucket in +// production. +type ObjectStore interface { + PutStream(ctx context.Context, bucket, key string, r io.Reader, contentType string) error + GetObject(ctx context.Context, bucket, key string) (io.ReadCloser, error) + DeleteObject(ctx context.Context, bucket, key string) error + ListObjects(ctx context.Context, prefix string) ([]string, error) + PresignGetURL(ctx context.Context, bucket, key string, expiry time.Duration) (string, error) + Bucket() string +} + +type s3ObjectStore struct { + client *s3client.Client +} + +// NewS3ObjectStore adapts the private S3 client to the diagnostics store +// interface. A nil client returns nil so interface-nil checks remain reliable. +func NewS3ObjectStore(client *s3client.Client) ObjectStore { + if client == nil { + return nil + } + return &s3ObjectStore{client: client} +} + +func (s *s3ObjectStore) PutStream(ctx context.Context, bucket, key string, r io.Reader, contentType string) error { + return normalizeObjectStoreError(s.client.PutObjectStream(ctx, bucket, key, r, contentType)) +} + +func (s *s3ObjectStore) GetObject(ctx context.Context, bucket, key string) (io.ReadCloser, error) { + body, err := s.client.GetObjectStream(ctx, bucket, key) + if err != nil { + return nil, normalizeObjectStoreError(err) + } + return body, nil +} + +func (s *s3ObjectStore) DeleteObject(ctx context.Context, bucket, key string) error { + return normalizeObjectStoreError(s.client.DeleteObject(ctx, bucket, key)) +} + +func (s *s3ObjectStore) ListObjects(ctx context.Context, prefix string) ([]string, error) { + keys, err := s.client.ListObjects(ctx, s.client.Bucket(), prefix) + if err != nil { + return nil, normalizeObjectStoreError(err) + } + return keys, nil +} + +func (s *s3ObjectStore) PresignGetURL(ctx context.Context, bucket, key string, expiry time.Duration) (string, error) { + url, err := s.client.PresignGetURL(ctx, bucket, key, expiry) + if err != nil { + return "", normalizeObjectStoreError(err) + } + return url, nil +} + +func (s *s3ObjectStore) EffectivePresignTTL(requested time.Duration) time.Duration { + return s.client.EffectivePresignTTL(requested) +} + +func (s *s3ObjectStore) Bucket() string { + return s.client.Bucket() +} + +func normalizeObjectStoreError(err error) error { + if err == nil { + return nil + } + if errors.Is(err, s3client.ErrNotFound) { + return ErrObjectNotFound + } + return err +} + +func IsObjectNotFound(err error) bool { + return errors.Is(err, ErrObjectNotFound) || errors.Is(err, s3client.ErrNotFound) +} diff --git a/internal/diagnostics/types.go b/internal/diagnostics/types.go new file mode 100644 index 00000000..29946e59 --- /dev/null +++ b/internal/diagnostics/types.go @@ -0,0 +1,150 @@ +package diagnostics + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + "unicode/utf8" +) + +const MaxCrashSummaryBytes = 8 * 1024 + +type ReportState string + +const ( + StateReceiving ReportState = "receiving" + StateReady ReportState = "ready" + StateFailed ReportState = "failed" +) + +var ( + ErrNotFound = errors.New("diagnostic report not found") + ErrQuotaExceeded = errors.New("diagnostic report quota exceeded") + ErrShortIDExhausted = errors.New("diagnostic short id collision retries exhausted") + ErrInvalidReportInput = errors.New("invalid diagnostic report input") + ErrInvalidCursor = errors.New("invalid diagnostics cursor") +) + +type ReportStore interface { + InsertReceiving(ctx context.Context, input InsertReceivingInput) (InsertReceivingResult, error) + MarkReady(ctx context.Context, id string, blob BlobInfo) error + MarkFailed(ctx context.Context, id string) error + GetByID(ctx context.Context, id string) (*Report, error) + ListForAdmin(ctx context.Context, filters ListFilters) (ListResult, error) + DeleteByID(ctx context.Context, id string) (*Report, error) + RetentionCandidates(ctx context.Context, olderThan time.Time, perUserByteCap int64) ([]Report, error) + StaleReceiving(ctx context.Context, grace time.Duration) ([]Report, error) +} + +type InsertReceivingInput struct { + UserID int + ProfileID *string + CapturedAt time.Time + ReportType string + Platform string + AppVersion string + CrashSummary *string + Manifest json.RawMessage + PlaybackSessionIDs []string + ExpectedBlobBytes int64 + MaxReportsPerUserDay int + MaxBytesPerUser int64 + Now time.Time + ShortIDGenerator func() (string, error) + ReportIDGenerator func() string + ShortIDCollisionRetries int +} + +type InsertReceivingResult struct { + ID string + ShortID string +} + +type BlobInfo struct { + Bucket string + Key string + Bytes int64 + UncompressedBytes int64 + SHA256 string +} + +type Report struct { + ID string `json:"id"` + ShortID string `json:"short_id"` + UserID int `json:"user_id"` + ProfileID *string `json:"profile_id,omitempty"` + State ReportState `json:"state"` + CapturedAt time.Time `json:"captured_at"` + ReceivedAt time.Time `json:"received_at"` + ReportType string `json:"report_type"` + Platform string `json:"platform"` + AppVersion string `json:"app_version"` + // AppBuild is projected from manifest->'report'->>'app_build' so list rows + // (which omit the full manifest) can still show the build number without + // dragging every report's manifest JSONB through the DB. Empty when absent. + AppBuild string `json:"app_build"` + CrashSummary *string `json:"crash_summary,omitempty"` + Manifest json.RawMessage `json:"manifest,omitempty"` + PlaybackSessionIDs []string `json:"playback_session_ids"` + BlobBucket *string `json:"blob_bucket,omitempty"` + BlobKey *string `json:"blob_key,omitempty"` + BlobBytes *int64 `json:"blob_bytes,omitempty"` + UncompressedBytes *int64 `json:"uncompressed_bytes,omitempty"` + BlobSHA256 *string `json:"blob_sha256,omitempty"` +} + +type ListFilters struct { + UserID *int + Platform string + ReportType string + From *time.Time + To *time.Time + ShortID string + Limit int + Cursor string +} + +type ListResult struct { + Reports []Report `json:"reports"` + NextCursor string `json:"next_cursor,omitempty"` +} + +type QuotaKind string + +const ( + QuotaKindReportsPerDay QuotaKind = "reports_per_day" + QuotaKindBytesPerUser QuotaKind = "bytes_per_user" +) + +type QuotaError struct { + Kind QuotaKind + Limit int64 +} + +func (e *QuotaError) Error() string { + if e == nil { + return ErrQuotaExceeded.Error() + } + return fmt.Sprintf("%s: %s limit %d", ErrQuotaExceeded, e.Kind, e.Limit) +} + +func (e *QuotaError) Unwrap() error { + return ErrQuotaExceeded +} + +func truncateCrashSummary(summary *string) *string { + if summary == nil { + return nil + } + value := *summary + if len(value) <= MaxCrashSummaryBytes { + return &value + } + value = value[:MaxCrashSummaryBytes] + for !utf8.ValidString(value) && len(value) > 0 { + value = value[:len(value)-1] + } + return &value +} diff --git a/internal/ebooks/enrichment.go b/internal/ebooks/enrichment.go index a9a387a9..e6d9062e 100644 --- a/internal/ebooks/enrichment.go +++ b/internal/ebooks/enrichment.go @@ -527,7 +527,7 @@ func (e *Enricher) cacheRemotePoster(ctx context.Context, contentID string, resu return } - if storedPath := cachedOriginalImagePath(cached.BasePath, cached.Ext); storedPath != "" { + if storedPath := metadata.CachedImageOriginalPath(cached); storedPath != "" { result.PosterPath = storedPath } if cached.Thumbhash != "" { @@ -589,19 +589,6 @@ func isNilImageCacher(cacher metadata.ImageCacher) bool { } } -func cachedOriginalImagePath(basePath, ext string) string { - if basePath == "" { - return "" - } - if strings.Contains(basePath, "/original.") { - return basePath - } - if ext == "" { - ext = ".jpg" - } - return strings.TrimRight(basePath, "/") + "/original" + ext -} - func (e *Enricher) persist(ctx context.Context, contentID string, providerIDs map[string]string, result *metadata.MetadataResult) error { upd := &catalog.MetadataUpdate{} diff --git a/internal/historyimport/emby_client.go b/internal/historyimport/emby_client.go index 9d949cce..f3fd27c9 100644 --- a/internal/historyimport/emby_client.go +++ b/internal/historyimport/emby_client.go @@ -200,11 +200,19 @@ func (c *EmbyClient) AuthenticateServerUser(ctx context.Context, baseURL, userna } func (c *EmbyClient) FetchItems(ctx context.Context, auth embyLocalAuth, filter string) ([]embyItem, error) { + return c.fetchItems(ctx, auth, filter, "Movie,Episode") +} + +func (c *EmbyClient) FetchFavoriteItems(ctx context.Context, auth embyLocalAuth) ([]embyItem, error) { + return c.fetchItems(ctx, auth, "IsFavorite", "Movie,Series") +} + +func (c *EmbyClient) fetchItems(ctx context.Context, auth embyLocalAuth, filter, includeItemTypes string) ([]embyItem, error) { query := url.Values{} query.Set("Recursive", "true") query.Set("EnableUserData", "true") query.Set("Fields", "ProviderIds") - query.Set("IncludeItemTypes", "Movie,Episode") + query.Set("IncludeItemTypes", includeItemTypes) query.Set("Filters", filter) path := fmt.Sprintf("%s/Users/%s/Items?%s", auth.BaseURL, url.PathEscape(auth.UserID), query.Encode()) req, err := http.NewRequestWithContext(ctx, http.MethodGet, path, nil) diff --git a/internal/historyimport/emby_favorites_test.go b/internal/historyimport/emby_favorites_test.go new file mode 100644 index 00000000..f482204f --- /dev/null +++ b/internal/historyimport/emby_favorites_test.go @@ -0,0 +1,124 @@ +package historyimport + +import ( + "context" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/userstore" + "github.com/Silo-Server/silo-server/internal/userstore/pgstore" +) + +func TestAddFavoriteUsesAtomicInsertResult(t *testing.T) { + store := &favoriteInsertTestStore{} + service := &Service{stores: favoriteInsertTestProvider{store: store}} + + inserted, err := service.addFavorite(context.Background(), 42, "profile-1", "movie-1") + if err != nil { + t.Fatalf("addFavorite: %v", err) + } + if inserted { + t.Fatal("addFavorite reported an existing favorite as newly inserted") + } + if store.addCalls != 1 { + t.Fatalf("AddFavoriteAt calls = %d, want 1", store.addCalls) + } + if store.isFavoriteCalls != 0 { + t.Fatalf("IsFavorite calls = %d, want 0", store.isFavoriteCalls) + } +} + +func TestEmbyFavoriteOnlyRecordAddsFavoriteWithoutProgress(t *testing.T) { + ctx := context.Background() + pool := newPlexWatchlistImportTestPool(t) + repo := NewRepository(pool, nil) + service := &Service{ + repo: repo, + matcher: NewMatcher(repo), + stores: pgstore.NewPostgresProvider(pool), + bgContext: ctx, + runSemaphore: make(chan struct{}, maxConcurrentRuns), + runCancels: make(map[string]context.CancelFunc), + } + + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, year, tvdb_id, status) + VALUES ($1, $2, $3, $4, $5, $6)`, + "series-371980", KindSeries, "Severance", 2022, "371980", "matched", + ); err != nil { + t.Fatalf("seed media item: %v", err) + } + run, err := repo.CreateRun(ctx, Run{ + ID: "emby-favorites-run", + UserID: 42, + ProfileID: "profile-1", + SourceType: SourceTypeEmby, + ConnectionMode: ConnectionModeCustom, + Status: RunStatusQueued, + Warnings: []string{}, + UnmatchedSamples: []UnmatchedSample{}, + }) + if err != nil { + t.Fatalf("CreateRun: %v", err) + } + + record := Record{ + ExternalID: "emby-series-1", + Kind: KindSeries, + Title: "Severance", + Year: 2022, + TVDBID: "371980", + Favorite: true, + FavoriteOnly: true, + } + service.executeRun(run, staticWatchlistProvider{records: []Record{record, record}}) + + completed, err := repo.GetRunForUser(ctx, 42, run.ID) + if err != nil { + t.Fatalf("GetRunForUser: %v", err) + } + if completed.Status != RunStatusCompleted { + t.Fatalf("run status = %q, want %q; warnings=%v", completed.Status, RunStatusCompleted, completed.Warnings) + } + if completed.FavoritesImported != 1 || completed.ProgressUpdated != 0 || completed.HistoryCreated != 0 { + t.Fatalf("run counters = favorites:%d progress:%d history:%d, want 1/0/0", + completed.FavoritesImported, completed.ProgressUpdated, completed.HistoryCreated) + } + var rows int + if err := pool.QueryRow(ctx, ` + SELECT COUNT(*) FROM user_favorites + WHERE user_id = $1 AND profile_id = $2 AND media_item_id = $3`, + 42, "profile-1", "series-371980", + ).Scan(&rows); err != nil { + t.Fatalf("count favorite rows: %v", err) + } + if rows != 1 { + t.Fatalf("favorite rows = %d, want 1", rows) + } +} + +type favoriteInsertTestProvider struct { + store userstore.UserStore +} + +func (p favoriteInsertTestProvider) ForUser(context.Context, int) (userstore.UserStore, error) { + return p.store, nil +} + +func (favoriteInsertTestProvider) Close() error { return nil } + +type favoriteInsertTestStore struct { + userstore.UserStore + addCalls int + isFavoriteCalls int +} + +func (s *favoriteInsertTestStore) AddFavoriteAt(context.Context, string, string, time.Time) (bool, error) { + s.addCalls++ + return false, nil +} + +func (s *favoriteInsertTestStore) IsFavorite(context.Context, string, string) (bool, error) { + s.isFavoriteCalls++ + return false, nil +} diff --git a/internal/historyimport/emby_provider.go b/internal/historyimport/emby_provider.go index b4f6c395..4d68ae18 100644 --- a/internal/historyimport/emby_provider.go +++ b/internal/historyimport/emby_provider.go @@ -27,13 +27,19 @@ func (p *EmbyProvider) Fetch(ctx context.Context) ([]Record, []string, error) { if err != nil { return nil, nil, err } + favoriteItems, err := p.client.FetchFavoriteItems(ctx, p.auth) + var warnings []string + if err != nil { + warnings = append(warnings, "fetching Emby favorites: "+err.Error()) + favoriteItems = nil + } seriesMeta, err := p.fetchSeriesMetadata(ctx, append(playedItems, resumableItems...)) if err != nil { return nil, nil, err } - merged := make(map[string]Record, len(playedItems)+len(resumableItems)) + merged := make(map[string]Record, len(playedItems)+len(resumableItems)+len(favoriteItems)) for _, item := range append(playedItems, resumableItems...) { record := normalizeEmbyItem(item, seriesMeta[item.SeriesID]) if record.ExternalID == "" { @@ -46,12 +52,24 @@ func (p *EmbyProvider) Fetch(ctx context.Context) ([]Record, []string, error) { } merged[record.ExternalID] = mergeRecords(existing, record) } + for _, item := range favoriteItems { + record := normalizeEmbyItem(item, embyItem{}) + record.Favorite = true + record.FavoriteOnly = true + record.PreferTMDB = true + existing, ok := merged[record.ExternalID] + if !ok { + merged[record.ExternalID] = record + continue + } + merged[record.ExternalID] = mergeRecords(existing, record) + } records := make([]Record, 0, len(merged)) for _, record := range merged { records = append(records, record) } - return records, nil, nil + return records, warnings, nil } func (p *EmbyProvider) fetchSeriesMetadata(ctx context.Context, items []embyItem) (map[string]embyItem, error) { @@ -126,6 +144,13 @@ func mergeRecords(a, b Record) Record { if b.Played { result.Played = true } + if b.Favorite { + result.Favorite = true + } + if b.PreferTMDB { + result.PreferTMDB = true + } + result.FavoriteOnly = result.FavoriteOnly && b.FavoriteOnly if b.PlayCount > result.PlayCount { result.PlayCount = b.PlayCount } diff --git a/internal/historyimport/emby_provider_test.go b/internal/historyimport/emby_provider_test.go index 2f5069ae..89f5fc93 100644 --- a/internal/historyimport/emby_provider_test.go +++ b/internal/historyimport/emby_provider_test.go @@ -1,10 +1,125 @@ package historyimport import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" "testing" "time" ) +func TestEmbyProviderFetchIncludesMovieAndSeriesFavorites(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if got := r.URL.Query().Get("EnableUserData"); got != "true" { + t.Errorf("EnableUserData = %q, want true", got) + } + var items []embyItem + switch filter := r.URL.Query().Get("Filters"); filter { + case "IsPlayed": + items = []embyItem{{ + ID: "movie-1", Name: "Arrival", Type: "Movie", ProductionYear: 2016, + ProviderIDs: map[string]string{"Tmdb": "329865"}, + }} + case "IsResumable": + items = []embyItem{} + case "IsFavorite": + if got := r.URL.Query().Get("IncludeItemTypes"); got != "Movie,Series" { + t.Errorf("favorite IncludeItemTypes = %q, want Movie,Series", got) + } + items = []embyItem{ + {ID: "movie-1", Name: "Arrival", Type: "Movie", ProductionYear: 2016, ProviderIDs: map[string]string{"Tmdb": "329865"}}, + {ID: "series-1", Name: "Severance", Type: "Series", ProductionYear: 2022, ProviderIDs: map[string]string{"Tvdb": "371980"}}, + } + default: + t.Errorf("unexpected filter %q", filter) + w.WriteHeader(http.StatusBadRequest) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(embyItemsResponse{Items: items}); err != nil { + t.Errorf("encode response: %v", err) + } + })) + defer server.Close() + + provider := NewEmbyProvider(NewEmbyClient(), embyLocalAuth{ + BaseURL: server.URL, UserID: "emby-user", AccessToken: "token", + }) + records, warnings, err := provider.Fetch(context.Background()) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if len(warnings) != 0 { + t.Fatalf("warnings = %v, want none", warnings) + } + if len(records) != 2 { + t.Fatalf("records = %d, want 2", len(records)) + } + + byID := make(map[string]Record, len(records)) + for _, record := range records { + byID[record.ExternalID] = record + } + movie := byID["movie-1"] + if !movie.Favorite || movie.FavoriteOnly { + t.Fatalf("merged movie flags = favorite:%v favoriteOnly:%v, want true/false", movie.Favorite, movie.FavoriteOnly) + } + if !movie.PreferTMDB { + t.Fatalf("movie PreferTMDB = %v, want true", movie.PreferTMDB) + } + series := byID["series-1"] + if series.Kind != KindSeries || !series.Favorite || !series.FavoriteOnly { + t.Fatalf("series record = %+v, want favorite-only series", series) + } + if !series.PreferTMDB { + t.Fatalf("series PreferTMDB = %v, want true", series.PreferTMDB) + } +} + +func TestEmbyProviderFetchContinuesWhenFavoritesFail(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Query().Get("Filters") { + case "IsPlayed": + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(embyItemsResponse{Items: []embyItem{{ + ID: "movie-1", Name: "Arrival", Type: "Movie", ProductionYear: 2016, + ProviderIDs: map[string]string{"Tmdb": "329865"}, + }}}); err != nil { + t.Errorf("encode response: %v", err) + } + case "IsResumable": + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(embyItemsResponse{}); err != nil { + t.Errorf("encode response: %v", err) + } + case "IsFavorite": + http.Error(w, "favorites unavailable", http.StatusInternalServerError) + default: + w.WriteHeader(http.StatusBadRequest) + } + })) + defer server.Close() + + provider := NewEmbyProvider(NewEmbyClient(), embyLocalAuth{ + BaseURL: server.URL, UserID: "emby-user", AccessToken: "token", + }) + records, warnings, err := provider.Fetch(context.Background()) + if err != nil { + t.Fatalf("Fetch: %v", err) + } + if len(records) != 1 || records[0].ExternalID != "movie-1" { + t.Fatalf("records = %+v, want played movie preserved", records) + } + if len(warnings) != 1 { + t.Fatalf("warnings = %v, want one favorites warning", warnings) + } +} + func TestNormalizeEmbyItemWithoutLastPlayedDateHasNoFreshnessTimestamp(t *testing.T) { t.Parallel() diff --git a/internal/historyimport/matcher.go b/internal/historyimport/matcher.go index af33b029..d7361233 100644 --- a/internal/historyimport/matcher.go +++ b/internal/historyimport/matcher.go @@ -35,7 +35,7 @@ func (m *Matcher) Match(ctx context.Context, record Record) (*Match, string, err } func (m *Matcher) matchMovie(ctx context.Context, record Record) (*Match, string, error) { - match, reason, err := m.matchMedia(ctx, KindMovie, record.TMDBID, record.IMDbID, "", record.Title, record.Year) + match, reason, err := m.matchMedia(ctx, KindMovie, record.TMDBID, record.IMDbID, "", record.Title, record.Year, false) if err != nil { return nil, "", err } @@ -43,7 +43,7 @@ func (m *Matcher) matchMovie(ctx context.Context, record Record) (*Match, string } func (m *Matcher) matchSeries(ctx context.Context, record Record) (*Match, string, error) { - match, reason, err := m.matchMedia(ctx, KindSeries, record.TMDBID, record.IMDbID, record.TVDBID, record.Title, record.Year) + match, reason, err := m.matchMedia(ctx, KindSeries, record.TMDBID, record.IMDbID, record.TVDBID, record.Title, record.Year, record.PreferTMDB) if err != nil { return nil, "", err } @@ -115,7 +115,7 @@ func (m *Matcher) matchEpisode(ctx context.Context, record Record) (*Match, stri return match, "", nil } -func (m *Matcher) matchMedia(ctx context.Context, kind, tmdbID, imdbID, tvdbID, title string, year int) (*Match, string, error) { +func (m *Matcher) matchMedia(ctx context.Context, kind, tmdbID, imdbID, tvdbID, title string, year int, preferTMDB bool) (*Match, string, error) { attempts := make([]string, 0, 4) candidates := []struct { column string @@ -135,6 +135,9 @@ func (m *Matcher) matchMedia(ctx context.Context, kind, tmdbID, imdbID, tvdbID, {column: "tmdb_id", value: tmdbID, label: "tmdb_id"}, {column: "imdb_id", value: imdbID, label: "imdb_id"}, } + if preferTMDB { + candidates[0], candidates[1] = candidates[1], candidates[0] + } } for _, candidate := range candidates { diff --git a/internal/historyimport/matcher_test.go b/internal/historyimport/matcher_test.go index 53265eb6..07831607 100644 --- a/internal/historyimport/matcher_test.go +++ b/internal/historyimport/matcher_test.go @@ -120,3 +120,33 @@ func TestMatcherMatchEpisode_AllowsSeasonZeroSpecials(t *testing.T) { repo.episodeBySeriesID, repo.episodeBySeriesSeason, repo.episodeBySeriesEpisode) } } + +func TestMatcherMatchSeries_PrefersTMDBForEmbyFavorite(t *testing.T) { + t.Parallel() + + repo := &matcherRepoStub{ + mediaByExternal: map[string][]mediaLookupRow{ + "series:tmdb_id:94997": {{ContentID: "house-of-the-dragon", Title: "House of the Dragon", Year: 2022}}, + "series:tvdb_id:425793": {{ContentID: "making-of", Title: "The House That Dragons Built", Year: 2022}}, + }, + } + matcher := NewMatcher(repo) + + match, reason, err := matcher.Match(context.Background(), Record{ + Kind: KindSeries, + Title: "House of the Dragon", + Year: 2022, + TMDBID: "94997", + TVDBID: "425793", + PreferTMDB: true, + }) + if err != nil { + t.Fatalf("Match returned error: %v", err) + } + if reason != "" { + t.Fatalf("reason = %q, want empty string", reason) + } + if match == nil || match.MediaItemID != "house-of-the-dragon" { + t.Fatalf("match = %+v, want TMDB-backed House of the Dragon", match) + } +} diff --git a/internal/historyimport/plex_watchlist_test.go b/internal/historyimport/plex_watchlist_test.go index ea8d757d..02a0ece2 100644 --- a/internal/historyimport/plex_watchlist_test.go +++ b/internal/historyimport/plex_watchlist_test.go @@ -344,6 +344,7 @@ func newPlexWatchlistImportTestPool(t *testing.T) *pgxpool.Pool { progress_updated integer NOT NULL DEFAULT 0, history_created integer NOT NULL DEFAULT 0, watchlist_added integer NOT NULL DEFAULT 0, + favorites_imported integer NOT NULL DEFAULT 0, skipped integer NOT NULL DEFAULT 0, warnings jsonb NOT NULL DEFAULT '[]'::jsonb, unmatched_samples jsonb NOT NULL DEFAULT '[]'::jsonb, @@ -353,6 +354,13 @@ func newPlexWatchlistImportTestPool(t *testing.T) *pgxpool.Pool { completed_at timestamptz, last_heartbeat_at timestamptz ) ON COMMIT PRESERVE ROWS`, + `CREATE TEMP TABLE user_favorites ( + user_id integer NOT NULL, + profile_id text NOT NULL, + media_item_id text NOT NULL, + added_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (user_id, profile_id, media_item_id) + ) ON COMMIT PRESERVE ROWS`, } { if _, err := pool.Exec(ctx, stmt); err != nil { t.Fatalf("create temp test table: %v", err) diff --git a/internal/historyimport/repo.go b/internal/historyimport/repo.go index 097d3d2f..1177cf33 100644 --- a/internal/historyimport/repo.go +++ b/internal/historyimport/repo.go @@ -597,21 +597,21 @@ func (r *Repository) CreateRun(ctx context.Context, run Run) (*Run, error) { INSERT INTO history_import_runs ( id, user_id, profile_id, source_type, connection_mode, status, mapping_id, - fetched, matched, unmatched, progress_updated, history_created, watchlist_added, skipped, + fetched, matched, unmatched, progress_updated, history_created, watchlist_added, favorites_imported, skipped, warnings, unmatched_samples, error_message ) VALUES ( $1, $2, $3, $4, $5, $6, $7, - $8, $9, $10, $11, $12, $13, $14, - $15, $16, NULLIF($17, '') + $8, $9, $10, $11, $12, $13, $14, $15, + $16, $17, NULLIF($18, '') ) RETURNING id, user_id, profile_id, source_type, connection_mode, status, mapping_id, - fetched, matched, unmatched, progress_updated, history_created, watchlist_added, skipped, + fetched, matched, unmatched, progress_updated, history_created, watchlist_added, favorites_imported, skipped, warnings, unmatched_samples, COALESCE(error_message, ''), created_at, started_at, completed_at`, run.ID, run.UserID, run.ProfileID, run.SourceType, run.ConnectionMode, run.Status, run.MappingID, - run.Fetched, run.Matched, run.Unmatched, run.ProgressUpdated, run.HistoryCreated, run.WatchlistAdded, run.Skipped, + run.Fetched, run.Matched, run.Unmatched, run.ProgressUpdated, run.HistoryCreated, run.WatchlistAdded, run.FavoritesImported, run.Skipped, warningsJSON, unmatchedJSON, run.ErrorMessage, ) created, err := scanRunWithMappingID(row) @@ -668,15 +668,16 @@ func (r *Repository) CompleteRun(ctx context.Context, runID string, summary Exec progress_updated = $6, history_created = $7, watchlist_added = $8, - skipped = $9, - warnings = $10, - unmatched_samples = $11, + favorites_imported = $9, + skipped = $10, + warnings = $11, + unmatched_samples = $12, error_message = NULL, completed_at = NOW(), last_heartbeat_at = NOW() WHERE id = $1`, runID, RunStatusCompleted, summary.Fetched, summary.Matched, summary.Unmatched, - summary.ProgressUpdated, summary.HistoryCreated, summary.WatchlistAdded, summary.Skipped, warningsJSON, unmatchedJSON, + summary.ProgressUpdated, summary.HistoryCreated, summary.WatchlistAdded, summary.FavoritesImported, summary.Skipped, warningsJSON, unmatchedJSON, ) if err != nil { return fmt.Errorf("completing run %s: %w", runID, err) @@ -705,11 +706,12 @@ func (r *Repository) UpdateRunProgress(ctx context.Context, runID string, summar progress_updated = $5, history_created = $6, watchlist_added = $7, - skipped = $8, - warnings = $9, - unmatched_samples = $10, + favorites_imported = $8, + skipped = $9, + warnings = $10, + unmatched_samples = $11, last_heartbeat_at = NOW() - WHERE id = $1 AND status = $11`, + WHERE id = $1 AND status = $12`, runID, summary.Fetched, summary.Matched, @@ -717,6 +719,7 @@ func (r *Repository) UpdateRunProgress(ctx context.Context, runID string, summar summary.ProgressUpdated, summary.HistoryCreated, summary.WatchlistAdded, + summary.FavoritesImported, summary.Skipped, warningsJSON, unmatchedJSON, @@ -771,15 +774,16 @@ func (r *Repository) FailRun(ctx context.Context, runID string, summary Executio progress_updated = $6, history_created = $7, watchlist_added = $8, - skipped = $9, - warnings = $10, - unmatched_samples = $11, - error_message = NULLIF($12, ''), + favorites_imported = $9, + skipped = $10, + warnings = $11, + unmatched_samples = $12, + error_message = NULLIF($13, ''), completed_at = NOW(), last_heartbeat_at = NOW() WHERE id = $1`, runID, RunStatusFailed, summary.Fetched, summary.Matched, summary.Unmatched, - summary.ProgressUpdated, summary.HistoryCreated, summary.WatchlistAdded, summary.Skipped, warningsJSON, unmatchedJSON, errorMessage, + summary.ProgressUpdated, summary.HistoryCreated, summary.WatchlistAdded, summary.FavoritesImported, summary.Skipped, warningsJSON, unmatchedJSON, errorMessage, ) if err != nil { return fmt.Errorf("failing run %s: %w", runID, err) @@ -794,7 +798,7 @@ func (r *Repository) ListRunsForUser(ctx context.Context, userID, limit int) ([] rows, err := r.pool.Query(ctx, ` SELECT id, user_id, profile_id, source_type, connection_mode, status, mapping_id, - fetched, matched, unmatched, progress_updated, history_created, watchlist_added, skipped, + fetched, matched, unmatched, progress_updated, history_created, watchlist_added, favorites_imported, skipped, warnings, unmatched_samples, COALESCE(error_message, ''), created_at, started_at, completed_at FROM history_import_runs WHERE user_id = $1 @@ -811,7 +815,7 @@ func (r *Repository) GetRunForUser(ctx context.Context, userID int, runID string row := r.pool.QueryRow(ctx, ` SELECT id, user_id, profile_id, source_type, connection_mode, status, mapping_id, - fetched, matched, unmatched, progress_updated, history_created, watchlist_added, skipped, + fetched, matched, unmatched, progress_updated, history_created, watchlist_added, favorites_imported, skipped, warnings, unmatched_samples, COALESCE(error_message, ''), created_at, started_at, completed_at FROM history_import_runs WHERE id = $1 AND user_id = $2`, runID, userID) @@ -829,7 +833,7 @@ func (r *Repository) ListActiveRunsForUser(ctx context.Context, userID int) ([]R rows, err := r.pool.Query(ctx, ` SELECT id, user_id, profile_id, source_type, connection_mode, status, mapping_id, - fetched, matched, unmatched, progress_updated, history_created, watchlist_added, skipped, + fetched, matched, unmatched, progress_updated, history_created, watchlist_added, favorites_imported, skipped, warnings, unmatched_samples, COALESCE(error_message, ''), created_at, started_at, completed_at FROM history_import_runs WHERE user_id = $1 @@ -1105,7 +1109,7 @@ func scanRun(scanner interface{ Scan(dest ...any) error }) (*Run, error) { var unmatchedJSON []byte if err := scanner.Scan( &run.ID, &run.UserID, &run.ProfileID, &run.SourceType, &run.ConnectionMode, &run.Status, - &run.Fetched, &run.Matched, &run.Unmatched, &run.ProgressUpdated, &run.HistoryCreated, &run.WatchlistAdded, &run.Skipped, + &run.Fetched, &run.Matched, &run.Unmatched, &run.ProgressUpdated, &run.HistoryCreated, &run.WatchlistAdded, &run.FavoritesImported, &run.Skipped, &warningsJSON, &unmatchedJSON, &run.ErrorMessage, &run.CreatedAt, &run.StartedAt, &run.CompletedAt, ); err != nil { return nil, err @@ -1121,7 +1125,7 @@ func scanRunWithMappingID(scanner interface{ Scan(dest ...any) error }) (*Run, e if err := scanner.Scan( &run.ID, &run.UserID, &run.ProfileID, &run.SourceType, &run.ConnectionMode, &run.Status, &run.MappingID, - &run.Fetched, &run.Matched, &run.Unmatched, &run.ProgressUpdated, &run.HistoryCreated, &run.WatchlistAdded, &run.Skipped, + &run.Fetched, &run.Matched, &run.Unmatched, &run.ProgressUpdated, &run.HistoryCreated, &run.WatchlistAdded, &run.FavoritesImported, &run.Skipped, &warningsJSON, &unmatchedJSON, &run.ErrorMessage, &run.CreatedAt, &run.StartedAt, &run.CompletedAt, ); err != nil { return nil, err diff --git a/internal/historyimport/repo_admin.go b/internal/historyimport/repo_admin.go index 3ffd7212..dd51bfbc 100644 --- a/internal/historyimport/repo_admin.go +++ b/internal/historyimport/repo_admin.go @@ -271,7 +271,7 @@ func (r *Repository) ListAllRuns(ctx context.Context, sourceID *int, limit int) rows, err = r.pool.Query(ctx, ` SELECT r.id, r.user_id, r.profile_id, r.source_type, r.connection_mode, r.status, r.mapping_id, - r.fetched, r.matched, r.unmatched, r.progress_updated, r.history_created, r.watchlist_added, r.skipped, + r.fetched, r.matched, r.unmatched, r.progress_updated, r.history_created, r.watchlist_added, r.favorites_imported, r.skipped, r.warnings, r.unmatched_samples, COALESCE(r.error_message, ''), r.created_at, r.started_at, r.completed_at FROM history_import_runs r @@ -283,7 +283,7 @@ func (r *Repository) ListAllRuns(ctx context.Context, sourceID *int, limit int) rows, err = r.pool.Query(ctx, ` SELECT id, user_id, profile_id, source_type, connection_mode, status, mapping_id, - fetched, matched, unmatched, progress_updated, history_created, watchlist_added, skipped, + fetched, matched, unmatched, progress_updated, history_created, watchlist_added, favorites_imported, skipped, warnings, unmatched_samples, COALESCE(error_message, ''), created_at, started_at, completed_at FROM history_import_runs @@ -302,7 +302,7 @@ func (r *Repository) GetRunByID(ctx context.Context, runID string) (*Run, error) row := r.pool.QueryRow(ctx, ` SELECT id, user_id, profile_id, source_type, connection_mode, status, mapping_id, - fetched, matched, unmatched, progress_updated, history_created, watchlist_added, skipped, + fetched, matched, unmatched, progress_updated, history_created, watchlist_added, favorites_imported, skipped, warnings, unmatched_samples, COALESCE(error_message, ''), created_at, started_at, completed_at FROM history_import_runs @@ -330,7 +330,7 @@ func (r *Repository) ListActiveRuns(ctx context.Context, sourceID *int) ([]Run, rows, err = r.pool.Query(ctx, ` SELECT r.id, r.user_id, r.profile_id, r.source_type, r.connection_mode, r.status, r.mapping_id, - r.fetched, r.matched, r.unmatched, r.progress_updated, r.history_created, r.watchlist_added, r.skipped, + r.fetched, r.matched, r.unmatched, r.progress_updated, r.history_created, r.watchlist_added, r.favorites_imported, r.skipped, r.warnings, r.unmatched_samples, COALESCE(r.error_message, ''), r.created_at, r.started_at, r.completed_at FROM history_import_runs r @@ -342,7 +342,7 @@ func (r *Repository) ListActiveRuns(ctx context.Context, sourceID *int) ([]Run, rows, err = r.pool.Query(ctx, ` SELECT id, user_id, profile_id, source_type, connection_mode, status, mapping_id, - fetched, matched, unmatched, progress_updated, history_created, watchlist_added, skipped, + fetched, matched, unmatched, progress_updated, history_created, watchlist_added, favorites_imported, skipped, warnings, unmatched_samples, COALESCE(error_message, ''), created_at, started_at, completed_at FROM history_import_runs diff --git a/internal/historyimport/service.go b/internal/historyimport/service.go index 1e210588..91e87deb 100644 --- a/internal/historyimport/service.go +++ b/internal/historyimport/service.go @@ -422,6 +422,21 @@ func (s *Service) addToWatchlist(ctx context.Context, userID int, profileID, med return inserted, nil } +func (s *Service) addFavorite(ctx context.Context, userID int, profileID, mediaItemID string) (bool, error) { + if s.stores == nil { + return false, fmt.Errorf("favorites import: user store provider not configured") + } + store, err := s.stores.ForUser(ctx, userID) + if err != nil { + return false, fmt.Errorf("favorites import: open user store: %w", err) + } + inserted, err := store.AddFavoriteAt(ctx, profileID, mediaItemID, time.Now().UTC()) + if err != nil { + return false, fmt.Errorf("favorites import: add %s: %w", mediaItemID, err) + } + return inserted, nil +} + func (s *Service) executeRun(run *Run, provider Provider) { ctx, cancel := newRunContext(s.bgContext) s.registerRunCancel(run.ID, cancel) @@ -505,6 +520,14 @@ func (s *Service) executeRun(run *Run, provider Provider) { } summary.Matched++ + if record.Favorite { + inserted, err := s.addFavorite(ctx, run.UserID, run.ProfileID, match.MediaItemID) + if err != nil { + summary.Warnings = append(summary.Warnings, err.Error()) + } else if inserted { + summary.FavoritesImported++ + } + } // Watchlist entries carry no watch state: the matched item joins the // importing profile's watchlist and the progress pipeline is skipped. if record.Watchlisted { @@ -517,6 +540,10 @@ func (s *Service) executeRun(run *Run, provider Provider) { s.persistProgressMaybe(ctx, run.ID, summary, i+1, len(records)) continue } + if record.FavoriteOnly { + s.persistProgressMaybe(ctx, run.ID, summary, i+1, len(records)) + continue + } shouldWriteProgress := true localProgress, err := s.repo.GetProgress(ctx, run.UserID, run.ProfileID, match.MediaItemID) @@ -585,6 +612,7 @@ func (s *Service) executeRun(run *Run, provider Provider) { "unmatched", summary.Unmatched, "progress_updated", summary.ProgressUpdated, "history_created", summary.HistoryCreated, + "favorites_imported", summary.FavoritesImported, "skipped", summary.Skipped, "warnings", len(summary.Warnings), ) @@ -622,7 +650,7 @@ func userFacingRunError(summary ExecutionSummary, err error) string { if UpstreamHTTPStatus(err) == http.StatusUnauthorized { return "Couldn't connect to that server. Check the URL, username, and password and try again." } - if summary.Fetched > 0 || summary.Matched > 0 || summary.ProgressUpdated > 0 || summary.HistoryCreated > 0 { + if summary.Fetched > 0 || summary.Matched > 0 || summary.ProgressUpdated > 0 || summary.HistoryCreated > 0 || summary.FavoritesImported > 0 { return "Import stopped early. Some history may already be imported." } return "Import couldn't be completed. Please try again." diff --git a/internal/historyimport/types.go b/internal/historyimport/types.go index 3262177c..539c8e96 100644 --- a/internal/historyimport/types.go +++ b/internal/historyimport/types.go @@ -133,26 +133,27 @@ type PlexServerPublic struct { // Run represents one history import execution. type Run struct { - ID string `json:"id"` - UserID int `json:"user_id"` - ProfileID string `json:"profile_id"` - SourceType string `json:"source_type"` - ConnectionMode string `json:"connection_mode"` - Status string `json:"status"` - MappingID *int `json:"mapping_id,omitempty"` - Fetched int `json:"fetched"` - Matched int `json:"matched"` - Unmatched int `json:"unmatched"` - ProgressUpdated int `json:"progress_updated"` - HistoryCreated int `json:"history_created"` - WatchlistAdded int `json:"watchlist_added"` - Skipped int `json:"skipped"` - Warnings []string `json:"warnings"` - UnmatchedSamples []UnmatchedSample `json:"unmatched_samples"` - ErrorMessage string `json:"error_message,omitempty"` - CreatedAt time.Time `json:"created_at"` - StartedAt *time.Time `json:"started_at,omitempty"` - CompletedAt *time.Time `json:"completed_at,omitempty"` + ID string `json:"id"` + UserID int `json:"user_id"` + ProfileID string `json:"profile_id"` + SourceType string `json:"source_type"` + ConnectionMode string `json:"connection_mode"` + Status string `json:"status"` + MappingID *int `json:"mapping_id,omitempty"` + Fetched int `json:"fetched"` + Matched int `json:"matched"` + Unmatched int `json:"unmatched"` + ProgressUpdated int `json:"progress_updated"` + HistoryCreated int `json:"history_created"` + WatchlistAdded int `json:"watchlist_added"` + FavoritesImported int `json:"favorites_imported"` + Skipped int `json:"skipped"` + Warnings []string `json:"warnings"` + UnmatchedSamples []UnmatchedSample `json:"unmatched_samples"` + ErrorMessage string `json:"error_message,omitempty"` + CreatedAt time.Time `json:"created_at"` + StartedAt *time.Time `json:"started_at,omitempty"` + CompletedAt *time.Time `json:"completed_at,omitempty"` } type UnmatchedSample struct { @@ -186,7 +187,13 @@ type Record struct { // matched item is added to the importing profile's watchlist instead of // receiving watch state. Watchlisted bool - UpdatedAt time.Time + // FavoriteOnly prevents a favorite with no played/resumable counterpart + // from creating an empty progress row. PreferTMDB avoids stale secondary + // TVDB IDs winning over Emby's primary TMDB identity for favorite series. + Favorite bool + FavoriteOnly bool + PreferTMDB bool + UpdatedAt time.Time } type Match struct { @@ -250,6 +257,7 @@ type ExecutionSummary struct { ProgressUpdated int HistoryCreated int WatchlistAdded int + FavoritesImported int Skipped int Warnings []string UnmatchedSamples []UnmatchedSample diff --git a/internal/imagecache/imagecache.go b/internal/imagecache/imagecache.go index ab6047ea..17679d19 100644 --- a/internal/imagecache/imagecache.go +++ b/internal/imagecache/imagecache.go @@ -4,16 +4,21 @@ package imagecache import ( "context" + "crypto/sha256" + "encoding/binary" + "encoding/hex" "fmt" "io" "net" "net/http" "net/netip" "net/url" + "sort" "strings" "sync" "time" + "github.com/Silo-Server/silo-server/internal/artworkkey" "github.com/Silo-Server/silo-server/internal/imageutil" "github.com/Silo-Server/silo-server/internal/metadata" ) @@ -29,8 +34,14 @@ type ObjectPutter interface { Bucket() string } -type objectExister interface { - ObjectExists(ctx context.Context, bucket, key string) (bool, error) +type objectMatcher interface { + ObjectMatches(ctx context.Context, bucket, key string, data []byte) (bool, error) +} + +// ArtworkRevisionTracker persists the exact object manifest for an immutable +// revision before any object is uploaded. +type ArtworkRevisionTracker interface { + TrackArtworkRevision(ctx context.Context, originalPath, imageType string, objectKeys []string) error } // ImageURLResolver resolves plugin:// paths to HTTP URLs. @@ -52,11 +63,18 @@ type CacheRequest struct { EpisodeNumber *int Language string ImageResolver ImageURLResolver // optional; used when SourceURL is a plugin:// path + // KeyDiscriminator, when set, is inserted into the S3 key between the + // content ID and the image type (local sidecar art uses the file's 8-hex + // content hash) so re-cached art rotates to a fresh key. + KeyDiscriminator string } // CacheResult is returned by Cache on success. type CacheResult struct { BasePath string // S3 key prefix, e.g. "tmdb/movies/550/poster" + OriginalPath string // exact immutable original-variant object key + Revision string // content revision shared by generated variants + VariantPaths map[string]string Thumbhash string // base64-encoded Ext string // file extension including dot (e.g. ".jpg", ".png") UploadedVariants int @@ -66,6 +84,7 @@ type CacheResult struct { // Cacher downloads and stores image variants to S3. type Cacher struct { s3 ObjectPutter + revisionTracker ArtworkRevisionTracker httpClient *http.Client enforcePublicURLs bool } @@ -75,6 +94,14 @@ func New(s3 ObjectPutter) *Cacher { return &Cacher{s3: s3, httpClient: newSecureHTTPClient(), enforcePublicURLs: true} } +// SetArtworkRevisionTracker wires durable revision lifecycle tracking. The +// production server configures this whenever object storage is available. +func (c *Cacher) SetArtworkRevisionTracker(tracker ArtworkRevisionTracker) { + if c != nil { + c.revisionTracker = tracker + } +} + func newWithHTTPClient(s3 ObjectPutter, client *http.Client) *Cacher { if client == nil { client = http.DefaultClient @@ -97,13 +124,38 @@ func (c *Cacher) CacheImage(ctx context.Context, req metadata.CacheImageRequest) if err != nil { return nil, err } + return cacheImageResultFromCacheResult(result), nil +} + +// CacheImageBytes implements metadata.ImageByteCacher using CacheBytes. Used +// by the image cache processor for file:// sources that it reads itself. +func (c *Cacher) CacheImageBytes(ctx context.Context, data []byte, req metadata.CacheImageRequest) (*metadata.CacheImageResult, error) { + result, err := c.CacheBytes(ctx, data, CacheRequest{ + ProviderID: req.ProviderID, + ContentType: req.ContentType, + ContentID: req.ContentID, + ImageType: req.ImageType, + SeasonNumber: req.SeasonNumber, + EpisodeNumber: req.EpisodeNumber, + Language: req.Language, + KeyDiscriminator: req.KeyDiscriminator, + }) + if err != nil { + return nil, err + } + return cacheImageResultFromCacheResult(result), nil +} + +func cacheImageResultFromCacheResult(result *CacheResult) *metadata.CacheImageResult { return &metadata.CacheImageResult{ BasePath: result.BasePath, + OriginalPath: result.OriginalPath, + Revision: result.Revision, Thumbhash: result.Thumbhash, Ext: result.Ext, UploadedVariants: result.UploadedVariants, ExistingVariants: result.ExistingVariants, - }, nil + } } // CacheAudiobookCover is a thin convenience over CacheBytes specifically @@ -111,7 +163,7 @@ func (c *Cacher) CacheImage(ctx context.Context, req metadata.CacheImageRequest) // struct to the scanner package (which would create an import cycle // scanner -> imagecache -> metadata -> scanner). Stores under // "local/audiobooks/{contentID}/poster/...". -func (c *Cacher) CacheAudiobookCover(ctx context.Context, data []byte, contentID string) (basePath string, ext string, thumbhash string, err error) { +func (c *Cacher) CacheAudiobookCover(ctx context.Context, data []byte, contentID string) (storedPath string, thumbhash string, err error) { res, err := c.CacheBytes(ctx, data, CacheRequest{ ProviderID: "local", ContentType: "audiobooks", @@ -119,15 +171,15 @@ func (c *Cacher) CacheAudiobookCover(ctx context.Context, data []byte, contentID ImageType: metadata.ImagePoster, }) if err != nil { - return "", "", "", err + return "", "", err } - return res.BasePath, res.Ext, res.Thumbhash, nil + return res.OriginalPath, res.Thumbhash, nil } // CacheEbookCover stores an embedded ebook cover under // "local/ebooks/{contentID}/poster/..." using the same poster variants as // provider-hosted book artwork. -func (c *Cacher) CacheEbookCover(ctx context.Context, data []byte, contentID string) (basePath string, ext string, thumbhash string, err error) { +func (c *Cacher) CacheEbookCover(ctx context.Context, data []byte, contentID string) (storedPath string, thumbhash string, err error) { res, err := c.CacheBytes(ctx, data, CacheRequest{ ProviderID: "local", ContentType: "ebooks", @@ -135,9 +187,31 @@ func (c *Cacher) CacheEbookCover(ctx context.Context, data []byte, contentID str ImageType: metadata.ImagePoster, }) if err != nil { - return "", "", "", err + return "", "", err } - return res.BasePath, res.Ext, res.Thumbhash, nil + return res.OriginalPath, res.Thumbhash, nil +} + +// validateCacheRequest checks the required identity fields and the +// episode/season invariant shared by Cache and CacheBytes. Keeping it in one +// place prevents the season/episode guard from drifting between the two paths: +// buildBasePath only appends the episode segment inside the SeasonNumber branch, +// so an episode without a season would silently collide distinct episodes' art +// under the same S3 key. +func validateCacheRequest(req CacheRequest) error { + if strings.TrimSpace(req.ProviderID) == "" { + return fmt.Errorf("imagecache: provider ID is required") + } + if strings.TrimSpace(req.ContentType) == "" { + return fmt.Errorf("imagecache: content type is required") + } + if strings.TrimSpace(req.ContentID) == "" { + return fmt.Errorf("imagecache: content ID is required") + } + if req.EpisodeNumber != nil && req.SeasonNumber == nil { + return fmt.Errorf("imagecache: episode number requires a season number") + } + return nil } // CacheBytes performs the same variant generation, thumbhash, and S3 upload as @@ -145,14 +219,8 @@ func (c *Cacher) CacheEbookCover(ctx context.Context, data []byte, contentID str // audiobook scanner to push embedded M4B cover art into S3 without round- // tripping through HTTP. func (c *Cacher) CacheBytes(ctx context.Context, data []byte, req CacheRequest) (*CacheResult, error) { - if strings.TrimSpace(req.ProviderID) == "" { - return nil, fmt.Errorf("imagecache: provider ID is required") - } - if strings.TrimSpace(req.ContentType) == "" { - return nil, fmt.Errorf("imagecache: content type is required") - } - if strings.TrimSpace(req.ContentID) == "" { - return nil, fmt.Errorf("imagecache: content ID is required") + if err := validateCacheRequest(req); err != nil { + return nil, err } if len(data) == 0 { return nil, fmt.Errorf("imagecache: image data is empty") @@ -166,15 +234,23 @@ func (c *Cacher) CacheBytes(ctx context.Context, data []byte, req CacheRequest) if err != nil { return nil, fmt.Errorf("imagecache: generate variants: %w", err) } - basePath := buildBasePath(req.ProviderID, req.ContentType, req.ContentID, req.ImageType, req.Language, req.SeasonNumber, req.EpisodeNumber) + basePath := buildBasePath(req) bucket := c.s3.Bucket() + revision := variantRevision(result) + variantPaths := buildVariantPaths(basePath, revision, result) + if err := c.trackRevision(ctx, req.ImageType, variantPaths); err != nil { + return nil, err + } - uploadStats, err := c.uploadVariants(ctx, bucket, basePath, result) + uploadStats, err := c.uploadVariants(ctx, bucket, result, variantPaths) if err != nil { return nil, err } return &CacheResult{ BasePath: basePath, + OriginalPath: variantPaths[artworkkey.OriginalVariant], + Revision: revision, + VariantPaths: variantPaths, Thumbhash: thumbhash, Ext: result.Ext, UploadedVariants: uploadStats.uploaded, @@ -182,20 +258,11 @@ func (c *Cacher) CacheBytes(ctx context.Context, data []byte, req CacheRequest) }, nil } -// Cache downloads the image at req.SourceURL, generates variants, computes a -// thumbhash, uploads all variants to S3, and returns the base path and thumbhash. +// Cache downloads the image at req.SourceURL and stores it through the same +// variant, revision-tracking, and upload pipeline as CacheBytes. func (c *Cacher) Cache(ctx context.Context, req CacheRequest) (*CacheResult, error) { - if strings.TrimSpace(req.ProviderID) == "" { - return nil, fmt.Errorf("imagecache: provider ID is required") - } - if strings.TrimSpace(req.ContentType) == "" { - return nil, fmt.Errorf("imagecache: content type is required") - } - if strings.TrimSpace(req.ContentID) == "" { - return nil, fmt.Errorf("imagecache: content ID is required") - } - if req.EpisodeNumber != nil && req.SeasonNumber == nil { - return nil, fmt.Errorf("imagecache: episode number requires a season number") + if err := validateCacheRequest(req); err != nil { + return nil, err } url := req.SourceURL @@ -216,53 +283,14 @@ func (c *Cacher) Cache(ctx context.Context, req CacheRequest) (*CacheResult, err return nil, fmt.Errorf("imagecache: download %s: %w", url, err) } - // Compute thumbhash from the original downloaded data (JPEG/PNG) before - // converting to WebP, since Go's image.Decode doesn't support WebP. - thumbhash, err := imageutil.Thumbhash(data) - if err != nil { - return nil, fmt.Errorf("imagecache: thumbhash: %w", err) - } - - widths := variantWidths(req.ImageType) - - result, err := imageutil.GenerateVariants(data, widths) - if err != nil { - return nil, fmt.Errorf("imagecache: generate variants: %w", err) - } - - basePath := buildBasePath(req.ProviderID, req.ContentType, req.ContentID, req.ImageType, req.Language, req.SeasonNumber, req.EpisodeNumber) - bucket := c.s3.Bucket() - - uploadStats, err := c.uploadVariants(ctx, bucket, basePath, result) - if err != nil { - return nil, err - } - - return &CacheResult{ - BasePath: basePath, - Thumbhash: thumbhash, - Ext: result.Ext, - UploadedVariants: uploadStats.uploaded, - ExistingVariants: uploadStats.existing, - }, nil + return c.CacheBytes(ctx, data, req) } -// variantWidths returns the resize widths for the given image type. +// variantWidths returns the resize widths for the given image type. The +// ladder itself is owned by artworkkey so key expansion and GC manifests can +// never drift from what is generated here. func variantWidths(t metadata.ImageType) []int { - switch t { - case metadata.ImagePoster: - return []int{500, 300} - case metadata.ImageBackdrop: - return []int{1920, 1280, 300} - case metadata.ImageLogo: - return []int{500} - case metadata.ImageStill: - return []int{500, 300} - case metadata.ImageProfile: - return []int{500, 300} - default: - return []int{500, 300} - } + return artworkkey.VariantWidths(metadata.ImageTypeToString(t)) } type uploadVariantStats struct { @@ -270,7 +298,7 @@ type uploadVariantStats struct { existing int } -func (c *Cacher) uploadVariants(ctx context.Context, bucket, basePath string, result *imageutil.VariantResult) (uploadVariantStats, error) { +func (c *Cacher) uploadVariants(ctx context.Context, bucket string, result *imageutil.VariantResult, variantPaths map[string]string) (uploadVariantStats, error) { var wg sync.WaitGroup uploadErrs := make([]error, len(result.Variants)) stats := make([]uploadVariantStats, len(result.Variants)) @@ -278,8 +306,8 @@ func (c *Cacher) uploadVariants(ctx context.Context, bucket, basePath string, re wg.Add(1) go func(idx int, variant imageutil.Variant) { defer wg.Done() - key := basePath + "/" + variant.Key + result.Ext - if exists, err := objectExists(ctx, c.s3, bucket, key); err != nil { + key := variantPaths[variant.Key] + if exists, err := objectMatches(ctx, c.s3, bucket, key, variant.Data); err != nil { uploadErrs[idx] = fmt.Errorf("imagecache: check existing %s: %w", key, err) return } else if exists { @@ -307,12 +335,57 @@ func (c *Cacher) uploadVariants(ctx context.Context, bucket, basePath string, re return total, nil } -func objectExists(ctx context.Context, putter ObjectPutter, bucket, key string) (bool, error) { - exister, ok := putter.(objectExister) +func variantRevision(result *imageutil.VariantResult) string { + h := sha256.New() + _, _ = io.WriteString(h, "silo-artwork-v1\x00") + _, _ = io.WriteString(h, result.Ext) + _, _ = h.Write([]byte{0}) + variants := append([]imageutil.Variant(nil), result.Variants...) + sort.Slice(variants, func(i, j int) bool { return variants[i].Key < variants[j].Key }) + var size [8]byte + for _, variant := range variants { + _, _ = io.WriteString(h, variant.Key) + _, _ = h.Write([]byte{0}) + binary.BigEndian.PutUint64(size[:], uint64(len(variant.Data))) + _, _ = h.Write(size[:]) + _, _ = h.Write(variant.Data) + } + return hex.EncodeToString(h.Sum(nil)) +} + +func buildVariantPaths(basePath, revision string, result *imageutil.VariantResult) map[string]string { + paths := make(map[string]string, len(result.Variants)) + for _, variant := range result.Variants { + paths[variant.Key] = artworkkey.Build(basePath, variant.Key, revision, result.Ext) + } + return paths +} + +func (c *Cacher) trackRevision(ctx context.Context, imageType metadata.ImageType, variantPaths map[string]string) error { + if c == nil || c.revisionTracker == nil { + return nil + } + originalPath := variantPaths[artworkkey.OriginalVariant] + keys := make([]string, 0, len(variantPaths)) + for _, key := range variantPaths { + keys = append(keys, key) + } + sort.Strings(keys) + if err := c.revisionTracker.TrackArtworkRevision(ctx, originalPath, metadata.ImageTypeToString(imageType), keys); err != nil { + return fmt.Errorf("imagecache: track artwork revision: %w", err) + } + return nil +} + +// objectMatches reports whether the object at key already holds exactly data. +// Backends that cannot verify content report false so the immutable object is +// rewritten; bare existence must never be accepted as a content match. +func objectMatches(ctx context.Context, putter ObjectPutter, bucket, key string, data []byte) (bool, error) { + matcher, ok := putter.(objectMatcher) if !ok { return false, nil } - return exister.ObjectExists(ctx, bucket, key) + return matcher.ObjectMatches(ctx, bucket, key, data) } func putObjectWithRetry(ctx context.Context, putter ObjectPutter, bucket, key string, data []byte) error { @@ -348,18 +421,25 @@ func putObjectWithRetry(ctx context.Context, putter ObjectPutter, bucket, key st // season: {provider}/{type}/{id}/seasons/{n}/{imageType} // localized season: {provider}/{type}/{id}/localizations/{lang}/seasons/{n}/{imageType} // episode: {provider}/{type}/{id}/seasons/{n}/episodes/{m}/{imageType} -func buildBasePath(providerID, contentType, contentID string, t metadata.ImageType, language string, seasonNumber, episodeNumber *int) string { - imageTypeName := imageTypeName(t) - base := fmt.Sprintf("%s/%s/%s", providerID, contentType, contentID) - if lang := normalizeImageLanguage(language); lang != "" { +// +// A non-empty KeyDiscriminator (local sidecar content hash) is inserted +// immediately before the image type so the variant's parent directory stays +// the image type segment (the imageTypeFromCachedPath contract). +func buildBasePath(req CacheRequest) string { + imageTypeName := imageTypeName(req.ImageType) + base := fmt.Sprintf("%s/%s/%s", req.ProviderID, req.ContentType, req.ContentID) + if lang := normalizeImageLanguage(req.Language); lang != "" { base = fmt.Sprintf("%s/localizations/%s", base, lang) } - if seasonNumber != nil { - base = fmt.Sprintf("%s/seasons/%d", base, *seasonNumber) - if episodeNumber != nil { - base = fmt.Sprintf("%s/episodes/%d", base, *episodeNumber) + if req.SeasonNumber != nil { + base = fmt.Sprintf("%s/seasons/%d", base, *req.SeasonNumber) + if req.EpisodeNumber != nil { + base = fmt.Sprintf("%s/episodes/%d", base, *req.EpisodeNumber) } } + if discriminator := strings.TrimSpace(req.KeyDiscriminator); discriminator != "" { + base = base + "/" + discriminator + } return base + "/" + imageTypeName } diff --git a/internal/imagecache/imagecache_test.go b/internal/imagecache/imagecache_test.go index b504dffb..20ce810e 100644 --- a/internal/imagecache/imagecache_test.go +++ b/internal/imagecache/imagecache_test.go @@ -4,13 +4,15 @@ import ( "bytes" "context" "errors" - "fmt" "image" "image/color" "image/jpeg" "image/png" "net/http" "net/http/httptest" + "slices" + "sort" + "strings" "sync" "testing" @@ -19,6 +21,11 @@ import ( "github.com/Silo-Server/silo-server/internal/metadata" ) +const ( + testTMDBProviderID = "tmdb" + testMoviesContentType = "movies" +) + // makeTestJPEG generates a minimal solid-color JPEG for use in tests. func makeTestJPEG(t *testing.T) []byte { t.Helper() @@ -69,6 +76,37 @@ type putCall struct { data []byte } +type trackedRevision struct { + originalPath string + imageType string + objectKeys []string +} + +type recordingRevisionTracker struct { + mu sync.Mutex + calls []trackedRevision + err error +} + +func (t *recordingRevisionTracker) TrackArtworkRevision(_ context.Context, originalPath, imageType string, objectKeys []string) error { + t.mu.Lock() + defer t.mu.Unlock() + t.calls = append(t.calls, trackedRevision{ + originalPath: originalPath, + imageType: imageType, + objectKeys: append([]string(nil), objectKeys...), + }) + return t.err +} + +func (t *recordingRevisionTracker) recorded() []trackedRevision { + t.mu.Lock() + defer t.mu.Unlock() + result := make([]trackedRevision, len(t.calls)) + copy(result, t.calls) + return result +} + func (m *mockS3) PutObject(_ context.Context, bucket, key string, data []byte) error { m.mu.Lock() defer m.mu.Unlock() @@ -86,7 +124,9 @@ func (m *mockS3) PutObject(_ context.Context, bucket, key string, data []byte) e func (m *mockS3) Bucket() string { return m.bucket } -func (m *mockS3) ObjectExists(_ context.Context, _ string, key string) (bool, error) { +// ObjectMatches treats keys registered via setExisting as content matches; +// real content verification is exercised against the s3client implementation. +func (m *mockS3) ObjectMatches(_ context.Context, _ string, key string, _ []byte) (bool, error) { m.mu.Lock() defer m.mu.Unlock() m.existsCalls = append(m.existsCalls, key) @@ -125,6 +165,80 @@ func (m *mockS3) checkedKeys() []string { return keys } +func (m *mockS3) resetCalls() { + m.mu.Lock() + defer m.mu.Unlock() + m.calls = nil + m.existsCalls = nil +} + +func TestCacheBytesTracksExactRevisionBeforeUpload(t *testing.T) { + s3 := &mockS3{bucket: "artwork"} + tracker := &recordingRevisionTracker{} + cacher := newWithHTTPClient(s3, nil) + cacher.SetArtworkRevisionTracker(tracker) + + result, err := cacher.CacheBytes(context.Background(), makeTestJPEG(t), CacheRequest{ + ProviderID: testTMDBProviderID, + ContentType: testMoviesContentType, + ContentID: "335984", + ImageType: metadata.ImagePoster, + }) + if err != nil { + t.Fatalf("CacheBytes: %v", err) + } + + calls := tracker.recorded() + if len(calls) != 1 { + t.Fatalf("tracker calls = %d, want 1", len(calls)) + } + if calls[0].originalPath != result.OriginalPath { + t.Fatalf("tracked original = %q, want %q", calls[0].originalPath, result.OriginalPath) + } + if calls[0].imageType != "poster" { + t.Fatalf("tracked image type = %q, want poster", calls[0].imageType) + } + wantKeys := make([]string, 0, len(result.VariantPaths)) + for _, key := range result.VariantPaths { + wantKeys = append(wantKeys, key) + } + sort.Strings(wantKeys) + if !slices.Equal(calls[0].objectKeys, wantKeys) { + t.Fatalf("tracked keys = %v, want %v", calls[0].objectKeys, wantKeys) + } +} + +func TestCacheBytesDoesNotUploadWhenRevisionTrackingFails(t *testing.T) { + s3 := &mockS3{bucket: "artwork"} + tracker := &recordingRevisionTracker{err: errors.New("registry unavailable")} + cacher := newWithHTTPClient(s3, nil) + cacher.SetArtworkRevisionTracker(tracker) + + _, err := cacher.CacheBytes(context.Background(), makeTestJPEG(t), CacheRequest{ + ProviderID: testTMDBProviderID, + ContentType: testMoviesContentType, + ContentID: "335984", + ImageType: metadata.ImagePoster, + }) + if err == nil || !strings.Contains(err.Error(), "track artwork revision") { + t.Fatalf("CacheBytes error = %v, want tracking failure", err) + } + if got := len(s3.keys()); got != 0 { + t.Fatalf("uploaded objects = %d, want 0", got) + } +} + +func (m *mockS3) setExisting(keys ...string) { + m.mu.Lock() + defer m.mu.Unlock() + if m.existing == nil { + m.existing = make(map[string]bool) + } + for _, key := range keys { + m.existing[key] = true + } +} + func containsKey(keys []string, suffix string) bool { for _, k := range keys { if len(k) >= len(suffix) && k[len(k)-len(suffix):] == suffix { @@ -195,7 +309,7 @@ func TestCache_Poster(t *testing.T) { t.Errorf("expected 3 uploaded variants, got %d: %v", len(keys), keys) } for _, variant := range []string{"original", "w500", "w300"} { - want := wantBase + "/" + variant + ".webp" + want := result.VariantPaths[variant] if !hasKey(keys, want) { t.Errorf("missing S3 key %q in %v", want, keys) } @@ -207,23 +321,23 @@ func TestCacheSkipsUploadingVariantsThatAlreadyExist(t *testing.T) { srv := startImageServer(t, jpeg, http.StatusOK) wantBase := "tmdb/movies/550/poster" - s3 := &mockS3{ - bucket: "media", - existing: map[string]bool{ - wantBase + "/original.webp": true, - wantBase + "/w500.webp": true, - wantBase + "/w300.webp": true, - }, - } + s3 := &mockS3{bucket: "media"} c := newWithHTTPClient(s3, srv.Client()) - result, err := c.Cache(context.Background(), CacheRequest{ + req := CacheRequest{ SourceURL: srv.URL + "/poster.jpg", ProviderID: "tmdb", ContentType: "movies", ContentID: "550", ImageType: metadata.ImagePoster, - }) + } + first, err := c.Cache(context.Background(), req) + if err != nil { + t.Fatalf("prime immutable variants: %v", err) + } + s3.setExisting(first.VariantPaths["original"], first.VariantPaths["w500"], first.VariantPaths["w300"]) + s3.resetCalls() + result, err := c.Cache(context.Background(), req) if err != nil { t.Fatalf("Cache poster with existing variants: %v", err) } @@ -236,38 +350,60 @@ func TestCacheSkipsUploadingVariantsThatAlreadyExist(t *testing.T) { if result.UploadedVariants != 0 || result.ExistingVariants != 3 { t.Fatalf("upload stats = uploaded %d existing %d, want uploaded 0 existing 3", result.UploadedVariants, result.ExistingVariants) } - for _, key := range []string{wantBase + "/original.webp", wantBase + "/w500.webp", wantBase + "/w300.webp"} { + for _, key := range []string{result.VariantPaths["original"], result.VariantPaths["w500"], result.VariantPaths["w300"]} { if !hasKey(s3.checkedKeys(), key) { t.Fatalf("ObjectExists was not checked for %q; checked %v", key, s3.checkedKeys()) } } } +func TestCacheDifferentContentCreatesDifferentImmutableRevision(t *testing.T) { + jpeg := makeTestJPEG(t) + png := makeTestPNG(t, 400, 600) + s3 := &mockS3{bucket: "media"} + c := newWithHTTPClient(s3, http.DefaultClient) + req := CacheRequest{ProviderID: testTMDBProviderID, ContentType: testMoviesContentType, ContentID: "550", ImageType: metadata.ImagePoster} + first, err := c.CacheBytes(context.Background(), jpeg, req) + if err != nil { + t.Fatalf("cache first poster: %v", err) + } + second, err := c.CacheBytes(context.Background(), png, req) + if err != nil { + t.Fatalf("cache replacement poster: %v", err) + } + if first.Revision == second.Revision || first.OriginalPath == second.OriginalPath { + t.Fatalf("different content reused revision: first=%q second=%q", first.OriginalPath, second.OriginalPath) + } + if got := s3.keys(); len(got) != 6 { + t.Fatalf("uploaded keys = %v, want both immutable three-variant revisions", got) + } +} + func TestCacheUploadsOnlyMissingVariants(t *testing.T) { jpeg := makeTestJPEG(t) srv := startImageServer(t, jpeg, http.StatusOK) - wantBase := "tmdb/movies/550/poster" - s3 := &mockS3{ - bucket: "media", - existing: map[string]bool{ - wantBase + "/original.webp": true, - wantBase + "/w500.webp": true, - }, - } + s3 := &mockS3{bucket: "media"} c := newWithHTTPClient(s3, srv.Client()) - result, err := c.Cache(context.Background(), CacheRequest{ + req := CacheRequest{ SourceURL: srv.URL + "/poster.jpg", ProviderID: "tmdb", ContentType: "movies", ContentID: "550", ImageType: metadata.ImagePoster, - }) + } + first, err := c.Cache(context.Background(), req) + if err != nil { + t.Fatalf("prime immutable variants: %v", err) + } + s3.setExisting(first.VariantPaths["original"], first.VariantPaths["w500"]) + s3.resetCalls() + result, err := c.Cache(context.Background(), req) if err != nil { t.Fatalf("Cache poster with partial existing variants: %v", err) } - if got := s3.keys(); len(got) != 1 || got[0] != wantBase+"/w300.webp" { + if got := s3.keys(); len(got) != 1 || got[0] != result.VariantPaths["w300"] { t.Fatalf("uploaded keys = %v, want only missing w300 variant", got) } if result.UploadedVariants != 1 || result.ExistingVariants != 2 { @@ -304,13 +440,13 @@ func TestCache_Backdrop(t *testing.T) { t.Errorf("expected 4 uploaded variants, got %d: %v", len(keys), keys) } for _, variant := range []string{"original", "w1920", "w1280", "w300"} { - want := wantBase + "/" + variant + ".webp" + want := result.VariantPaths[variant] if !hasKey(keys, want) { t.Errorf("missing S3 key %q in %v", want, keys) } } // Must not have w500 - if hasKey(keys, wantBase+"/w500.webp") { + if _, ok := result.VariantPaths["w500"]; ok { t.Error("backdrop should not have w500 variant") } } @@ -344,13 +480,13 @@ func TestCache_Logo(t *testing.T) { t.Errorf("expected 2 uploaded variants, got %d: %v", len(keys), keys) } for _, variant := range []string{"original", "w500"} { - want := wantBase + "/" + variant + ".webp" + want := result.VariantPaths[variant] if !hasKey(keys, want) { t.Errorf("missing S3 key %q in %v", want, keys) } } for _, forbidden := range []string{"w300", "w1280"} { - if hasKey(keys, wantBase+"/"+forbidden+".webp") { + if _, ok := result.VariantPaths[forbidden]; ok { t.Errorf("logo should not have %s variant", forbidden) } } @@ -380,9 +516,8 @@ func TestCache_ConvertsSVGLogo(t *testing.T) { if result.Thumbhash == "" { t.Fatal("Thumbhash is empty") } - wantBase := "tmdb/series/1396/logo" for _, variant := range []string{"original", "w500"} { - want := wantBase + "/" + variant + ".webp" + want := result.VariantPaths[variant] if !hasKey(s3.keys(), want) { t.Errorf("missing S3 key %q in %v", want, s3.keys()) } @@ -400,7 +535,7 @@ func TestCache_CapsLargeOriginalVariant(t *testing.T) { s3 := &mockS3{bucket: "media"} c := newWithHTTPClient(s3, srv.Client()) - _, err := c.Cache(context.Background(), CacheRequest{ + result, err := c.Cache(context.Background(), CacheRequest{ SourceURL: srv.URL + "/logo.png", ProviderID: "tmdb", ContentType: "series", @@ -410,7 +545,7 @@ func TestCache_CapsLargeOriginalVariant(t *testing.T) { if err != nil { t.Fatalf("Cache large logo: %v", err) } - original := s3.objectData("tmdb/series/1396/logo/original.webp") + original := s3.objectData(result.OriginalPath) if len(original) == 0 { t.Fatal("missing original.webp upload") } @@ -449,7 +584,7 @@ func TestCache_LocalizedPosterUsesLanguageScopedPath(t *testing.T) { if result.BasePath != wantBase { t.Errorf("BasePath = %q, want %q", result.BasePath, wantBase) } - if !hasKey(s3.keys(), wantBase+"/original.webp") { + if !hasKey(s3.keys(), result.OriginalPath) { t.Errorf("missing localized original in %v", s3.keys()) } } @@ -477,7 +612,7 @@ func TestCache_ProfileUsesProfileImagePath(t *testing.T) { t.Errorf("BasePath = %q, want %q", result.BasePath, wantBase) } for _, variant := range []string{"original", "w500", "w300"} { - want := wantBase + "/" + variant + ".webp" + want := result.VariantPaths[variant] if !hasKey(s3.keys(), want) { t.Errorf("missing S3 key %q in %v", want, s3.keys()) } @@ -571,7 +706,7 @@ func TestCache_SeasonPoster_NestsUnderSeries(t *testing.T) { t.Errorf("BasePath = %q, want %q", result.BasePath, wantBase) } for _, variant := range []string{"original", "w500", "w300"} { - want := wantBase + "/" + variant + ".webp" + want := result.VariantPaths[variant] if !hasKey(s3.keys(), want) { t.Errorf("missing S3 key %q in %v", want, s3.keys()) } @@ -604,7 +739,7 @@ func TestCache_EpisodeStill_NestsUnderSeasonAndEpisode(t *testing.T) { t.Errorf("BasePath = %q, want %q", result.BasePath, wantBase) } for _, variant := range []string{"original", "w500", "w300"} { - want := wantBase + "/" + variant + ".webp" + want := result.VariantPaths[variant] if !hasKey(s3.keys(), want) { t.Errorf("missing S3 key %q in %v", want, s3.keys()) } @@ -618,9 +753,10 @@ func TestCache_SeasonsDoNotCollide(t *testing.T) { s3 := &mockS3{bucket: "media"} c := newWithHTTPClient(s3, srv.Client()) + originalPaths := make(map[int]string) for _, season := range []int{1, 2, 3} { s := season - _, err := c.Cache(context.Background(), CacheRequest{ + result, err := c.Cache(context.Background(), CacheRequest{ SourceURL: srv.URL + "/season.jpg", ProviderID: "tmdb", ContentType: "series", @@ -631,11 +767,12 @@ func TestCache_SeasonsDoNotCollide(t *testing.T) { if err != nil { t.Fatalf("Cache season %d: %v", season, err) } + originalPaths[season] = result.OriginalPath } keys := s3.keys() for _, season := range []int{1, 2, 3} { - want := fmt.Sprintf("tmdb/series/1396/seasons/%d/poster/original.webp", season) + want := originalPaths[season] if !hasKey(keys, want) { t.Errorf("missing S3 key %q in %v", want, keys) } diff --git a/internal/imagecache/local_key_test.go b/internal/imagecache/local_key_test.go new file mode 100644 index 00000000..07408382 --- /dev/null +++ b/internal/imagecache/local_key_test.go @@ -0,0 +1,76 @@ +package imagecache + +import ( + "context" + "strings" + "testing" + + "github.com/Silo-Server/silo-server/internal/metadata" +) + +func TestCacheBytesKeyDiscriminatorPrecedesImageType(t *testing.T) { + s3 := &mockS3{bucket: "images"} + cacher := New(s3) + data := makeTestJPEG(t) + + res, err := cacher.CacheBytes(context.Background(), data, CacheRequest{ + ProviderID: "local", + ContentType: "movies", + ContentID: "movie-1", + ImageType: metadata.ImagePoster, + KeyDiscriminator: "deadbeef", + }) + if err != nil { + t.Fatalf("CacheBytes: %v", err) + } + // Hash segment sits BEFORE the image type so the variant's parent + // directory stays the image type (imageTypeFromCachedPath contract). + if res.BasePath != "local/movies/movie-1/deadbeef/poster" { + t.Fatalf("BasePath = %q", res.BasePath) + } + for _, call := range s3.calls { + if !strings.HasPrefix(call.key, "local/movies/movie-1/deadbeef/poster/") { + t.Fatalf("uploaded key %q outside discriminated prefix", call.key) + } + } +} + +func TestCacheBytesWithoutDiscriminatorKeepsLegacyLayout(t *testing.T) { + s3 := &mockS3{bucket: "images"} + cacher := New(s3) + + res, err := cacher.CacheBytes(context.Background(), makeTestJPEG(t), CacheRequest{ + ProviderID: "local", + ContentType: "ebooks", + ContentID: "book-1", + ImageType: metadata.ImagePoster, + }) + if err != nil { + t.Fatalf("CacheBytes: %v", err) + } + if res.BasePath != "local/ebooks/book-1/poster" { + t.Fatalf("BasePath = %q", res.BasePath) + } +} + +func TestCacheImageBytesAdapter(t *testing.T) { + s3 := &mockS3{bucket: "images"} + cacher := New(s3) + + res, err := cacher.CacheImageBytes(context.Background(), makeTestJPEG(t), metadata.CacheImageRequest{ + ProviderID: "local", + ContentType: "movies", + ContentID: "movie-1", + ImageType: metadata.ImageBackdrop, + KeyDiscriminator: "cafef00d", + }) + if err != nil { + t.Fatalf("CacheImageBytes: %v", err) + } + if res.BasePath != "local/movies/movie-1/cafef00d/backdrop" { + t.Fatalf("BasePath = %q", res.BasePath) + } + if res.Thumbhash == "" { + t.Fatal("thumbhash missing") + } +} diff --git a/internal/jellycompat/audio_selection_test.go b/internal/jellycompat/audio_selection_test.go index 0cafd106..8543d874 100644 --- a/internal/jellycompat/audio_selection_test.go +++ b/internal/jellycompat/audio_selection_test.go @@ -66,9 +66,12 @@ func (m *testCompatSessionManager) UpdateProgress(sessionID string, position flo isPaused: isPaused, }) if m.sessions != nil { - if _, ok := m.sessions[sessionID]; !ok { + session, ok := m.sessions[sessionID] + if !ok { return playback.ErrSessionNotFound } + session.Position = position + session.IsPaused = isPaused } return nil } @@ -111,6 +114,7 @@ func (m *testCompatSessionManager) UpdateAudioTrack(sessionID string, audioTrack func (m *testCompatSessionManager) StopSession(sessionID string) error { m.stopCalls = append(m.stopCalls, sessionID) + delete(m.sessions, sessionID) return nil } diff --git a/internal/jellycompat/auth_api_key.go b/internal/jellycompat/auth_api_key.go index 534d685d..4940ee13 100644 --- a/internal/jellycompat/auth_api_key.go +++ b/internal/jellycompat/auth_api_key.go @@ -9,6 +9,7 @@ import ( "sync" "time" + "github.com/Silo-Server/silo-server/internal/auth" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/userstore" ) @@ -36,8 +37,7 @@ type AdminAPIKeyAuthenticator struct { // authorization decision, which is re-checked on every request. profiles *apiKeyProfileCache - lastUsedMu sync.Mutex - lastUsedAt map[int64]time.Time + apiKeyLastUsed *auth.APIKeyLastUsedTracker } type adminAPIKeyAuthResult struct { @@ -61,12 +61,12 @@ func NewAdminAPIKeyAuthenticator(keys apiKeyValidator, users apiKeyUserLoader, p now = time.Now } return &AdminAPIKeyAuthenticator{ - keys: keys, - users: users, - provider: provider, - now: now, - profiles: newAPIKeyProfileCache(), - lastUsedAt: make(map[int64]time.Time), + keys: keys, + users: users, + provider: provider, + now: now, + profiles: newAPIKeyProfileCache(), + apiKeyLastUsed: auth.NewAPIKeyLastUsedTracker(keys, now), } } @@ -138,7 +138,7 @@ func (a *AdminAPIKeyAuthenticator) authenticate(r *http.Request) adminAPIKeyAuth if !res.ok { return res } - a.touchLastUsed(apiKey.ID) + a.apiKeyLastUsed.Touch(apiKey.ID) return adminAPIKeyAuthResult{ ctx: context.WithValue(r.Context(), adminAPIKeyKey, true), status: http.StatusOK, @@ -182,28 +182,6 @@ func (a *AdminAPIKeyAuthenticator) validate(ctx context.Context, token string) ( return apiKey, user, adminAPIKeyAuthResult{ok: true} } -// touchLastUsed records key usage without blocking the request, throttled to at -// most once per apiKeyLastUsedInterval per key so per-request validation on the -// hot path (e.g. HLS segments) does not issue a DB write each time. -func (a *AdminAPIKeyAuthenticator) touchLastUsed(id int64) { - now := a.now() - a.lastUsedMu.Lock() - if last, ok := a.lastUsedAt[id]; ok && now.Sub(last) < apiKeyLastUsedInterval { - a.lastUsedMu.Unlock() - return - } - a.lastUsedAt[id] = now - a.lastUsedMu.Unlock() - - go func(id int64) { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - if err := a.keys.UpdateLastUsed(ctx, id); err != nil { - slog.Debug("jellycompat api key last-used update failed", "id", id, "error", err) - } - }(id) -} - // resolveSession authenticates an sa_ API key and returns a compat session // synthesized for the key user's primary profile, so ordinary browse/stream // handlers (which read SessionFromContext) work unchanged for an API key. @@ -224,7 +202,7 @@ func (a *AdminAPIKeyAuthenticator) resolveSession(ctx context.Context, token str if !res.ok { return nil, res, true } - a.touchLastUsed(apiKey.ID) + a.apiKeyLastUsed.Touch(apiKey.ID) profile, err := a.primaryProfile(ctx, user.ID) if err != nil { @@ -298,9 +276,6 @@ const ( // reassigned/renamed primary profile is picked up, never authorization // (the key and user are re-validated on every request). apiKeyProfileCacheTTL = 5 * time.Minute - // apiKeyLastUsedInterval throttles api_keys.last_used_at writes so that - // per-request validation on the hot path does not write each time. - apiKeyLastUsedInterval = time.Minute ) type apiKeyProfileCacheEntry struct { diff --git a/internal/jellycompat/content_direct_test.go b/internal/jellycompat/content_direct_test.go index ae56fbe0..995efc89 100644 --- a/internal/jellycompat/content_direct_test.go +++ b/internal/jellycompat/content_direct_test.go @@ -359,7 +359,7 @@ func (s *progressCountingStore) DeleteHomeDismissal(context.Context, string, str panic("unused") } func (s *progressCountingStore) AddFavorite(context.Context, string, string) error { panic("unused") } -func (s *progressCountingStore) AddFavoriteAt(context.Context, string, string, time.Time) error { +func (s *progressCountingStore) AddFavoriteAt(context.Context, string, string, time.Time) (bool, error) { panic("unused") } func (s *progressCountingStore) RemoveFavorite(context.Context, string, string) error { diff --git a/internal/jellycompat/deviceprofile_conditions_test.go b/internal/jellycompat/deviceprofile_conditions_test.go index 56eddaee..154ef9b4 100644 --- a/internal/jellycompat/deviceprofile_conditions_test.go +++ b/internal/jellycompat/deviceprofile_conditions_test.go @@ -1,6 +1,7 @@ package jellycompat import ( + "encoding/json" "strings" "testing" @@ -273,6 +274,55 @@ func TestBuildMediaStreamsUsesJellyfinVideoRangeType(t *testing.T) { } } +func TestBuildMediaStreamsPreservesColorRange(t *testing.T) { + tests := []struct { + name string + colorRange string + wantColorRange string + wantJSONField bool + }{ + {name: "limited", colorRange: "tv", wantColorRange: "tv", wantJSONField: true}, + {name: "full", colorRange: "pc", wantColorRange: "pc", wantJSONField: true}, + {name: "internal unknown sentinel", colorRange: "unknown"}, + {name: "missing"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + version := catalog.FileVersion{ + VideoTracks: []models.VideoTrack{{ + Codec: "h264", + ColorRange: tt.colorRange, + }}, + } + + streams := buildMediaStreams("item", "source", version) + if len(streams) != 1 { + t.Fatalf("streams length = %d, want 1", len(streams)) + } + if got := streams[0].ColorRange; got != tt.wantColorRange { + t.Fatalf("ColorRange = %q, want %q", got, tt.wantColorRange) + } + + payload, err := json.Marshal(streams[0]) + if err != nil { + t.Fatalf("marshal media stream: %v", err) + } + var decoded map[string]any + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("unmarshal media stream: %v", err) + } + got, present := decoded["ColorRange"] + if present != tt.wantJSONField { + t.Fatalf("JSON ColorRange present = %v, want %v (value %#v)", present, tt.wantJSONField, got) + } + if tt.wantJSONField && got != tt.wantColorRange { + t.Fatalf("JSON ColorRange = %#v, want %q", got, tt.wantColorRange) + } + }) + } +} + func TestCodecProfileAVCRefFramesConstraint(t *testing.T) { version := catalog.FileVersion{ FileID: 1, diff --git a/internal/jellycompat/dto.go b/internal/jellycompat/dto.go index aa8bc35f..06ead21c 100644 --- a/internal/jellycompat/dto.go +++ b/internal/jellycompat/dto.go @@ -264,6 +264,7 @@ type mediaStreamDTO struct { AspectRatio string `json:"AspectRatio,omitempty"` VideoRange string `json:"VideoRange,omitempty"` VideoRangeType string `json:"VideoRangeType,omitempty"` + ColorRange string `json:"ColorRange,omitempty"` ColorPrimaries string `json:"ColorPrimaries,omitempty"` ColorSpace string `json:"ColorSpace,omitempty"` ColorTransfer string `json:"ColorTransfer,omitempty"` diff --git a/internal/jellycompat/handlers_playback.go b/internal/jellycompat/handlers_playback.go index 85a017fe..c178cf2a 100644 --- a/internal/jellycompat/handlers_playback.go +++ b/internal/jellycompat/handlers_playback.go @@ -29,6 +29,7 @@ import ( "github.com/Silo-Server/silo-server/internal/subtitles" "github.com/Silo-Server/silo-server/internal/transcodenode" "github.com/Silo-Server/silo-server/internal/userstore" + "github.com/Silo-Server/silo-server/internal/watchsync" ) type playbackInfoRequest struct { @@ -101,6 +102,38 @@ type sessionStarterContext interface { StartSessionWithContext(ctx context.Context, userID int, profileID string, fileID int, method playback.PlayMethod, transcodeAudio bool) (*playback.Session, error) } +// transcodeStreamDetailsSetter is implemented by the native SessionManager. +// Optional (like sessionStarterContext) so lightweight test fakes don't have +// to; without it the session keeps transport-level defaults only. +type transcodeStreamDetailsSetter interface { + SetTranscodeStreamDetails(sessionID, targetVideoCodec, targetAudioCodec string, transcodeAudio bool) error +} + +// recordTranscodeStreamDetails mirrors the encode decisions of a started +// transcode onto the upstream session. StartSession only records the transport +// method (play_method "transcode", transcodeAudio false), so without this an +// audio-only re-encode — video copied — syncs to session sync and the admin +// activity views as a full video transcode. Shared by the local +// (ensureTranscodeSession) and remote (startRemoteTranscode) paths. +func (h *PlaybackHandler) recordTranscodeStreamDetails(ctx context.Context, upstreamSessionID string, opts playback.TranscodeOpts) { + setter, ok := h.sessionMgr.(transcodeStreamDetailsSetter) + if !ok { + return + } + transcodeAudio := playback.TranscodesAudio(opts.TargetCodecAudio) + if err := setter.SetTranscodeStreamDetails(upstreamSessionID, opts.TargetCodecVideo, opts.TargetCodecAudio, transcodeAudio); err != nil { + slog.WarnContext(ctx, "record transcode stream details failed", "component", "jellycompat", + "error", err, "playback_session_id", upstreamSessionID) + return + } + // The "compat_start" sync inside ensureUpstreamPlayback already flushed + // this session with transport-level defaults, so flush again now that the + // real encode decisions are on it — otherwise the admin view shows a + // video-copy stream as a full video transcode until the next reconciler + // tick. + h.syncSessionsNow(ctx, "compat_transcode_details") +} + // FilePathResolver looks up media files by ID. type FilePathResolver interface { GetByID(ctx context.Context, id int) (*models.MediaFile, error) @@ -119,6 +152,21 @@ type PlaybackSessionSyncer interface { SyncNow(ctx context.Context) error } +// PlaybackWatchScrobbler forwards a playback lifecycle to connected watch +// providers. The watchsync service implements this interface. +type PlaybackWatchScrobbler interface { + ScrobbleStart(ctx context.Context, event watchsync.ScrobbleEvent) error + ScrobblePause(ctx context.Context, event watchsync.ScrobbleEvent) error + ScrobbleStop(ctx context.Context, event watchsync.ScrobbleEvent) error +} + +// PlaybackWatchStopConfirmer is implemented by watch-sync services that can +// wait for a terminal stop to be accepted by every provider. Jellycompat uses +// it before deleting its durable terminal record. +type PlaybackWatchStopConfirmer interface { + ScrobbleStopConfirmed(ctx context.Context, event watchsync.ScrobbleEvent) error +} + // PlaybackHandler serves Jellyfin playback negotiation endpoints. type PlaybackHandler struct { cfg *config.Config @@ -141,12 +189,15 @@ type PlaybackHandler struct { // and node-affinity rule for free. The reconstruction recipe is carried in the // compat playback store (PlaybackSession.Recipe), since Jellyfin clients cannot // round-trip a native stream token. - tm *playback.TranscodeManager - SubtitleRepo subtitles.Repository // optional; enables downloaded subtitles - S3Client subtitles.S3Client // optional; for serving S3 subtitles - S3Bucket string // bucket for subtitle storage - SettingsRepo SettingsReader // optional; reads watched threshold setting - SessionSyncer PlaybackSessionSyncer // optional; enables immediate session sync to shared admin view + tm *playback.TranscodeManager + SubtitleRepo subtitles.Repository // optional; enables downloaded subtitles + S3Client subtitles.S3Client // optional; for serving S3 subtitles + S3Bucket string // bucket for subtitle storage + SettingsRepo SettingsReader // optional; reads watched threshold setting + SessionSyncer PlaybackSessionSyncer // optional; enables immediate session sync to shared admin view + WatchScrobbler PlaybackWatchScrobbler + StableIdentityResolver watchsync.ScrobbleIdentityResolver + terminalFallbackDelay time.Duration // RecipeNodeStore hands a remote transcode's reconstruction recipe to the // control-plane recipe store (Redis) so a dedicated transcode node that // restarts can rebuild ffmpeg from it. The node-hop token is server-minted and @@ -260,8 +311,10 @@ func NewPlaybackHandler( // ffmpeg crash: drop the dead transcode and stop the upstream native // session. The recipe stays in the compat store so a resume reconstructs. nodeURL := "" + var upstreamSession *playback.Session if h.sessionMgr != nil { if up, err := h.sessionMgr.GetSession(sessionID); err == nil && up != nil { + upstreamSession = up nodeURL = up.TranscodeNodeURL } } @@ -273,6 +326,11 @@ func NewPlaybackHandler( // the dead transcode. The recipe stays in the compat store either way so a // resume reconstructs. if h.sessionMgr != nil && h.tm.CloseTranscodeSessionIf(sessionID, dead, nodeURL) { + if h.playbackStore != nil { + if playSession, ok := h.playbackStore.FindByUpstreamSessionID(sessionID); ok { + h.dispatchCompatScrobble(ctx, compatScrobblePause, playSession, upstreamSession, nil) + } + } _ = h.sessionMgr.StopSession(sessionID) } } @@ -443,6 +501,10 @@ func (h *PlaybackHandler) startRemoteTranscode( opts.TargetCodecVideo = "copy" } + // Same mirror as the local path: the node runs the encode, but the + // upstream session here is what session sync and admin views read. + h.recordTranscodeStreamDetails(ctx, upstreamSessionID, opts) + if err := h.persistTranscodeRecipe(ctx, playSessionID, upstreamSessionID, opts); err != nil { // Roll back the already-started node ffmpeg so it isn't leaked. h.tm.CloseTranscodeSession(upstreamSessionID, transcodeNodeURL) @@ -479,6 +541,12 @@ func (h *PlaybackHandler) persistTranscodeRecipe( if h.sessionMgr != nil { if upstream, err := h.sessionMgr.GetSession(upstreamSessionID); err == nil && upstream != nil { card := playback.NewRecipeCard(upstream.UserID, upstream.ProfileID, upstream.MediaFileID, upstream.TranscodeNodeURL, opts) + // Mirror the client metadata into the card so a session rebuilt + // after a restart keeps its client label and Jellyfin + // identification instead of syncing with empty client fields. + card.ClientName = upstream.ClientName + card.ClientVersion = upstream.ClientVersion + card.ClientUserAgent = upstream.ClientUserAgent recipe = &card } } @@ -649,9 +717,14 @@ func (h *PlaybackHandler) HandlePlaybackInfo(w http.ResponseWriter, r *http.Requ return } - h.playbackStore.Put(PlaybackSession{ + clientDeviceID := firstNonEmpty( + firstMediaBrowserAuthorizationValue(r, "DeviceId"), + newCaseInsensitiveQuery(r.URL.Query()).Get("DeviceId"), + ) + h.playbackStore.PutNegotiated(PlaybackSession{ ID: playSessionID, CompatToken: session.Token, + ClientDeviceID: clientDeviceID, ItemID: detail.ContentID, RouteItemID: routeItemID, UserID: session.PseudoUserID.String(), @@ -848,6 +921,7 @@ func buildMediaStreamsWithSelection(routeItemID, mediaSourceID string, version c AspectRatio: track.AspectRatio, VideoRange: compatVideoRange(track, version.HDR), VideoRangeType: compatVideoRangeType(track, version.HDR), + ColorRange: compatColorRange(track.ColorRange), ColorPrimaries: track.ColorPrimaries, ColorSpace: track.ColorSpace, ColorTransfer: track.ColorTransfer, @@ -928,6 +1002,16 @@ func buildMediaStreamsWithSelection(routeItemID, mediaSourceID string, version c return streams } +func compatColorRange(colorRange string) string { + colorRange = strings.TrimSpace(colorRange) + if strings.EqualFold(colorRange, "unknown") { + // The scanner persists this sentinel so legacy probe repair converges, + // but Jellyfin omits ColorRange when ffprobe did not provide a value. + return "" + } + return colorRange +} + func mediaSourceETag(version catalog.FileVersion) string { sum := sha1.Sum(fmt.Appendf(nil, "%d:%s:%s:%d", version.FileID, version.Container, version.CodecVideo, version.Bitrate)) return hex.EncodeToString(sum[:8]) diff --git a/internal/jellycompat/playback_negotiation_dedup_test.go b/internal/jellycompat/playback_negotiation_dedup_test.go new file mode 100644 index 00000000..99786a7f --- /dev/null +++ b/internal/jellycompat/playback_negotiation_dedup_test.go @@ -0,0 +1,139 @@ +package jellycompat + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/go-chi/chi/v5" +) + +func TestPlaybackSessionStorePutNegotiatedReplacesUnstartedSameDevice(t *testing.T) { + store := NewPlaybackSessionStore(0, nil) + store.PutNegotiated(PlaybackSession{ + ID: "first", + CompatToken: "token", + ClientDeviceID: "web-device", + RouteItemID: "route", + }) + store.PutNegotiated(PlaybackSession{ + ID: "second", + CompatToken: "token", + ClientDeviceID: "web-device", + RouteItemID: "route", + }) + + if _, ok := store.Get("first"); ok { + t.Fatal("superseded unstarted negotiation was retained") + } + if _, ok := store.Get("second"); !ok { + t.Fatal("new negotiation was not stored") + } +} + +func TestPlaybackSessionStorePutNegotiatedPreservesDistinctOrStartedPlays(t *testing.T) { + tests := []struct { + name string + first PlaybackSession + }{ + { + name: "different device", + first: PlaybackSession{ + ID: "first", + CompatToken: "token", + ClientDeviceID: "other-device", + RouteItemID: "route", + }, + }, + { + name: "already started", + first: PlaybackSession{ + ID: "first", + CompatToken: "token", + ClientDeviceID: "web-device", + RouteItemID: "route", + UpstreamSessionID: "upstream-first", + }, + }, + { + name: "terminal session", + first: PlaybackSession{ + ID: "first", + CompatToken: "token", + ClientDeviceID: "web-device", + RouteItemID: "route", + Terminal: true, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + store := NewPlaybackSessionStore(0, nil) + store.PutNegotiated(tc.first) + store.PutNegotiated(PlaybackSession{ + ID: "second", + CompatToken: "token", + ClientDeviceID: "web-device", + RouteItemID: "route", + }) + + if _, ok := store.GetFinalizable("first", "token"); !ok { + t.Fatal("distinct or already-started play was replaced") + } + if _, ok := store.Get("second"); !ok { + t.Fatal("new negotiation was not stored") + } + }) + } +} + +func TestHandlePlaybackInfoReplacesDuplicateJellyfinWebNegotiation(t *testing.T) { + handler, routeID := newSubtitleSelectionHandler(t) + first := postPlaybackInfoForDevice(t, handler, routeID, "web-device") + second := postPlaybackInfoForDevice(t, handler, routeID, "web-device") + + store := handler.playbackStore.(*PlaybackSessionStore) + if _, ok := store.Get(first.PlaySessionID); ok { + t.Fatal("first Jellyfin Web negotiation remained routable") + } + stored, ok := store.Get(second.PlaySessionID) + if !ok { + t.Fatal("second Jellyfin Web negotiation was not routable") + } + if stored.ClientDeviceID != "web-device" { + t.Fatalf("ClientDeviceID = %q, want web-device", stored.ClientDeviceID) + } +} + +func postPlaybackInfoForDevice( + t *testing.T, + handler *PlaybackHandler, + routeID string, + deviceID string, +) playbackInfoResponseDTO { + t.Helper() + req := httptest.NewRequest(http.MethodPost, "/Items/"+routeID+"/PlaybackInfo", strings.NewReader(`{}`)) + req.Header.Set( + "X-Emby-Authorization", + `MediaBrowser Client="Jellyfin Web", Device="Chrome", DeviceId="`+deviceID+`", Version="10.11.6"`, + ) + routeCtx := chi.NewRouteContext() + routeCtx.URLParams.Add("id", routeID) + req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx)) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, &Session{Token: "token-1"})) + + recorder := httptest.NewRecorder() + handler.HandlePlaybackInfo(recorder, req) + if recorder.Code != http.StatusOK { + t.Fatalf("status = %d, body = %s", recorder.Code, recorder.Body.String()) + } + var response playbackInfoResponseDTO + if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil { + t.Fatalf("unmarshal response: %v", err) + } + return response +} diff --git a/internal/jellycompat/playback_report_liveness_test.go b/internal/jellycompat/playback_report_liveness_test.go index 6707552b..4114125c 100644 --- a/internal/jellycompat/playback_report_liveness_test.go +++ b/internal/jellycompat/playback_report_liveness_test.go @@ -38,14 +38,15 @@ func newReportLivenessHandler(upstreamID string, registerUpstream bool) (*Playba playbackStore := NewPlaybackSessionStore(time.Hour, nil) playbackStore.Put(PlaybackSession{ - ID: "play-1", - CompatToken: "token-1", - ItemID: "movie-1", - RouteItemID: encodedItemID, - UserID: "user-1", - UpstreamSessionID: upstreamID, - UpstreamPlayMethod: "direct", - MediaSources: []PlaybackMediaSource{source}, + ID: "play-1", + CompatToken: "token-1", + ItemID: "movie-1", + RouteItemID: encodedItemID, + UserID: "user-1", + UpstreamSessionID: upstreamID, + UpstreamPlayMethod: "direct", + ProgressPersistenceKnown: true, + MediaSources: []PlaybackMediaSource{source}, }) sessions := map[string]*playback.Session{} @@ -336,11 +337,9 @@ func TestHandlePlaybackReport_StopViaAliasTearsDown(t *testing.T) { } } -// TestHandlePlaybackReport_StopViaRouteMatchDoesNotTearDown proves a Stopped -// report that only matched by item/source route (an ambiguous match when the -// same item plays twice under one token) must not tear down the session it -// happened to hit; stale cleanup owns that session's end of life. -func TestHandlePlaybackReport_StopViaRouteMatchDoesNotTearDown(t *testing.T) { +// TestHandlePlaybackReport_StopViaUniqueRouteDoesNotTearDown proves a delayed +// Stopped report cannot use a same-item route to tear down a newer play. +func TestHandlePlaybackReport_StopViaUniqueRouteDoesNotTearDown(t *testing.T) { handler, mgr, encodedItemID, sourceID := newReportLivenessHandler("upstream-1", true) req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( @@ -354,14 +353,85 @@ func TestHandlePlaybackReport_StopViaRouteMatchDoesNotTearDown(t *testing.T) { t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) } if len(mgr.stopCalls) != 0 { - t.Fatalf("StopSession calls = %v, want none for an ambiguous route-only match", mgr.stopCalls) + t.Fatalf("StopSession calls = %v, want none", mgr.stopCalls) } if _, ok := handler.playbackStore.Get("play-1"); !ok { - t.Fatal("expected route-only Stopped report to leave the play session in place") + t.Fatal("route-only Stopped report tore down the active play session") } - // The final position still lands on the upstream session. - if len(mgr.progressUpdates) != 1 || mgr.progressUpdates[0].sessionID != "upstream-1" { - t.Fatalf("progress updates = %+v, want one update on upstream-1", mgr.progressUpdates) + if len(mgr.progressUpdates) != 0 { + t.Fatalf("progress updates = %+v, want none", mgr.progressUpdates) + } +} + +// TestHandlePlaybackReport_StopViaAmbiguousRouteDoesNotTearDown proves route +// fallback refuses to choose when the same item/source is playing twice under +// one token. +func TestHandlePlaybackReport_StopViaAmbiguousRouteDoesNotTearDown(t *testing.T) { + handler, mgr, encodedItemID, sourceID := newReportLivenessHandler("upstream-1", true) + original, ok := handler.playbackStore.Get("play-1") + if !ok { + t.Fatal("original play session missing") + } + sibling := *original + sibling.ID = "play-2" + sibling.UpstreamSessionID = "upstream-2" + handler.playbackStore.Put(sibling) + mgr.sessions["upstream-2"] = &playback.Session{ID: "upstream-2", PlayMethod: playback.PlayDirect} + + req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( + `{"PlaySessionId":"never-seen-psid","ItemId":"`+encodedItemID+`","MediaSourceId":"`+sourceID+`","PositionTicks":9000000000}`)) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 1, ProfileID: "profile-1"})) + rec := httptest.NewRecorder() + handler.HandleSessionPlayingStopped(rec, req) + + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + if len(mgr.stopCalls) != 0 || len(mgr.progressUpdates) != 0 { + t.Fatalf("ambiguous route mutated playback: stops=%v progress=%+v", mgr.stopCalls, mgr.progressUpdates) + } + if _, ok := handler.playbackStore.Get("play-1"); !ok { + t.Fatal("ambiguous route removed play-1") + } + if _, ok := handler.playbackStore.Get("play-2"); !ok { + t.Fatal("ambiguous route removed play-2") + } +} + +func TestHandlePlaybackReport_StopRejectsMixedRouteIdentifiers(t *testing.T) { + handler, mgr, encodedItemID, _ := newReportLivenessHandler("upstream-1", true) + original, ok := handler.playbackStore.Get("play-1") + if !ok { + t.Fatal("original play session missing") + } + sibling := *original + sibling.ID = "play-2" + sibling.ItemID = "movie-2" + sibling.RouteItemID = handler.codec.EncodeStringID(EncodedIDItem, "movie-2") + sibling.UpstreamSessionID = "upstream-2" + sibling.MediaSources = []PlaybackMediaSource{{ID: "source-2", FileID: 43}} + handler.playbackStore.Put(sibling) + mgr.sessions["upstream-2"] = &playback.Session{ID: "upstream-2", PlayMethod: playback.PlayDirect} + + req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( + `{"PlaySessionId":"stale-play-id","ItemId":"`+encodedItemID+`","MediaSourceId":"source-2","PositionTicks":9000000000}`)) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 1, ProfileID: "profile-1"})) + rec := httptest.NewRecorder() + handler.HandleSessionPlayingStopped(rec, req) + + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + if len(mgr.stopCalls) != 0 || len(mgr.progressUpdates) != 0 { + t.Fatalf("mixed route identifiers mutated playback: stops=%v progress=%+v", mgr.stopCalls, mgr.progressUpdates) + } + if _, ok := handler.playbackStore.Get("play-1"); !ok { + t.Fatal("mixed route identifiers removed play-1") + } + if _, ok := handler.playbackStore.Get("play-2"); !ok { + t.Fatal("mixed route identifiers removed play-2") } } diff --git a/internal/jellycompat/playback_scrobble.go b/internal/jellycompat/playback_scrobble.go new file mode 100644 index 00000000..6643bab9 --- /dev/null +++ b/internal/jellycompat/playback_scrobble.go @@ -0,0 +1,187 @@ +package jellycompat + +import ( + "context" + "errors" + "log/slog" + "time" + + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/userstore" + "github.com/Silo-Server/silo-server/internal/watchsync" +) + +type compatScrobbleAction string + +const ( + compatScrobbleStart compatScrobbleAction = "start" + compatScrobblePause compatScrobbleAction = "pause" + compatScrobbleStop compatScrobbleAction = "stop" +) + +func (h *PlaybackHandler) dispatchCompatScrobble( + ctx context.Context, + action compatScrobbleAction, + playSession *PlaybackSession, + upstreamSession *playback.Session, + preferredSource *PlaybackMediaSource, +) error { + return h.dispatchCompatScrobbleAt(ctx, action, playSession, upstreamSession, preferredSource, nil) +} + +func (h *PlaybackHandler) dispatchCompatScrobbleAt( + ctx context.Context, + action compatScrobbleAction, + playSession *PlaybackSession, + upstreamSession *playback.Session, + preferredSource *PlaybackMediaSource, + positionOverride *float64, +) error { + event, ok := h.compatScrobbleEvent( + ctx, action, playSession, upstreamSession, preferredSource, positionOverride, + ) + if !ok { + return nil + } + return h.dispatchCompatScrobbleEvent(ctx, action, event) +} + +func (h *PlaybackHandler) compatScrobbleEvent( + ctx context.Context, + action compatScrobbleAction, + playSession *PlaybackSession, + upstreamSession *playback.Session, + preferredSource *PlaybackMediaSource, + positionOverride *float64, +) (watchsync.ScrobbleEvent, bool) { + if h == nil || h.WatchScrobbler == nil || playSession == nil || upstreamSession == nil || + upstreamSession.DisableProgressPersistence || playSession.ItemID == "" { + return watchsync.ScrobbleEvent{}, false + } + scrobbleCtx, cancel := compatDetachedContext(ctx) + defer cancel() + + source := compatScrobbleSource(playSession, upstreamSession, preferredSource) + duration := 0.0 + if source != nil { + duration = float64(source.Version.Duration) + } + position := upstreamSession.Position + if positionOverride != nil { + position = *positionOverride + } else if position <= 0 && playSession.InitialSeekSeconds > 0 { + position = playSession.InitialSeekSeconds + } + completed := false + if action == compatScrobbleStop && duration > 0 { + _, completed, _ = userstore.ResolveProgressState(position, duration, h.playbackThresholds(scrobbleCtx)) + } + event := watchsync.ResolveScrobbleIdentity(scrobbleCtx, h.StableIdentityResolver, watchsync.ScrobbleEvent{ + PlaybackSessionID: upstreamSession.ID, + UserID: upstreamSession.UserID, + ProfileID: upstreamSession.ProfileID, + MediaItemID: playSession.ItemID, + PositionSeconds: position, + DurationSeconds: duration, + Completed: completed, + OccurredAt: time.Now().UTC(), + }) + return event, true +} + +func (h *PlaybackHandler) dispatchCompatScrobbleEvent( + ctx context.Context, + action compatScrobbleAction, + event watchsync.ScrobbleEvent, +) error { + return h.dispatchCompatScrobbleEventConfirmed(ctx, action, event, false) +} + +func (h *PlaybackHandler) dispatchCompatScrobbleEventConfirmed( + ctx context.Context, + action compatScrobbleAction, + event watchsync.ScrobbleEvent, + confirmStop bool, +) error { + if h == nil || h.WatchScrobbler == nil { + return nil + } + scrobbleCtx, cancel := compatDetachedContext(ctx) + defer cancel() + var err error + switch action { + case compatScrobblePause: + err = h.WatchScrobbler.ScrobblePause(scrobbleCtx, event) + case compatScrobbleStop: + if confirmer, ok := h.WatchScrobbler.(PlaybackWatchStopConfirmer); confirmStop && ok { + err = confirmer.ScrobbleStopConfirmed(scrobbleCtx, event) + } else if confirmStop { + err = errors.New("watch scrobbler does not support confirmed stops") + } else { + err = h.WatchScrobbler.ScrobbleStop(scrobbleCtx, event) + } + default: + err = h.WatchScrobbler.ScrobbleStart(scrobbleCtx, event) + } + if err != nil { + slog.WarnContext(scrobbleCtx, "failed to queue jellycompat watch provider scrobble", + "component", "jellycompat", + "action", action, + "playback_session_id", event.PlaybackSessionID, + "error", err, + ) + } + return err +} + +func compatScrobbleSource( + playSession *PlaybackSession, + upstreamSession *playback.Session, + preferredSource *PlaybackMediaSource, +) *PlaybackMediaSource { + if preferredSource != nil { + return preferredSource + } + if playSession == nil { + return nil + } + if upstreamSession != nil { + for _, source := range playSession.MediaSources { + if source.FileID == upstreamSession.MediaFileID { + copy := source + return © + } + } + } + return firstMediaSource(playSession) +} + +// compatScrobbleFallbackSession preserves enough authenticated report state to +// emit a terminal event after the in-memory upstream session has already been +// reaped. The compat play session remains the source of media identity. +func compatScrobbleFallbackSession( + compatSession *Session, + playSession *PlaybackSession, + preferredSource *PlaybackMediaSource, + position float64, + positionKnown bool, + isPaused bool, +) *playback.Session { + if compatSession == nil || playSession == nil || playSession.UpstreamSessionID == "" || !positionKnown { + return nil + } + source := compatScrobbleSource(playSession, nil, preferredSource) + fileID := 0 + if source != nil { + fileID = source.FileID + } + return &playback.Session{ + ID: playSession.UpstreamSessionID, + UserID: compatSession.StreamAppUserID, + ProfileID: compatSession.ProfileID, + MediaFileID: fileID, + Position: position, + IsPaused: isPaused, + DisableProgressPersistence: !playSession.ProgressPersistenceKnown || playSession.DisableProgressPersistence, + } +} diff --git a/internal/jellycompat/playback_scrobble_test.go b/internal/jellycompat/playback_scrobble_test.go new file mode 100644 index 00000000..221868bb --- /dev/null +++ b/internal/jellycompat/playback_scrobble_test.go @@ -0,0 +1,1154 @@ +package jellycompat + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/watchsync" +) + +type compatScrobbleCall struct { + action string + event watchsync.ScrobbleEvent +} + +type recordingCompatWatchScrobbler struct { + calls []compatScrobbleCall +} + +type channelCompatWatchScrobbler struct { + stopEvents chan watchsync.ScrobbleEvent + failStops int +} + +type failingCompatWatchScrobbler struct { + stopCalls atomic.Int32 +} + +type poisonBatchCompatWatchScrobbler struct { + deliverableSessionID string + stopCalls atomic.Int32 +} + +type confirmingCompatWatchScrobbler struct { + confirmedErr error + confirmedCalls int + normalStopCalls int +} + +type unconfirmedCompatWatchScrobbler struct{} + +type contextErrorFileResolver struct{} + +func (contextErrorFileResolver) GetByID(ctx context.Context, _ int) (*models.MediaFile, error) { + return nil, ctx.Err() +} + +type cancelingErrorFileResolver struct { + cancel context.CancelFunc + err error +} + +func (r cancelingErrorFileResolver) GetByID(context.Context, int) (*models.MediaFile, error) { + r.cancel() + return nil, r.err +} + +func (*failingCompatWatchScrobbler) ScrobbleStart(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +func (*failingCompatWatchScrobbler) ScrobblePause(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +func (s *failingCompatWatchScrobbler) ScrobbleStop(context.Context, watchsync.ScrobbleEvent) error { + s.stopCalls.Add(1) + return errors.New("queue unavailable") +} + +func (s *failingCompatWatchScrobbler) ScrobbleStopConfirmed(ctx context.Context, event watchsync.ScrobbleEvent) error { + return s.ScrobbleStop(ctx, event) +} + +func (*poisonBatchCompatWatchScrobbler) ScrobbleStart(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +func (*poisonBatchCompatWatchScrobbler) ScrobblePause(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +func (s *poisonBatchCompatWatchScrobbler) ScrobbleStop(_ context.Context, event watchsync.ScrobbleEvent) error { + s.stopCalls.Add(1) + if event.PlaybackSessionID != s.deliverableSessionID { + return errors.New("poison terminal event") + } + return nil +} + +func (s *poisonBatchCompatWatchScrobbler) ScrobbleStopConfirmed(ctx context.Context, event watchsync.ScrobbleEvent) error { + return s.ScrobbleStop(ctx, event) +} + +func (*confirmingCompatWatchScrobbler) ScrobbleStart(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +func (*confirmingCompatWatchScrobbler) ScrobblePause(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +func (s *confirmingCompatWatchScrobbler) ScrobbleStop(context.Context, watchsync.ScrobbleEvent) error { + s.normalStopCalls++ + return nil +} + +func (s *confirmingCompatWatchScrobbler) ScrobbleStopConfirmed(context.Context, watchsync.ScrobbleEvent) error { + s.confirmedCalls++ + return s.confirmedErr +} + +func (unconfirmedCompatWatchScrobbler) ScrobbleStart(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +func (unconfirmedCompatWatchScrobbler) ScrobblePause(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +func (unconfirmedCompatWatchScrobbler) ScrobbleStop(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +type flakyTerminalPlaybackStore struct { + *PlaybackSessionStore + mu sync.Mutex + failStages int + stageCalls int +} + +func (s *flakyTerminalPlaybackStore) StageTerminal( + id string, + compatToken string, + event watchsync.ScrobbleEvent, + authoritative bool, +) (*PlaybackSession, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.stageCalls++ + if s.failStages > 0 { + s.failStages-- + return nil, errors.New("terminal store unavailable") + } + return s.PlaybackSessionStore.StageTerminal(id, compatToken, event, authoritative) +} + +func (s *flakyTerminalPlaybackStore) calls() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.stageCalls +} + +func (s *channelCompatWatchScrobbler) ScrobbleStart(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +func (s *channelCompatWatchScrobbler) ScrobblePause(context.Context, watchsync.ScrobbleEvent) error { + return nil +} + +func (s *channelCompatWatchScrobbler) ScrobbleStop(_ context.Context, event watchsync.ScrobbleEvent) error { + if s.failStops > 0 { + s.failStops-- + return errors.New("queue unavailable") + } + s.stopEvents <- event + return nil +} + +func (s *channelCompatWatchScrobbler) ScrobbleStopConfirmed(ctx context.Context, event watchsync.ScrobbleEvent) error { + return s.ScrobbleStop(ctx, event) +} + +func (s *recordingCompatWatchScrobbler) ScrobbleStart(_ context.Context, event watchsync.ScrobbleEvent) error { + s.calls = append(s.calls, compatScrobbleCall{action: "start", event: event}) + return nil +} + +func (s *recordingCompatWatchScrobbler) ScrobblePause(_ context.Context, event watchsync.ScrobbleEvent) error { + s.calls = append(s.calls, compatScrobbleCall{action: "pause", event: event}) + return nil +} + +func (s *recordingCompatWatchScrobbler) ScrobbleStop(_ context.Context, event watchsync.ScrobbleEvent) error { + s.calls = append(s.calls, compatScrobbleCall{action: "stop", event: event}) + return nil +} + +func (s *recordingCompatWatchScrobbler) ScrobbleStopConfirmed(ctx context.Context, event watchsync.ScrobbleEvent) error { + return s.ScrobbleStop(ctx, event) +} + +func TestEnsureUpstreamPlaybackStartsWatchProviderScrobble(t *testing.T) { + mgr := &testCompatSessionManager{} + h, store := newActiveEncodingsHandler(mgr) + scrobbler := &recordingCompatWatchScrobbler{} + h.WatchScrobbler = scrobbler + source := PlaybackMediaSource{ID: "source-1", FileID: 42, Version: testCompatVersion()} + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + ItemID: "movie-1", + InitialSeekSeconds: 125, + MediaSources: []PlaybackMediaSource{source}, + }) + compatSession := &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"} + + if _, err := h.ensureUpstreamPlayback(context.Background(), compatSession, "play-1", source, "direct"); err != nil { + t.Fatalf("ensureUpstreamPlayback: %v", err) + } + if len(scrobbler.calls) != 1 { + t.Fatalf("scrobble calls = %d, want 1", len(scrobbler.calls)) + } + call := scrobbler.calls[0] + if call.action != "start" || call.event.PlaybackSessionID != "upstream-started" { + t.Fatalf("start call = %+v", call) + } + if call.event.UserID != 7 || call.event.ProfileID != "profile-1" || call.event.MediaItemID != "movie-1" { + t.Fatalf("start scope = %+v", call.event) + } + if call.event.PositionSeconds != 125 || call.event.DurationSeconds != 3600 { + t.Fatalf("start progress = %v/%v, want 125/3600", call.event.PositionSeconds, call.event.DurationSeconds) + } + + if _, err := h.ensureUpstreamPlayback(context.Background(), compatSession, "play-1", source, "direct"); err != nil { + t.Fatalf("ensureUpstreamPlayback reuse: %v", err) + } + if len(scrobbler.calls) != 1 { + t.Fatalf("reuse emitted %d scrobbles, want the original start only", len(scrobbler.calls)) + } +} + +func TestFailedTranscodeStartupClosesWatchProviderScrobble(t *testing.T) { + mgr := &testCompatSessionManager{} + h, store := newActiveEncodingsHandler(mgr) + scrobbler := &recordingCompatWatchScrobbler{} + h.WatchScrobbler = scrobbler + source := PlaybackMediaSource{ID: "source-1", FileID: 42, Version: testCompatVersion()} + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + ItemID: "movie-1", + MediaSources: []PlaybackMediaSource{source}, + }) + compatSession := &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"} + + if _, err := h.ensureUpstreamPlayback( + context.Background(), compatSession, "play-1", source, "transcode", + ); err != nil { + t.Fatalf("ensure upstream playback: %v", err) + } + if len(scrobbler.calls) != 1 || scrobbler.calls[0].action != "start" { + t.Fatalf("initial transcode scrobbles = %+v, want one start", scrobbler.calls) + } + if _, err := h.ensureTranscodeManifest( + context.Background(), compatSession, "play-1", source, + ); err == nil { + t.Fatal("transcode unexpectedly started without a file resolver") + } + if len(scrobbler.calls) != 2 || scrobbler.calls[1].action != "stop" { + t.Fatalf("failed transcode scrobbles = %+v, want start then stop", scrobbler.calls) + } +} + +func TestCanceledTranscodeStartupKeepsWatchProviderSessionRetryable(t *testing.T) { + mgr := &testCompatSessionManager{} + h, store := newActiveEncodingsHandler(mgr) + scrobbler := &recordingCompatWatchScrobbler{} + h.WatchScrobbler = scrobbler + h.fileResolver = contextErrorFileResolver{} + source := PlaybackMediaSource{ID: "source-1", FileID: 42, Version: testCompatVersion()} + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + ItemID: "movie-1", + MediaSources: []PlaybackMediaSource{source}, + }) + compatSession := &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"} + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := h.ensureTranscodeManifest(ctx, compatSession, "play-1", source); !errors.Is(err, context.Canceled) { + t.Fatalf("ensureTranscodeManifest error = %v, want context canceled", err) + } + if len(scrobbler.calls) != 1 || scrobbler.calls[0].action != "start" { + t.Fatalf("canceled transcode scrobbles = %+v, want start without terminal stop", scrobbler.calls) + } + if _, ok := store.Get("play-1"); !ok { + t.Fatal("canceled transcode attempt removed the retryable play session") + } + if len(mgr.stopCalls) != 0 { + t.Fatalf("canceled transcode stopped upstream sessions: %v", mgr.stopCalls) + } +} + +func TestFailedTranscodeStartupStillClosesWhenRequestCancelsAfterFailure(t *testing.T) { + mgr := &testCompatSessionManager{} + h, store := newActiveEncodingsHandler(mgr) + scrobbler := &recordingCompatWatchScrobbler{} + h.WatchScrobbler = scrobbler + source := PlaybackMediaSource{ID: "source-1", FileID: 42, Version: testCompatVersion()} + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + ItemID: "movie-1", + MediaSources: []PlaybackMediaSource{source}, + }) + compatSession := &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"} + startupErr := errors.New("file lookup failed") + ctx, cancel := context.WithCancel(context.Background()) + h.fileResolver = cancelingErrorFileResolver{cancel: cancel, err: startupErr} + + if _, err := h.ensureTranscodeManifest(ctx, compatSession, "play-1", source); !errors.Is(err, startupErr) { + t.Fatalf("ensureTranscodeManifest error = %v, want file lookup failure", err) + } + if len(scrobbler.calls) != 2 || scrobbler.calls[0].action != "start" || + scrobbler.calls[1].action != "stop" { + t.Fatalf("failed transcode scrobbles = %+v, want start then stop", scrobbler.calls) + } + if _, ok := store.Get("play-1"); ok { + t.Fatal("failed transcode left the play session retryable after a genuine startup error") + } +} + +func TestEnsureUpstreamPlaybackStopsDiscardedMethodScrobble(t *testing.T) { + mgr := &testCompatSessionManager{sessions: map[string]*playback.Session{ + "upstream-old": { + ID: "upstream-old", + UserID: 7, + ProfileID: "profile-1", + MediaFileID: 42, + Position: 45, + }, + }} + h, store := newActiveEncodingsHandler(mgr) + scrobbler := &recordingCompatWatchScrobbler{} + h.WatchScrobbler = scrobbler + source := PlaybackMediaSource{ID: "source-1", FileID: 42, Version: testCompatVersion()} + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + ItemID: "movie-1", + UpstreamSessionID: "upstream-old", + UpstreamPlayMethod: "direct", + MediaSources: []PlaybackMediaSource{source}, + }) + compatSession := &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"} + + if _, err := h.ensureUpstreamPlayback( + context.Background(), compatSession, "play-1", source, "transcode", + ); err != nil { + t.Fatalf("switch playback method: %v", err) + } + if len(scrobbler.calls) != 2 || scrobbler.calls[0].action != "stop" || + scrobbler.calls[0].event.PlaybackSessionID != "upstream-old" || + scrobbler.calls[1].action != "start" || + scrobbler.calls[1].event.PlaybackSessionID != "upstream-started" { + t.Fatalf("method-switch scrobbles = %+v, want old stop then new start", scrobbler.calls) + } +} + +func TestHandlePlaybackReportScrobblesPauseAndResumeTransitions(t *testing.T) { + handler, mgr, _, sourceID := newReportLivenessHandler("upstream-1", true) + scrobbler := &recordingCompatWatchScrobbler{} + handler.WatchScrobbler = scrobbler + mgr.sessions["upstream-1"].UserID = 7 + mgr.sessions["upstream-1"].ProfileID = "profile-1" + mgr.sessions["upstream-1"].MediaFileID = 42 + + post := func(paused bool, ticks int64) { + body := strings.NewReader(`{"PlaySessionId":"play-1","MediaSourceId":"` + sourceID + + `","PositionTicks":` + strconv.FormatInt(ticks, 10) + `,"IsPaused":` + strconv.FormatBool(paused) + `}`) + req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Progress", body) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"})) + rec := httptest.NewRecorder() + handler.HandleSessionPlayingProgress(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + } + + post(true, 600_000_000) + post(true, 700_000_000) + post(false, 800_000_000) + + if len(scrobbler.calls) != 2 { + t.Fatalf("scrobble calls = %+v, want pause and resume only", scrobbler.calls) + } + if scrobbler.calls[0].action != "pause" || scrobbler.calls[0].event.PositionSeconds != 60 { + t.Fatalf("pause call = %+v", scrobbler.calls[0]) + } + if scrobbler.calls[1].action != "start" || scrobbler.calls[1].event.PositionSeconds != 80 { + t.Fatalf("resume call = %+v", scrobbler.calls[1]) + } +} + +func TestHandlePlaybackReportPreservesExplicitZeroOnPause(t *testing.T) { + handler, mgr, _, sourceID := newReportLivenessHandler("upstream-1", true) + scrobbler := &recordingCompatWatchScrobbler{} + handler.WatchScrobbler = scrobbler + mgr.sessions["upstream-1"].UserID = 7 + mgr.sessions["upstream-1"].ProfileID = "profile-1" + mgr.sessions["upstream-1"].MediaFileID = 42 + if err := handler.playbackStore.Update("play-1", func(session *PlaybackSession) error { + session.InitialSeekSeconds = 125 + return nil + }); err != nil { + t.Fatalf("set initial seek: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Progress", strings.NewReader( + `{"PlaySessionId":"play-1","MediaSourceId":"`+sourceID+`","PositionTicks":0,"IsPaused":true}`)) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"})) + rec := httptest.NewRecorder() + handler.HandleSessionPlayingProgress(rec, req) + + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + if len(scrobbler.calls) != 1 || scrobbler.calls[0].action != "pause" || + scrobbler.calls[0].event.PositionSeconds != 0 { + t.Fatalf("explicit-zero pause scrobble = %+v", scrobbler.calls) + } +} + +func TestCompatTeardownScrobblesAuthoritativeStopExactlyOnce(t *testing.T) { + tests := []struct { + name string + stoppedFirst bool + wantPositions []float64 + }{ + {name: "stopped report first", stoppedFirst: true, wantPositions: []float64{90}}, + {name: "active encodings first", stoppedFirst: false, wantPositions: []float64{90}}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mgr := &testCompatSessionManager{sessions: map[string]*playback.Session{ + "upstream-1": { + ID: "upstream-1", + UserID: 7, + ProfileID: "profile-1", + MediaFileID: 42, + Position: 45, + }, + }} + h, store := newActiveEncodingsHandler(mgr) + scrobbler := &recordingCompatWatchScrobbler{} + h.WatchScrobbler = scrobbler + source := PlaybackMediaSource{ID: "source-1", FileID: 42, Version: testCompatVersion()} + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + ItemID: "movie-1", + UpstreamSessionID: "upstream-1", + UpstreamPlayMethod: "direct", + ProgressPersistenceKnown: true, + MediaSources: []PlaybackMediaSource{source}, + }) + + stopped := func() { + req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", + strings.NewReader(`{"PlaySessionId":"play-1","MediaSourceId":"source-1","PositionTicks":900000000}`)) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"})) + rec := httptest.NewRecorder() + h.HandleSessionPlayingStopped(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("stopped status = %d", rec.Code) + } + } + activeEncodings := func() { + req := withCompatSession(httptest.NewRequest(http.MethodDelete, + "/Videos/ActiveEncodings?PlaySessionId=play-1", nil), "token-1") + rec := httptest.NewRecorder() + h.HandleDeleteActiveEncodings(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("active encodings status = %d", rec.Code) + } + } + + if tt.stoppedFirst { + stopped() + activeEncodings() + } else { + activeEncodings() + stopped() + } + + if len(scrobbler.calls) != len(tt.wantPositions) { + t.Fatalf("scrobble calls = %+v, want positions %v", scrobbler.calls, tt.wantPositions) + } + for i, wantPosition := range tt.wantPositions { + if scrobbler.calls[i].action != "stop" || scrobbler.calls[i].event.PositionSeconds != wantPosition { + t.Fatalf("stop call %d = %+v, want position %v", i, scrobbler.calls[i], wantPosition) + } + } + }) + } +} + +func TestActiveEncodingsOnNonOwnerDefersScrobbleToStoppedReport(t *testing.T) { + mgr := &testCompatSessionManager{sessions: map[string]*playback.Session{}} + h, store := newActiveEncodingsHandler(mgr) + scrobbler := &recordingCompatWatchScrobbler{} + h.WatchScrobbler = scrobbler + h.terminalFallbackDelay = 10 * time.Millisecond + source := PlaybackMediaSource{ID: "source-1", FileID: 42, Version: testCompatVersion()} + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + ClientPlaySessionID: "client-replaced-play-id", + ItemID: "movie-1", + UpstreamSessionID: "upstream-1", + ProgressPersistenceKnown: true, + MediaSources: []PlaybackMediaSource{source}, + }) + + activeReq := withCompatSession(httptest.NewRequest(http.MethodDelete, + "/Videos/ActiveEncodings?PlaySessionId=play-1", nil), "token-1") + activeRec := httptest.NewRecorder() + h.HandleDeleteActiveEncodings(activeRec, activeReq) + if activeRec.Code != http.StatusNoContent { + t.Fatalf("active encodings status = %d", activeRec.Code) + } + time.Sleep(3 * h.terminalFallbackDelay) + if len(scrobbler.calls) != 0 { + t.Fatalf("non-owner cleanup emitted stale scrobble: %+v", scrobbler.calls) + } + if _, ok := store.Get("play-1"); ok { + t.Fatal("terminal session remained routable after encoder cleanup") + } + if _, ok := store.GetFinalizable("play-1", "token-1"); !ok { + t.Fatal("terminal session was not retained for the final report") + } + + stoppedReq := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( + `{"PlaySessionId":"client-replaced-play-id","MediaSourceId":"source-1","PositionTicks":900000000}`)) + stoppedReq = stoppedReq.WithContext(context.WithValue(stoppedReq.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"})) + stoppedRec := httptest.NewRecorder() + h.HandleSessionPlayingStopped(stoppedRec, stoppedReq) + + if stoppedRec.Code != http.StatusNoContent { + t.Fatalf("stopped status = %d, body = %s", stoppedRec.Code, stoppedRec.Body.String()) + } + if len(scrobbler.calls) != 1 || scrobbler.calls[0].action != "stop" || + scrobbler.calls[0].event.PositionSeconds != 90 { + t.Fatalf("authoritative stopped scrobble = %+v, want one stop at 90s", scrobbler.calls) + } +} + +func TestActiveEncodingsFallbackAllowsLaterAuthoritativeStop(t *testing.T) { + mgr := &testCompatSessionManager{sessions: map[string]*playback.Session{ + "upstream-1": { + ID: "upstream-1", + UserID: 7, + ProfileID: "profile-1", + MediaFileID: 42, + Position: 45, + }, + }} + h, store := newActiveEncodingsHandler(mgr) + scrobbler := &channelCompatWatchScrobbler{stopEvents: make(chan watchsync.ScrobbleEvent, 2)} + h.WatchScrobbler = scrobbler + h.terminalFallbackDelay = 10 * time.Millisecond + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + ItemID: "movie-1", + UpstreamSessionID: "upstream-1", + ProgressPersistenceKnown: true, + MediaSources: []PlaybackMediaSource{{ + ID: "source-1", FileID: 42, Version: testCompatVersion(), + }}, + }) + + req := withCompatSession(httptest.NewRequest(http.MethodDelete, + "/Videos/ActiveEncodings?PlaySessionId=play-1", nil), "token-1") + rec := httptest.NewRecorder() + h.HandleDeleteActiveEncodings(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d", rec.Code) + } + + select { + case event := <-scrobbler.stopEvents: + if event.PositionSeconds != 45 || event.PlaybackSessionID != "upstream-1" { + t.Fatalf("fallback stop = %+v, want upstream-1 at 45s", event) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for ActiveEncodings terminal fallback") + } + terminal, ok := store.GetFinalizable("play-1", "token-1") + if !ok || !terminal.TerminalFallbackSent || terminal.TerminalAuthoritative { + t.Fatalf("fallback terminal state = ok=%v session=%+v", ok, terminal) + } + + stoppedReq := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( + `{"PlaySessionId":"play-1","MediaSourceId":"source-1","PositionTicks":900000000}`)) + stoppedReq = stoppedReq.WithContext(context.WithValue(stoppedReq.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"})) + stoppedRec := httptest.NewRecorder() + h.HandleSessionPlayingStopped(stoppedRec, stoppedReq) + if stoppedRec.Code != http.StatusNoContent { + t.Fatalf("stopped status = %d", stoppedRec.Code) + } + select { + case event := <-scrobbler.stopEvents: + if event.PositionSeconds != 90 { + t.Fatalf("authoritative stop = %+v, want 90s", event) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for authoritative stop after fallback") + } + if _, ok := store.GetFinalizable("play-1", "token-1"); ok { + t.Fatal("authoritative terminal event remained after delivery") + } +} + +func TestPositionlessLateStopPreservesAndDeliversPendingFallback(t *testing.T) { + mgr := &testCompatSessionManager{sessions: map[string]*playback.Session{ + "upstream-1": { + ID: "upstream-1", + UserID: 7, + ProfileID: "profile-1", + MediaFileID: 42, + Position: 45, + }, + }} + h, store := newActiveEncodingsHandler(mgr) + scrobbler := &channelCompatWatchScrobbler{stopEvents: make(chan watchsync.ScrobbleEvent, 1)} + h.WatchScrobbler = scrobbler + h.terminalFallbackDelay = time.Hour + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + ItemID: "movie-1", + UpstreamSessionID: "upstream-1", + ProgressPersistenceKnown: true, + MediaSources: []PlaybackMediaSource{{ + ID: "source-1", FileID: 42, Version: testCompatVersion(), + }}, + }) + + activeReq := withCompatSession(httptest.NewRequest( + http.MethodDelete, "/Videos/ActiveEncodings?PlaySessionId=play-1", nil, + ), "token-1") + h.HandleDeleteActiveEncodings(httptest.NewRecorder(), activeReq) + terminal, ok := store.GetFinalizable("play-1", "token-1") + if !ok || terminal.TerminalScrobbleEvent == nil || terminal.TerminalFallbackSent { + t.Fatalf("pending fallback = ok=%v session=%+v", ok, terminal) + } + + stoppedReq := httptest.NewRequest( + http.MethodPost, + "/Sessions/Playing/Stopped", + strings.NewReader(`{"PlaySessionId":"play-1","MediaSourceId":"source-1"}`), + ) + stoppedReq = stoppedReq.WithContext(context.WithValue( + stoppedReq.Context(), + compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"}, + )) + h.HandleSessionPlayingStopped(httptest.NewRecorder(), stoppedReq) + + select { + case event := <-scrobbler.stopEvents: + if event.PositionSeconds != 45 { + t.Fatalf("preserved fallback = %+v, want 45s", event) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for preserved terminal fallback") + } + terminal, ok = store.GetFinalizable("play-1", "token-1") + if !ok || !terminal.TerminalFallbackSent || terminal.TerminalAuthoritative { + t.Fatalf("delivered fallback state = ok=%v session=%+v", ok, terminal) + } +} + +func TestStoppedScrobbleQueueFailureRetainsAndRetriesTerminalEvent(t *testing.T) { + handler, mgr, _, sourceID := newReportLivenessHandler("upstream-1", true) + scrobbler := &channelCompatWatchScrobbler{ + stopEvents: make(chan watchsync.ScrobbleEvent, 1), + failStops: 1, + } + handler.WatchScrobbler = scrobbler + mgr.sessions["upstream-1"].UserID = 7 + mgr.sessions["upstream-1"].ProfileID = "profile-1" + mgr.sessions["upstream-1"].MediaFileID = 42 + + req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( + `{"PlaySessionId":"play-1","MediaSourceId":"`+sourceID+`","PositionTicks":900000000}`)) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"})) + rec := httptest.NewRecorder() + handler.HandleSessionPlayingStopped(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d", rec.Code) + } + if _, ok := handler.playbackStore.GetFinalizable("play-1", "token-1"); !ok { + t.Fatal("terminal event was deleted after queue failure") + } + + select { + case event := <-scrobbler.stopEvents: + if event.PositionSeconds != 90 { + t.Fatalf("retried stop = %+v, want 90s", event) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for terminal queue retry") + } + if _, ok := handler.playbackStore.GetFinalizable("play-1", "token-1"); ok { + t.Fatal("authoritative terminal event remained after successful retry") + } +} + +func TestStoppedScrobbleRestagesAfterTerminalPersistenceFailure(t *testing.T) { + handler, mgr, _, sourceID := newReportLivenessHandler("upstream-1", true) + baseStore := handler.playbackStore.(*PlaybackSessionStore) + flakyStore := &flakyTerminalPlaybackStore{PlaybackSessionStore: baseStore, failStages: 1} + handler.playbackStore = flakyStore + scrobbler := &channelCompatWatchScrobbler{stopEvents: make(chan watchsync.ScrobbleEvent, 1)} + handler.WatchScrobbler = scrobbler + mgr.sessions["upstream-1"].UserID = 7 + mgr.sessions["upstream-1"].ProfileID = "profile-1" + mgr.sessions["upstream-1"].MediaFileID = 42 + + req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( + `{"PlaySessionId":"play-1","MediaSourceId":"`+sourceID+`","PositionTicks":900000000}`)) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"})) + rec := httptest.NewRecorder() + handler.HandleSessionPlayingStopped(rec, req) + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d", rec.Code) + } + if calls := flakyStore.calls(); calls != 1 { + t.Fatalf("synchronous stage calls = %d, want 1", calls) + } + if _, ok := flakyStore.Get("play-1"); ok { + t.Fatal("failed durable terminal stage left the stopped session routable") + } + if len(mgr.stopCalls) != 1 || mgr.stopCalls[0] != "upstream-1" { + t.Fatalf("cleanup after failed stage = %v, want upstream-1 stopped immediately", mgr.stopCalls) + } + + select { + case event := <-scrobbler.stopEvents: + if event.PositionSeconds != 90 { + t.Fatalf("restaged stop = %+v, want 90s", event) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for terminal restage retry") + } + if calls := flakyStore.calls(); calls < 2 { + t.Fatalf("stage calls = %d, want persistence retry", calls) + } + if _, ok := flakyStore.GetFinalizable("play-1", "token-1"); ok { + t.Fatal("restaged authoritative event remained after delivery") + } +} + +func TestStoppedScrobblePreservesExplicitZeroPosition(t *testing.T) { + handler, mgr, _, sourceID := newReportLivenessHandler("upstream-1", true) + scrobbler := &channelCompatWatchScrobbler{stopEvents: make(chan watchsync.ScrobbleEvent, 1)} + handler.WatchScrobbler = scrobbler + mgr.sessions["upstream-1"].Position = 45 + mgr.sessions["upstream-1"].UserID = 7 + mgr.sessions["upstream-1"].ProfileID = "profile-1" + mgr.sessions["upstream-1"].MediaFileID = 42 + + req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( + `{"PlaySessionId":"play-1","MediaSourceId":"`+sourceID+`","PositionTicks":0}`)) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"})) + rec := httptest.NewRecorder() + handler.HandleSessionPlayingStopped(rec, req) + + select { + case event := <-scrobbler.stopEvents: + if event.PositionSeconds != 0 { + t.Fatalf("stop position = %v, want explicit zero", event.PositionSeconds) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for zero-position stop") + } +} + +func TestTerminalScrobbleRecoveryDeliversPersistedEventAfterRestart(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "token-1"}) + event := watchsync.ScrobbleEvent{ + PlaybackSessionID: "upstream-1", + UserID: 7, + ProfileID: "profile-1", + MediaItemID: "movie-1", + PositionSeconds: 90, + } + if _, err := store.StageTerminal("play-1", "token-1", event, true); err != nil { + t.Fatalf("stage terminal event: %v", err) + } + scrobbler := &channelCompatWatchScrobbler{stopEvents: make(chan watchsync.ScrobbleEvent, 1)} + handler := &PlaybackHandler{playbackStore: store, WatchScrobbler: scrobbler} + + if err := recoverPendingTerminalScrobbles(context.Background(), handler); err != nil { + t.Fatalf("recover terminal events: %v", err) + } + select { + case got := <-scrobbler.stopEvents: + if got.PlaybackSessionID != "upstream-1" || got.PositionSeconds != 90 { + t.Fatalf("recovered event = %+v", got) + } + case <-time.After(time.Second): + t.Fatal("timed out waiting for recovered terminal event") + } + if _, ok := store.GetFinalizable("play-1", "token-1"); ok { + t.Fatal("recovered authoritative event remained pending") + } +} + +func TestTerminalScrobbleRecoveryWaitsForConfirmedProviderStop(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "token-1"}) + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1"} + if _, err := store.StageTerminal("play-1", "token-1", event, true); err != nil { + t.Fatalf("stage terminal event: %v", err) + } + scrobbler := &confirmingCompatWatchScrobbler{confirmedErr: errors.New("provider unavailable")} + handler := &PlaybackHandler{playbackStore: store, WatchScrobbler: scrobbler} + + if err := recoverPendingTerminalScrobbles(context.Background(), handler); err != nil { + t.Fatalf("recover terminal events: %v", err) + } + if scrobbler.confirmedCalls != 1 { + t.Fatalf("confirmed stop calls = %d, want 1", scrobbler.confirmedCalls) + } + if _, ok := store.GetFinalizable("play-1", "token-1"); !ok { + t.Fatal("failed confirmed stop deleted the durable terminal event") + } + + scrobbler.confirmedErr = nil + if err := recoverPendingTerminalScrobbles(context.Background(), handler); err != nil { + t.Fatalf("retry terminal event: %v", err) + } + if _, ok := store.GetFinalizable("play-1", "token-1"); ok { + t.Fatal("successful confirmed stop left the terminal event pending") + } +} + +func TestTerminalScrobbleRecoveryRetainsEventWithoutConfirmationSupport(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "token-1"}) + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1"} + if _, err := store.StageTerminal("play-1", "token-1", event, true); err != nil { + t.Fatalf("stage terminal event: %v", err) + } + handler := &PlaybackHandler{ + playbackStore: store, + WatchScrobbler: unconfirmedCompatWatchScrobbler{}, + } + + if err := recoverPendingTerminalScrobbles(context.Background(), handler); err != nil { + t.Fatalf("recover terminal events: %v", err) + } + if _, ok := store.GetFinalizable("play-1", "token-1"); !ok { + t.Fatal("authoritative event was deleted without confirmed provider delivery") + } +} + +func TestTerminalScrobbleRecoveryKeepsFallbackReplaceable(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "token-1"}) + event := watchsync.ScrobbleEvent{ + PlaybackSessionID: "upstream-1", + OccurredAt: time.Now().Add(-time.Minute), + } + if _, err := store.StageTerminal("play-1", "token-1", event, false); err != nil { + t.Fatalf("stage fallback terminal event: %v", err) + } + scrobbler := &confirmingCompatWatchScrobbler{} + handler := &PlaybackHandler{ + playbackStore: store, + WatchScrobbler: scrobbler, + terminalFallbackDelay: time.Second, + } + + if err := recoverPendingTerminalScrobbles(context.Background(), handler); err != nil { + t.Fatalf("recover fallback terminal event: %v", err) + } + if scrobbler.normalStopCalls != 1 || scrobbler.confirmedCalls != 0 { + t.Fatalf( + "fallback stop calls = normal:%d confirmed:%d, want ordinary delivery only", + scrobbler.normalStopCalls, scrobbler.confirmedCalls, + ) + } + terminal, ok := store.GetFinalizable("play-1", "token-1") + if !ok || !terminal.TerminalFallbackSent || terminal.TerminalAuthoritative { + t.Fatalf("fallback terminal state = %+v, %v; want replaceable fallback", terminal, ok) + } +} + +func TestStartTerminalScrobbleRecoverySignalsInitialScanCompletion(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "token-1"}) + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1"} + if _, err := store.StageTerminal("play-1", "token-1", event, true); err != nil { + t.Fatalf("stage terminal event: %v", err) + } + scrobbler := &channelCompatWatchScrobbler{stopEvents: make(chan watchsync.ScrobbleEvent, 1)} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + initialScanDone := StartTerminalScrobbleRecovery(ctx, store, scrobbler, time.Hour) + select { + case <-initialScanDone: + case <-time.After(time.Second): + t.Fatal("timed out waiting for initial terminal recovery") + } + + if _, ok := store.GetFinalizable("play-1", "token-1"); ok { + t.Fatal("initial terminal recovery was still pending after completion signal") + } +} + +func TestInitialTerminalScrobbleRecoveryDrainsMultipleBatches(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + for i := 0; i <= compatTerminalRecoveryBatchSize; i++ { + id := fmt.Sprintf("play-%03d", i) + store.Put(PlaybackSession{ID: id, CompatToken: "token-1"}) + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-" + id} + if _, err := store.StageTerminal(id, "token-1", event, true); err != nil { + t.Fatalf("stage terminal event %s: %v", id, err) + } + } + scrobbler := &recordingCompatWatchScrobbler{} + handler := &PlaybackHandler{playbackStore: store, WatchScrobbler: scrobbler} + + if err := recoverInitialPendingTerminalScrobbles(context.Background(), handler); err != nil { + t.Fatalf("recover initial terminal events: %v", err) + } + if len(scrobbler.calls) != compatTerminalRecoveryBatchSize+1 { + t.Fatalf( + "recovered terminal calls = %d, want %d", + len(scrobbler.calls), compatTerminalRecoveryBatchSize+1, + ) + } +} + +func TestTerminalScrobbleRecoveryHonorsFallbackGracePeriod(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "token-1"}) + event := watchsync.ScrobbleEvent{ + PlaybackSessionID: "upstream-1", + OccurredAt: time.Now(), + } + if _, err := store.StageTerminal("play-1", "token-1", event, false); err != nil { + t.Fatalf("stage fallback event: %v", err) + } + scrobbler := &channelCompatWatchScrobbler{stopEvents: make(chan watchsync.ScrobbleEvent, 1)} + handler := &PlaybackHandler{ + playbackStore: store, + WatchScrobbler: scrobbler, + terminalFallbackDelay: time.Second, + } + + if err := recoverPendingTerminalScrobbles(context.Background(), handler); err != nil { + t.Fatalf("recover fallback event: %v", err) + } + select { + case got := <-scrobbler.stopEvents: + t.Fatalf("fallback delivered during grace period: %+v", got) + case <-time.After(25 * time.Millisecond): + } + select { + case <-scrobbler.stopEvents: + case <-time.After(2 * time.Second): + t.Fatal("fallback was not delivered after grace period") + } +} + +func TestTerminalScrobbleRecoveryLeavesRetryToNextScan(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "token-1"}) + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1"} + if _, err := store.StageTerminal("play-1", "token-1", event, true); err != nil { + t.Fatalf("stage terminal event: %v", err) + } + scrobbler := &failingCompatWatchScrobbler{} + handler := &PlaybackHandler{playbackStore: store, WatchScrobbler: scrobbler} + + if err := recoverPendingTerminalScrobbles(context.Background(), handler); err != nil { + t.Fatalf("recover terminal events: %v", err) + } + time.Sleep(compatTerminalInitialRetryDelay + 100*time.Millisecond) + if calls := scrobbler.stopCalls.Load(); calls != 1 { + t.Fatalf("recovery stop attempts = %d, want one attempt per scan", calls) + } + if _, ok := store.GetFinalizable("play-1", "token-1"); !ok { + t.Fatal("failed recovery did not retain the terminal event for the next scan") + } +} + +func TestTerminalScrobbleRecoveryRotatesPastPoisonBatch(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + for i := 0; i <= compatTerminalRecoveryBatchSize; i++ { + id := fmt.Sprintf("play-%03d", i) + store.Put(PlaybackSession{ID: id, CompatToken: "token-1"}) + if _, err := store.StageTerminal( + id, + "token-1", + watchsync.ScrobbleEvent{PlaybackSessionID: fmt.Sprintf("upstream-%03d", i)}, + true, + ); err != nil { + t.Fatalf("stage terminal event %s: %v", id, err) + } + } + scrobbler := &poisonBatchCompatWatchScrobbler{deliverableSessionID: "upstream-100"} + handler := &PlaybackHandler{playbackStore: store, WatchScrobbler: scrobbler} + + if err := recoverPendingTerminalScrobbles(context.Background(), handler); err != nil { + t.Fatalf("recover poison batch: %v", err) + } + if _, ok := store.GetFinalizable("play-100", "token-1"); !ok { + t.Fatal("first bounded scan unexpectedly reached the event after the poison batch") + } + if err := recoverPendingTerminalScrobbles(context.Background(), handler); err != nil { + t.Fatalf("recover after poison batch: %v", err) + } + if _, ok := store.GetFinalizable("play-100", "token-1"); ok { + t.Fatal("event after poison batch remained starved on the next scan") + } + if calls := scrobbler.stopCalls.Load(); calls != compatTerminalRecoveryBatchSize+1 { + t.Fatalf("recovery attempts = %d, want %d", calls, compatTerminalRecoveryBatchSize+1) + } +} + +func TestReapedSessionFallbackHonorsProgressPersistencePolicy(t *testing.T) { + tests := []struct { + name string + known bool + disabled bool + }{ + {name: "disabled", known: true, disabled: true}, + {name: "legacy row with unknown policy", known: false, disabled: false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + handler, _, _, sourceID := newReportLivenessHandler("upstream-reaped", false) + scrobbler := &recordingCompatWatchScrobbler{} + handler.WatchScrobbler = scrobbler + if err := handler.playbackStore.Update("play-1", func(session *PlaybackSession) error { + session.ProgressPersistenceKnown = tt.known + session.DisableProgressPersistence = tt.disabled + return nil + }); err != nil { + t.Fatalf("set progress policy: %v", err) + } + + req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( + `{"PlaySessionId":"play-1","MediaSourceId":"`+sourceID+`","PositionTicks":900000000}`)) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"})) + rec := httptest.NewRecorder() + handler.HandleSessionPlayingStopped(rec, req) + + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + if len(scrobbler.calls) != 0 { + t.Fatalf("privacy-suppressed fallback emitted scrobble: %+v", scrobbler.calls) + } + }) + } +} + +func TestHandleSessionStoppedScrobblesAfterUpstreamSessionWasReaped(t *testing.T) { + handler, _, _, sourceID := newReportLivenessHandler("upstream-reaped", false) + scrobbler := &recordingCompatWatchScrobbler{} + handler.WatchScrobbler = scrobbler + + req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Stopped", strings.NewReader( + `{"PlaySessionId":"play-1","MediaSourceId":"`+sourceID+`","PositionTicks":900000000}`)) + req = req.WithContext(context.WithValue(req.Context(), compatSessionKey, + &Session{Token: "token-1", StreamAppUserID: 7, ProfileID: "profile-1"})) + rec := httptest.NewRecorder() + handler.HandleSessionPlayingStopped(rec, req) + + if rec.Code != http.StatusNoContent { + t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String()) + } + if len(scrobbler.calls) != 1 || scrobbler.calls[0].action != "stop" { + t.Fatalf("scrobble calls = %+v, want one stop", scrobbler.calls) + } + event := scrobbler.calls[0].event + if event.PlaybackSessionID != "upstream-reaped" || event.UserID != 7 || event.ProfileID != "profile-1" { + t.Fatalf("stop scope = %+v", event) + } + if event.MediaItemID != "movie-1" || event.PositionSeconds != 90 || event.DurationSeconds != 3600 { + t.Fatalf("stop progress = %+v", event) + } + if _, ok := handler.playbackStore.GetFinalizable("play-1", "token-1"); ok { + t.Fatal("stopped compat session should be consumed") + } +} + +func TestTeardownStillCleansLocalPlaybackAfterAnotherCallerClaimsStop(t *testing.T) { + mgr := &testCompatSessionManager{sessions: map[string]*playback.Session{ + "upstream-1": {ID: "upstream-1", UserID: 7, ProfileID: "profile-1", MediaFileID: 42}, + }} + h, store := newActiveEncodingsHandler(mgr) + scrobbler := &recordingCompatWatchScrobbler{} + h.WatchScrobbler = scrobbler + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + ItemID: "movie-1", + UpstreamSessionID: "upstream-1", + MediaSources: []PlaybackMediaSource{{ID: "source-1", FileID: 42, Version: testCompatVersion()}}, + }) + candidate, ok := store.Get("play-1") + if !ok { + t.Fatal("playback session missing") + } + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1", UserID: 7, ProfileID: "profile-1"} + if _, err := store.StageTerminal("play-1", "token-1", event, true); err != nil { + t.Fatal("failed to stage competing terminal event") + } + if _, err := store.ClaimTerminal("play-1", "token-1", time.Now().Add(compatTerminalClaimLease)); err != nil { + t.Fatal("failed to simulate a competing terminal delivery claim") + } + + h.teardownPlaySession(context.Background(), candidate, nil, nil) + + if _, err := mgr.GetSession("upstream-1"); err == nil { + t.Fatal("local upstream session was not cleaned up after losing the terminal claim") + } + if len(scrobbler.calls) != 0 { + t.Fatalf("losing teardown emitted provider event: %+v", scrobbler.calls) + } +} diff --git a/internal/jellycompat/playback_sessions.go b/internal/jellycompat/playback_sessions.go index 584f7086..fbac880b 100644 --- a/internal/jellycompat/playback_sessions.go +++ b/internal/jellycompat/playback_sessions.go @@ -1,31 +1,56 @@ package jellycompat import ( + "context" + "errors" + "sort" "sync" "time" "github.com/Silo-Server/silo-server/internal/catalog" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/watchsync" ) +// ErrTerminalClaimUnavailable means a staged terminal event still exists but +// another process owns its delivery lease (or already sent its fallback). +var ErrTerminalClaimUnavailable = errors.New("compat terminal event claim unavailable") + // PlaybackSession stores compat-owned playback negotiation state before the // native Silo playback session starts. type PlaybackSession struct { ID string CompatToken string - ItemID string - RouteItemID string + // ClientDeviceID identifies the Jellyfin client installation that created + // this negotiation. Stock Jellyfin Web can issue a second PlaybackInfo + // request for the same play before it starts either response; the newer + // negotiation replaces an older, still-unstarted one from the same device. + ClientDeviceID string + ItemID string + RouteItemID string // ClientPlaySessionID records the client's own generated PlaySessionId // when it differs from ours (Static=true direct play skips PlaybackInfo, // so the client never learns the server id). Playback reports carrying // that id resolve to this session directly instead of by ambiguous route. - ClientPlaySessionID string - UserID string - InitialSeekSeconds float64 - MediaSources []PlaybackMediaSource - UpstreamSessionID string - UpstreamPlayMethod string - TranscodeStarted bool + ClientPlaySessionID string + UserID string + InitialSeekSeconds float64 + MediaSources []PlaybackMediaSource + UpstreamSessionID string + UpstreamPlayMethod string + TranscodeStarted bool + ProgressPersistenceKnown bool + DisableProgressPersistence bool + // Terminal hides a play session from stream and progress routing after + // ActiveEncodings cleanup while retaining the authenticated mapping long + // enough for a later Stopped report to publish its authoritative position. + Terminal bool + TerminalAuthoritative bool + TerminalFallbackSent bool + TerminalClaimUntil time.Time + TerminalEventVersion int64 + TerminalClaimVersion int64 + TerminalScrobbleEvent *watchsync.ScrobbleEvent // Recipe is the transcode reconstruction descriptor for this session. Jellyfin // clients cannot round-trip a native stream token, so jellycompat carries the // recipe in its own durable compat store (this struct, persisted as JSONB) @@ -61,10 +86,34 @@ type PlaybackMediaSource struct { type CompatPlaybackStore interface { // Put stores or replaces a compat playback session. Put(session PlaybackSession) + // PutNegotiated stores a PlaybackInfo negotiation and atomically replaces + // older, still-unstarted negotiations for the same client device and item. + PutNegotiated(session PlaybackSession) // Get returns a session when it exists and is not expired. Get(id string) (*PlaybackSession, bool) // Delete removes a session. Delete(id string) + // HideFromRouting immediately makes a caller-owned session unavailable to + // stream/progress routing before slower durable terminal staging begins. + HideFromRouting(id, compatToken string) error + // StageTerminal hides a session from playback routing and durably records + // the provider event. An authoritative Stopped event replaces a fallback; + // a later fallback can never replace an authoritative event. + StageTerminal(id, compatToken string, event watchsync.ScrobbleEvent, authoritative bool) (*PlaybackSession, error) + // ClaimTerminal leases one staged event for delivery across server processes. + ClaimTerminal(id, compatToken string, claimUntil time.Time) (*PlaybackSession, error) + // ReleaseTerminalClaim releases an exact lease after delivery failure, or + // records a delivered fallback while retaining the row for a later Stopped. + ReleaseTerminalClaim(id, compatToken string, claimUntil time.Time, claimVersion int64, fallbackSent bool) + // CompleteTerminal deletes an authoritatively delivered terminal row only + // when the caller still owns the exact lease. + CompleteTerminal(id, compatToken string, claimUntil time.Time, claimVersion int64) + // ListPendingTerminals returns retryable authoritative events and unsent + // fallbacks for startup/periodic delivery recovery. + ListPendingTerminals(ctx context.Context, limit int) ([]PlaybackSession, error) + // GetFinalizable reads an active or terminal caller-owned session for report + // validation before an atomic Take. + GetFinalizable(id, compatToken string) (*PlaybackSession, bool) // Update modifies a session in place under the store's lock. Update(id string, fn func(*PlaybackSession) error) error // FindByRoute resolves a route item / media-source id to a session. @@ -73,6 +122,18 @@ type CompatPlaybackStore interface { // alias recorded for plays that skipped PlaybackInfo. The alias must // identify exactly one live session; ambiguity returns not-found. FindByClientPlaySessionID(compatToken, clientPlaySessionID string) (*PlaybackSession, bool) + // FindFinalizableByClientPlaySessionID is the stop-report variant of alias + // lookup and includes terminal sessions retained by Deactivate. The report + // identifiers disambiguate clients that reuse an alias across plays. + FindFinalizableByClientPlaySessionID( + compatToken, clientPlaySessionID, routeItemID, mediaSourceID string, + ) (*PlaybackSession, bool) + // FindFinalizableByRoute resolves exactly one active or terminal session for + // a caller-owned route. Ambiguous matches return not-found. + FindFinalizableByRoute(compatToken, routeID string) (*PlaybackSession, *PlaybackMediaSource, bool) + // FindByUpstreamSessionID resolves the local upstream session that owns a + // compat play. It is used for process-local failure lifecycle handling. + FindByUpstreamSessionID(upstreamSessionID string) (*PlaybackSession, bool) } // PlaybackSessionStore keeps compat playback sessions in memory. It is the @@ -82,6 +143,9 @@ type PlaybackSessionStore struct { sessions map[string]PlaybackSession ttl time.Duration now func() time.Time + // pendingCursor rotates bounded recovery scans through the full queue so a + // permanently failing first batch cannot starve later terminal events. + pendingCursor string } // NewPlaybackSessionStore creates a new playback session store. @@ -108,6 +172,16 @@ func (s *PlaybackSessionStore) Put(session PlaybackSession) { s.putNormalized(session) } +// PutNegotiated stores a freshly-created PlaybackInfo session. Jellyfin Web +// may negotiate the same play twice and then request both manifests; retaining +// both creates two native sessions, while only the newer one receives progress +// and Stopped reports. Replacing only unstarted sessions keeps real concurrent +// playback intact while preventing the abandoned negotiation from later +// publishing a stale pause. +func (s *PlaybackSessionStore) PutNegotiated(session PlaybackSession) { + s.putNegotiatedNormalized(session) +} + // putNormalized stores or replaces a compat playback session and returns the // stored copy with normalized timestamps (CreatedAt/UpdatedAt/ExpiresAt). The // durable wrapper uses the return value to persist the same timestamps the cache @@ -117,14 +191,43 @@ func (s *PlaybackSessionStore) putNormalized(session PlaybackSession) PlaybackSe s.mu.Lock() defer s.mu.Unlock() - if session.CreatedAt.IsZero() { - session.CreatedAt = s.now() + session = s.normalizeSession(session) + s.sessions[session.ID] = session + return session +} + +func (s *PlaybackSessionStore) putNegotiatedNormalized(session PlaybackSession) (PlaybackSession, []string) { + s.mu.Lock() + defer s.mu.Unlock() + + session = s.normalizeSession(session) + removed := make([]string, 0, 1) + if session.CompatToken != "" && session.ClientDeviceID != "" && session.RouteItemID != "" { + for id, existing := range s.sessions { + if id == session.ID || existing.Terminal || existing.UpstreamSessionID != "" { + continue + } + if existing.CompatToken == session.CompatToken && + existing.ClientDeviceID == session.ClientDeviceID && + mediaSourceIDsEqual(existing.RouteItemID, session.RouteItemID) { + delete(s.sessions, id) + removed = append(removed, id) + } + } } - session.UpdatedAt = s.now() + s.sessions[session.ID] = session + return session, removed +} + +func (s *PlaybackSessionStore) normalizeSession(session PlaybackSession) PlaybackSession { + now := s.now() + if session.CreatedAt.IsZero() { + session.CreatedAt = now + } + session.UpdatedAt = now if session.ExpiresAt.IsZero() { session.ExpiresAt = session.CreatedAt.Add(s.ttl) } - s.sessions[session.ID] = session return session } @@ -140,6 +243,9 @@ func (s *PlaybackSessionStore) Get(id string) (*PlaybackSession, bool) { s.Delete(id) return nil, false } + if session.Terminal { + return nil, false + } cp := session return &cp, true } @@ -151,13 +257,254 @@ func (s *PlaybackSessionStore) Delete(id string) { delete(s.sessions, id) } +func (s *PlaybackSessionStore) compatTokenForID(id string) string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.sessions[id].CompatToken +} + +// HideFromRouting marks a session terminal without requiring its provider event +// to have been staged yet. Final-report lookups remain available for retries. +func (s *PlaybackSessionStore) HideFromRouting(id, compatToken string) error { + s.mu.Lock() + defer s.mu.Unlock() + session, ok := s.sessions[id] + if !ok || session.CompatToken != compatToken || !session.ExpiresAt.After(s.now()) { + return ErrSessionNotFound + } + session.Terminal = true + session.UpdatedAt = s.now() + s.sessions[id] = session + return nil +} + +func (s *PlaybackSessionStore) terminalIDsByCompatToken(compatToken string) map[string]struct{} { + s.mu.Lock() + defer s.mu.Unlock() + result := make(map[string]struct{}) + now := s.now() + for id, session := range s.sessions { + if !session.ExpiresAt.After(now) { + delete(s.sessions, id) + continue + } + if session.CompatToken == compatToken && session.Terminal { + result[id] = struct{}{} + } + } + return result +} + +func (s *PlaybackSessionStore) replaceByCompatToken( + compatToken string, + replacements []PlaybackSession, + preserveIDs map[string]struct{}, +) []string { + s.mu.Lock() + defer s.mu.Unlock() + affected := make(map[string]struct{}, len(replacements)) + for id, session := range s.sessions { + _, preserve := preserveIDs[id] + if session.CompatToken == compatToken && !preserve { + delete(s.sessions, id) + affected[id] = struct{}{} + } + } + for _, session := range replacements { + if _, preserve := preserveIDs[session.ID]; preserve { + continue + } + if session.CreatedAt.IsZero() { + session.CreatedAt = s.now() + } + if session.ExpiresAt.IsZero() { + session.ExpiresAt = session.CreatedAt.Add(s.ttl) + } + s.sessions[session.ID] = session + affected[session.ID] = struct{}{} + } + result := make([]string, 0, len(affected)) + for id := range affected { + result = append(result, id) + } + return result +} + +// StageTerminal hides a playback session and records the event that must reach +// watch providers. Authoritative final reports replace provisional cleanup +// events; provisional events never overwrite an authoritative report. +func (s *PlaybackSessionStore) StageTerminal( + id string, + compatToken string, + event watchsync.ScrobbleEvent, + authoritative bool, +) (*PlaybackSession, error) { + s.mu.Lock() + defer s.mu.Unlock() + + session, ok := s.sessions[id] + if !ok || session.CompatToken != compatToken { + return nil, ErrSessionNotFound + } + if !session.ExpiresAt.After(s.now()) { + delete(s.sessions, id) + return nil, ErrSessionNotFound + } + if !session.TerminalAuthoritative || authoritative { + eventCopy := event + session.TerminalScrobbleEvent = &eventCopy + session.TerminalAuthoritative = authoritative + session.TerminalEventVersion++ + } + session.Terminal = true + session.UpdatedAt = s.now() + s.sessions[id] = session + return &session, nil +} + +// ClaimTerminal leases one staged event. Expired leases may be reclaimed after +// a process dies; a delivered fallback is skipped unless Stopped subsequently +// staged an authoritative replacement. +func (s *PlaybackSessionStore) ClaimTerminal(id, compatToken string, claimUntil time.Time) (*PlaybackSession, error) { + s.mu.Lock() + defer s.mu.Unlock() + + session, ok := s.sessions[id] + if !ok || session.CompatToken != compatToken || !session.Terminal || session.TerminalScrobbleEvent == nil { + return nil, ErrSessionNotFound + } + if !session.ExpiresAt.After(s.now()) { + delete(s.sessions, id) + return nil, ErrSessionNotFound + } + if session.TerminalClaimUntil.After(s.now()) || (session.TerminalFallbackSent && !session.TerminalAuthoritative) { + return nil, ErrTerminalClaimUnavailable + } + session.TerminalClaimUntil = claimUntil + session.TerminalClaimVersion = session.TerminalEventVersion + session.UpdatedAt = s.now() + s.sessions[id] = session + return &session, nil +} + +// ReleaseTerminalClaim releases an exact delivery lease. A stale caller cannot +// clear a successor's lease after its own lease expires. +func (s *PlaybackSessionStore) ReleaseTerminalClaim( + id string, + compatToken string, + claimUntil time.Time, + claimVersion int64, + fallbackSent bool, +) { + s.mu.Lock() + defer s.mu.Unlock() + + session, ok := s.sessions[id] + if !ok || session.CompatToken != compatToken || !session.TerminalClaimUntil.Equal(claimUntil) || + session.TerminalClaimVersion != claimVersion { + return + } + session.TerminalClaimUntil = time.Time{} + session.TerminalClaimVersion = 0 + if fallbackSent { + session.TerminalFallbackSent = true + } + session.UpdatedAt = s.now() + s.sessions[id] = session +} + +// CompleteTerminal removes an authoritatively delivered event while protecting +// a newer lease from a stale completion. +func (s *PlaybackSessionStore) CompleteTerminal(id, compatToken string, claimUntil time.Time, claimVersion int64) { + s.mu.Lock() + defer s.mu.Unlock() + + session, ok := s.sessions[id] + if !ok || session.CompatToken != compatToken || !session.TerminalAuthoritative || + !session.TerminalClaimUntil.Equal(claimUntil) || session.TerminalClaimVersion != claimVersion || + session.TerminalEventVersion != claimVersion { + return + } + delete(s.sessions, id) +} + +// ListPendingTerminals returns staged events that still need delivery. A +// successfully sent fallback remains retained for a possible authoritative +// replacement but is not itself pending. +func (s *PlaybackSessionStore) ListPendingTerminals(_ context.Context, limit int) ([]PlaybackSession, error) { + s.mu.Lock() + defer s.mu.Unlock() + if limit <= 0 { + limit = 100 + } + now := s.now() + eligible := make([]PlaybackSession, 0, len(s.sessions)) + for id, session := range s.sessions { + if !session.ExpiresAt.After(now) { + delete(s.sessions, id) + continue + } + if !session.Terminal || session.TerminalScrobbleEvent == nil || + (session.TerminalFallbackSent && !session.TerminalAuthoritative) { + continue + } + eligible = append(eligible, session) + } + sort.Slice(eligible, func(i, j int) bool { return eligible[i].ID < eligible[j].ID }) + start := sort.Search(len(eligible), func(i int) bool { return eligible[i].ID > s.pendingCursor }) + if start == len(eligible) && s.pendingCursor != "" { + start = 0 + s.pendingCursor = "" + } + end := min(start+limit, len(eligible)) + result := append([]PlaybackSession(nil), eligible[start:end]...) + if len(result) > 0 { + s.pendingCursor = result[len(result)-1].ID + } else { + s.pendingCursor = "" + } + return result, nil +} + +func (s *PlaybackSessionStore) deleteExpired() map[string]string { + s.mu.Lock() + defer s.mu.Unlock() + now := s.now() + removed := make(map[string]string) + for id, session := range s.sessions { + if !session.ExpiresAt.After(now) { + removed[id] = session.CompatToken + delete(s.sessions, id) + } + } + return removed +} + +// GetFinalizable returns an active or terminal caller-owned session so a stop +// report can validate its media fields before atomically consuming it. +func (s *PlaybackSessionStore) GetFinalizable(id, compatToken string) (*PlaybackSession, bool) { + s.mu.Lock() + defer s.mu.Unlock() + + session, ok := s.sessions[id] + if !ok || session.CompatToken != compatToken { + return nil, false + } + if !session.ExpiresAt.After(s.now()) { + delete(s.sessions, id) + return nil, false + } + copy := session + return ©, true +} + // Update modifies a playback session in place. func (s *PlaybackSessionStore) Update(id string, fn func(*PlaybackSession) error) error { s.mu.Lock() defer s.mu.Unlock() session, ok := s.sessions[id] - if !ok { + if !ok || session.Terminal { return ErrSessionNotFound } if !session.ExpiresAt.After(s.now()) { @@ -178,6 +525,27 @@ func (s *PlaybackSessionStore) Update(id string, fn func(*PlaybackSession) error // PlaySessionId across plays makes the alias ambiguous, and the caller should // fall back to route matching instead of binding an arbitrary session. func (s *PlaybackSessionStore) FindByClientPlaySessionID(compatToken, clientPlaySessionID string) (*PlaybackSession, bool) { + return s.findByClientPlaySessionID(compatToken, clientPlaySessionID, "", "", false) +} + +// FindFinalizableByClientPlaySessionID includes terminal sessions retained for +// an authoritative final report. Route identifiers narrow reused client aliases +// to the play described by that report before the uniqueness check runs. +func (s *PlaybackSessionStore) FindFinalizableByClientPlaySessionID( + compatToken, clientPlaySessionID, routeItemID, mediaSourceID string, +) (*PlaybackSession, bool) { + return s.findByClientPlaySessionID( + compatToken, clientPlaySessionID, routeItemID, mediaSourceID, true, + ) +} + +func (s *PlaybackSessionStore) findByClientPlaySessionID( + compatToken string, + clientPlaySessionID string, + routeItemID string, + mediaSourceID string, + includeTerminal bool, +) (*PlaybackSession, bool) { if clientPlaySessionID == "" { return nil, false } @@ -187,12 +555,18 @@ func (s *PlaybackSessionStore) FindByClientPlaySessionID(compatToken, clientPlay now := s.now() var match *PlaybackSession for _, session := range s.sessions { - if !session.ExpiresAt.After(now) { + if !session.ExpiresAt.After(now) || (!includeTerminal && session.Terminal) { continue } if session.CompatToken != compatToken { continue } + if routeItemID != "" && !mediaSourceIDsEqual(session.RouteItemID, routeItemID) { + continue + } + if mediaSourceID != "" && findMediaSource(&session, mediaSourceID) == nil { + continue + } if session.ClientPlaySessionID == clientPlaySessionID { if match != nil { return nil, false @@ -204,14 +578,50 @@ func (s *PlaybackSessionStore) FindByClientPlaySessionID(compatToken, clientPlay return match, match != nil } -// FindByRoute resolves a route item/media-source identifier to a compat playback session. -func (s *PlaybackSessionStore) FindByRoute(compatToken, routeID string) (*PlaybackSession, *PlaybackMediaSource, bool) { +// FindByUpstreamSessionID resolves the unique compat play attached to a local +// upstream session. +func (s *PlaybackSessionStore) FindByUpstreamSessionID(upstreamSessionID string) (*PlaybackSession, bool) { + if upstreamSessionID == "" { + return nil, false + } s.mu.RLock() defer s.mu.RUnlock() now := s.now() for _, session := range s.sessions { - if !session.ExpiresAt.After(now) { + if session.ExpiresAt.After(now) && !session.Terminal && session.UpstreamSessionID == upstreamSessionID { + copy := session + return ©, true + } + } + return nil, false +} + +// FindByRoute resolves a route item/media-source identifier to a compat playback session. +func (s *PlaybackSessionStore) FindByRoute(compatToken, routeID string) (*PlaybackSession, *PlaybackMediaSource, bool) { + return s.findByRoute(compatToken, routeID, false, false) +} + +// FindFinalizableByRoute includes terminal sessions retained for a stopped +// report, but only returns a unique token-scoped match. +func (s *PlaybackSessionStore) FindFinalizableByRoute( + compatToken, routeID string, +) (*PlaybackSession, *PlaybackMediaSource, bool) { + return s.findByRoute(compatToken, routeID, true, true) +} + +func (s *PlaybackSessionStore) findByRoute( + compatToken, routeID string, + includeTerminal, requireUnique bool, +) (*PlaybackSession, *PlaybackMediaSource, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + now := s.now() + var matchedSession *PlaybackSession + var matchedSource *PlaybackMediaSource + for _, session := range s.sessions { + if !session.ExpiresAt.After(now) || (!includeTerminal && session.Terminal) { continue } if compatToken != "" && session.CompatToken != compatToken { @@ -221,17 +631,33 @@ func (s *PlaybackSessionStore) FindByRoute(compatToken, routeID string) (*Playba // whatever casing/dash format the client model uses, which may differ // from the raw route param captured at stream time. if mediaSourceIDsEqual(session.RouteItemID, routeID) { + if requireUnique && matchedSession != nil { + return nil, nil, false + } cp := session - return &cp, nil, true + matchedSession = &cp + matchedSource = nil + if !requireUnique { + return matchedSession, nil, true + } + continue } for _, source := range session.MediaSources { if mediaSourceIDsEqual(source.ID, routeID) { + if requireUnique && matchedSession != nil { + return nil, nil, false + } cp := session sourceCopy := source - return &cp, &sourceCopy, true + matchedSession = &cp + matchedSource = &sourceCopy + if !requireUnique { + return matchedSession, matchedSource, true + } + break } } } - return nil, nil, false + return matchedSession, matchedSource, matchedSession != nil } diff --git a/internal/jellycompat/playback_sessions_postgres.go b/internal/jellycompat/playback_sessions_postgres.go index 926cf004..a8cf174d 100644 --- a/internal/jellycompat/playback_sessions_postgres.go +++ b/internal/jellycompat/playback_sessions_postgres.go @@ -6,12 +6,34 @@ import ( "errors" "fmt" "log/slog" + "sync" "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/watchsync" ) +// compatCacheRevalidationInterval bounds how long one process may retain an +// active routing view after another process terminalizes the durable row. It +// also keeps two-second HLS segment requests on the in-memory hot path instead +// of adding a Postgres round trip to every segment. +const compatCacheRevalidationInterval = 5 * time.Second + +const compatValidationCacheLimit = 16_384 + +type pendingCompatPlaybackUpdate struct { + sequence uint64 + compatToken string + apply func(*PlaybackSession) error +} + +type compatCacheGeneration struct { + epoch uint64 + value uint64 +} + var ( _ CompatPlaybackStore = (*PlaybackSessionStore)(nil) _ CompatPlaybackStore = (*DurableCompatPlaybackStore)(nil) @@ -19,16 +41,37 @@ var ( // DurableCompatPlaybackStore is a CompatPlaybackStore that persists compat // playback sessions to Postgres so the PlaySessionId -> upstream-session mapping -// (and the negotiated media sources) survives a server restart. It wraps an -// in-memory PlaybackSessionStore as a write-through cache so the hot segment path -// (Get on every segment request) stays in-process; a cache miss falls back to a -// DB read and repopulates the cache. A Redis swap would reimplement this same -// interface, leaving every caller unchanged. +// (and the negotiated media sources) survives a server restart. The in-memory +// store remains a write-through working set. Active routing periodically +// revalidates durable rows so cross-process terminal transitions invalidate a +// stale cache within a bounded window without putting Postgres on every segment +// request's hot path. type DurableCompatPlaybackStore struct { mem *PlaybackSessionStore pool *pgxpool.Pool ttl time.Duration now func() time.Time + + validationMu sync.Mutex + validatedIDs map[string]time.Time + validatedTokens map[string]time.Time + unpersistedIDs map[string]struct{} + pendingUpdateMu sync.Mutex + pendingUpdateSequence uint64 + pendingUpdates map[string][]pendingCompatPlaybackUpdate + pendingCursorMu sync.Mutex + pendingCursor string + + // cacheMutationMu lets unrelated writes proceed concurrently while durable + // read snapshots take an exclusive lock only for their in-memory apply step. + // Per-ID/token generations discard snapshots that overlapped a mutation in + // the same routing scope. + cacheMutationMu sync.RWMutex + sessionMutations [256]sync.Mutex + generationMu sync.Mutex + generationEpoch uint64 + idGenerations map[string]uint64 + tokenGenerations map[string]uint64 } // NewDurableCompatPlaybackStore returns a Postgres-backed compat store. pool must @@ -41,10 +84,16 @@ func NewDurableCompatPlaybackStore(pool *pgxpool.Pool, ttl time.Duration, now fu ttl = 6 * time.Hour } return &DurableCompatPlaybackStore{ - mem: NewPlaybackSessionStore(ttl, now), - pool: pool, - ttl: ttl, - now: now, + mem: NewPlaybackSessionStore(ttl, now), + pool: pool, + ttl: ttl, + now: now, + validatedIDs: make(map[string]time.Time), + validatedTokens: make(map[string]time.Time), + unpersistedIDs: make(map[string]struct{}), + pendingUpdates: make(map[string][]pendingCompatPlaybackUpdate), + idGenerations: make(map[string]uint64), + tokenGenerations: make(map[string]uint64), } } @@ -59,35 +108,211 @@ func NewDurableCompatPlaybackStore(pool *pgxpool.Pool, ttl time.Duration, now fu // across a fresh instance. The upsert itself is still best-effort (a DB failure // is logged, not propagated); the cache holds the authoritative in-process state. func (d *DurableCompatPlaybackStore) Put(session PlaybackSession) { - stored := d.mem.putNormalized(session) if d.pool == nil { + d.mem.Put(session) return } + unlockSession := d.lockSessionMutation(session.ID) + defer unlockSession() + d.cacheMutationMu.RLock() + stored := d.mem.putNormalized(session) + defer d.finishCacheMutation(stored.ID, stored.CompatToken) + d.markIDValidated(stored.ID) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - d.upsert(ctx, stored) + if err := d.upsert(ctx, stored); err != nil { + d.markUnpersisted(stored.ID) + } else { + d.clearUnpersisted(stored.ID) + } } -// Get returns the cached session, falling back to Postgres on a miss (e.g. after -// a restart) and repopulating the cache. -func (d *DurableCompatPlaybackStore) Get(id string) (*PlaybackSession, bool) { - if s, ok := d.mem.Get(id); ok { - return s, true +// PutNegotiated persists a new PlaybackInfo session while removing older, +// unstarted negotiations for the same compat token, client device, and item. +// The advisory transaction lock makes the replacement atomic across Silo +// processes; the in-memory mutation is likewise atomic for the local process. +func (d *DurableCompatPlaybackStore) PutNegotiated(session PlaybackSession) { + if d.pool == nil { + d.mem.PutNegotiated(session) + return + } + + scope := "negotiated\x00" + session.CompatToken + "\x00" + session.ClientDeviceID + "\x00" + session.RouteItemID + unlockSession := d.lockSessionMutation(scope) + defer unlockSession() + d.cacheMutationMu.RLock() + stored, locallyRemoved := d.mem.putNegotiatedNormalized(session) + defer d.finishCacheMutation(stored.ID, stored.CompatToken) + d.markIDValidated(stored.ID) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + durablyRemoved, err := d.replaceUnstartedNegotiation(ctx, stored) + if err != nil { + d.markUnpersisted(stored.ID) + slog.WarnContext(ctx, "persist negotiated compat playback session failed", + "component", "jellycompat", + "error", err, + "play_session_id", stored.ID, + ) + } else { + d.clearUnpersisted(stored.ID) + } + + removed := make(map[string]struct{}, len(locallyRemoved)+len(durablyRemoved)) + for _, id := range locallyRemoved { + removed[id] = struct{}{} + } + for _, id := range durablyRemoved { + removed[id] = struct{}{} + } + for id := range removed { + d.invalidateValidation(id, "") + d.clearUnpersisted(id) + d.clearPendingUpdates(id) + d.bumpCacheGenerations(id, "") + } + d.invalidateValidation("", stored.CompatToken) +} + +func (d *DurableCompatPlaybackStore) replaceUnstartedNegotiation( + ctx context.Context, + session PlaybackSession, +) ([]string, error) { + tx, err := d.pool.Begin(ctx) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback(ctx) }() + + var removed []string + if session.CompatToken != "" && session.ClientDeviceID != "" && session.RouteItemID != "" { + scope := session.CompatToken + "\x00" + session.ClientDeviceID + "\x00" + session.RouteItemID + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, scope); err != nil { + return nil, err + } + rows, err := tx.Query(ctx, ` + DELETE FROM jellycompat_playback_sessions + WHERE id <> $1 + AND compat_token = $2 + AND data->>'ClientDeviceID' = $3 + AND data->>'RouteItemID' = $4 + AND COALESCE(data->>'UpstreamSessionID', '') = '' + AND COALESCE((data->>'Terminal')::boolean, false) = false + AND expires_at > $5 + RETURNING id + `, session.ID, session.CompatToken, session.ClientDeviceID, session.RouteItemID, d.now()) + if err != nil { + return nil, err + } + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + rows.Close() + return nil, err + } + removed = append(removed, id) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, err + } + rows.Close() + } + + data, err := json.Marshal(session) + if err != nil { + return nil, err + } + expiresAt := session.ExpiresAt + if expiresAt.IsZero() { + expiresAt = d.now().Add(d.ttl) + } + if _, err := tx.Exec( + ctx, upsertSessionQuery, + session.ID, session.CompatToken, session.UserID, data, expiresAt, + ); err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, err + } + return removed, nil +} + +// Get periodically revalidates the durable row before returning an active +// session. Query failures preserve a still-valid cache entry: a temporary DB +// outage must not interrupt an already-playing stream. +func (d *DurableCompatPlaybackStore) Get(id string) (*PlaybackSession, bool) { + if d.pool == nil { + return d.mem.Get(id) + } + cached, cachedOK := d.mem.Get(id) + if cachedOK && cached.UpstreamSessionID != "" && !d.shouldRevalidateID(id) { + return cached, cachedOK + } + if !cachedOK { + // Cold callers must load (or share a future single-flight); another + // request reserving the throttle window is not proof the row is absent. + _ = d.shouldRevalidateID(id) + } + generation := d.idGenerationSnapshot(id) + s, ok, err := d.load(id) + if err != nil { + return cached, cachedOK + } + d.cacheMutationMu.Lock() + defer d.cacheMutationMu.Unlock() + if generation != d.idGenerationSnapshot(id) { + d.invalidateValidation(id, "") + return d.mem.Get(id) + } + if ok && s.Terminal { + d.clearPendingUpdates(id) + } else if ok && d.hasPendingUpdates(id) { + if current, currentOK := d.mem.Get(id); currentOK { + return current, true + } + } + if ok && !s.Terminal { + if local, localOK := d.mem.GetFinalizable(id, s.CompatToken); localOK && local.Terminal { + return nil, false + } } - s, ok := d.load(id) if !ok { + compatToken := d.mem.compatTokenForID(id) + if current, currentOK := d.mem.GetFinalizable(id, compatToken); currentOK && d.repairUnpersisted(current) { + return d.mem.Get(id) + } + d.clearUnpersisted(id) + d.clearPendingUpdates(id) + d.mem.Delete(id) + if cachedOK || compatToken != "" { + d.bumpCacheGenerations(id, "") + } return nil, false } + d.clearUnpersisted(id) d.mem.Put(*s) + d.bumpCacheGenerations(id, s.CompatToken) return d.mem.Get(id) } // Delete removes the session from both the cache and Postgres. func (d *DurableCompatPlaybackStore) Delete(id string) { - d.mem.Delete(id) if d.pool == nil { + d.mem.Delete(id) return } + unlockSession := d.lockSessionMutation(id) + defer unlockSession() + d.cacheMutationMu.RLock() + compatToken := d.mem.compatTokenForID(id) + defer d.finishCacheMutation(id, compatToken) + d.invalidateValidation(id, compatToken) + d.clearUnpersisted(id) + d.clearPendingUpdates(id) + d.mem.Delete(id) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() if _, err := d.pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id); err != nil { @@ -95,6 +320,377 @@ func (d *DurableCompatPlaybackStore) Delete(id string) { } } +// HideFromRouting immediately makes the local routing cache terminal. It is +// intentionally independent of Postgres so a staging outage cannot let a +// stopped client reconstruct a fresh upstream session. +func (d *DurableCompatPlaybackStore) HideFromRouting(id, compatToken string) error { + if d.pool == nil { + return d.mem.HideFromRouting(id, compatToken) + } + unlockSession := d.lockSessionMutation(id) + defer unlockSession() + d.cacheMutationMu.RLock() + defer d.finishCacheMutation(id, compatToken) + if err := d.mem.HideFromRouting(id, compatToken); err != nil { + return err + } + d.clearPendingUpdates(id) + if d.isUnpersisted(id) { + cached, ok := d.mem.GetFinalizable(id, compatToken) + if !ok { + return ErrSessionNotFound + } + inserted, err := d.insertIfAbsent(cached) + if err != nil { + return err + } + d.clearUnpersisted(id) + if inserted { + return nil + } + // A row appeared after the failed Put. Fall through and terminalize that + // durable row rather than leaving another process able to route it. + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + tag, err := d.pool.Exec(ctx, ` + UPDATE jellycompat_playback_sessions + SET data = jsonb_set(data, '{Terminal}', 'true'::jsonb, true) + WHERE id = $1 AND compat_token = $2 AND expires_at > $3 + `, id, compatToken, d.now()) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return ErrSessionNotFound + } + return nil +} + +// StageTerminal hides a session and persists the provider event under a row +// lock. This merge keeps an authoritative Stopped event from being overwritten +// by a later ActiveEncodings fallback on another server process. +func (d *DurableCompatPlaybackStore) StageTerminal( + id string, + compatToken string, + event watchsync.ScrobbleEvent, + authoritative bool, +) (*PlaybackSession, error) { + if d.pool == nil { + return d.mem.StageTerminal(id, compatToken, event, authoritative) + } + unlockSession := d.lockSessionMutation(id) + defer unlockSession() + d.cacheMutationMu.RLock() + defer d.finishCacheMutation(id, compatToken) + if err := d.mem.HideFromRouting(id, compatToken); err != nil { + return nil, err + } + d.clearPendingUpdates(id) + if d.isUnpersisted(id) { + cached, ok := d.mem.GetFinalizable(id, compatToken) + if !ok { + return nil, ErrSessionNotFound + } + candidate := *cached + eventCopy := event + candidate.Terminal = true + candidate.TerminalAuthoritative = authoritative + candidate.TerminalScrobbleEvent = &eventCopy + candidate.TerminalEventVersion++ + if _, err := d.insertIfAbsent(&candidate); err != nil { + return nil, err + } + d.clearUnpersisted(id) + } + + committed, err := d.stageTerminalDB(id, compatToken, event, authoritative) + if err != nil { + slog.Warn("stage durable compat terminal event failed", "error", err, "play_session_id", id) + return nil, err + } + if committed == nil { + d.mem.Delete(id) + return nil, ErrSessionNotFound + } + d.mem.Delete(id) + d.mem.Put(*committed) + d.markIDValidated(committed.ID) + return committed, nil +} + +func (d *DurableCompatPlaybackStore) stageTerminalDB( + id string, + compatToken string, + event watchsync.ScrobbleEvent, + authoritative bool, +) (*PlaybackSession, error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + tx, err := d.pool.Begin(ctx) + if err != nil { + return nil, err + } + defer func() { _ = tx.Rollback(ctx) }() + + var raw []byte + err = tx.QueryRow(ctx, ` + SELECT data + FROM jellycompat_playback_sessions + WHERE id = $1 AND compat_token = $2 AND expires_at > $3 + FOR UPDATE + `, id, compatToken, d.now()).Scan(&raw) + if errors.Is(err, pgx.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + + var session PlaybackSession + if err := json.Unmarshal(raw, &session); err != nil { + return nil, err + } + if !session.TerminalAuthoritative || authoritative { + eventCopy := event + session.TerminalScrobbleEvent = &eventCopy + session.TerminalAuthoritative = authoritative + session.TerminalEventVersion++ + } + session.Terminal = true + session.UpdatedAt = d.now() + data, err := json.Marshal(session) + if err != nil { + return nil, err + } + if _, err := tx.Exec(ctx, `UPDATE jellycompat_playback_sessions SET data = $2 WHERE id = $1`, id, data); err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, err + } + return &session, nil +} + +// ClaimTerminal leases one pending terminal event across processes without +// deleting its retry state. Expired leases can be reclaimed after a crash. +func (d *DurableCompatPlaybackStore) ClaimTerminal(id, compatToken string, claimUntil time.Time) (*PlaybackSession, error) { + if d.pool == nil { + return d.mem.ClaimTerminal(id, compatToken, claimUntil) + } + unlockSession := d.lockSessionMutation(id) + defer unlockSession() + d.cacheMutationMu.RLock() + defer d.finishCacheMutation(id, compatToken) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + leaseDuration := claimUntil.Sub(d.now()) + if leaseDuration <= 0 { + leaseDuration = compatTerminalClaimLease + } + var dbNow time.Time + if err := d.pool.QueryRow(ctx, `SELECT clock_timestamp()`).Scan(&dbNow); err != nil { + return nil, err + } + durableClaimUntil := dbNow.Add(leaseDuration).UTC().Truncate(time.Microsecond) + claimText := durableClaimUntil.Format(time.RFC3339Nano) + var raw []byte + err := d.pool.QueryRow(ctx, ` + UPDATE jellycompat_playback_sessions + SET data = jsonb_set( + jsonb_set(data, '{TerminalClaimUntil}', to_jsonb($4::text), true), + '{TerminalClaimVersion}', + to_jsonb(COALESCE((data->>'TerminalEventVersion')::bigint, 0)), + true + ) + WHERE id = $1 + AND compat_token = $2 + AND expires_at > $5 + AND COALESCE((data->>'Terminal')::boolean, false) = true + AND COALESCE(data->'TerminalScrobbleEvent' <> 'null'::jsonb, false) + AND COALESCE( + NULLIF(data->>'TerminalClaimUntil', '0001-01-01T00:00:00Z')::timestamptz, + '-infinity'::timestamptz + ) <= $3 + AND ( + COALESCE((data->>'TerminalFallbackSent')::boolean, false) = false + OR COALESCE((data->>'TerminalAuthoritative')::boolean, false) = true + ) + RETURNING data + `, id, compatToken, dbNow, claimText, d.now()).Scan(&raw) + if errors.Is(err, pgx.ErrNoRows) { + var exists bool + existsErr := d.pool.QueryRow(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM jellycompat_playback_sessions + WHERE id = $1 AND compat_token = $2 AND expires_at > $3 + ) + `, id, compatToken, d.now()).Scan(&exists) + if existsErr != nil { + return nil, existsErr + } + if exists { + return nil, ErrTerminalClaimUnavailable + } + return nil, ErrSessionNotFound + } + if err != nil { + slog.Warn("claim compat terminal event failed", "error", err, "play_session_id", id) + return nil, err + } + + var session PlaybackSession + if err := json.Unmarshal(raw, &session); err != nil { + slog.Warn("unmarshal claimed compat terminal event failed", "error", err, "play_session_id", id) + return nil, err + } + d.mem.Delete(id) + d.mem.Put(session) + d.markIDValidated(session.ID) + return &session, nil +} + +// ReleaseTerminalClaim releases an exact lease and optionally records that the +// provisional ActiveEncodings fallback reached the durable watch-sync queue. +func (d *DurableCompatPlaybackStore) ReleaseTerminalClaim( + id string, + compatToken string, + claimUntil time.Time, + claimVersion int64, + fallbackSent bool, +) { + if d.pool == nil { + d.mem.ReleaseTerminalClaim(id, compatToken, claimUntil, claimVersion, fallbackSent) + return + } + unlockSession := d.lockSessionMutation(id) + defer unlockSession() + d.cacheMutationMu.RLock() + defer d.finishCacheMutation(id, compatToken) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + var raw []byte + err := d.pool.QueryRow(ctx, ` + UPDATE jellycompat_playback_sessions + SET data = CASE WHEN $5 + THEN jsonb_set(data - 'TerminalClaimUntil' - 'TerminalClaimVersion', '{TerminalFallbackSent}', 'true'::jsonb, true) + ELSE data - 'TerminalClaimUntil' - 'TerminalClaimVersion' + END + WHERE id = $1 + AND compat_token = $2 + AND (data->>'TerminalClaimUntil')::timestamptz = $3 + AND COALESCE((data->>'TerminalClaimVersion')::bigint, 0) = $4 + RETURNING data + `, id, compatToken, claimUntil, claimVersion, fallbackSent).Scan(&raw) + if err != nil { + if !errors.Is(err, pgx.ErrNoRows) { + slog.Warn("release compat terminal event claim failed", "error", err, "play_session_id", id) + } + d.mem.ReleaseTerminalClaim(id, compatToken, claimUntil, claimVersion, fallbackSent) + return + } + var session PlaybackSession + if err := json.Unmarshal(raw, &session); err == nil { + d.mem.Delete(id) + d.mem.Put(session) + d.markIDValidated(session.ID) + } +} + +// CompleteTerminal deletes an authoritatively queued event only while the +// caller still owns its exact lease. +func (d *DurableCompatPlaybackStore) CompleteTerminal( + id string, + compatToken string, + claimUntil time.Time, + claimVersion int64, +) { + if d.pool == nil { + d.mem.CompleteTerminal(id, compatToken, claimUntil, claimVersion) + return + } + unlockSession := d.lockSessionMutation(id) + defer unlockSession() + d.cacheMutationMu.RLock() + defer d.finishCacheMutation(id, compatToken) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + tag, err := d.pool.Exec(ctx, ` + DELETE FROM jellycompat_playback_sessions + WHERE id = $1 + AND compat_token = $2 + AND COALESCE((data->>'TerminalAuthoritative')::boolean, false) = true + AND (data->>'TerminalClaimUntil')::timestamptz = $3 + AND COALESCE((data->>'TerminalClaimVersion')::bigint, 0) = $4 + AND COALESCE((data->>'TerminalEventVersion')::bigint, 0) = $4 + `, id, compatToken, claimUntil, claimVersion) + if err != nil { + slog.Warn("complete compat terminal event failed", "error", err, "play_session_id", id) + return + } + if tag.RowsAffected() > 0 { + d.mem.Delete(id) + d.invalidateValidation(id, compatToken) + } +} + +// GetFinalizable reads an active or terminal caller-owned session, checking the +// process cache before the durable row. +func (d *DurableCompatPlaybackStore) GetFinalizable(id, compatToken string) (*PlaybackSession, bool) { + if d.pool == nil { + return d.mem.GetFinalizable(id, compatToken) + } + cached, cachedOK := d.mem.GetFinalizable(id, compatToken) + if cachedOK && !d.shouldRevalidateID(id) { + return cached, cachedOK + } + if !cachedOK { + _ = d.shouldRevalidateID(id) + } + generation := d.idGenerationSnapshot(id) + session, ok, err := d.load(id) + if err != nil { + return cached, cachedOK + } + d.cacheMutationMu.Lock() + defer d.cacheMutationMu.Unlock() + if generation != d.idGenerationSnapshot(id) { + d.invalidateValidation(id, "") + return d.mem.GetFinalizable(id, compatToken) + } + if ok && session.Terminal { + d.clearPendingUpdates(id) + } else if ok && d.hasPendingUpdates(id) { + if current, currentOK := d.mem.GetFinalizable(id, compatToken); currentOK { + return current, true + } + } + if ok && !session.Terminal { + if local, localOK := d.mem.GetFinalizable(id, compatToken); localOK && local.Terminal { + return local, true + } + } + if !ok { + if current, currentOK := d.mem.GetFinalizable(id, compatToken); currentOK && d.repairUnpersisted(current) { + return d.mem.GetFinalizable(id, compatToken) + } + d.clearUnpersisted(id) + d.clearPendingUpdates(id) + d.mem.Delete(id) + if cachedOK { + d.bumpCacheGenerations(id, "") + } + return nil, false + } + d.clearUnpersisted(id) + d.mem.Put(*session) + d.bumpCacheGenerations(id, session.CompatToken) + return d.mem.GetFinalizable(id, compatToken) +} + // Update modifies the session in place under the cache's lock (in-process // atomicity), then persists the result. The session is loaded from Postgres into // the cache first when absent so an update after a restart still applies. @@ -108,21 +704,44 @@ func (d *DurableCompatPlaybackStore) Delete(id string) { // best-effort for availability: a DB failure is logged and the in-memory mutation // stands, but a successful DB read-modify-write is never silently lost. func (d *DurableCompatPlaybackStore) Update(id string, fn func(*PlaybackSession) error) error { + if d.pool == nil { + return d.mem.Update(id, fn) + } + unlockSession := d.lockSessionMutation(id) + defer unlockSession() + d.cacheMutationMu.RLock() + defer func() { + d.finishCacheMutation(id, d.mem.compatTokenForID(id)) + }() if _, ok := d.mem.Get(id); !ok { - if s, ok := d.load(id); ok { + if s, ok, err := d.load(id); err == nil && ok { d.mem.Put(*s) + d.markIDValidated(s.ID) } } if err := d.mem.Update(id, fn); err != nil { return err } - committed, err := d.updateDB(id, fn) + pending := d.pendingUpdatesSnapshot(id) + committed, err := d.updateDB(id, func(session *PlaybackSession) error { + for _, update := range pending { + if err := update.apply(session); err != nil { + return err + } + } + return fn(session) + }) if committed != nil { // Refresh the cache from the DB-authoritative committed row so the cache // reflects any concurrent writer's fields that fn merged on top of. d.mem.Put(*committed) + d.markIDValidated(committed.ID) + if len(pending) > 0 { + d.consumePendingUpdates(id, pending[len(pending)-1].sequence) + } } if err != nil { + d.appendPendingUpdate(id, d.mem.compatTokenForID(id), fn) // The in-memory mutation stands (live state is correct), but the durable // row was NOT updated: surface the failure so durability-sensitive callers // (recipe/upstream-session writes that promise restart resilience) can roll @@ -207,12 +826,9 @@ func (d *DurableCompatPlaybackStore) updateDB(id string, fn func(*PlaybackSessio return &session, nil } -// FindByRoute resolves a route id, checking the cache first and falling back to -// loading the matching compat-token rows from Postgres into the cache. +// FindByRoute periodically refreshes the caller's bounded durable row set before +// resolving a route. A refresh failure leaves the cached routing set intact. func (d *DurableCompatPlaybackStore) FindByRoute(compatToken, routeID string) (*PlaybackSession, *PlaybackMediaSource, bool) { - if s, src, ok := d.mem.FindByRoute(compatToken, routeID); ok { - return s, src, ok - } // An empty compat token cannot be pushed into a bounded, indexed DB query, so // the only DB fallback would be loading every live row and scanning it on this // request goroutine — an O(table) cliff. The sole caller @@ -220,30 +836,166 @@ func (d *DurableCompatPlaybackStore) FindByRoute(compatToken, routeID string) (* // empty-token DB fallback is never load-bearing for route resolution; return // the in-memory result rather than incurring a full-table scan. if compatToken == "" { - return nil, nil, false + return d.mem.FindByRoute(compatToken, routeID) } - d.loadByCompatToken(compatToken) + cachedSession, cachedSource, cachedOK := d.mem.FindByRoute(compatToken, routeID) + if cachedOK && !d.shouldRevalidateToken(compatToken) { + return cachedSession, cachedSource, true + } + _ = d.loadByCompatToken(compatToken) return d.mem.FindByRoute(compatToken, routeID) } +// FindFinalizableByRoute is the terminal-aware, uniqueness-enforcing route +// lookup used only by authenticated Stopped reports. +func (d *DurableCompatPlaybackStore) FindFinalizableByRoute( + compatToken, routeID string, +) (*PlaybackSession, *PlaybackMediaSource, bool) { + if compatToken == "" { + return d.mem.FindFinalizableByRoute(compatToken, routeID) + } + cachedSession, cachedSource, cachedOK := d.mem.FindFinalizableByRoute(compatToken, routeID) + if cachedOK && !d.shouldRevalidateToken(compatToken) { + return cachedSession, cachedSource, true + } + _ = d.loadByCompatToken(compatToken) + return d.mem.FindFinalizableByRoute(compatToken, routeID) +} + // FindByClientPlaySessionID resolves the client-generated PlaySessionId alias, // checking the cache first and falling back to loading the compat token's live // rows from Postgres into the cache (same bounded fallback as FindByRoute; the // alias uniqueness check runs against the repopulated cache). func (d *DurableCompatPlaybackStore) FindByClientPlaySessionID(compatToken, clientPlaySessionID string) (*PlaybackSession, bool) { - if s, ok := d.mem.FindByClientPlaySessionID(compatToken, clientPlaySessionID); ok { - return s, ok - } if compatToken == "" { + return d.mem.FindByClientPlaySessionID(compatToken, clientPlaySessionID) + } + cached, cachedOK := d.mem.FindByClientPlaySessionID(compatToken, clientPlaySessionID) + if cachedOK && !d.shouldRevalidateToken(compatToken) { + return cached, true + } + _ = d.loadByCompatToken(compatToken) + return d.mem.FindByClientPlaySessionID(compatToken, clientPlaySessionID) +} + +// FindFinalizableByClientPlaySessionID is the terminal-aware alias lookup used +// only by authenticated Stopped reports. +func (d *DurableCompatPlaybackStore) FindFinalizableByClientPlaySessionID( + compatToken, clientPlaySessionID, routeItemID, mediaSourceID string, +) (*PlaybackSession, bool) { + if compatToken == "" { + return d.mem.FindFinalizableByClientPlaySessionID( + compatToken, clientPlaySessionID, routeItemID, mediaSourceID, + ) + } + cached, cachedOK := d.mem.FindFinalizableByClientPlaySessionID( + compatToken, clientPlaySessionID, routeItemID, mediaSourceID, + ) + if cachedOK && !d.shouldRevalidateToken(compatToken) { + return cached, true + } + _ = d.loadByCompatToken(compatToken) + return d.mem.FindFinalizableByClientPlaySessionID( + compatToken, clientPlaySessionID, routeItemID, mediaSourceID, + ) +} + +// FindByUpstreamSessionID serves process-local lifecycle callbacks. A local +// ffmpeg crash can only belong to a session already present in this process's +// cache, so no unindexed JSON scan of the durable table is needed. +func (d *DurableCompatPlaybackStore) FindByUpstreamSessionID(upstreamSessionID string) (*PlaybackSession, bool) { + candidate, ok := d.mem.FindByUpstreamSessionID(upstreamSessionID) + if !ok || d.pool == nil { + return candidate, ok + } + validated, ok := d.Get(candidate.ID) + if !ok || validated.UpstreamSessionID != upstreamSessionID { return nil, false } - d.loadByCompatToken(compatToken) - return d.mem.FindByClientPlaySessionID(compatToken, clientPlaySessionID) + return validated, true +} + +// ListPendingTerminals loads durable terminal events that need first delivery +// or an authoritative retry. Successfully delivered provisional fallbacks stay +// stored for late Stopped replacement but are excluded from recovery scans. +func (d *DurableCompatPlaybackStore) ListPendingTerminals(ctx context.Context, limit int) ([]PlaybackSession, error) { + if d.pool == nil { + return d.mem.ListPendingTerminals(ctx, limit) + } + if limit <= 0 { + limit = 100 + } + d.pendingCursorMu.Lock() + defer d.pendingCursorMu.Unlock() + + query := func(afterID string) ([]PlaybackSession, error) { + rows, err := d.pool.Query(ctx, ` + SELECT data + FROM jellycompat_playback_sessions + WHERE expires_at > $1 + AND ($2 = '' OR id > $2) + AND COALESCE((data->>'Terminal')::boolean, false) = true + AND COALESCE(data->'TerminalScrobbleEvent' <> 'null'::jsonb, false) + AND ( + COALESCE((data->>'TerminalFallbackSent')::boolean, false) = false + OR COALESCE((data->>'TerminalAuthoritative')::boolean, false) = true + ) + ORDER BY id ASC + LIMIT $3 + `, d.now(), afterID, limit) + if err != nil { + return nil, err + } + defer rows.Close() + result := make([]PlaybackSession, 0, limit) + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + return nil, err + } + var session PlaybackSession + if err := json.Unmarshal(raw, &session); err != nil { + return nil, err + } + result = append(result, session) + } + if err := rows.Err(); err != nil { + return nil, err + } + return result, nil + } + + result, err := query(d.pendingCursor) + if err != nil { + return nil, err + } + if len(result) == 0 && d.pendingCursor != "" { + d.pendingCursor = "" + result, err = query("") + if err != nil { + return nil, err + } + } + if len(result) > 0 { + d.pendingCursor = result[len(result)-1].ID + } else { + d.pendingCursor = "" + } + return result, nil } // DeleteExpired physically removes lapsed rows. Reads already filter on // expires_at, so this only bounds table growth; run it on the janitor cadence. func (d *DurableCompatPlaybackStore) DeleteExpired(ctx context.Context) (int64, error) { + d.cacheMutationMu.Lock() + expired := d.mem.deleteExpired() + for id, compatToken := range expired { + d.clearUnpersisted(id) + d.clearPendingUpdates(id) + d.invalidateValidation(id, compatToken) + d.bumpCacheGenerations(id, compatToken) + } + d.cacheMutationMu.Unlock() if d.pool == nil { return 0, nil } @@ -263,17 +1015,16 @@ const upsertSessionQuery = ` data = EXCLUDED.data, expires_at = EXCLUDED.expires_at` -// upsert persists a session on the given context. It is best-effort: a DB -// failure is logged and swallowed (the cache holds the authoritative in-process -// state). Callers own the context and its timeout. -func (d *DurableCompatPlaybackStore) upsert(ctx context.Context, session PlaybackSession) { +// upsert persists a session on the given context. The caller records failures +// while retaining the cache as the authoritative in-process state. +func (d *DurableCompatPlaybackStore) upsert(ctx context.Context, session PlaybackSession) error { if d.pool == nil { - return + return nil } data, err := json.Marshal(session) if err != nil { slog.WarnContext(ctx, "marshal compat playback session failed", "component", "jellycompat", "error", err, "play_session_id", session.ID) - return + return err } expiresAt := session.ExpiresAt if expiresAt.IsZero() { @@ -281,12 +1032,74 @@ func (d *DurableCompatPlaybackStore) upsert(ctx context.Context, session Playbac } if _, err := d.pool.Exec(ctx, upsertSessionQuery, session.ID, session.CompatToken, session.UserID, data, expiresAt); err != nil { slog.WarnContext(ctx, "persist compat playback session failed", "component", "jellycompat", "error", err, "play_session_id", session.ID) + return err } + return nil } -func (d *DurableCompatPlaybackStore) load(id string) (*PlaybackSession, bool) { +// insertIfAbsent repairs a session whose initial upsert failed without reviving +// a row that another process created or terminalized in the meantime. +func (d *DurableCompatPlaybackStore) insertIfAbsent(session *PlaybackSession) (bool, error) { + data, err := json.Marshal(session) + if err != nil { + return false, err + } + expiresAt := session.ExpiresAt + if expiresAt.IsZero() { + expiresAt = d.now().Add(d.ttl) + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + tag, err := d.pool.Exec(ctx, ` + INSERT INTO jellycompat_playback_sessions (id, compat_token, user_id, data, expires_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (id) DO NOTHING + `, session.ID, session.CompatToken, session.UserID, data, expiresAt) + if err != nil { + slog.Warn("repair unpersisted compat playback session failed", "error", err, "play_session_id", session.ID) + return false, err + } + return tag.RowsAffected() > 0, nil +} + +// repairUnpersisted retries the creation write retained after Put failed. The +// insert never overwrites a row another process created or terminalized. The +// caller must hold cacheMutationMu. +func (d *DurableCompatPlaybackStore) repairUnpersisted(session *PlaybackSession) bool { + if session == nil || !d.isUnpersisted(session.ID) { + return false + } + if session.Terminal { + // Terminal staging owns persistence because it also carries the provider + // event needed for crash recovery. Never repair a terminal shell without + // that event from an ordinary routing revalidation. + return true + } + inserted, err := d.insertIfAbsent(session) + if err != nil { + return true + } + d.clearUnpersisted(session.ID) + d.bumpCacheGenerations(session.ID, session.CompatToken) + if inserted { + return true + } + // A concurrent process created the row after the first load. Read its + // authoritative state instead of overwriting it. + durable, ok, err := d.load(session.ID) + if err != nil { + return true + } + if !ok { + return false + } + d.mem.Put(*durable) + return true +} + +func (d *DurableCompatPlaybackStore) load(id string) (*PlaybackSession, bool, error) { if d.pool == nil { - return nil, false + return nil, false, nil } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() @@ -295,27 +1108,29 @@ func (d *DurableCompatPlaybackStore) load(id string) (*PlaybackSession, bool) { `SELECT data FROM jellycompat_playback_sessions WHERE id = $1 AND expires_at > $2`, id, d.now(), ).Scan(&raw) if err != nil { - if !errors.Is(err, pgx.ErrNoRows) { - slog.Warn("load compat playback session failed", "error", err, "play_session_id", id) + if errors.Is(err, pgx.ErrNoRows) { + return nil, false, nil } - return nil, false + slog.Warn("load compat playback session failed", "error", err, "play_session_id", id) + return nil, false, err } var session PlaybackSession if err := json.Unmarshal(raw, &session); err != nil { slog.Warn("unmarshal compat playback session failed", "error", err, "play_session_id", id) - return nil, false + return nil, false, err } - return &session, true + return &session, true, nil } // loadByCompatToken loads the live rows for a (non-empty) compat token into the // cache so a subsequent cache scan can resolve the route. The query is bounded by // the indexed compat_token predicate; FindByRoute never calls it with an empty // token (that would be an unbounded full-table load). -func (d *DurableCompatPlaybackStore) loadByCompatToken(compatToken string) { +func (d *DurableCompatPlaybackStore) loadByCompatToken(compatToken string) error { if d.pool == nil || compatToken == "" { - return + return nil } + generation := d.tokenGenerationSnapshot(compatToken) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() rows, err := d.pool.Query(ctx, @@ -323,19 +1138,324 @@ func (d *DurableCompatPlaybackStore) loadByCompatToken(compatToken string) { compatToken, d.now()) if err != nil { slog.Warn("load compat playback sessions by token failed", "error", err) - return + return err } defer rows.Close() + var sessions []PlaybackSession for rows.Next() { var raw []byte if err := rows.Scan(&raw); err != nil { slog.Warn("scan compat playback session failed", "error", err) - return + return err } var session PlaybackSession if err := json.Unmarshal(raw, &session); err != nil { + slog.Warn("unmarshal compat playback session by token failed", "error", err) + return err + } + sessions = append(sessions, session) + } + if err := rows.Err(); err != nil { + slog.Warn("iterate compat playback sessions failed", "error", err) + return err + } + d.applyCompatTokenSnapshot(compatToken, sessions, generation) + return nil +} + +func (d *DurableCompatPlaybackStore) applyCompatTokenSnapshot( + compatToken string, + sessions []PlaybackSession, + generation compatCacheGeneration, +) bool { + d.cacheMutationMu.Lock() + defer d.cacheMutationMu.Unlock() + if generation != d.tokenGenerationSnapshot(compatToken) { + d.invalidateValidation("", compatToken) + return false + } + preserveIDs := d.preservedSnapshotIDs(compatToken, sessions) + affectedIDs := d.mem.replaceByCompatToken(compatToken, sessions, preserveIDs) + for _, session := range sessions { + if _, preserve := preserveIDs[session.ID]; preserve { + d.clearUnpersisted(session.ID) continue } - d.mem.Put(session) + d.markIDValidated(session.ID) + } + d.markTokenValidated(compatToken) + d.bumpCacheGenerations("", compatToken) + for _, id := range affectedIDs { + d.bumpCacheGenerations(id, "") + } + return true +} + +// shouldRevalidateID and shouldRevalidateToken reserve one validation attempt +// per bounded interval. Reserving before I/O prevents concurrent segment +// requests from stampeding Postgres; failed attempts are also briefly throttled +// while callers continue from their last known-good in-memory state. +func (d *DurableCompatPlaybackStore) shouldRevalidateID(id string) bool { + return d.shouldRevalidate(d.validatedIDs, id) +} + +func (d *DurableCompatPlaybackStore) shouldRevalidateToken(compatToken string) bool { + return d.shouldRevalidate(d.validatedTokens, compatToken) +} + +func (d *DurableCompatPlaybackStore) shouldRevalidate(validated map[string]time.Time, key string) bool { + if key == "" { + return true + } + d.validationMu.Lock() + defer d.validationMu.Unlock() + now := d.now() + if checkedAt, ok := validated[key]; ok { + elapsed := now.Sub(checkedAt) + if elapsed >= 0 && elapsed < compatCacheRevalidationInterval { + return false + } + } + d.makeValidationRoom(validated, key, now) + validated[key] = now + return true +} + +func (d *DurableCompatPlaybackStore) markIDValidated(id string) { + d.validationMu.Lock() + defer d.validationMu.Unlock() + now := d.now() + if id != "" { + d.makeValidationRoom(d.validatedIDs, id, now) + d.validatedIDs[id] = now } } + +func (d *DurableCompatPlaybackStore) markTokenValidated(compatToken string) { + d.validationMu.Lock() + defer d.validationMu.Unlock() + now := d.now() + if compatToken != "" { + d.makeValidationRoom(d.validatedTokens, compatToken, now) + d.validatedTokens[compatToken] = now + } +} + +// makeValidationRoom keeps attacker-controlled missing IDs from growing the +// throttling maps without bound. Expired entries go first; at capacity an +// arbitrary old slot is reused. +func (d *DurableCompatPlaybackStore) makeValidationRoom(validated map[string]time.Time, key string, now time.Time) { + if _, exists := validated[key]; exists || len(validated) < compatValidationCacheLimit { + return + } + for candidate, checkedAt := range validated { + if now.Sub(checkedAt) >= compatCacheRevalidationInterval { + delete(validated, candidate) + } + } + if len(validated) < compatValidationCacheLimit { + return + } + for candidate := range validated { + delete(validated, candidate) + break + } +} + +func (d *DurableCompatPlaybackStore) invalidateValidation(id, compatToken string) { + d.validationMu.Lock() + defer d.validationMu.Unlock() + delete(d.validatedIDs, id) + if compatToken != "" { + delete(d.validatedTokens, compatToken) + } +} + +func (d *DurableCompatPlaybackStore) markUnpersisted(id string) { + d.validationMu.Lock() + defer d.validationMu.Unlock() + d.unpersistedIDs[id] = struct{}{} +} + +func (d *DurableCompatPlaybackStore) clearUnpersisted(id string) { + d.validationMu.Lock() + defer d.validationMu.Unlock() + delete(d.unpersistedIDs, id) +} + +func (d *DurableCompatPlaybackStore) isUnpersisted(id string) bool { + d.validationMu.Lock() + defer d.validationMu.Unlock() + _, ok := d.unpersistedIDs[id] + return ok +} + +func (d *DurableCompatPlaybackStore) unpersistedSnapshot() map[string]struct{} { + d.validationMu.Lock() + defer d.validationMu.Unlock() + result := make(map[string]struct{}, len(d.unpersistedIDs)) + for id := range d.unpersistedIDs { + result[id] = struct{}{} + } + return result +} + +func (d *DurableCompatPlaybackStore) appendPendingUpdate( + id string, + compatToken string, + update func(*PlaybackSession) error, +) { + d.pendingUpdateMu.Lock() + defer d.pendingUpdateMu.Unlock() + d.pendingUpdateSequence++ + d.pendingUpdates[id] = append(d.pendingUpdates[id], pendingCompatPlaybackUpdate{ + sequence: d.pendingUpdateSequence, + compatToken: compatToken, + apply: update, + }) +} + +func (d *DurableCompatPlaybackStore) pendingUpdatesSnapshot(id string) []pendingCompatPlaybackUpdate { + d.pendingUpdateMu.Lock() + defer d.pendingUpdateMu.Unlock() + pending := d.pendingUpdates[id] + result := make([]pendingCompatPlaybackUpdate, len(pending)) + copy(result, pending) + return result +} + +func (d *DurableCompatPlaybackStore) pendingUpdateIDsSnapshot(compatToken string) map[string]struct{} { + d.pendingUpdateMu.Lock() + defer d.pendingUpdateMu.Unlock() + result := make(map[string]struct{}, len(d.pendingUpdates)) + for id, pending := range d.pendingUpdates { + for _, update := range pending { + if update.compatToken == compatToken { + result[id] = struct{}{} + break + } + } + } + return result +} + +func (d *DurableCompatPlaybackStore) hasPendingUpdates(id string) bool { + d.pendingUpdateMu.Lock() + defer d.pendingUpdateMu.Unlock() + return len(d.pendingUpdates[id]) > 0 +} + +func (d *DurableCompatPlaybackStore) consumePendingUpdates(id string, throughSequence uint64) { + if throughSequence == 0 { + return + } + d.pendingUpdateMu.Lock() + defer d.pendingUpdateMu.Unlock() + pending := d.pendingUpdates[id] + firstRemaining := 0 + for firstRemaining < len(pending) && pending[firstRemaining].sequence <= throughSequence { + firstRemaining++ + } + if firstRemaining == len(pending) { + delete(d.pendingUpdates, id) + return + } + d.pendingUpdates[id] = pending[firstRemaining:] +} + +func (d *DurableCompatPlaybackStore) clearPendingUpdates(id string) { + d.pendingUpdateMu.Lock() + defer d.pendingUpdateMu.Unlock() + delete(d.pendingUpdates, id) +} + +// preservedSnapshotIDs keeps uncertain local creations and monotonic local +// terminal markers from being replaced by an older active DB snapshot. A +// durable terminal row is safe to apply because it can only advance terminal +// event/claim state. +func (d *DurableCompatPlaybackStore) preservedSnapshotIDs( + compatToken string, + durable []PlaybackSession, +) map[string]struct{} { + preserve := d.unpersistedSnapshot() + pendingIDs := d.pendingUpdateIDsSnapshot(compatToken) + durableIDs := make(map[string]struct{}, len(durable)) + durableTerminal := make(map[string]bool, len(durable)) + for _, session := range durable { + durableIDs[session.ID] = struct{}{} + durableTerminal[session.ID] = session.Terminal + if session.Terminal { + delete(preserve, session.ID) + d.clearPendingUpdates(session.ID) + } else if _, pending := pendingIDs[session.ID]; pending { + preserve[session.ID] = struct{}{} + } + } + for id := range pendingIDs { + if _, exists := durableIDs[id]; !exists { + d.clearPendingUpdates(id) + } + } + for id := range d.mem.terminalIDsByCompatToken(compatToken) { + if !durableTerminal[id] { + preserve[id] = struct{}{} + } + } + return preserve +} + +func (d *DurableCompatPlaybackStore) idGenerationSnapshot(id string) compatCacheGeneration { + d.generationMu.Lock() + defer d.generationMu.Unlock() + return compatCacheGeneration{epoch: d.generationEpoch, value: d.idGenerations[id]} +} + +func (d *DurableCompatPlaybackStore) tokenGenerationSnapshot(compatToken string) compatCacheGeneration { + d.generationMu.Lock() + defer d.generationMu.Unlock() + return compatCacheGeneration{epoch: d.generationEpoch, value: d.tokenGenerations[compatToken]} +} + +func (d *DurableCompatPlaybackStore) bumpCacheGenerations(id, compatToken string) { + d.generationMu.Lock() + defer d.generationMu.Unlock() + if id != "" { + d.makeGenerationRoomLocked(d.idGenerations, id) + d.idGenerations[id]++ + } + if compatToken != "" { + d.makeGenerationRoomLocked(d.tokenGenerations, compatToken) + d.tokenGenerations[compatToken]++ + } +} + +// makeGenerationRoomLocked bounds tombstones without letting an in-flight +// snapshot mistake an evicted generation for its original zero value. Advancing +// the epoch invalidates every captured stamp before the maps are reset. +func (d *DurableCompatPlaybackStore) makeGenerationRoomLocked(generations map[string]uint64, key string) { + if _, exists := generations[key]; exists || len(generations) < compatValidationCacheLimit { + return + } + d.generationEpoch++ + clear(d.idGenerations) + clear(d.tokenGenerations) +} + +// finishCacheMutation must be deferred only while cacheMutationMu is read-held. +func (d *DurableCompatPlaybackStore) finishCacheMutation(id, compatToken string) { + d.bumpCacheGenerations(id, compatToken) + d.cacheMutationMu.RUnlock() +} + +func (d *DurableCompatPlaybackStore) lockSessionMutation(id string) func() { + const fnvOffset32 = uint32(2166136261) + const fnvPrime32 = uint32(16777619) + hash := fnvOffset32 + for i := 0; i < len(id); i++ { + hash ^= uint32(id[i]) + hash *= fnvPrime32 + } + lock := &d.sessionMutations[int(hash%uint32(len(d.sessionMutations)))] + lock.Lock() + return lock.Unlock +} diff --git a/internal/jellycompat/playback_sessions_postgres_test.go b/internal/jellycompat/playback_sessions_postgres_test.go index 37d1a970..ea59879c 100644 --- a/internal/jellycompat/playback_sessions_postgres_test.go +++ b/internal/jellycompat/playback_sessions_postgres_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/Silo-Server/silo-server/internal/watchsync" "github.com/jackc/pgx/v5/pgxpool" ) @@ -85,6 +86,77 @@ func TestDurableCompatPlaybackStore_SurvivesRestart(t *testing.T) { } } +func TestDurableCompatPlaybackStoreRevalidatesUnstartedNegotiationAcrossInstances(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + firstID := "compat-first-" + suffix + secondID := "compat-second-" + suffix + t.Cleanup(func() { + _, _ = pool.Exec( + ctx, + `DELETE FROM jellycompat_playback_sessions WHERE id = ANY($1)`, + []string{firstID, secondID}, + ) + }) + first := PlaybackSession{ + ID: firstID, + CompatToken: "owner-" + suffix, + ClientDeviceID: "device-1", + RouteItemID: "route-1", + } + seed := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + seed.PutNegotiated(first) + + stale := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + if _, ok := stale.Get(firstID); !ok { + t.Fatal("first negotiation was not cached") + } + replacer := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + second := first + second.ID = secondID + replacer.PutNegotiated(second) + + if _, ok := stale.Get(firstID); ok { + t.Fatal("superseded unstarted negotiation remained routable from another instance's cache") + } +} + +func TestDurableCompatPlaybackStorePutNegotiatedReplacesAcrossInstances(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + firstID := "compat-negotiated-first-" + suffix + secondID := "compat-negotiated-second-" + suffix + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = ANY($1)`, []string{firstID, secondID}) + }) + + first := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + first.PutNegotiated(PlaybackSession{ + ID: firstID, + CompatToken: "negotiated-token-" + suffix, + ClientDeviceID: "web-device", + RouteItemID: "route-1", + }) + + second := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + second.PutNegotiated(PlaybackSession{ + ID: secondID, + CompatToken: "negotiated-token-" + suffix, + ClientDeviceID: "web-device", + RouteItemID: "route-1", + }) + + fresh := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + if _, ok := fresh.Get(firstID); ok { + t.Fatal("superseded negotiation remained durable") + } + if _, ok := fresh.Get(secondID); !ok { + t.Fatal("replacement negotiation was not durable") + } +} + // M5: an empty compat token must never trigger a DB scan — FindByRoute returns // the in-memory result only. With a non-nil pool but no live DB, a scan attempt // would block/error on the pool; instead the empty-token path returns cleanly @@ -216,3 +288,668 @@ func TestDurableCompatPlaybackStore_NilPoolInMemory(t *testing.T) { t.Fatal("nil-pool Delete failed") } } + +func TestPlaybackSessionStoreTerminalClaimIsAtomicAndOwnerScoped(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "owner", UpstreamSessionID: "upstream-1"}) + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1"} + + if _, err := store.StageTerminal("play-1", "other", event, true); err == nil { + t.Fatal("foreign token staged terminal event") + } + if _, err := store.StageTerminal("play-1", "owner", event, true); err != nil { + t.Fatal("owner failed to stage terminal event") + } + claimUntil := time.Now().Add(time.Minute) + if _, err := store.ClaimTerminal("play-1", "other", claimUntil); err == nil { + t.Fatal("foreign token claimed terminal event") + } + got, err := store.ClaimTerminal("play-1", "owner", claimUntil) + if err != nil || got.UpstreamSessionID != "upstream-1" { + t.Fatalf("owner claim failed: err=%v session=%+v", err, got) + } + if _, err := store.ClaimTerminal("play-1", "owner", claimUntil.Add(time.Minute)); err == nil { + t.Fatal("terminal event was claimed twice") + } +} + +func TestPlaybackSessionStoreDeactivateRetainsOnlyFinalReportLookup(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "owner", + ClientPlaySessionID: "client-play-1", + UpstreamSessionID: "upstream-1", + }) + + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1"} + terminal, err := store.StageTerminal("play-1", "owner", event, true) + if err != nil || !terminal.Terminal { + t.Fatalf("terminal stage failed: err=%v session=%+v", err, terminal) + } + if _, ok := store.Get("play-1"); ok { + t.Fatal("terminal session remained available to ordinary lookup") + } + if _, ok := store.FindByClientPlaySessionID("owner", "client-play-1"); ok { + t.Fatal("terminal session remained available to ordinary alias lookup") + } + if _, ok := store.GetFinalizable("play-1", "owner"); !ok { + t.Fatal("terminal session was unavailable to final report lookup") + } + if _, ok := store.FindFinalizableByClientPlaySessionID("owner", "client-play-1", "", ""); !ok { + t.Fatal("terminal session was unavailable to final alias lookup") + } + claimUntil := time.Now().Add(time.Minute) + claimed, err := store.ClaimTerminal("play-1", "owner", claimUntil) + if err != nil || !claimed.Terminal { + t.Fatalf("final report could not claim terminal session: err=%v session=%+v", err, claimed) + } + store.CompleteTerminal("play-1", "owner", claimUntil, claimed.TerminalClaimVersion) + if _, ok := store.GetFinalizable("play-1", "owner"); ok { + t.Fatal("authoritatively completed terminal session was retained") + } +} + +func TestPlaybackSessionStoreFinalAliasLookupDisambiguatesReusedAlias(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ + ID: "old-play", + CompatToken: "owner", + ClientPlaySessionID: "reused-client-play", + RouteItemID: "old-item", + MediaSources: []PlaybackMediaSource{{ID: "old-source"}}, + UpstreamSessionID: "old-upstream", + }) + if _, err := store.StageTerminal( + "old-play", + "owner", + watchsync.ScrobbleEvent{PlaybackSessionID: "old-upstream"}, + false, + ); err != nil { + t.Fatalf("stage old terminal play: %v", err) + } + store.Put(PlaybackSession{ + ID: "current-play", + CompatToken: "owner", + ClientPlaySessionID: "reused-client-play", + RouteItemID: "current-item", + MediaSources: []PlaybackMediaSource{{ID: "current-source"}}, + UpstreamSessionID: "current-upstream", + }) + + if _, ok := store.FindFinalizableByClientPlaySessionID( + "owner", "reused-client-play", "", "", + ); ok { + t.Fatal("unscoped reused alias unexpectedly selected an arbitrary play") + } + current, ok := store.FindFinalizableByClientPlaySessionID( + "owner", "reused-client-play", "current-item", "current-source", + ) + if !ok || current.ID != "current-play" { + t.Fatalf("current report resolved to ok=%v session=%+v", ok, current) + } + old, ok := store.FindFinalizableByClientPlaySessionID( + "owner", "reused-client-play", "old-item", "old-source", + ) + if !ok || old.ID != "old-play" { + t.Fatalf("late old report resolved to ok=%v session=%+v", ok, old) + } + oldByRoute, _, ok := store.FindFinalizableByRoute("owner", "old-source") + if !ok || oldByRoute.ID != "old-play" { + t.Fatalf("terminal route resolved to ok=%v session=%+v", ok, oldByRoute) + } +} + +func TestPlaybackSessionStoreExpiredTerminalLeaseCanBeReclaimed(t *testing.T) { + now := time.Date(2026, 7, 22, 12, 0, 0, 0, time.UTC) + store := NewPlaybackSessionStore(time.Hour, func() time.Time { return now }) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "owner"}) + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1"} + if _, err := store.StageTerminal("play-1", "owner", event, true); err != nil { + t.Fatalf("stage terminal: %v", err) + } + firstLease := now.Add(10 * time.Second) + firstClaim, err := store.ClaimTerminal("play-1", "owner", firstLease) + if err != nil { + t.Fatalf("first claim: %v", err) + } + now = firstLease.Add(time.Microsecond) + secondLease := now.Add(10 * time.Second) + secondClaim, err := store.ClaimTerminal("play-1", "owner", secondLease) + if err != nil { + t.Fatalf("reclaim expired lease: %v", err) + } + store.CompleteTerminal("play-1", "owner", firstLease, firstClaim.TerminalClaimVersion) + if _, ok := store.GetFinalizable("play-1", "owner"); !ok { + t.Fatal("stale first lease completed the successor's terminal row") + } + store.CompleteTerminal("play-1", "owner", secondLease, secondClaim.TerminalClaimVersion) + if _, ok := store.GetFinalizable("play-1", "owner"); ok { + t.Fatal("successor lease did not complete terminal row") + } +} + +func TestPlaybackSessionStorePendingScanEvictsExpiredTerminal(t *testing.T) { + now := time.Date(2026, 7, 22, 12, 0, 0, 0, time.UTC) + store := NewPlaybackSessionStore(time.Minute, func() time.Time { return now }) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "owner"}) + if _, err := store.StageTerminal( + "play-1", "owner", watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1"}, false, + ); err != nil { + t.Fatalf("stage terminal: %v", err) + } + now = now.Add(2 * time.Minute) + pending, err := store.ListPendingTerminals(context.Background(), 100) + if err != nil { + t.Fatalf("list pending terminals: %v", err) + } + if len(pending) != 0 { + t.Fatalf("expired pending terminals = %d, want 0", len(pending)) + } + store.mu.RLock() + remaining := len(store.sessions) + store.mu.RUnlock() + if remaining != 0 { + t.Fatalf("expired terminal entries retained in memory = %d", remaining) + } +} + +func TestDurableCompatPlaybackStoreExpiryClearsFailureBookkeeping(t *testing.T) { + now := time.Date(2026, 7, 22, 12, 0, 0, 0, time.UTC) + store := NewDurableCompatPlaybackStore(nil, time.Minute, func() time.Time { return now }) + store.mem.Put(PlaybackSession{ID: "play-1", CompatToken: "owner"}) + store.markUnpersisted("play-1") + store.appendPendingUpdate("play-1", "owner", func(*PlaybackSession) error { return nil }) + now = now.Add(2 * time.Minute) + + if _, err := store.DeleteExpired(context.Background()); err != nil { + t.Fatalf("delete expired: %v", err) + } + if store.isUnpersisted("play-1") { + t.Fatal("expired session retained its unpersisted marker") + } + if store.hasPendingUpdates("play-1") { + t.Fatal("expired session retained pending update closures") + } +} + +func TestDurableCompatPlaybackStoreBoundsGenerationTombstones(t *testing.T) { + store := NewDurableCompatPlaybackStore(nil, time.Hour, nil) + before := store.idGenerationSnapshot("old-session") + for i := 0; i <= compatValidationCacheLimit; i++ { + store.bumpCacheGenerations(fmt.Sprintf("missing-%d", i), "") + } + store.generationMu.Lock() + count := len(store.idGenerations) + epoch := store.generationEpoch + store.generationMu.Unlock() + if count > compatValidationCacheLimit { + t.Fatalf("generation tombstones = %d, limit = %d", count, compatValidationCacheLimit) + } + if epoch == 0 || before == store.idGenerationSnapshot("old-session") { + t.Fatal("generation eviction did not invalidate an older captured stamp") + } +} + +func TestDurableCompatPlaybackStoreIDWriteDoesNotRefreshTokenSnapshot(t *testing.T) { + now := time.Date(2026, 7, 22, 12, 0, 0, 0, time.UTC) + store := NewDurableCompatPlaybackStore(nil, time.Hour, func() time.Time { return now }) + store.markTokenValidated("owner") + now = now.Add(4 * time.Second) + store.markIDValidated("play-2") + now = now.Add(2 * time.Second) + if !store.shouldRevalidateToken("owner") { + t.Fatal("single-row validation incorrectly extended the token-wide cache window") + } +} + +func TestPlaybackSessionStoreNewAuthoritativeEventSurvivesStaleCompletion(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ID: "play-1", CompatToken: "owner"}) + firstEvent := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1", PositionSeconds: 45} + if _, err := store.StageTerminal("play-1", "owner", firstEvent, true); err != nil { + t.Fatalf("stage first event: %v", err) + } + firstLease := time.Now().Add(time.Minute) + firstClaim, err := store.ClaimTerminal("play-1", "owner", firstLease) + if err != nil { + t.Fatalf("claim first event: %v", err) + } + secondEvent := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1", PositionSeconds: 90} + if _, err := store.StageTerminal("play-1", "owner", secondEvent, true); err != nil { + t.Fatalf("stage replacement event: %v", err) + } + + store.CompleteTerminal("play-1", "owner", firstLease, firstClaim.TerminalClaimVersion) + store.ReleaseTerminalClaim("play-1", "owner", firstLease, firstClaim.TerminalClaimVersion, false) + pending, ok := store.GetFinalizable("play-1", "owner") + if !ok || pending.TerminalScrobbleEvent == nil || pending.TerminalScrobbleEvent.PositionSeconds != 90 { + t.Fatalf("replacement event was lost: ok=%v session=%+v", ok, pending) + } + secondLease := firstLease.Add(time.Minute) + secondClaim, err := store.ClaimTerminal("play-1", "owner", secondLease) + if err != nil { + t.Fatalf("claim replacement event: %v", err) + } + store.CompleteTerminal("play-1", "owner", secondLease, secondClaim.TerminalClaimVersion) + if _, ok := store.GetFinalizable("play-1", "owner"); ok { + t.Fatal("replacement event was not completed") + } +} + +func TestDurableCompatPlaybackStoreTerminalClaimAcrossInstances(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + id := fmt.Sprintf("compat-take-%d", time.Now().UnixNano()) + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id) }) + + seed := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + seed.Put(PlaybackSession{ID: id, CompatToken: "owner", UpstreamSessionID: "upstream-1"}) + skewedNow := time.Now().Add(-6 * time.Hour) + first := NewDurableCompatPlaybackStore(pool, time.Hour, func() time.Time { return skewedNow }) + second := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + if _, ok := first.Get(id); !ok { + t.Fatal("first instance did not load session") + } + if _, ok := second.Get(id); !ok { + t.Fatal("second instance did not load session") + } + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1"} + if _, err := seed.StageTerminal(id, "owner", event, true); err != nil { + t.Fatal("failed to stage durable terminal event") + } + + var dbBefore time.Time + if err := pool.QueryRow(ctx, `SELECT clock_timestamp()`).Scan(&dbBefore); err != nil { + t.Fatalf("read database clock: %v", err) + } + claimUntil := skewedNow.Add(time.Minute) + claimed, err := first.ClaimTerminal(id, "owner", claimUntil) + if err != nil { + t.Fatal("first instance did not claim session") + } + var dbAfter time.Time + if err := pool.QueryRow(ctx, `SELECT clock_timestamp()`).Scan(&dbAfter); err != nil { + t.Fatalf("read database clock after claim: %v", err) + } + if claimed.TerminalClaimUntil.Before(dbBefore.Add(55*time.Second)) || + claimed.TerminalClaimUntil.After(dbAfter.Add(65*time.Second)) { + t.Fatalf( + "claim deadline %s was not anchored to database clock interval [%s, %s]", + claimed.TerminalClaimUntil, + dbBefore.Add(55*time.Second), + dbAfter.Add(65*time.Second), + ) + } + if _, err := second.ClaimTerminal(id, "owner", time.Now().Add(time.Minute)); err == nil { + t.Fatal("second instance claimed an already leased durable session") + } +} + +func TestDurableCompatPlaybackStoreTerminalStageSurvivesInstanceBoundary(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + id := fmt.Sprintf("compat-deactivate-%d", time.Now().UnixNano()) + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id) }) + now := time.Now() + clock := func() time.Time { return now } + + seed := NewDurableCompatPlaybackStore(pool, time.Hour, clock) + seed.Put(PlaybackSession{ + ID: id, + CompatToken: "owner", + ClientPlaySessionID: "client-play-1", + UpstreamSessionID: "upstream-1", + }) + fresh := NewDurableCompatPlaybackStore(pool, time.Hour, clock) + if _, ok := fresh.Get(id); !ok { + t.Fatal("fresh instance did not preload the active session") + } + if _, ok := fresh.FindByClientPlaySessionID("owner", "client-play-1"); !ok { + t.Fatal("fresh instance did not preload the active alias") + } + event := watchsync.ScrobbleEvent{PlaybackSessionID: "upstream-1"} + if terminal, err := seed.StageTerminal(id, "owner", event, false); err != nil || !terminal.Terminal { + t.Fatalf("durable terminal stage failed: err=%v session=%+v", err, terminal) + } + // Cross-process invalidation is deliberately bounded rather than putting a + // DB query on every segment request. Advance past that window before the + // other instance revalidates its preloaded cache. + now = now.Add(compatCacheRevalidationInterval) + + if _, ok := fresh.Get(id); ok { + t.Fatal("instance routed a terminal session from its stale active cache") + } + if _, ok := fresh.FindByClientPlaySessionID("owner", "client-play-1"); ok { + t.Fatal("instance routed a terminal alias from its stale active cache") + } + if _, ok := fresh.GetFinalizable(id, "owner"); !ok { + t.Fatal("fresh instance could not resolve terminal session for final report") + } + if _, ok := fresh.FindFinalizableByClientPlaySessionID("owner", "client-play-1", "", ""); !ok { + t.Fatal("fresh instance could not resolve terminal alias for final report") + } + claimUntil := time.Now().UTC().Truncate(time.Microsecond).Add(time.Minute) + claimed, err := fresh.ClaimTerminal(id, "owner", claimUntil) + if err != nil || !claimed.Terminal { + t.Fatalf("fresh instance could not claim terminal session: err=%v session=%+v", err, claimed) + } + fresh.ReleaseTerminalClaim(id, "owner", claimed.TerminalClaimUntil, claimed.TerminalClaimVersion, true) + if claimed, err := fresh.ClaimTerminal(id, "owner", claimUntil.Add(time.Minute)); err == nil || claimed != nil { + t.Fatalf("delivered fallback was claimed again: err=%v session=%+v", err, claimed) + } +} + +func TestDurableCompatPlaybackStorePreservesCachedSessionOnValidationFailure(t *testing.T) { + pool := newCompatTestPool(t) + now := time.Now() + clock := func() time.Time { return now } + store := NewDurableCompatPlaybackStore(pool, time.Hour, clock) + pool.Close() + store.Put(PlaybackSession{ID: "cached-session", CompatToken: "owner"}) + + // A request inside the validation window must be served entirely from cache. + if _, ok := store.Get("cached-session"); !ok { + t.Fatal("hot cache lookup consulted the closed database") + } + + // Once validation is due, the failed query must not evict a last-known-good + // active session and interrupt playback. + now = now.Add(compatCacheRevalidationInterval) + if _, ok := store.Get("cached-session"); !ok { + t.Fatal("database validation failure evicted the cached active session") + } +} + +func TestDurableCompatPlaybackStoreRepairsFailedInitialPersistence(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + id := fmt.Sprintf("compat-repair-%d", time.Now().UnixNano()) + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id) }) + now := time.Now() + clock := func() time.Time { return now } + store := NewDurableCompatPlaybackStore(pool, time.Hour, clock) + stored := store.mem.putNormalized(PlaybackSession{ID: id, CompatToken: "owner", UpstreamSessionID: "upstream-1"}) + store.markIDValidated(id) + store.markUnpersisted(id) + + now = now.Add(compatCacheRevalidationInterval) + if got, ok := store.Get(id); !ok || got.UpstreamSessionID != "upstream-1" { + t.Fatalf("unpersisted cache entry was not retained and repaired: ok=%v session=%+v", ok, got) + } + if store.isUnpersisted(id) { + t.Fatal("successfully repaired session remained marked unpersisted") + } + fresh := NewDurableCompatPlaybackStore(pool, time.Hour, clock) + if got, ok := fresh.Get(stored.ID); !ok || got.UpstreamSessionID != "upstream-1" { + t.Fatalf("repaired session was not durable: ok=%v session=%+v", ok, got) + } +} + +func TestDurableCompatPlaybackStorePreservesUnpersistedTerminalOnRevalidation(t *testing.T) { + pool := newCompatTestPool(t) + id := fmt.Sprintf("compat-unpersisted-terminal-%d", time.Now().UnixNano()) + now := time.Now() + clock := func() time.Time { return now } + store := NewDurableCompatPlaybackStore(pool, time.Hour, clock) + store.mem.Put(PlaybackSession{ID: id, CompatToken: "owner"}) + if err := store.mem.HideFromRouting(id, "owner"); err != nil { + t.Fatalf("hide terminal: %v", err) + } + store.markUnpersisted(id) + store.markIDValidated(id) + now = now.Add(compatCacheRevalidationInterval) + + if _, ok := store.Get(id); ok { + t.Fatal("terminal session became routable during missing-row revalidation") + } + terminal, ok := store.GetFinalizable(id, "owner") + if !ok || !terminal.Terminal || !store.isUnpersisted(id) { + t.Fatalf("unpersisted terminal state was lost: ok=%v session=%+v", ok, terminal) + } +} + +func TestDurableCompatPlaybackStoreDiscardsTokenSnapshotAfterLocalMutation(t *testing.T) { + store := NewDurableCompatPlaybackStore(nil, time.Hour, nil) + generation := store.tokenGenerationSnapshot("owner") + store.cacheMutationMu.Lock() + store.mem.Put(PlaybackSession{ID: "new-session", CompatToken: "owner", RouteItemID: "route-1"}) + store.bumpCacheGenerations("new-session", "owner") + store.cacheMutationMu.Unlock() + + if store.applyCompatTokenSnapshot("owner", nil, generation) { + t.Fatal("stale token snapshot applied after a concurrent local mutation") + } + if _, _, ok := store.mem.FindByRoute("owner", "route-1"); !ok { + t.Fatal("stale token snapshot deleted the concurrent local session") + } +} + +func TestDurableCompatPlaybackStoreAppliesTokenSnapshotAfterUnrelatedMutation(t *testing.T) { + store := NewDurableCompatPlaybackStore(nil, time.Hour, nil) + store.mem.Put(PlaybackSession{ID: "stale-session", CompatToken: "owner", RouteItemID: "route-1"}) + generation := store.tokenGenerationSnapshot("owner") + store.cacheMutationMu.Lock() + store.mem.Put(PlaybackSession{ID: "other-session", CompatToken: "other"}) + store.bumpCacheGenerations("other-session", "other") + store.cacheMutationMu.Unlock() + + if !store.applyCompatTokenSnapshot("owner", nil, generation) { + t.Fatal("unrelated local mutation incorrectly invalidated the token snapshot") + } + if _, _, ok := store.mem.FindByRoute("owner", "route-1"); ok { + t.Fatal("valid token snapshot was not applied after unrelated mutation") + } +} + +func TestDurableCompatPlaybackStoreDoesNotReviveLocalTerminalFromActiveSnapshot(t *testing.T) { + store := NewDurableCompatPlaybackStore(nil, time.Hour, nil) + active := PlaybackSession{ID: "play-1", CompatToken: "owner", RouteItemID: "route-1"} + store.mem.Put(active) + if err := store.mem.HideFromRouting("play-1", "owner"); err != nil { + t.Fatalf("hide local terminal: %v", err) + } + generation := store.tokenGenerationSnapshot("owner") + + if !store.applyCompatTokenSnapshot("owner", []PlaybackSession{active}, generation) { + t.Fatal("active durable snapshot was unexpectedly discarded") + } + if _, ok := store.mem.Get("play-1"); ok { + t.Fatal("active durable snapshot revived a locally terminal session") + } + if terminal, ok := store.mem.GetFinalizable("play-1", "owner"); !ok || !terminal.Terminal { + t.Fatalf("local terminal marker was lost: ok=%v session=%+v", ok, terminal) + } +} + +func TestDurableCompatPlaybackStorePreservesPendingUpdateFromOlderSnapshot(t *testing.T) { + store := NewDurableCompatPlaybackStore(nil, time.Hour, nil) + local := PlaybackSession{ID: "play-1", CompatToken: "owner", UpstreamSessionID: "upstream-new"} + store.mem.Put(local) + store.appendPendingUpdate("play-1", "owner", func(session *PlaybackSession) error { + session.UpstreamSessionID = "upstream-new" + return nil + }) + generation := store.tokenGenerationSnapshot("owner") + durable := local + durable.UpstreamSessionID = "upstream-old" + + if !store.applyCompatTokenSnapshot("owner", []PlaybackSession{durable}, generation) { + t.Fatal("durable snapshot was unexpectedly discarded") + } + if got, ok := store.mem.Get("play-1"); !ok || got.UpstreamSessionID != "upstream-new" { + t.Fatalf("pending local update was overwritten: ok=%v session=%+v", ok, got) + } +} + +func TestDurableCompatPlaybackStoreDropsPendingSessionMissingFromSnapshot(t *testing.T) { + store := NewDurableCompatPlaybackStore(nil, time.Hour, nil) + store.mem.Put(PlaybackSession{ID: "play-1", CompatToken: "owner", RouteItemID: "route-1"}) + store.appendPendingUpdate("play-1", "owner", func(session *PlaybackSession) error { + session.UpstreamSessionID = "upstream-new" + return nil + }) + generation := store.tokenGenerationSnapshot("owner") + + if !store.applyCompatTokenSnapshot("owner", nil, generation) { + t.Fatal("durable deletion snapshot was unexpectedly discarded") + } + if _, _, ok := store.mem.FindByRoute("owner", "route-1"); ok { + t.Fatal("missing durable row remained routable due to a pending update") + } + if store.hasPendingUpdates("play-1") { + t.Fatal("pending update survived authoritative durable deletion") + } +} + +func TestDurableCompatPlaybackStoreTokenSnapshotKeepsOtherTokenPendingUpdates(t *testing.T) { + store := NewDurableCompatPlaybackStore(nil, time.Hour, nil) + store.mem.Put(PlaybackSession{ID: "play-a", CompatToken: "owner-a"}) + store.mem.Put(PlaybackSession{ID: "play-b", CompatToken: "owner-b"}) + store.appendPendingUpdate("play-b", "owner-b", func(session *PlaybackSession) error { + session.UpstreamSessionID = "upstream-b" + return nil + }) + generation := store.tokenGenerationSnapshot("owner-a") + + if !store.applyCompatTokenSnapshot( + "owner-a", []PlaybackSession{{ID: "play-a", CompatToken: "owner-a"}}, generation, + ) { + t.Fatal("token A snapshot was unexpectedly discarded") + } + if !store.hasPendingUpdates("play-b") { + t.Fatal("token A snapshot discarded token B's pending update") + } +} + +func TestDurableCompatPlaybackStoreSerializesSameSessionMutations(t *testing.T) { + store := NewDurableCompatPlaybackStore(nil, time.Hour, nil) + unlock := store.lockSessionMutation("play-1") + acquired := make(chan struct{}) + released := make(chan struct{}) + go func() { + secondUnlock := store.lockSessionMutation("play-1") + close(acquired) + <-released + secondUnlock() + }() + select { + case <-acquired: + t.Fatal("same-session mutation lock was acquired concurrently") + case <-time.After(20 * time.Millisecond): + } + unlock() + select { + case <-acquired: + close(released) + case <-time.After(time.Second): + t.Fatal("same-session mutation did not resume after release") + } +} + +func TestDurableCompatPlaybackStorePersistsTerminalShell(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + id := fmt.Sprintf("compat-terminal-shell-%d", time.Now().UnixNano()) + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id) }) + store := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + store.Put(PlaybackSession{ID: id, CompatToken: "owner", UpstreamSessionID: "upstream-1"}) + + if err := store.HideFromRouting(id, "owner"); err != nil { + t.Fatalf("persist terminal shell: %v", err) + } + fresh := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + if _, ok := fresh.Get(id); ok { + t.Fatal("fresh process routed a durably hidden terminal shell") + } + terminal, ok := fresh.GetFinalizable(id, "owner") + if !ok || !terminal.Terminal || terminal.TerminalScrobbleEvent != nil { + t.Fatalf("terminal shell = ok=%v session=%+v", ok, terminal) + } + pending, err := fresh.ListPendingTerminals(context.Background(), 100) + if err != nil { + t.Fatalf("list terminal shells: %v", err) + } + if len(pending) != 0 { + t.Fatalf("terminal shell appeared as a pending event: %+v", pending) + } + claimUntil := time.Now().UTC().Truncate(time.Microsecond).Add(time.Minute) + if claimed, err := fresh.ClaimTerminal(id, "owner", claimUntil); err == nil || claimed != nil { + t.Fatalf("terminal shell without an event was claimable: err=%v session=%+v", err, claimed) + } +} + +func TestDurableCompatPlaybackStoreTerminalizesConflictingUnpersistedShell(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + id := fmt.Sprintf("compat-terminal-conflict-%d", time.Now().UnixNano()) + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id) }) + seed := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + seed.Put(PlaybackSession{ID: id, CompatToken: "owner", UpstreamSessionID: "upstream-1"}) + local := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + local.mem.Put(PlaybackSession{ID: id, CompatToken: "owner", UpstreamSessionID: "upstream-1"}) + local.markUnpersisted(id) + + if err := local.HideFromRouting(id, "owner"); err != nil { + t.Fatalf("hide conflicting terminal shell: %v", err) + } + fresh := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + if _, ok := fresh.Get(id); ok { + t.Fatal("insert conflict left the durable session routable") + } +} + +func TestDurableCompatPlaybackStoreReplaysPendingUpdates(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + id := fmt.Sprintf("compat-pending-update-%d", time.Now().UnixNano()) + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id) }) + store := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + store.Put(PlaybackSession{ID: id, CompatToken: "owner", UpstreamSessionID: "upstream-old"}) + if err := store.mem.Update(id, func(session *PlaybackSession) error { + session.UpstreamSessionID = "upstream-new" + return nil + }); err != nil { + t.Fatalf("seed pending cache update: %v", err) + } + store.appendPendingUpdate(id, "owner", func(session *PlaybackSession) error { + session.UpstreamSessionID = "upstream-new" + return nil + }) + + if err := store.Update(id, func(session *PlaybackSession) error { + session.TranscodeStarted = true + return nil + }); err != nil { + t.Fatalf("update with pending replay: %v", err) + } + if store.hasPendingUpdates(id) { + t.Fatal("successfully replayed update remained pending") + } + fresh := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + got, ok := fresh.Get(id) + if !ok || got.UpstreamSessionID != "upstream-new" || !got.TranscodeStarted { + t.Fatalf("durable replay lost updates: ok=%v session=%+v", ok, got) + } +} + +func TestDurableCompatPlaybackStoreColdLookupIgnoresReservedThrottle(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + id := fmt.Sprintf("compat-cold-load-%d", time.Now().UnixNano()) + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id) }) + seed := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + seed.Put(PlaybackSession{ID: id, CompatToken: "owner", RouteItemID: "route-1"}) + fresh := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + _ = fresh.shouldRevalidateID(id) + _ = fresh.shouldRevalidateToken("owner") + + if _, ok := fresh.Get(id); !ok { + t.Fatal("cold ID lookup treated an in-flight validation reservation as not-found") + } + other := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + _ = other.shouldRevalidateToken("owner") + if _, _, ok := other.FindByRoute("owner", "route-1"); !ok { + t.Fatal("cold token lookup treated an in-flight validation reservation as not-found") + } +} diff --git a/internal/jellycompat/router.go b/internal/jellycompat/router.go index 8f3f0d54..c1e32f12 100644 --- a/internal/jellycompat/router.go +++ b/internal/jellycompat/router.go @@ -111,15 +111,17 @@ func NewRouter(deps Dependencies) chi.Router { // Compat transcode reconstruct is driven by the recipe carried in the durable // compat playback store (jellycompat_playback_sessions); no separate native // recipe table is needed. - if cleaned, err := playbackHandler.CleanupOrphanedTranscodes(); err != nil { - slog.Warn("jellycompat transcode cleanup failed", "dir", playbackHandler.TranscodeDir, "error", err) - } else if cleaned > 0 { - slog.Info("jellycompat transcode cleanup removed orphaned dirs", "dir", playbackHandler.TranscodeDir, "count", cleaned) - } + // + // This shares TranscodeDir with the native api sweep but snapshots only this + // manager's live set; see the api NewRouter call site for why cross-manager + // reaping of a >24h idle dir is bounded and safe. + playback.StartPeriodicOrphanCleanup(deps.AppContext, "jellycompat", playbackHandler.TranscodeDir, playbackHandler.CleanupOrphanedTranscodes, playback.OrphanCleanupInterval) playbackHandler.profileRefreshRequester = deps.RecWorker playbackHandler.SettingsRepo = deps.SettingsRepo playbackHandler.RecipeNodeStore = deps.RecipeNodeStore playbackHandler.SessionSyncer = deps.SessionSyncer + playbackHandler.WatchScrobbler = deps.WatchScrobbler + playbackHandler.StableIdentityResolver = deps.StableIdentityResolver if subtitleRepo != nil { playbackHandler.SubtitleRepo = subtitleRepo playbackHandler.S3Client = deps.S3Client diff --git a/internal/jellycompat/server.go b/internal/jellycompat/server.go index 0e70a0ca..59b41754 100644 --- a/internal/jellycompat/server.go +++ b/internal/jellycompat/server.go @@ -20,11 +20,16 @@ import ( "github.com/Silo-Server/silo-server/internal/subtitles" "github.com/Silo-Server/silo-server/internal/userstore" "github.com/Silo-Server/silo-server/internal/watchstate" + "github.com/Silo-Server/silo-server/internal/watchsync" ) // Dependencies holds the pluggable pieces used by the compat server. type Dependencies struct { Config *config.Config + // AppContext is the process lifecycle context. When set, it bounds the + // periodic orphan-transcode sweep so it stops on shutdown; nil (tests) makes + // the sweep a single boot-time run instead of a long-lived ticker. + AppContext context.Context // LiveConfig returns the current hot-reloaded config. May be nil (tests, // worker modes); read through CurrentConfig(), which falls back to Config. LiveConfig func() *config.Config @@ -88,14 +93,16 @@ type Dependencies struct { // admin live-session table right after compat playback starts/stops, so // the activity dashboard doesn't wait for the periodic reconciler tick. // Optional. - SessionSyncer PlaybackSessionSyncer - FileResolver FilePathResolver - UserStoreProvider userstore.UserStoreProvider - AccessFilterFn AccessFilterResolver - NodePlanner nodepool.SessionPlanner - JWTSecret string - Recommender recommendations.Recommender - RecWorker *recommendations.Worker + SessionSyncer PlaybackSessionSyncer + FileResolver FilePathResolver + UserStoreProvider userstore.UserStoreProvider + WatchScrobbler PlaybackWatchScrobbler + StableIdentityResolver watchsync.ScrobbleIdentityResolver + AccessFilterFn AccessFilterResolver + NodePlanner nodepool.SessionPlanner + JWTSecret string + Recommender recommendations.Recommender + RecWorker *recommendations.Worker // Settings (optional; reads server_settings for watched threshold, etc.) SettingsRepo SettingsReader @@ -164,11 +171,15 @@ func (s *Server) SessionStore() *SessionStore { // StartBackgroundTasks starts background goroutines tied to the server lifecycle. // Call this once after constructing the server; goroutines stop when ctx is cancelled. -func (s *Server) StartBackgroundTasks(ctx context.Context) { +// The returned channel closes after the initial terminal recovery scan so +// callers can order broader stale-session sweeps behind authoritative compat +// recovery without delaying HTTP startup. +func (s *Server) StartBackgroundTasks(ctx context.Context) <-chan struct{} { if s.deps.DB != nil { repo := NewSessionRepository(s.deps.DB, s.deps.SecretCipher) StartSessionCleanupWithPlaybackStore(ctx, repo, s.deps.PlaybackStore, 1*time.Hour) } + return StartTerminalScrobbleRecovery(ctx, s.deps.PlaybackStore, s.deps.WatchScrobbler, 30*time.Second) } // NewDependencies fills in sensible defaults for optional compat dependencies. diff --git a/internal/jellycompat/streams.go b/internal/jellycompat/streams.go index 78cbb544..1ac34fbe 100644 --- a/internal/jellycompat/streams.go +++ b/internal/jellycompat/streams.go @@ -25,6 +25,7 @@ import ( "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/subtitles" + "github.com/Silo-Server/silo-server/internal/watchsync" ) // Jellyfin Web is sensitive to startup latency. Use shorter compat segments @@ -40,7 +41,7 @@ type sessionReportRequest struct { ItemID string `json:"ItemId"` MediaSourceID string `json:"MediaSourceId"` PlaySessionID string `json:"PlaySessionId"` - PositionTicks int64 `json:"PositionTicks"` + PositionTicks *int64 `json:"PositionTicks,omitempty"` IsPaused bool `json:"IsPaused"` AudioStreamIndex *compatIntValue `json:"AudioStreamIndex,omitempty"` SubtitleStreamIndex *compatIntValue `json:"SubtitleStreamIndex,omitempty"` @@ -222,24 +223,31 @@ func (h *PlaybackHandler) HandleMasterManifest(w http.ResponseWriter, r *http.Re writeCompatUpstreamError(w, err) return } + failRemoteStart := func() { + h.teardownPlaySession(context.WithoutCancel(r.Context()), playSession, nil, nil) + } upstreamSession, upstreamErr := h.sessionMgr.GetSession(playSession.UpstreamSessionID) if upstreamErr == nil { plan := h.NodePlanner.PlanSession(playSession.UpstreamSessionID, upstreamSession.TranscodeNodeURL, true, source.Version.Bitrate) if tcNode := plan.TranscodeNode; tcNode != nil { if h.fileResolver == nil { + failRemoteStart() writeError(w, http.StatusInternalServerError, "ServerError", "File resolver not available") return } file, fileErr := h.fileResolver.GetByID(r.Context(), source.FileID) if fileErr != nil { + failRemoteStart() writeError(w, http.StatusNotFound, "NotFound", "Media file not found") return } if err := h.sessionMgr.SetTranscodeNodeURL(playSession.UpstreamSessionID, tcNode.URL); err != nil { + failRemoteStart() writeError(w, http.StatusInternalServerError, "ServerError", "Failed to bind transcode node") return } if err := h.startRemoteTranscode(r.Context(), playSession.ID, playSession.UpstreamSessionID, *source, file, playSession.InitialSeekSeconds, tcNode.URL); err != nil { + failRemoteStart() if errors.Is(err, errTranscode4KDisallowed) { writeError(w, http.StatusForbidden, "Forbidden", "4K video transcoding is disabled on this server") return @@ -249,6 +257,7 @@ func (h *PlaybackHandler) HandleMasterManifest(w http.ResponseWriter, r *http.Re } redirectURL, redirectErr := h.buildProxyRedirectURL(playSession.ID, playSession.UpstreamSessionID, string(playback.PlayTranscode), file, *source, tcNode.URL, 0, plan.ProxyNode) if redirectErr != nil { + failRemoteStart() writeError(w, http.StatusInternalServerError, "ServerError", "Failed to sign proxy stream URL") return } @@ -261,6 +270,9 @@ func (h *PlaybackHandler) HandleMasterManifest(w http.ResponseWriter, r *http.Re // In distributed mode admins can disable the local fallback so the API // server never transcodes when no eligible node exists. if h.NodePlanner != nil && !nodepool.LocalTranscodeFallbackAllowed(r.Context(), h.SettingsRepo) { + if playSession.UpstreamSessionID != "" { + h.teardownPlaySession(context.WithoutCancel(r.Context()), playSession, nil, nil) + } writeError(w, http.StatusServiceUnavailable, "NoTranscodeNode", "No transcode node is available and local transcode fallback is disabled") return @@ -778,22 +790,98 @@ func (h *PlaybackHandler) HandleDeleteActiveEncodings(w http.ResponseWriter, r * return } - h.teardownPlaySession(r.Context(), playSession) + fallback := compatScrobbleFallbackSession(session, playSession, nil, 0, false, false) + upstreamSession, transcodeNodeURL := h.compatStopSnapshot(playSession, fallback) + if event, ok := h.compatScrobbleEvent( + r.Context(), compatScrobbleStop, playSession, upstreamSession, nil, nil, + ); ok { + h.stageCompatTerminal(r.Context(), playSession, upstreamSession, transcodeNodeURL, event, false, false, 0) + } else if upstreamSession == nil { + // With no native session and no reported position, publishing a zero-value + // fallback could move provider progress backwards. Keep only the terminal + // authenticated mapping for a possible later Stopped report. + if err := h.playbackStore.HideFromRouting(playSession.ID, playSession.CompatToken); err != nil && + !errors.Is(err, ErrSessionNotFound) { + h.scheduleCompatTerminalHide(playSession.ID, playSession.CompatToken, playSession.ExpiresAt, 1) + } + h.cleanupPlaySession(r.Context(), playSession, nil, transcodeNodeURL) + } else { + h.playbackStore.Delete(playSession.ID) + h.cleanupPlaySession(r.Context(), playSession, upstreamSession, transcodeNodeURL) + } w.WriteHeader(http.StatusNoContent) } -// teardownPlaySession stops the upstream playback session and removes the compat -// play session from the store. Every step is idempotent, so it is safe to call -// from both the explicit ActiveEncodings teardown and a Stopped playback report -// (which may race). It is a no-op-safe teardown for an already-gone session. -func (h *PlaybackHandler) teardownPlaySession(ctx context.Context, playSession *PlaybackSession) { +// teardownPlaySession stages the authoritative stop before resource cleanup, +// then delivers it through a leased durable record. The record is removed only +// after watch-sync accepts the event, so a provider-queue failure remains +// retryable by the client or the delayed ActiveEncodings fallback. +func (h *PlaybackHandler) teardownPlaySession( + ctx context.Context, + playSession *PlaybackSession, + fallbackSession *playback.Session, + positionOverride *float64, +) { + upstreamSession, transcodeNodeURL := h.compatStopSnapshot(playSession, fallbackSession) + if event, ok := h.compatScrobbleEvent( + ctx, compatScrobbleStop, playSession, upstreamSession, nil, positionOverride, + ); ok { + h.stageCompatTerminal(ctx, playSession, upstreamSession, transcodeNodeURL, event, true, false, 0) + } else if playSession.Terminal { + // A late Stopped report without PositionTicks cannot replace a staged + // fallback after ActiveEncodings already removed the native session. Keep + // that durable event (or terminal shell) and retry its delivery instead of + // deleting the only recoverable stop position. + h.cleanupPlaySession(ctx, playSession, upstreamSession, transcodeNodeURL) + if playSession.TerminalScrobbleEvent != nil { + h.deliverCompatTerminal( + ctx, + playSession.ID, + playSession.CompatToken, + playSession.TerminalAuthoritative, + playSession.ExpiresAt, + 0, + true, + ) + } + } else { + h.playbackStore.Delete(playSession.ID) + h.cleanupPlaySession(ctx, playSession, upstreamSession, transcodeNodeURL) + } +} + +func (h *PlaybackHandler) compatStopSnapshot( + playSession *PlaybackSession, + fallbackSession *playback.Session, +) (*playback.Session, string) { transcodeNodeURL := "" + var upstreamSession *playback.Session if h.sessionMgr != nil { - if upstreamSession, err := h.sessionMgr.GetSession(playSession.UpstreamSessionID); err == nil { + if current, err := h.sessionMgr.GetSession(playSession.UpstreamSessionID); err == nil { + upstreamSession = current transcodeNodeURL = upstreamSession.TranscodeNodeURL } } + if upstreamSession == nil && fallbackSession != nil { + copy := *fallbackSession + copy.ID = playSession.UpstreamSessionID + if source := compatScrobbleSource(playSession, ©, nil); source != nil { + copy.MediaFileID = source.FileID + } + upstreamSession = © + } + return upstreamSession, transcodeNodeURL +} + +// cleanupPlaySession performs idempotent process/resource cleanup after the +// terminal provider event has been staged (or intentionally omitted). +func (h *PlaybackHandler) cleanupPlaySession( + ctx context.Context, + playSession *PlaybackSession, + upstreamSession *playback.Session, + transcodeNodeURL string, +) { h.tm.CloseTranscodeSession(playSession.UpstreamSessionID, transcodeNodeURL) if h.sessionMgr != nil { _ = h.sessionMgr.StopSession(playSession.UpstreamSessionID) @@ -809,17 +897,237 @@ func (h *PlaybackHandler) teardownPlaySession(ctx context.Context, playSession * "playback_session_id", playSession.UpstreamSessionID) } } - h.playbackStore.Delete(playSession.ID) // Clients often drop the connection right after reporting a stop, so detach // the sync from request cancellation to keep the admin view accurate. h.syncSessionsNow(context.WithoutCancel(ctx), "compat_stop") } +const ( + compatTerminalClaimLease = 10 * time.Second + compatTerminalInitialRetryDelay = 250 * time.Millisecond + compatTerminalMaxRetryDelay = 30 * time.Second + defaultCompatTerminalFallbackDelay = 2 * time.Second +) + +func (h *PlaybackHandler) compatTerminalFallbackDelay() time.Duration { + if h != nil && h.terminalFallbackDelay > 0 { + return h.terminalFallbackDelay + } + return defaultCompatTerminalFallbackDelay +} + +func compatTerminalRetryDelay(attempt int) time.Duration { + delay := compatTerminalInitialRetryDelay + for i := 0; i < attempt && delay < compatTerminalMaxRetryDelay; i++ { + delay *= 2 + if delay > compatTerminalMaxRetryDelay { + return compatTerminalMaxRetryDelay + } + } + return delay +} + +func (h *PlaybackHandler) stageCompatTerminal( + ctx context.Context, + playSession *PlaybackSession, + upstreamSession *playback.Session, + transcodeNodeURL string, + event watchsync.ScrobbleEvent, + authoritative bool, + cleanupDone bool, + attempt int, +) { + staged, err := h.playbackStore.StageTerminal(playSession.ID, playSession.CompatToken, event, authoritative) + if err != nil { + // Production durable staging installs its local marker before I/O. Keep + // the interface invariant for alternate stores that fail before doing so. + _ = h.playbackStore.HideFromRouting(playSession.ID, playSession.CompatToken) + if errors.Is(err, ErrSessionNotFound) { + if !cleanupDone { + h.cleanupPlaySession(ctx, playSession, upstreamSession, transcodeNodeURL) + } + return + } + if !cleanupDone { + h.cleanupPlaySession(ctx, playSession, upstreamSession, transcodeNodeURL) + cleanupDone = true + } + if playSession.ExpiresAt.IsZero() || time.Now().Before(playSession.ExpiresAt) { + h.scheduleCompatTerminalStage( + playSession, upstreamSession, transcodeNodeURL, event, authoritative, cleanupDone, attempt+1, + ) + } else if !cleanupDone { + h.cleanupPlaySession(ctx, playSession, upstreamSession, transcodeNodeURL) + } + return + } + if !cleanupDone { + h.cleanupPlaySession(ctx, staged, upstreamSession, transcodeNodeURL) + } + if authoritative { + h.deliverCompatTerminal(ctx, staged.ID, staged.CompatToken, true, staged.ExpiresAt, 0, true) + return + } + h.scheduleCompatTerminalDelivery( + staged.ID, staged.CompatToken, false, staged.ExpiresAt, h.compatTerminalFallbackDelay(), 0, + ) +} + +func (h *PlaybackHandler) scheduleCompatTerminalHide( + playSessionID string, + compatToken string, + expiresAt time.Time, + attempt int, +) { + time.AfterFunc(compatTerminalRetryDelay(attempt), func() { + if !expiresAt.IsZero() && !time.Now().Before(expiresAt) { + return + } + err := h.playbackStore.HideFromRouting(playSessionID, compatToken) + if err != nil && !errors.Is(err, ErrSessionNotFound) { + h.scheduleCompatTerminalHide(playSessionID, compatToken, expiresAt, attempt+1) + } + }) +} + +func (h *PlaybackHandler) scheduleCompatTerminalStage( + playSession *PlaybackSession, + upstreamSession *playback.Session, + transcodeNodeURL string, + event watchsync.ScrobbleEvent, + authoritative bool, + cleanupDone bool, + attempt int, +) { + playSessionCopy := *playSession + var upstreamCopy *playback.Session + if upstreamSession != nil { + copy := *upstreamSession + upstreamCopy = © + } + time.AfterFunc(compatTerminalRetryDelay(attempt), func() { + h.stageCompatTerminal( + context.Background(), &playSessionCopy, upstreamCopy, transcodeNodeURL, + event, authoritative, cleanupDone, attempt, + ) + }) +} + +func (h *PlaybackHandler) scheduleCompatTerminalDelivery( + playSessionID string, + compatToken string, + requireAuthoritative bool, + expiresAt time.Time, + delay time.Duration, + attempt int, +) { + time.AfterFunc(delay, func() { + h.deliverCompatTerminal( + context.Background(), playSessionID, compatToken, requireAuthoritative, expiresAt, attempt, true, + ) + }) +} + +// deliverCompatTerminal leases the staged event, persists it into watch-sync's +// durable queue, and only then completes the compat terminal record. A +// provisional ActiveEncodings fallback remains available for a later +// authoritative Stopped replacement. +func (h *PlaybackHandler) deliverCompatTerminal( + ctx context.Context, + playSessionID string, + compatToken string, + requireAuthoritative bool, + expiresAt time.Time, + attempt int, + retry bool, +) { + if h == nil || h.playbackStore == nil || h.WatchScrobbler == nil { + return + } + if !expiresAt.IsZero() && !time.Now().Before(expiresAt) { + return + } + now := time.Now().UTC().Truncate(time.Microsecond) + claimUntil := now.Add(compatTerminalClaimLease) + playSession, err := h.playbackStore.ClaimTerminal(playSessionID, compatToken, claimUntil) + if err != nil { + if !requireAuthoritative && errors.Is(err, ErrTerminalClaimUnavailable) { + if pending, ok := h.playbackStore.GetFinalizable(playSessionID, compatToken); ok && + pending.TerminalFallbackSent && !pending.TerminalAuthoritative { + return + } + } + if retry && !errors.Is(err, ErrSessionNotFound) { + h.scheduleCompatTerminalDelivery( + playSessionID, compatToken, requireAuthoritative, expiresAt, + compatTerminalRetryDelay(attempt), attempt+1, + ) + } + return + } + ownedClaimUntil := playSession.TerminalClaimUntil + if playSession.TerminalScrobbleEvent == nil || (requireAuthoritative && !playSession.TerminalAuthoritative) { + h.playbackStore.ReleaseTerminalClaim( + playSessionID, compatToken, ownedClaimUntil, playSession.TerminalClaimVersion, false, + ) + if retry { + h.scheduleCompatTerminalDelivery( + playSessionID, compatToken, requireAuthoritative, expiresAt, + compatTerminalRetryDelay(attempt), attempt+1, + ) + } + return + } + + err = h.dispatchCompatScrobbleEventConfirmed( + ctx, + compatScrobbleStop, + *playSession.TerminalScrobbleEvent, + playSession.TerminalAuthoritative, + ) + if err != nil { + h.playbackStore.ReleaseTerminalClaim( + playSessionID, compatToken, ownedClaimUntil, playSession.TerminalClaimVersion, false, + ) + if retry { + h.scheduleCompatTerminalDelivery( + playSessionID, compatToken, requireAuthoritative, expiresAt, + compatTerminalRetryDelay(attempt), attempt+1, + ) + } + return + } + if playSession.TerminalAuthoritative { + h.playbackStore.CompleteTerminal( + playSessionID, compatToken, ownedClaimUntil, playSession.TerminalClaimVersion, + ) + // If a newer authoritative report replaced this event while it was in + // flight, completion intentionally failed. Release the old lease so the + // replacement can be claimed immediately instead of waiting for expiry. + h.playbackStore.ReleaseTerminalClaim( + playSessionID, compatToken, ownedClaimUntil, playSession.TerminalClaimVersion, false, + ) + return + } + h.playbackStore.ReleaseTerminalClaim( + playSessionID, compatToken, ownedClaimUntil, playSession.TerminalClaimVersion, true, + ) +} + // compatSessionSyncTimeout bounds the immediate session sync issued from // request paths, so a stalled database degrades to the periodic reconciler // tick instead of pinning request goroutines. const compatSessionSyncTimeout = 5 * time.Second +func compatDetachedContext(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } else { + ctx = context.WithoutCancel(ctx) + } + return context.WithTimeout(ctx, compatSessionSyncTimeout) +} + // syncSessionsNow flushes the native-session snapshot to the shared admin // live-session table so compat start/stop events are visible immediately // instead of on the next reconciler tick. @@ -851,11 +1159,16 @@ func (h *PlaybackHandler) handlePlaybackReport(w http.ResponseWriter, r *http.Re return } - playSession, ok := h.playbackStore.Get(req.PlaySessionID) - if ok && playSession.CompatToken != session.Token { - playSession, ok = nil, false + var playSession *PlaybackSession + var ok bool + if stop { + playSession, ok = h.playbackStore.GetFinalizable(req.PlaySessionID, session.Token) + } else { + playSession, ok = h.playbackStore.Get(req.PlaySessionID) + if ok && playSession.CompatToken != session.Token { + playSession, ok = nil, false + } } - matchedByRouteOnly := false if !ok { // Static=true direct play (Infuse, SenPlayer) skips PlaybackInfo, so the // client reports progress under its own generated PlaySessionId. The @@ -864,20 +1177,27 @@ func (h *PlaybackHandler) handlePlaybackReport(w http.ResponseWriter, r *http.Re // route-scoped lookup the stream path uses (see resolvePlaybackRoute). // Without either, these reports silently no-op, the admin activity view // position freezes, and stale cleanup drops the still-active session. - playSession, ok = h.playbackStore.FindByClientPlaySessionID(session.Token, req.PlaySessionID) + if stop { + playSession, ok = h.playbackStore.FindFinalizableByClientPlaySessionID( + session.Token, req.PlaySessionID, req.ItemID, req.MediaSourceID, + ) + } else { + playSession, ok = h.playbackStore.FindByClientPlaySessionID(session.Token, req.PlaySessionID) + } if ok && !reportMatchesPlaySession(playSession, req) { playSession, ok = nil, false } } - if !ok { + if !ok && !stop { for _, routeID := range []string{req.ItemID, req.MediaSourceID} { if routeID == "" { continue } - if playSession, _, ok = h.playbackStore.FindByRoute(session.Token, routeID); ok { - matchedByRouteOnly = true + playSession, _, ok = h.playbackStore.FindByRoute(session.Token, routeID) + if ok && reportMatchesPlaySession(playSession, req) { break } + playSession, ok = nil, false } } if !ok || playSession.UpstreamSessionID == "" { @@ -885,7 +1205,14 @@ func (h *PlaybackHandler) handlePlaybackReport(w http.ResponseWriter, r *http.Re return } - positionSeconds := float64(req.PositionTicks) / 10_000_000 + positionSeconds := 0.0 + positionReported := req.PositionTicks != nil + if positionReported { + positionSeconds = float64(*req.PositionTicks) / 10_000_000 + if positionSeconds < 0 { + positionSeconds = 0 + } + } audioTrackIndex := 0 audioRestarted := false // Jellyfin web/mobile clients send AudioStreamIndex on every progress @@ -928,8 +1255,15 @@ func (h *PlaybackHandler) handlePlaybackReport(w http.ResponseWriter, r *http.Re ) } } - if positionSeconds > 0 && h.sessionMgr != nil { + var previousSession *playback.Session + progressUpdated := false + if positionReported && h.sessionMgr != nil { + if current, err := h.sessionMgr.GetSession(playSession.UpstreamSessionID); err == nil && current != nil { + copy := *current + previousSession = © + } err := h.sessionMgr.UpdateProgress(playSession.UpstreamSessionID, positionSeconds, req.IsPaused) + progressUpdated = err == nil if errors.Is(err, playback.ErrSessionNotFound) && !stop { // The upstream session was reaped as stale (e.g. the client buffered // far ahead and went quiet between range requests). The report proves @@ -937,10 +1271,24 @@ func (h *PlaybackHandler) handlePlaybackReport(w http.ResponseWriter, r *http.Re // dropping it from session tracking for the rest of playback. if revived := h.reviveUpstreamForReport(r.Context(), session, playSession, req.MediaSourceID); revived != nil { playSession = revived - _ = h.sessionMgr.UpdateProgress(playSession.UpstreamSessionID, positionSeconds, req.IsPaused) + progressUpdated = h.sessionMgr.UpdateProgress(playSession.UpstreamSessionID, positionSeconds, req.IsPaused) == nil + previousSession = nil } } } + if progressUpdated && !stop && previousSession != nil && previousSession.IsPaused != req.IsPaused { + updatedSession := *previousSession + updatedSession.Position = positionSeconds + updatedSession.IsPaused = req.IsPaused + action := compatScrobbleStart + if req.IsPaused { + action = compatScrobblePause + } + h.dispatchCompatScrobbleAt( + r.Context(), action, playSession, &updatedSession, + findMediaSource(playSession, req.MediaSourceID), &positionSeconds, + ) + } // Persist progress to user store if positionSeconds > 0 && h.storeProvider != nil && playSession.ItemID != "" { if store, storeErr := h.storeProvider.ForUser(r.Context(), session.StreamAppUserID); storeErr == nil { @@ -957,12 +1305,20 @@ func (h *PlaybackHandler) handlePlaybackReport(w http.ResponseWriter, r *http.Re } } } - if stop && !matchedByRouteOnly { - // A bare item/source route match is ambiguous when the same item plays - // twice under one token, so never tear down a session the report may - // not own. A session that really stopped emits no further reports or - // transport, so stale cleanup reaps it shortly anyway. - h.teardownPlaySession(r.Context(), playSession) + if stop { + // Direct ids and recorded aliases are per-play, caller-owned identifiers. + // Route-only matching is intentionally excluded for Stopped reports: a + // delayed stop for an earlier play of the same item must never tear down + // the current play. + source := findMediaSource(playSession, req.MediaSourceID) + fallback := compatScrobbleFallbackSession( + session, playSession, source, positionSeconds, positionReported, req.IsPaused, + ) + var positionOverride *float64 + if positionReported { + positionOverride = &positionSeconds + } + h.teardownPlaySession(r.Context(), playSession, fallback, positionOverride) } w.WriteHeader(http.StatusNoContent) @@ -1041,8 +1397,22 @@ func (h *PlaybackHandler) ensureUpstreamPlayback(ctx context.Context, compatSess return nil, err } if h.tm != nil { - if reconstructed := h.tm.ReconstructSession(ctx, playSession.UpstreamSessionID, compatSession.StreamAppUserID, h.upstreamRecipeCard(playSession, compatSession, source, method)); reconstructed != nil { + card := h.upstreamRecipeCard(playSession, compatSession, source, method) + // Cards minted before client metadata was recorded (and the + // direct/remux fallback cards built here from scratch) carry + // none; the current compat request identifies the client, so + // the reconstructed session keeps its label and JF pill. + if card.ClientName == "" && card.ClientUserAgent == "" { + info := playback.ClientInfoFromContext(ctx) + card.ClientName, card.ClientVersion, card.ClientUserAgent = info.Name, info.Version, info.UserAgent + } + if reconstructed := h.tm.ReconstructSession(ctx, playSession.UpstreamSessionID, compatSession.StreamAppUserID, card); reconstructed != nil { + if !playSession.ProgressPersistenceKnown || + playSession.DisableProgressPersistence != reconstructed.DisableProgressPersistence { + h.recordCompatProgressPersistence(playSession.ID, reconstructed.DisableProgressPersistence) + } _ = h.syncUpstreamAudioSelection(playSession, source) + h.dispatchCompatScrobble(ctx, compatScrobbleStart, playSession, reconstructed, &source) return playSession, nil } } @@ -1058,6 +1428,11 @@ func (h *PlaybackHandler) ensureUpstreamPlayback(ctx context.Context, compatSess playSession.UpstreamPlayMethod = "" playSession.TranscodeStarted = false } else { + if current, currentErr := h.sessionMgr.GetSession(playSession.UpstreamSessionID); currentErr == nil && + (!playSession.ProgressPersistenceKnown || + playSession.DisableProgressPersistence != current.DisableProgressPersistence) { + h.recordCompatProgressPersistence(playSession.ID, current.DisableProgressPersistence) + } _ = h.syncUpstreamAudioSelection(playSession, source) return playSession, nil } @@ -1084,6 +1459,7 @@ func (h *PlaybackHandler) ensureUpstreamPlayback(ctx context.Context, compatSess transcodeNodeURL := "" if current, err := h.sessionMgr.GetSession(oldUpstreamSessionID); err == nil { transcodeNodeURL = current.TranscodeNodeURL + h.dispatchCompatScrobble(ctx, compatScrobbleStop, playSession, current, nil) } _ = h.sessionMgr.StopSession(oldUpstreamSessionID) h.tm.CloseTranscodeSession(oldUpstreamSessionID, transcodeNodeURL) @@ -1125,6 +1501,8 @@ func (h *PlaybackHandler) ensureUpstreamPlayback(ctx context.Context, compatSess current.UpstreamSessionID = session.ID current.UpstreamPlayMethod = method current.TranscodeStarted = false + current.ProgressPersistenceKnown = true + current.DisableProgressPersistence = session.DisableProgressPersistence return nil }); updateErr != nil { _ = h.sessionMgr.StopSession(session.ID) @@ -1145,9 +1523,21 @@ func (h *PlaybackHandler) ensureUpstreamPlayback(ctx context.Context, compatSess return nil, ErrSessionNotFound } h.syncSessionsNow(ctx, "compat_start") + h.dispatchCompatScrobble(ctx, compatScrobbleStart, updated, session, &source) return updated, nil } +func (h *PlaybackHandler) recordCompatProgressPersistence(playSessionID string, disabled bool) { + if h == nil || h.playbackStore == nil || playSessionID == "" { + return + } + _ = h.playbackStore.Update(playSessionID, func(session *PlaybackSession) error { + session.ProgressPersistenceKnown = true + session.DisableProgressPersistence = disabled + return nil + }) +} + func (h *PlaybackHandler) ensureTranscodeManifest(ctx context.Context, compatSession *Session, playSessionID string, source PlaybackMediaSource) ([]byte, error) { playSession, err := h.ensureUpstreamPlayback(ctx, compatSession, playSessionID, source, "transcode") if err != nil { @@ -1156,6 +1546,10 @@ func (h *PlaybackHandler) ensureTranscodeManifest(ctx context.Context, compatSes transcodeSession, err := h.ensureTranscodeSession(ctx, playSessionID, playSession.UpstreamSessionID, source) if err != nil { + requestErr := ctx.Err() + if requestErr == nil || !errors.Is(err, requestErr) { + h.teardownPlaySession(ctx, playSession, nil, nil) + } return nil, err } @@ -1183,6 +1577,7 @@ func (h *PlaybackHandler) ensureTranscodeManifest(ctx context.Context, compatSes case <-ctx.Done(): return nil, ctx.Err() case <-deadline: + h.teardownPlaySession(ctx, playSession, nil, nil) return nil, playback.ErrManifestNotReady case <-time.After(pollInterval): } @@ -1266,6 +1661,10 @@ func (h *PlaybackHandler) ensureTranscodeSession(ctx context.Context, playSessio h.tm.RegisterTranscodeSession(upstreamSessionID, transcodeSession) unlock() + // Mirror the actual encode decisions onto the upstream session before the + // recipe is persisted — video-copy HLS must not sync as a video transcode. + h.recordTranscodeStreamDetails(ctx, upstreamSessionID, opts) + // Register the exit monitor and persist the reconstruction recipe (shared with // the remote path). On a failed compat-store write roll back this abandoned // transcode rather than leaking it. diff --git a/internal/jellycompat/streams_test.go b/internal/jellycompat/streams_test.go index 8ce3cfd4..9eafc44f 100644 --- a/internal/jellycompat/streams_test.go +++ b/internal/jellycompat/streams_test.go @@ -198,7 +198,7 @@ func TestTeardownPlaySession_DeletesNodeRecipe(t *testing.T) { if !ok { t.Fatal("expected play session") } - h.teardownPlaySession(context.Background(), playSession) + h.teardownPlaySession(context.Background(), playSession, nil, nil) if _, ok := recipeStore.Get("upstream-1"); ok { t.Fatal("node recipe should be deleted on deliberate teardown") diff --git a/internal/jellycompat/terminal_scrobble_recovery.go b/internal/jellycompat/terminal_scrobble_recovery.go new file mode 100644 index 00000000..af18e101 --- /dev/null +++ b/internal/jellycompat/terminal_scrobble_recovery.go @@ -0,0 +1,122 @@ +package jellycompat + +import ( + "context" + "log/slog" + "time" +) + +const compatTerminalRecoveryBatchSize = 100 + +// StartTerminalScrobbleRecovery resumes terminal provider events that survived +// a server exit after staging but before delivery. The first scan runs at +// startup; periodic scans also recover expired leases from crashed peers. +func StartTerminalScrobbleRecovery( + ctx context.Context, + store CompatPlaybackStore, + scrobbler PlaybackWatchScrobbler, + interval time.Duration, +) <-chan struct{} { + initialScanDone := make(chan struct{}) + if store == nil || scrobbler == nil { + close(initialScanDone) + return initialScanDone + } + if interval <= 0 { + interval = 30 * time.Second + } + handler := &PlaybackHandler{playbackStore: store, WatchScrobbler: scrobbler} + run := func(runCtx context.Context) { + if err := recoverPendingTerminalScrobbles(runCtx, handler); err != nil { + slog.WarnContext(runCtx, "recover jellycompat terminal scrobbles failed", + "component", "jellycompat", "error", err) + } + } + go func() { + if err := recoverInitialPendingTerminalScrobbles(ctx, handler); err != nil { + slog.WarnContext(ctx, "initial jellycompat terminal scrobble recovery incomplete", + "component", "jellycompat", "error", err) + } + close(initialScanDone) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + run(ctx) + } + } + }() + return initialScanDone +} + +func recoverPendingTerminalScrobbles(ctx context.Context, handler *PlaybackHandler) error { + _, _, err := recoverPendingTerminalScrobbleBatch(ctx, handler, nil) + return err +} + +func recoverInitialPendingTerminalScrobbles(ctx context.Context, handler *PlaybackHandler) error { + seen := make(map[string]struct{}) + for { + processed, batchSize, err := recoverPendingTerminalScrobbleBatch(ctx, handler, seen) + if err != nil { + return err + } + // Both stores advance a rotating cursor between calls. A page with no + // unseen records means the cursor wrapped; a short page means it reached + // the current tail. + if processed == 0 || batchSize < compatTerminalRecoveryBatchSize { + return nil + } + } +} + +func recoverPendingTerminalScrobbleBatch( + ctx context.Context, + handler *PlaybackHandler, + seen map[string]struct{}, +) (int, int, error) { + pending, err := handler.playbackStore.ListPendingTerminals(ctx, compatTerminalRecoveryBatchSize) + if err != nil { + return 0, 0, err + } + processed := 0 + for _, session := range pending { + if err := ctx.Err(); err != nil { + return processed, len(pending), err + } + if seen != nil { + if _, ok := seen[session.ID]; ok { + continue + } + seen[session.ID] = struct{}{} + } + processed++ + if !session.TerminalAuthoritative && session.TerminalScrobbleEvent != nil { + deliverAfter := session.TerminalScrobbleEvent.OccurredAt.Add(handler.compatTerminalFallbackDelay()) + if delay := time.Until(deliverAfter); delay > 0 { + handler.scheduleCompatTerminalDelivery( + session.ID, + session.CompatToken, + false, + session.ExpiresAt, + delay, + 0, + ) + continue + } + } + handler.deliverCompatTerminal( + ctx, + session.ID, + session.CompatToken, + session.TerminalAuthoritative, + session.ExpiresAt, + 0, + false, + ) + } + return processed, len(pending), nil +} diff --git a/internal/manga/enrichment.go b/internal/manga/enrichment.go index 0b88fc9f..02f7d84d 100644 --- a/internal/manga/enrichment.go +++ b/internal/manga/enrichment.go @@ -625,7 +625,7 @@ func (e *Enricher) cacheRemoteImage(ctx context.Context, contentID, url string, return url, "" } - storedPath := cachedOriginalImagePath(cached.BasePath, cached.Ext) + storedPath := metadata.CachedImageOriginalPath(cached) if storedPath == "" { return url, "" } @@ -645,19 +645,6 @@ func isNilImageCacher(cacher metadata.ImageCacher) bool { } } -func cachedOriginalImagePath(basePath, ext string) string { - if basePath == "" { - return "" - } - if strings.Contains(basePath, "/original.") { - return basePath - } - if ext == "" { - ext = ".jpg" - } - return strings.TrimRight(basePath, "/") + "/original" + ext -} - func (e *Enricher) persist(ctx context.Context, contentID string, providerIDs map[string]string, result *metadata.MetadataResult) error { upd := &catalog.MetadataUpdate{} diff --git a/internal/metadata/apply_item_image_test.go b/internal/metadata/apply_item_image_test.go new file mode 100644 index 00000000..26eb4851 --- /dev/null +++ b/internal/metadata/apply_item_image_test.go @@ -0,0 +1,49 @@ +package metadata + +import ( + "context" + "testing" +) + +const ( + applyImageTestExt = ".webp" + applyImageTestProviderID = "tmdb" + applyImageTestContentType = "movie" +) + +type recordingImageCacher struct { + request CacheImageRequest +} + +func (c *recordingImageCacher) CacheImage(_ context.Context, req CacheImageRequest) (*CacheImageResult, error) { + c.request = req + return &CacheImageResult{ + BasePath: "tmdb/movies/335984/poster", + OriginalPath: "tmdb/movies/335984/poster/original.new-revision.webp", + Revision: "new-revision", + Thumbhash: "new-thumbhash", + Ext: applyImageTestExt, + }, nil +} + +func TestApplyItemImageReturnsExactImmutableRevision(t *testing.T) { + cacher := &recordingImageCacher{} + service := &MetadataService{imageCacher: cacher} + + result, err := service.ApplyItemImage(context.Background(), ApplyItemImageRequest{ + OriginalURL: "https://image.tmdb.org/t/p/original/new-poster.jpg", + ProviderID: applyImageTestProviderID, + ContentType: applyImageTestContentType, + ContentID: "335984", + ImageType: ImagePoster, + }) + if err != nil { + t.Fatalf("ApplyItemImage: %v", err) + } + if result.StoredPath != "tmdb/movies/335984/poster/original.new-revision.webp" { + t.Fatalf("StoredPath = %q, want exact immutable revision path", result.StoredPath) + } + if result.Revision != "new-revision" { + t.Fatalf("Revision = %q, want new-revision", result.Revision) + } +} diff --git a/internal/metadata/artwork_revision_gc.go b/internal/metadata/artwork_revision_gc.go new file mode 100644 index 00000000..96e1b564 --- /dev/null +++ b/internal/metadata/artwork_revision_gc.go @@ -0,0 +1,515 @@ +package metadata + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/artworkkey" +) + +const ( + artworkRevisionGCBatchSize = 100 + artworkRevisionGCLease = 15 * time.Minute + // artworkRevisionDormantRecheck bounds how stale a parked (referenced) + // revision may get before the sweep re-verifies it. Displacement triggers + // are the fast path; the sweep guarantees a reference that disappears + // through an untriggered surface still becomes collectible eventually. + artworkRevisionDormantRecheck = 24 * time.Hour +) + +// ArtworkRevisionDeleter is the object-storage surface used by revision GC. +type ArtworkRevisionDeleter interface { + DeleteObjects(ctx context.Context, bucket string, keys []string) (int, error) + Bucket() string +} + +// ArtworkRevisionGCStats summarizes one bounded cleanup pass. +type ArtworkRevisionGCStats struct { + Claimed int `json:"claimed"` + Deleted int `json:"deleted"` + Referenced int `json:"referenced"` + Retried int `json:"retried"` + DormantChecked int `json:"dormant_checked"` + DormantRequeued int `json:"dormant_requeued"` + Healed int `json:"healed"` +} + +// ArtworkRevisionGarbageCollector deletes unpublished or displaced immutable +// revisions only after their grace period and while no catalog surface +// references them. Work is leased with SKIP LOCKED so multiple workers are safe. +type ArtworkRevisionGarbageCollector struct { + pool *pgxpool.Pool + s3 ArtworkRevisionDeleter +} + +func NewArtworkRevisionGarbageCollector(pool *pgxpool.Pool, s3 ArtworkRevisionDeleter) *ArtworkRevisionGarbageCollector { + if pool == nil || s3 == nil { + return nil + } + return &ArtworkRevisionGarbageCollector{pool: pool, s3: s3} +} + +// Run processes one bounded batch. Failed deletions are retried with +// exponential backoff; an expired lease is recoverable by another worker. +func (g *ArtworkRevisionGarbageCollector) Run(ctx context.Context) (ArtworkRevisionGCStats, error) { + stats := ArtworkRevisionGCStats{} + if g == nil || g.pool == nil || g.s3 == nil { + return stats, fmt.Errorf("artwork revision GC is not configured") + } + + workerID := uuid.NewString() + candidates, err := g.claim(ctx, workerID, artworkRevisionGCBatchSize) + if err != nil { + return stats, err + } + + // Park referenced candidates with one batched reference check instead of a + // per-candidate transaction; most publish/re-cache churn lands here. The + // per-candidate path below re-verifies under the row lock before deleting, + // so a stale answer from this pre-check can only cost extra work, never a + // wrong deletion. + due := candidates + if len(candidates) > 0 { + if referenced, refErr := g.referencedPaths(ctx, candidatePaths(candidates)); refErr == nil { + var parked []int64 + due = due[:0] + for _, candidate := range candidates { + // Rows whose objects were already deleted must finish their + // pending heal; a reference to them is broken, not live. + if _, ok := referenced[candidate.originalPath]; ok && candidate.deletedAt == nil { + parked = append(parked, candidate.id) + continue + } + due = append(due, candidate) + } + if len(parked) > 0 { + if parkErr := g.parkClaimed(ctx, parked, workerID); parkErr != nil { + return stats, parkErr + } + stats.Referenced += len(parked) + } + } else { + slog.WarnContext(ctx, "artwork revision GC: batched reference pre-check failed; falling back to per-candidate checks", + "component", "metadata", "error", refErr) + } + } + + batchStats, err := processArtworkRevisionGCBatch( + due, + func(candidate artworkRevisionGCCandidate) (artworkRevisionGCOutcome, error) { + return g.processCandidate(ctx, candidate, workerID) + }, + func(candidate artworkRevisionGCCandidate, cause error) error { + return g.retry(ctx, candidate, workerID, cause) + }, + ) + stats.Claimed = len(candidates) + stats.Deleted = batchStats.Deleted + stats.Referenced += batchStats.Referenced + stats.Retried = batchStats.Retried + stats.Healed = batchStats.Healed + + checked, requeued, sweepErr := g.sweepDormant(ctx, artworkRevisionGCBatchSize) + stats.DormantChecked = checked + stats.DormantRequeued = requeued + if err == nil { + err = sweepErr + } + return stats, err +} + +func processArtworkRevisionGCBatch( + candidates []artworkRevisionGCCandidate, + process func(artworkRevisionGCCandidate) (artworkRevisionGCOutcome, error), + retry func(artworkRevisionGCCandidate, error) error, +) (ArtworkRevisionGCStats, error) { + stats := ArtworkRevisionGCStats{Claimed: len(candidates)} + var firstErr error + for _, candidate := range candidates { + outcome, err := process(candidate) + if err != nil { + stats.Retried++ + if retryErr := retry(candidate, err); retryErr != nil && firstErr == nil { + firstErr = retryErr + } + continue + } + switch outcome { + case artworkRevisionGCReferenced: + stats.Referenced++ + case artworkRevisionGCDeleted: + stats.Deleted++ + case artworkRevisionGCDeletedAndHealed: + stats.Deleted++ + stats.Healed++ + } + } + return stats, firstErr +} + +type artworkRevisionGCOutcome int + +const ( + artworkRevisionGCSuperseded artworkRevisionGCOutcome = iota + artworkRevisionGCReferenced + artworkRevisionGCDeleted + artworkRevisionGCDeletedAndHealed +) + +type artworkRevisionGCCandidate struct { + id int64 + originalPath string + imageType string + objectKeys []string + attemptCount int + deletedAt *time.Time +} + +func candidatePaths(candidates []artworkRevisionGCCandidate) []string { + paths := make([]string, 0, len(candidates)) + for _, candidate := range candidates { + paths = append(paths, candidate.originalPath) + } + return paths +} + +func (g *ArtworkRevisionGarbageCollector) claim(ctx context.Context, workerID string, limit int) ([]artworkRevisionGCCandidate, error) { + rows, err := g.pool.Query(ctx, ` + WITH due AS ( + SELECT id + FROM artwork_revision_gc_candidates + WHERE not_before <= NOW() + AND next_attempt_at <= NOW() + AND (locked_at IS NULL OR locked_at < NOW() - ($3 * interval '1 second')) + ORDER BY next_attempt_at, id + FOR UPDATE SKIP LOCKED + LIMIT $1 + ) + UPDATE artwork_revision_gc_candidates AS candidate + SET locked_at = NOW(), locked_by = $2, updated_at = NOW() + FROM due + WHERE candidate.id = due.id + RETURNING candidate.id, candidate.original_path, candidate.image_type, candidate.object_keys, candidate.attempt_count, candidate.deleted_at`, + limit, workerID, int64(artworkRevisionGCLease/time.Second)) + if err != nil { + return nil, fmt.Errorf("artwork revision GC: claim: %w", err) + } + defer rows.Close() + + var candidates []artworkRevisionGCCandidate + for rows.Next() { + var candidate artworkRevisionGCCandidate + if err := rows.Scan(&candidate.id, &candidate.originalPath, &candidate.imageType, &candidate.objectKeys, &candidate.attemptCount, &candidate.deletedAt); err != nil { + return nil, fmt.Errorf("artwork revision GC: scan claim: %w", err) + } + candidates = append(candidates, candidate) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("artwork revision GC: claims: %w", err) + } + return candidates, nil +} + +// parkClaimed releases claimed candidates back to the dormant state in one +// statement. Used when the batched pre-check already proved them referenced. +func (g *ArtworkRevisionGarbageCollector) parkClaimed(ctx context.Context, ids []int64, workerID string) error { + _, err := g.pool.Exec(ctx, ` + UPDATE artwork_revision_gc_candidates + SET next_attempt_at = NULL, + attempt_count = 0, + locked_at = NULL, + locked_by = '', + last_error = '', + updated_at = NOW() + WHERE id = ANY($1) AND locked_by = $2`, ids, workerID) + if err != nil { + return fmt.Errorf("artwork revision GC: park referenced revisions: %w", err) + } + return nil +} + +// processCandidate holds the registry row lock across the last reference check +// and object deletion. A concurrent cache attempt registers the revision before +// uploading and therefore waits here; once deletion commits, that attempt can +// safely recreate the complete object set before publication. +func (g *ArtworkRevisionGarbageCollector) processCandidate( + ctx context.Context, + candidate artworkRevisionGCCandidate, + workerID string, +) (artworkRevisionGCOutcome, error) { + tx, err := g.pool.Begin(ctx) + if err != nil { + return artworkRevisionGCSuperseded, fmt.Errorf("artwork revision GC: begin deletion: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + var originalPath, imageType string + var objectKeys []string + var deletedAt *time.Time + err = tx.QueryRow(ctx, ` + SELECT original_path, image_type, object_keys, deleted_at + FROM artwork_revision_gc_candidates + WHERE id = $1 AND locked_by = $2 + FOR UPDATE`, candidate.id, workerID).Scan(&originalPath, &imageType, &objectKeys, &deletedAt) + if errors.Is(err, pgx.ErrNoRows) { + return artworkRevisionGCSuperseded, nil + } + if err != nil { + return artworkRevisionGCSuperseded, fmt.Errorf("artwork revision GC: lock candidate: %w", err) + } + + // Once objects are deleted, a lingering reference is broken rather than + // live: skip parking and finish the pending heal instead. + if deletedAt == nil { + referenced, err := g.isReferenced(ctx, tx, originalPath) + if err != nil { + return artworkRevisionGCSuperseded, err + } + if referenced { + if _, err := tx.Exec(ctx, ` + UPDATE artwork_revision_gc_candidates + SET next_attempt_at = NULL, + attempt_count = 0, + locked_at = NULL, + locked_by = '', + last_error = '', + updated_at = NOW() + WHERE id = $1 AND locked_by = $2`, candidate.id, workerID); err != nil { + return artworkRevisionGCSuperseded, fmt.Errorf("artwork revision GC: park referenced revision: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return artworkRevisionGCSuperseded, fmt.Errorf("artwork revision GC: commit referenced revision: %w", err) + } + return artworkRevisionGCReferenced, nil + } + } + + // Rows queued by the displacement triggers carry no manifest; expand the + // expected object set from the image type so the variant ladder stays + // defined once, in artworkkey. + if len(objectKeys) == 0 { + objectKeys = artworkkey.ObjectKeys(originalPath, imageType) + } + if len(objectKeys) > 0 { + deleted, err := g.s3.DeleteObjects(ctx, g.s3.Bucket(), objectKeys) + if err == nil && deleted != len(objectKeys) { + err = fmt.Errorf("deleted %d of %d artwork objects", deleted, len(objectKeys)) + } + if err != nil { + return artworkRevisionGCSuperseded, err + } + } + // Keep the row until the post-delete heal succeeds: marking deleted_at + // (instead of deleting the row) preserves a durable retry if healing or + // the final row removal fails after the objects are already gone. + if _, err := tx.Exec(ctx, ` + UPDATE artwork_revision_gc_candidates + SET deleted_at = COALESCE(deleted_at, NOW()), updated_at = NOW() + WHERE id = $1 AND locked_by = $2`, candidate.id, workerID); err != nil { + return artworkRevisionGCSuperseded, fmt.Errorf("artwork revision GC: mark deleted: %w", err) + } + if err := tx.Commit(ctx); err != nil { + return artworkRevisionGCSuperseded, fmt.Errorf("artwork revision GC: commit deletion: %w", err) + } + + // Writers that assign an existing path without registering a revision + // (bulk upserts copying previously-read values) do not serialize with the + // row lock above. Detect that narrow race after the fact and reset the + // affected rows the same way the artwork reconciler handles missing + // objects, so pipelines re-cache instead of serving 404 artwork. + healed, healErr := g.healDeletedReferences(ctx, originalPath) + if healErr != nil { + return artworkRevisionGCSuperseded, fmt.Errorf("artwork revision GC: post-delete heal: %w", healErr) + } + // Healing can itself displace the path again (clearing a referencing row + // fires the trigger, which re-arms this row and drops our lease), so gate + // the removal on deleted_at instead of the lease: a concurrent tracker + // re-registration clears deleted_at and must survive; anything else with + // deleted_at set is a finished deletion. + if _, err := g.pool.Exec(ctx, ` + DELETE FROM artwork_revision_gc_candidates + WHERE id = $1 AND deleted_at IS NOT NULL`, candidate.id); err != nil { + return artworkRevisionGCSuperseded, fmt.Errorf("artwork revision GC: finish: %w", err) + } + if healed { + slog.WarnContext(ctx, "artwork revision GC: deleted revision was re-referenced concurrently; reset rows for re-cache", + "component", "metadata", "original_path", originalPath) + return artworkRevisionGCDeletedAndHealed, nil + } + return artworkRevisionGCDeleted, nil +} + +type artworkReferenceQuerier interface { + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row +} + +func artworkReferenceUnionSQL() string { + surfaces := artworkSweepSurfaces() + parts := make([]string, 0, len(surfaces)) + for _, surface := range surfaces { + parts = append(parts, fmt.Sprintf("SELECT %s AS path FROM %s WHERE %s = ANY($1)", surface.pathCol, surface.table, surface.pathCol)) + } + return strings.Join(parts, " UNION ALL ") +} + +func (g *ArtworkRevisionGarbageCollector) isReferenced(ctx context.Context, q artworkReferenceQuerier, originalPath string) (bool, error) { + var referenced bool + query := "SELECT EXISTS(" + artworkReferenceUnionSQL() + ")" + if err := q.QueryRow(ctx, query, []string{originalPath}).Scan(&referenced); err != nil { + return false, fmt.Errorf("artwork revision GC: check references: %w", err) + } + return referenced, nil +} + +// referencedPaths returns the subset of paths referenced by any catalog +// surface, using one query per run instead of one per candidate. +func (g *ArtworkRevisionGarbageCollector) referencedPaths(ctx context.Context, paths []string) (map[string]struct{}, error) { + referenced := make(map[string]struct{}) + if len(paths) == 0 { + return referenced, nil + } + rows, err := g.pool.Query(ctx, "SELECT DISTINCT path FROM ("+artworkReferenceUnionSQL()+") refs", paths) + if err != nil { + return nil, fmt.Errorf("artwork revision GC: batch reference check: %w", err) + } + defer rows.Close() + for rows.Next() { + var path string + if err := rows.Scan(&path); err != nil { + return nil, fmt.Errorf("artwork revision GC: scan reference: %w", err) + } + referenced[path] = struct{}{} + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("artwork revision GC: references: %w", err) + } + return referenced, nil +} + +// sweepDormant re-verifies a bounded batch of parked revisions whose last +// check is older than the recheck interval, re-arming any that lost every +// reference through a surface without a displacement trigger. +func (g *ArtworkRevisionGarbageCollector) sweepDormant(ctx context.Context, limit int) (checked, requeued int, err error) { + rows, err := g.pool.Query(ctx, ` + SELECT id, original_path + FROM artwork_revision_gc_candidates + WHERE next_attempt_at IS NULL + AND updated_at < NOW() - ($2 * interval '1 second') + ORDER BY updated_at, id + LIMIT $1`, limit, int64(artworkRevisionDormantRecheck/time.Second)) + if err != nil { + return 0, 0, fmt.Errorf("artwork revision GC: list dormant revisions: %w", err) + } + defer rows.Close() + + ids := make(map[string][]int64) + var paths []string + for rows.Next() { + var id int64 + var path string + if err := rows.Scan(&id, &path); err != nil { + return 0, 0, fmt.Errorf("artwork revision GC: scan dormant revision: %w", err) + } + if _, ok := ids[path]; !ok { + paths = append(paths, path) + } + ids[path] = append(ids[path], id) + checked++ + } + if err := rows.Err(); err != nil { + return 0, 0, fmt.Errorf("artwork revision GC: dormant revisions: %w", err) + } + if checked == 0 { + return 0, 0, nil + } + + referenced, err := g.referencedPaths(ctx, paths) + if err != nil { + return checked, 0, err + } + var touch, requeue []int64 + for path, pathIDs := range ids { + if _, ok := referenced[path]; ok { + touch = append(touch, pathIDs...) + continue + } + requeue = append(requeue, pathIDs...) + } + if len(requeue) > 0 { + if _, err := g.pool.Exec(ctx, ` + UPDATE artwork_revision_gc_candidates + SET next_attempt_at = GREATEST(not_before, NOW()), + updated_at = NOW() + WHERE id = ANY($1) AND next_attempt_at IS NULL`, requeue); err != nil { + return checked, 0, fmt.Errorf("artwork revision GC: requeue dormant revisions: %w", err) + } + requeued = len(requeue) + } + if len(touch) > 0 { + if _, err := g.pool.Exec(ctx, ` + UPDATE artwork_revision_gc_candidates + SET updated_at = NOW() + WHERE id = ANY($1) AND next_attempt_at IS NULL`, touch); err != nil { + return checked, requeued, fmt.Errorf("artwork revision GC: touch dormant revisions: %w", err) + } + } + return checked, requeued, nil +} + +// healDeletedReferences resets any row still pointing at a just-deleted +// revision, mirroring the reconciler: rows with a re-downloadable source are +// repointed at it (the image cache pipeline re-caches them); the rest are +// cleared for their owning pipeline to refill. +func (g *ArtworkRevisionGarbageCollector) healDeletedReferences(ctx context.Context, originalPath string) (bool, error) { + healed := false + for _, surface := range artworkSweepSurfaces() { + if surface.sourceCol != "" { + resetSQL := fmt.Sprintf(`UPDATE %s SET %s WHERE %s = $1 AND %s`, + surface.table, surface.resetSet(), surface.pathCol, surface.remoteSourcePredicate()) + tag, err := g.pool.Exec(ctx, resetSQL, originalPath) + if err != nil { + return healed, fmt.Errorf("artwork revision GC: heal %s: %w", surface.name, err) + } + healed = healed || tag.RowsAffected() > 0 + } + clearSQL := fmt.Sprintf(`UPDATE %s SET %s WHERE %s = $1 AND NOT (%s)`, + surface.table, surface.clearSet, surface.pathCol, surface.remoteSourcePredicate()) + tag, err := g.pool.Exec(ctx, clearSQL, originalPath) + if err != nil { + return healed, fmt.Errorf("artwork revision GC: heal %s: %w", surface.name, err) + } + healed = healed || tag.RowsAffected() > 0 + } + return healed, nil +} + +func (g *ArtworkRevisionGarbageCollector) retry(ctx context.Context, candidate artworkRevisionGCCandidate, workerID string, cause error) error { + delay := time.Minute << min(candidate.attemptCount, 10) + _, err := g.pool.Exec(ctx, ` + UPDATE artwork_revision_gc_candidates + SET attempt_count = attempt_count + 1, + next_attempt_at = NOW() + ($3 * interval '1 second'), + locked_at = NULL, + locked_by = '', + last_error = $4, + updated_at = NOW() + WHERE id = $1 AND locked_by = $2`, candidate.id, workerID, int64(delay/time.Second), cause.Error()) + if err != nil { + return fmt.Errorf("artwork revision GC: schedule retry: %w", err) + } + return nil +} + +func (s ArtworkRevisionGCStats) JSON() []byte { + data, _ := json.Marshal(s) + return data +} diff --git a/internal/metadata/artwork_revision_gc_test.go b/internal/metadata/artwork_revision_gc_test.go new file mode 100644 index 00000000..70e62625 --- /dev/null +++ b/internal/metadata/artwork_revision_gc_test.go @@ -0,0 +1,546 @@ +package metadata + +import ( + "context" + "errors" + "fmt" + "os" + "slices" + "sync" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/artworkkey" + "github.com/Silo-Server/silo-server/internal/catalog" +) + +type blockingArtworkRevisionDeleter struct { + started chan struct{} + release chan struct{} + once sync.Once + + mu sync.Mutex + deleted [][]string +} + +func (d *blockingArtworkRevisionDeleter) Bucket() string { return "artwork" } + +func (d *blockingArtworkRevisionDeleter) DeleteObjects(ctx context.Context, _ string, keys []string) (int, error) { + d.once.Do(func() { close(d.started) }) + if d.release != nil { + select { + case <-d.release: + case <-ctx.Done(): + return 0, ctx.Err() + } + } + d.mu.Lock() + d.deleted = append(d.deleted, append([]string(nil), keys...)) + d.mu.Unlock() + return len(keys), nil +} + +func artworkRevisionGCTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func TestProcessArtworkRevisionGCBatchContinuesAfterRetryFailure(t *testing.T) { + candidates := []artworkRevisionGCCandidate{{id: 1}, {id: 2}, {id: 3}} + processed := make([]int64, 0, len(candidates)) + retryErr := errors.New("schedule retry") + + stats, err := processArtworkRevisionGCBatch( + candidates, + func(candidate artworkRevisionGCCandidate) (artworkRevisionGCOutcome, error) { + processed = append(processed, candidate.id) + switch candidate.id { + case 1: + return artworkRevisionGCSuperseded, errors.New("delete object") + case 2: + return artworkRevisionGCReferenced, nil + default: + return artworkRevisionGCDeleted, nil + } + }, + func(candidate artworkRevisionGCCandidate, _ error) error { + if candidate.id == 1 { + return retryErr + } + return nil + }, + ) + + if !errors.Is(err, retryErr) { + t.Fatalf("process batch error = %v, want %v", err, retryErr) + } + if !slices.Equal(processed, []int64{1, 2, 3}) { + t.Fatalf("processed candidates = %v, want all candidates", processed) + } + want := ArtworkRevisionGCStats{Claimed: 3, Deleted: 1, Referenced: 1, Retried: 1} + if stats != want { + t.Fatalf("stats = %+v, want %+v", stats, want) + } +} + +func TestArtworkRevisionGCSerializesDeletionWithRetracking(t *testing.T) { + pool := artworkRevisionGCTestPool(t) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + suffix := time.Now().UnixNano() + originalPath := fmt.Sprintf("tmdb/movies/gc-%d/poster/original.old.webp", suffix) + oldKeys := []string{originalPath, fmt.Sprintf("tmdb/movies/gc-%d/poster/w500.old.webp", suffix)} + newKeys := []string{originalPath, fmt.Sprintf("tmdb/movies/gc-%d/poster/w500.old.webp", suffix), fmt.Sprintf("tmdb/movies/gc-%d/poster/w300.old.webp", suffix)} + workerID := fmt.Sprintf("gc-worker-%d", suffix) + + var candidateID int64 + if err := pool.QueryRow(ctx, ` + INSERT INTO artwork_revision_gc_candidates ( + original_path, object_keys, not_before, next_attempt_at, locked_at, locked_by + ) VALUES ($1, $2, NOW() - interval '1 hour', NOW() - interval '1 hour', NOW(), $3) + RETURNING id`, originalPath, oldKeys, workerID).Scan(&candidateID); err != nil { + t.Fatalf("seed revision: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM artwork_revision_gc_candidates WHERE original_path = $1`, originalPath) + }) + + deleter := &blockingArtworkRevisionDeleter{started: make(chan struct{}), release: make(chan struct{})} + collector := NewArtworkRevisionGarbageCollector(pool, deleter) + processDone := make(chan struct { + outcome artworkRevisionGCOutcome + err error + }, 1) + go func() { + outcome, err := collector.processCandidate(ctx, artworkRevisionGCCandidate{ + id: candidateID, originalPath: originalPath, objectKeys: oldKeys, + }, workerID) + processDone <- struct { + outcome artworkRevisionGCOutcome + err error + }{outcome: outcome, err: err} + }() + + select { + case <-deleter.started: + case <-ctx.Done(): + t.Fatalf("collector did not begin deletion: %v", ctx.Err()) + } + + tracker := catalog.NewArtworkRevisionTracker(pool) + trackDone := make(chan error, 1) + go func() { trackDone <- tracker.TrackArtworkRevision(ctx, originalPath, "poster", newKeys) }() + + select { + case err := <-trackDone: + t.Fatalf("tracking completed while deletion row was locked: %v", err) + case <-time.After(100 * time.Millisecond): + } + + close(deleter.release) + processed := <-processDone + if processed.err != nil { + t.Fatalf("processCandidate: %v", processed.err) + } + if processed.outcome != artworkRevisionGCDeleted { + t.Fatalf("outcome = %v, want deleted", processed.outcome) + } + if err := <-trackDone; err != nil { + t.Fatalf("retrack revision: %v", err) + } + + var storedKeys []string + var nextAttempt time.Time + if err := pool.QueryRow(ctx, ` + SELECT object_keys, next_attempt_at + FROM artwork_revision_gc_candidates + WHERE original_path = $1`, originalPath).Scan(&storedKeys, &nextAttempt); err != nil { + t.Fatalf("load retracked revision: %v", err) + } + if !slices.Equal(storedKeys, newKeys) { + t.Fatalf("stored manifest = %v, want %v", storedKeys, newKeys) + } + if !nextAttempt.After(time.Now().Add(23 * time.Hour)) { + t.Fatalf("next attempt = %v, want refreshed publication grace", nextAttempt) + } +} + +func TestArtworkRevisionGCParksReferencedRevision(t *testing.T) { + pool := artworkRevisionGCTestPool(t) + ctx := context.Background() + suffix := time.Now().UnixNano() + contentID := fmt.Sprintf("gc-referenced-%d", suffix) + originalPath := fmt.Sprintf("tmdb/movies/%d/poster/original.live.webp", suffix) + keys := []string{originalPath} + workerID := fmt.Sprintf("gc-worker-%d", suffix) + + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres, poster_path) + VALUES ($1, 'movie', 'GC Referenced', 'matched', '{}'::text[], $2)`, contentID, originalPath); err != nil { + t.Fatalf("seed referenced item: %v", err) + } + var candidateID int64 + if err := pool.QueryRow(ctx, ` + INSERT INTO artwork_revision_gc_candidates ( + original_path, object_keys, not_before, next_attempt_at, locked_at, locked_by + ) VALUES ($1, $2, NOW() - interval '1 hour', NOW() - interval '1 hour', NOW(), $3) + RETURNING id`, originalPath, keys, workerID).Scan(&candidateID); err != nil { + t.Fatalf("seed revision: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) + _, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path = $1`, originalPath) + }) + + deleter := &blockingArtworkRevisionDeleter{started: make(chan struct{})} + collector := NewArtworkRevisionGarbageCollector(pool, deleter) + outcome, err := collector.processCandidate(ctx, artworkRevisionGCCandidate{id: candidateID}, workerID) + if err != nil { + t.Fatalf("processCandidate: %v", err) + } + if outcome != artworkRevisionGCReferenced { + t.Fatalf("outcome = %v, want referenced", outcome) + } + select { + case <-deleter.started: + t.Fatal("referenced revision was sent to object deletion") + default: + } + + var lockedBy string + var nextAttempt *time.Time + if err := pool.QueryRow(ctx, ` + SELECT locked_by, next_attempt_at + FROM artwork_revision_gc_candidates + WHERE id = $1`, candidateID).Scan(&lockedBy, &nextAttempt); err != nil { + t.Fatalf("load parked revision: %v", err) + } + if lockedBy != "" { + t.Fatalf("locked_by = %q, want released", lockedBy) + } + if nextAttempt != nil { + t.Fatalf("next_attempt_at = %v, want dormant NULL", nextAttempt) + } +} + +func TestArtworkRevisionTriggerQueuesDisplacedRevision(t *testing.T) { + pool := artworkRevisionGCTestPool(t) + ctx := context.Background() + suffix := time.Now().UnixNano() + contentID := fmt.Sprintf("gc-trigger-%d", suffix) + oldPath := fmt.Sprintf("tmdb/movies/%d/poster/original.old.webp", suffix) + newPath := fmt.Sprintf("tmdb/movies/%d/poster/original.new.webp", suffix) + wantKeys := []string{ + oldPath, + fmt.Sprintf("tmdb/movies/%d/poster/w500.old.webp", suffix), + fmt.Sprintf("tmdb/movies/%d/poster/w300.old.webp", suffix), + fmt.Sprintf("tmdb/movies/%d/poster/future-variant.old.webp", suffix), + } + + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres, poster_path) + VALUES ($1, 'movie', 'GC Trigger', 'matched', '{}'::text[], $2)`, contentID, oldPath); err != nil { + t.Fatalf("seed artwork: %v", err) + } + if _, err := pool.Exec(ctx, ` + INSERT INTO artwork_revision_gc_candidates ( + original_path, object_keys, not_before, next_attempt_at + ) VALUES ($1, $2, NOW(), NULL)`, oldPath, wantKeys); err != nil { + t.Fatalf("seed dormant manifest: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) + _, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path IN ($1, $2)`, oldPath, newPath) + }) + + if _, err := pool.Exec(ctx, `UPDATE media_items SET poster_path = $2 WHERE content_id = $1`, contentID, newPath); err != nil { + t.Fatalf("replace artwork: %v", err) + } + + var objectKeys []string + var notBefore time.Time + if err := pool.QueryRow(ctx, ` + SELECT object_keys, not_before + FROM artwork_revision_gc_candidates + WHERE original_path = $1`, oldPath).Scan(&objectKeys, ¬Before); err != nil { + t.Fatalf("load displaced candidate: %v", err) + } + if !slices.Equal(objectKeys, wantKeys) { + t.Fatalf("object manifest = %v, want %v", objectKeys, wantKeys) + } + if !notBefore.After(time.Now().Add(23 * time.Hour)) { + t.Fatalf("not before = %v, want displacement grace period", notBefore) + } +} + +func TestArtworkRevisionTriggerRecordsImageTypeForCollectorExpansion(t *testing.T) { + pool := artworkRevisionGCTestPool(t) + ctx := context.Background() + suffix := time.Now().UnixNano() + contentID := fmt.Sprintf("gc-trigger-manifest-%d", suffix) + imageTypes := []string{ImageCacheImagePoster, ImageCacheImageBackdrop, ImageCacheImageLogo} + oldPaths := make(map[string]string, len(imageTypes)) + newPaths := make(map[string]string, len(imageTypes)) + for _, imageType := range imageTypes { + oldPaths[imageType] = fmt.Sprintf( + "tmdb/movies/original.segment/%d/%s/original.old.webp", suffix, imageType, + ) + newPaths[imageType] = fmt.Sprintf("tmdb/movies/%d/%s/original.new.webp", suffix, imageType) + } + + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items ( + content_id, type, title, status, genres, poster_path, backdrop_path, logo_path + ) VALUES ($1, 'movie', 'GC Trigger Manifest', 'matched', '{}'::text[], $2, $3, $4)`, + contentID, + oldPaths[ImageCacheImagePoster], + oldPaths[ImageCacheImageBackdrop], + oldPaths[ImageCacheImageLogo], + ); err != nil { + t.Fatalf("seed artwork: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) + paths := []string{ + oldPaths[ImageCacheImagePoster], oldPaths[ImageCacheImageBackdrop], oldPaths[ImageCacheImageLogo], + newPaths[ImageCacheImagePoster], newPaths[ImageCacheImageBackdrop], newPaths[ImageCacheImageLogo], + } + _, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path = ANY($1)`, paths) + }) + + if _, err := pool.Exec(ctx, ` + UPDATE media_items + SET poster_path = $2, backdrop_path = $3, logo_path = $4 + WHERE content_id = $1`, + contentID, + newPaths[ImageCacheImagePoster], + newPaths[ImageCacheImageBackdrop], + newPaths[ImageCacheImageLogo], + ); err != nil { + t.Fatalf("replace artwork: %v", err) + } + + // The trigger stores only the displaced path and its image type; the + // collector expands the manifest via artworkkey at deletion time. + for _, imageType := range imageTypes { + var objectKeys []string + var storedType string + if err := pool.QueryRow(ctx, ` + SELECT object_keys, image_type + FROM artwork_revision_gc_candidates + WHERE original_path = $1`, oldPaths[imageType]).Scan(&objectKeys, &storedType); err != nil { + t.Fatalf("load %s candidate: %v", imageType, err) + } + if len(objectKeys) != 0 { + t.Fatalf("%s object manifest = %v, want trigger to leave expansion to the collector", imageType, objectKeys) + } + if storedType != imageType { + t.Fatalf("stored image type = %q, want %q", storedType, imageType) + } + } +} + +func TestArtworkRevisionTriggerIgnoresUnchangedAssignment(t *testing.T) { + pool := artworkRevisionGCTestPool(t) + ctx := context.Background() + suffix := time.Now().UnixNano() + contentID := fmt.Sprintf("gc-trigger-noop-%d", suffix) + path := fmt.Sprintf("tmdb/movies/%d/poster/original.same.webp", suffix) + + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres, poster_path) + VALUES ($1, 'movie', 'GC Trigger Noop', 'matched', '{}'::text[], $2)`, contentID, path); err != nil { + t.Fatalf("seed artwork: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) + _, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path = $1`, path) + }) + + // Upsert-style write assigning the same value must not queue anything. + if _, err := pool.Exec(ctx, `UPDATE media_items SET poster_path = $2 WHERE content_id = $1`, contentID, path); err != nil { + t.Fatalf("no-op update: %v", err) + } + + var count int + if err := pool.QueryRow(ctx, ` + SELECT COUNT(*) FROM artwork_revision_gc_candidates WHERE original_path = $1`, path).Scan(&count); err != nil { + t.Fatalf("count candidates: %v", err) + } + if count != 0 { + t.Fatalf("unchanged assignment queued %d candidates, want 0", count) + } +} + +func TestArtworkRevisionGCExpandsTriggerManifestFromImageType(t *testing.T) { + pool := artworkRevisionGCTestPool(t) + ctx := context.Background() + suffix := time.Now().UnixNano() + originalPath := fmt.Sprintf("tmdb/movies/gc-expand-%d/backdrop/original.old.webp", suffix) + workerID := fmt.Sprintf("gc-worker-%d", suffix) + + var candidateID int64 + if err := pool.QueryRow(ctx, ` + INSERT INTO artwork_revision_gc_candidates ( + original_path, image_type, object_keys, not_before, next_attempt_at, locked_at, locked_by + ) VALUES ($1, 'backdrop', '{}', NOW() - interval '1 hour', NOW() - interval '1 hour', NOW(), $2) + RETURNING id`, originalPath, workerID).Scan(&candidateID); err != nil { + t.Fatalf("seed trigger-style candidate: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path = $1`, originalPath) + }) + + deleter := &blockingArtworkRevisionDeleter{started: make(chan struct{})} + collector := NewArtworkRevisionGarbageCollector(pool, deleter) + outcome, err := collector.processCandidate(ctx, artworkRevisionGCCandidate{id: candidateID}, workerID) + if err != nil { + t.Fatalf("processCandidate: %v", err) + } + if outcome != artworkRevisionGCDeleted { + t.Fatalf("outcome = %v, want deleted", outcome) + } + + deleter.mu.Lock() + deleted := deleter.deleted + deleter.mu.Unlock() + if len(deleted) != 1 { + t.Fatalf("DeleteObjects calls = %d, want 1", len(deleted)) + } + want := artworkkey.ObjectKeys(originalPath, "backdrop") + if !slices.Equal(deleted[0], want) { + t.Fatalf("deleted keys = %v, want expanded manifest %v", deleted[0], want) + } +} + +func TestArtworkRevisionGCHealsBrokenReferenceAfterObjectDeletion(t *testing.T) { + pool := artworkRevisionGCTestPool(t) + ctx := context.Background() + suffix := time.Now().UnixNano() + contentID := fmt.Sprintf("gc-heal-%d", suffix) + originalPath := fmt.Sprintf("tmdb/movies/%d/poster/original.gone.webp", suffix) + workerID := fmt.Sprintf("gc-worker-%d", suffix) + + // A racing writer re-referenced the path after its objects were deleted: + // the candidate row survived with deleted_at set (heal previously failed). + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres, poster_path) + VALUES ($1, 'movie', 'GC Heal', 'matched', '{}'::text[], $2)`, contentID, originalPath); err != nil { + t.Fatalf("seed re-referencing item: %v", err) + } + var candidateID int64 + if err := pool.QueryRow(ctx, ` + INSERT INTO artwork_revision_gc_candidates ( + original_path, image_type, object_keys, not_before, next_attempt_at, deleted_at, locked_at, locked_by + ) VALUES ($1, 'poster', '{}', NOW() - interval '1 hour', NOW() - interval '1 hour', NOW() - interval '1 hour', NOW(), $2) + RETURNING id`, originalPath, workerID).Scan(&candidateID); err != nil { + t.Fatalf("seed deleted candidate: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) + _, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path = $1`, originalPath) + }) + + deleter := &blockingArtworkRevisionDeleter{started: make(chan struct{})} + collector := NewArtworkRevisionGarbageCollector(pool, deleter) + outcome, err := collector.processCandidate(ctx, artworkRevisionGCCandidate{id: candidateID}, workerID) + if err != nil { + t.Fatalf("processCandidate: %v", err) + } + if outcome != artworkRevisionGCDeletedAndHealed { + t.Fatalf("outcome = %v, want deleted-and-healed (broken reference must not park)", outcome) + } + + var posterPath string + if err := pool.QueryRow(ctx, ` + SELECT poster_path FROM media_items WHERE content_id = $1`, contentID).Scan(&posterPath); err != nil { + t.Fatalf("load healed item: %v", err) + } + if posterPath == originalPath { + t.Fatalf("poster_path still references deleted revision %q", posterPath) + } + var remaining int + if err := pool.QueryRow(ctx, ` + SELECT COUNT(*) FROM artwork_revision_gc_candidates WHERE original_path = $1`, originalPath).Scan(&remaining); err != nil { + t.Fatalf("count candidates: %v", err) + } + if remaining != 0 { + t.Fatalf("candidate rows remaining = %d, want 0 after successful heal", remaining) + } +} + +func TestArtworkRevisionGCDormantSweep(t *testing.T) { + pool := artworkRevisionGCTestPool(t) + ctx := context.Background() + suffix := time.Now().UnixNano() + referencedContentID := fmt.Sprintf("gc-dormant-ref-%d", suffix) + referencedPath := fmt.Sprintf("tmdb/movies/%d/poster/original.ref.webp", suffix) + orphanPath := fmt.Sprintf("tmdb/movies/%d/poster/original.orphan.webp", suffix) + + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres, poster_path) + VALUES ($1, 'movie', 'GC Dormant Ref', 'matched', '{}'::text[], $2)`, referencedContentID, referencedPath); err != nil { + t.Fatalf("seed referenced item: %v", err) + } + for _, path := range []string{referencedPath, orphanPath} { + if _, err := pool.Exec(ctx, ` + INSERT INTO artwork_revision_gc_candidates ( + original_path, image_type, object_keys, not_before, next_attempt_at + ) VALUES ($1, 'poster', '{}', NOW() - interval '2 days', NULL)`, path); err != nil { + t.Fatalf("seed dormant candidate: %v", err) + } + } + // Age both rows past the recheck interval; a reference that vanished + // through an untriggered surface looks exactly like the orphan row. + if _, err := pool.Exec(ctx, ` + UPDATE artwork_revision_gc_candidates + SET updated_at = NOW() - interval '2 days' + WHERE original_path = ANY($1)`, []string{referencedPath, orphanPath}); err != nil { + t.Fatalf("age dormant candidates: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, referencedContentID) + _, _ = pool.Exec(ctx, `DELETE FROM artwork_revision_gc_candidates WHERE original_path = ANY($1)`, []string{referencedPath, orphanPath}) + }) + + deleter := &blockingArtworkRevisionDeleter{started: make(chan struct{})} + collector := NewArtworkRevisionGarbageCollector(pool, deleter) + checked, requeued, err := collector.sweepDormant(ctx, artworkRevisionGCBatchSize) + if err != nil { + t.Fatalf("sweepDormant: %v", err) + } + if checked < 2 { + t.Fatalf("checked = %d, want at least the two seeded rows", checked) + } + if requeued < 1 { + t.Fatalf("requeued = %d, want at least the orphan row", requeued) + } + + var orphanNext, referencedNext *time.Time + if err := pool.QueryRow(ctx, ` + SELECT next_attempt_at FROM artwork_revision_gc_candidates WHERE original_path = $1`, orphanPath).Scan(&orphanNext); err != nil { + t.Fatalf("load orphan candidate: %v", err) + } + if err := pool.QueryRow(ctx, ` + SELECT next_attempt_at FROM artwork_revision_gc_candidates WHERE original_path = $1`, referencedPath).Scan(&referencedNext); err != nil { + t.Fatalf("load referenced candidate: %v", err) + } + if orphanNext == nil { + t.Fatal("orphan dormant row was not re-armed by the sweep") + } + if referencedNext != nil { + t.Fatalf("referenced dormant row was re-armed: %v", *referencedNext) + } +} diff --git a/internal/metadata/builtin.go b/internal/metadata/builtin.go new file mode 100644 index 00000000..027f19dd --- /dev/null +++ b/internal/metadata/builtin.go @@ -0,0 +1,73 @@ +package metadata + +import ( + "context" + "sort" + "sync" +) + +// Built-in host metadata providers are registered as data: one reserved +// plugin_installations row (kind='builtin', plugin_id='silo.builtin') carries +// ordinary plugin_capabilities rows, so the chain store, seeding, defaults +// endpoint, and admin UI all work unchanged. This file is the single +// in-process registration point mapping those capability ids to provider +// constructors: buildProviders returns the registered provider instead of a +// gRPC PluginProvider when the resolved entry's installation is a builtin. +// +// Providers self-register from their package's init (e.g. internal/metadata/nfo) +// to avoid an import cycle — this package defines the Provider interfaces the +// builtin packages implement. cmd/silo blank-imports each builtin package. +// Activating a future builtin (e.g. xattr) is a registration call plus a +// capability row in a migration. + +// IdentityHintProvider is the narrow contract a built-in local provider +// implements to contribute trusted identity hints (tmdb/imdb/tvdb ids parsed +// from curated sidecar files) before Phase-1 search. Hints anchor candidate +// selection through the trusted-hint machinery, unlocking remote enrichment +// by ID even when remote search-by-title returns nothing. +type IdentityHintProvider interface { + IdentityHints(ctx context.Context, query SearchQuery) map[string]string +} + +var ( + builtinProvidersMu sync.RWMutex + builtinProviders = map[string]func() Provider{} +) + +// RegisterBuiltinProvider registers the constructor for a built-in provider +// under its plugin_capabilities capability_id. Later registrations for the +// same capability id replace earlier ones. +func RegisterBuiltinProvider(capabilityID string, construct func() Provider) { + // Registration only happens from package init(); empty id or nil constructor + // is a programmer error that would silently disable a provider, so fail loud. + if capabilityID == "" || construct == nil { + panic("metadata: RegisterBuiltinProvider requires a non-empty capabilityID and non-nil constructor") + } + builtinProvidersMu.Lock() + builtinProviders[capabilityID] = construct + builtinProvidersMu.Unlock() +} + +// BuiltinCapabilityIDs returns the registered builtin capability ids, sorted. +func BuiltinCapabilityIDs() []string { + builtinProvidersMu.RLock() + ids := make([]string, 0, len(builtinProviders)) + for id := range builtinProviders { + ids = append(ids, id) + } + builtinProvidersMu.RUnlock() + sort.Strings(ids) + return ids +} + +// builtinProvider constructs the registered in-process provider for a builtin +// capability id. +func builtinProvider(capabilityID string) (Provider, bool) { + builtinProvidersMu.RLock() + construct, ok := builtinProviders[capabilityID] + builtinProvidersMu.RUnlock() + if !ok { + return nil, false + } + return construct(), true +} diff --git a/internal/metadata/builtin_sync.go b/internal/metadata/builtin_sync.go new file mode 100644 index 00000000..14d3d147 --- /dev/null +++ b/internal/metadata/builtin_sync.go @@ -0,0 +1,150 @@ +package metadata + +import ( + "context" + "fmt" +) + +// ContentLevelsForLibraryType maps a media_folders.type to the metadata +// content levels it serves. It is the server-side source of truth shared by +// chain seeding, the provider-defaults endpoint, and the startup builtin +// chain sync. +func ContentLevelsForLibraryType(libraryType string) []string { + switch libraryType { + case "series": + return []string{"series", "season", "episode"} + case "movies", "movie": + return []string{"movie"} + case "audiobooks", "audiobook": + return []string{"audiobook"} + case "ebooks", "ebook": + return []string{"ebook"} + case "manga": + return []string{"manga"} + case "mixed": + return []string{"movie", "series", "season", "episode", "audiobook", "ebook"} + default: + return nil + } +} + +// SyncBuiltinProviderChains makes every existing library chain aware of the +// registered builtin host providers. It is idempotent and runs at startup +// before serving (callers must invalidate the chain cache afterwards when the +// service is already constructed). It also doubles as the repair path: if a +// stale chain-editor save drops a builtin row (SetChain is +// delete-all-and-reinsert), the row is re-appended disabled on next startup. +// +// Two steps, in order: +// +// 1. Legacy-” materialization: a library whose chain rows are only +// content_level=” predates per-level chains. AppendProviderToAllChains +// cannot insert anything for it (no per-level groups exist), yet the chain +// editor overlays per-level defaults for levels with no saved rows and the +// first save would materialize those defaults over the admin's legacy +// ordering. The sync therefore copies the ” rows to every content level +// the library type serves (same order/enabled) so the editor shows real +// rows; the ” rows stay in place for old binaries. GetChain's legacy +// fallback only applied when a level had zero rows, so resolved chains are +// unchanged. +// 2. For every builtin metadata capability (read from the database, the +// source of truth the registry mirrors), append it to all existing +// per-level chains it supports — disabled at MAX(priority)+1, +// ON CONFLICT DO NOTHING — exactly like a plugin install does. +func SyncBuiltinProviderChains(ctx context.Context, chainRepo *ChainRepository) error { + if chainRepo == nil { + return nil + } + pool := chainRepo.Pool() + + if err := materializeLegacyChains(ctx, chainRepo); err != nil { + return err + } + + rows, err := pool.Query(ctx, + `SELECT pc.plugin_installation_id, pc.capability_id + FROM plugin_capabilities pc + JOIN plugin_installations pi ON pi.id = pc.plugin_installation_id + WHERE pi.kind = 'builtin' AND pc.capability_type = 'metadata_provider.v1' + ORDER BY pc.plugin_installation_id, pc.capability_id`) + if err != nil { + return fmt.Errorf("listing builtin metadata capabilities: %w", err) + } + type builtinCap struct { + installationID int + capabilityID string + } + var caps []builtinCap + for rows.Next() { + var c builtinCap + if err := rows.Scan(&c.installationID, &c.capabilityID); err != nil { + rows.Close() + return fmt.Errorf("scanning builtin capability: %w", err) + } + caps = append(caps, c) + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating builtin capabilities: %w", err) + } + + for _, c := range caps { + if err := chainRepo.AppendProviderToAllChains(ctx, c.installationID, c.capabilityID, func(level string) SeedPlacement { + return LookupSeedPlacement(ctx, pool, c.installationID, c.capabilityID, level) + }); err != nil { + return fmt.Errorf("appending builtin capability %q to chains: %w", c.capabilityID, err) + } + } + return nil +} + +// materializeLegacyChains copies content_level=” chain rows to every content +// level the library type serves, for libraries that have only legacy rows. +// The copy preserves order and enabled state; existing rows win via +// ON CONFLICT DO NOTHING, and the ” rows are left untouched. +func materializeLegacyChains(ctx context.Context, chainRepo *ChainRepository) error { + pool := chainRepo.Pool() + rows, err := pool.Query(ctx, + `SELECT c.media_folder_id, f.type + FROM library_provider_chains c + JOIN media_folders f ON f.id = c.media_folder_id + GROUP BY c.media_folder_id, f.type + HAVING COUNT(*) FILTER (WHERE c.content_level <> '') = 0`) + if err != nil { + return fmt.Errorf("listing legacy-chain libraries: %w", err) + } + type legacyFolder struct { + folderID int + libraryType string + } + var folders []legacyFolder + for rows.Next() { + var f legacyFolder + if err := rows.Scan(&f.folderID, &f.libraryType); err != nil { + rows.Close() + return fmt.Errorf("scanning legacy-chain library: %w", err) + } + folders = append(folders, f) + } + rows.Close() + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating legacy-chain libraries: %w", err) + } + + for _, f := range folders { + for _, level := range ContentLevelsForLibraryType(f.libraryType) { + if _, err := pool.Exec(ctx, + `INSERT INTO library_provider_chains + (media_folder_id, plugin_installation_id, capability_id, capability_type, content_level, priority, enabled) + SELECT media_folder_id, plugin_installation_id, capability_id, capability_type, $2, priority, enabled + FROM library_provider_chains + WHERE media_folder_id = $1 AND content_level = '' + ON CONFLICT DO NOTHING`, + f.folderID, level, + ); err != nil { + return fmt.Errorf("materializing legacy chain for folder %d level %q: %w", f.folderID, level, err) + } + } + } + return nil +} diff --git a/internal/metadata/builtin_test.go b/internal/metadata/builtin_test.go new file mode 100644 index 00000000..a2691ffb --- /dev/null +++ b/internal/metadata/builtin_test.go @@ -0,0 +1,65 @@ +package metadata + +import ( + "slices" + "testing" +) + +type builtinStubProvider struct{ slug string } + +func (p *builtinStubProvider) Slug() string { return p.slug } +func (p *builtinStubProvider) Name() string { return p.slug } +func (p *builtinStubProvider) ForTypes() []string { return []string{"movie", "series"} } + +func TestBuiltinProviderRegistry(t *testing.T) { + // The test registers under a synthetic capability id so it cannot collide + // with real registrations (the nfo package self-registers on import). + RegisterBuiltinProvider("test-builtin-cap", func() Provider { + return &builtinStubProvider{slug: "test-builtin-cap"} + }) + + ids := BuiltinCapabilityIDs() + if !slices.Contains(ids, "test-builtin-cap") { + t.Fatalf("BuiltinCapabilityIDs() = %v, want to contain test-builtin-cap", ids) + } + + p, ok := builtinProvider("test-builtin-cap") + if !ok || p == nil { + t.Fatalf("builtinProvider(test-builtin-cap) = %v, %v; want constructed provider", p, ok) + } + if p.Slug() != "test-builtin-cap" { + t.Errorf("provider slug = %q", p.Slug()) + } + + if _, ok := builtinProvider("unregistered"); ok { + t.Error("builtinProvider(unregistered) = ok, want miss") + } +} + +// The chain-less fallback must skip capabilities that opt out via +// default_enabled=false (absent defaults to true) — otherwise a disabled +// seeded NFO row (or a default_enabled=false specialist plugin) would +// activate itself the moment a level's enabled-filter yields zero entries. +func TestProviderEligibleForFallback(t *testing.T) { + cases := []struct { + name string + metadataJSON string + contentLevel string + want bool + }{ + {"default enabled provider eligible", tmdbCapMetadata, "movie", true}, + {"no metadata eligible", ``, "movie", true}, + {"default_enabled=false excluded", `{"metadata":{"default_priority":{"movie":1},"default_enabled":false}}`, "movie", false}, + {"nfo capability metadata excluded", `{"display_name":"NFO Files","default_priority":{"movie":1,"series":1},"default_enabled":false}`, "movie", false}, + {"default_enabled=false excluded even on declared level", `{"metadata":{"default_priority":{"series":50},"default_enabled":false}}`, "series", false}, + {"unsupported level still excluded", tvdbCapMetadata, "movie", false}, + {"explicit true eligible", `{"metadata":{"default_priority":{"movie":2},"default_enabled":true}}`, "movie", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := providerEligibleForFallback([]byte(tc.metadataJSON), tc.contentLevel); got != tc.want { + t.Fatalf("providerEligibleForFallback(%q, %q) = %v, want %v", tc.metadataJSON, tc.contentLevel, got, tc.want) + } + }) + } +} diff --git a/internal/metadata/cache_result.go b/internal/metadata/cache_result.go new file mode 100644 index 00000000..3e1888dd --- /dev/null +++ b/internal/metadata/cache_result.go @@ -0,0 +1,31 @@ +package metadata + +import ( + "strings" + + "github.com/Silo-Server/silo-server/internal/artworkkey" +) + +// CachedImageOriginalPath returns the exact stored original key. The fallback +// keeps older ImageCacher test doubles and implementations source compatible +// while callers migrate away from reconstructing object keys themselves. +func CachedImageOriginalPath(result *CacheImageResult) string { + if result == nil { + return "" + } + if result.OriginalPath != "" { + return result.OriginalPath + } + if result.BasePath == "" { + return "" + } + if strings.Contains(result.BasePath, "/original.") { + return result.BasePath + } + ext := result.Ext + if ext == "" { + // Historical default for legacy cachers that predate WebP conversion. + ext = ".jpg" + } + return artworkkey.Original(result.BasePath, "", ext) +} diff --git a/internal/metadata/canonicalize.go b/internal/metadata/canonicalize.go index 03a01c06..8a0a4a34 100644 --- a/internal/metadata/canonicalize.go +++ b/internal/metadata/canonicalize.go @@ -52,6 +52,26 @@ func (s *MetadataService) canonicalizeLocalContentID( if s == nil || !contentid.IsLocal(from) { return from, nil } + return s.reanchorContentID(ctx, from, ids, itemType) +} + +// reanchorContentID moves `from` to the provider-anchored content_id derived +// from `ids` when the two differ, merging onto an existing target or renaming a +// free one. It is the shared core behind two callers: the local-skeleton +// promotion (canonicalizeLocalContentID) and the manual-refresh corrected- +// identity re-anchor (a fixed <uniqueid> in an NFO whose item was already +// anchored to the wrong provider id). Both must hold the provider-dedup lock so +// the target id cannot be claimed underneath us. When ids do not derive a +// provider-anchored id, or it equals `from`, this is a no-op. +func (s *MetadataService) reanchorContentID( + ctx context.Context, + from string, + ids contentid.ProviderIDs, + itemType string, +) (string, error) { + if s == nil { + return from, nil + } // Derive with no path fallback: we only want a provider-anchored id here, not // another local value. diff --git a/internal/metadata/chain.go b/internal/metadata/chain.go index 8022d774..c214fedb 100644 --- a/internal/metadata/chain.go +++ b/internal/metadata/chain.go @@ -345,7 +345,19 @@ func resolveEnabledProviders( if err != nil { return nil, err } - return buildProviders(ctx, caps, resolver, pool, checker), nil + // Exclude capabilities that opt out of default-enable (e.g. the seeded- + // disabled built-in NFO provider). Like the chain-less fallback, an + // installed-but-off provider must not activate itself through this + // content-level-agnostic path (person refresh); only an admin enabling its + // chain row does. Absent/unparseable metadata stays eligible (legacy). + eligible := make([]CapabilityInfo, 0, len(caps)) + for _, c := range caps { + if !extractDefaultEnabled(lookupCapabilityMetadata(ctx, pool, c.PluginInstallationID, c.CapabilityID)) { + continue + } + eligible = append(eligible, c) + } + return buildProviders(ctx, eligible, resolver, pool, checker), nil } // resolveEnabledProvidersByPriority returns all enabled providers sorted by @@ -375,8 +387,8 @@ func resolveEnabledProvidersByPriority( items := make([]ranked, 0, len(caps)) for _, c := range caps { metadataJSON := lookupCapabilityMetadata(ctx, pool, c.PluginInstallationID, c.CapabilityID) - if !providerSupportsLevel(metadataJSON, contentLevel) { - slog.DebugContext(ctx, "skipping metadata provider: does not declare support for content level", "component", "metadata", + if !providerEligibleForFallback(metadataJSON, contentLevel) { + slog.DebugContext(ctx, "skipping metadata provider: not eligible for chain-less fallback", "component", "metadata", "installation_id", c.PluginInstallationID, "capability_id", c.CapabilityID, "content_level", contentLevel) @@ -421,6 +433,17 @@ func providerSupportsLevel(metadataJSON []byte, contentLevel string) bool { return levels[contentLevel] > 0 } +// providerEligibleForFallback reports whether a capability may participate in +// the chain-less fallback for contentLevel. The fallback runs not only for +// chain-less libraries but whenever a level's enabled-filter yields zero +// entries, so a capability that opts out of being enabled by default +// (default_enabled=false — e.g. the seeded-disabled built-in NFO provider or a +// specialist plugin) must not activate itself through it. Absent or +// unparseable metadata defaults to eligible, preserving legacy behavior. +func providerEligibleForFallback(metadataJSON []byte, contentLevel string) bool { + return providerSupportsLevel(metadataJSON, contentLevel) && extractDefaultEnabled(metadataJSON) +} + // lookupCapabilityMetadata returns the raw plugin_capabilities.metadata JSON for // a provider's exact metadata_provider.v1 capability, or nil if absent. func lookupCapabilityMetadata(ctx context.Context, pool *pgxpool.Pool, pluginInstallationID int, capabilityID string) []byte { @@ -599,12 +622,30 @@ func buildProviders( providers := make([]Provider, 0, len(caps)) for _, c := range caps { enabled, err := installationEnabled(ctx, checker, pool, c.PluginInstallationID) - if err != nil || !enabled { + if err != nil { + slog.WarnContext(ctx, "skipping metadata provider: installation enabled-check failed", "component", "metadata", + "installation_id", c.PluginInstallationID, "capability_id", c.CapabilityID, "error", err) + continue + } + if !enabled { slog.DebugContext(ctx, "skipping metadata provider: plugin installation disabled", "component", "metadata", "installation_id", c.PluginInstallationID, "capability_id", c.CapabilityID) continue } + // A builtin installation's capabilities resolve to in-process providers + // from the builtin registry; everything else is a gRPC plugin provider. + if installationIsBuiltin(ctx, checker, pool, c.PluginInstallationID) { + provider, ok := builtinProvider(c.CapabilityID) + if !ok { + slog.WarnContext(ctx, "skipping builtin metadata provider: no registered constructor", "component", "metadata", + "installation_id", c.PluginInstallationID, "capability_id", c.CapabilityID) + continue + } + providers = append(providers, provider) + continue + } + provider, err := NewPluginProviderFromCapability(c.PluginInstallationID, c.CapabilityID, c.DisplayName, resolver) if err != nil { slog.WarnContext(ctx, "skipping metadata provider during chain resolution", "component", "metadata", @@ -619,6 +660,42 @@ func buildProviders( return providers } +// installationKindChecker is an optional extension of InstallationEnabledChecker +// that serves the installation kind from the plugins service's in-memory cache, +// letting chain resolution identify builtin rows without a per-capability DB +// query on the hot path. *plugins.Service implements it; when the checker does +// not (nil or a test fake), installationIsBuiltin falls back to a pool query. +type installationKindChecker interface { + InstallationKind(ctx context.Context, installationID int) (string, error) +} + +// installationIsBuiltin reports whether a plugin installation row is the +// reserved builtin-host installation (kind='builtin'). It prefers the cached +// checker (no DB read) and falls back to a direct query. Errors (including a +// pre-migration schema without the kind column) fail closed to "plugin" so +// resolution behavior is unchanged. +func installationIsBuiltin(ctx context.Context, checker InstallationEnabledChecker, pool *pgxpool.Pool, installationID int) bool { + if kc, ok := checker.(installationKindChecker); ok { + kind, err := kc.InstallationKind(ctx, installationID) + if err != nil { + return false + } + return kind == "builtin" + } + if pool == nil { + return false + } + var kind string + err := pool.QueryRow(ctx, + "SELECT kind FROM plugin_installations WHERE id = $1", + installationID, + ).Scan(&kind) + if err != nil { + return false + } + return kind == "builtin" +} + // lookupCapabilityDisplayName retrieves the display name from plugin capability metadata. func lookupCapabilityDisplayName(ctx context.Context, pool *pgxpool.Pool, installationID int, capabilityID string) string { var displayName string diff --git a/internal/metadata/chain_builtin_db_test.go b/internal/metadata/chain_builtin_db_test.go new file mode 100644 index 00000000..eb41c4ee --- /dev/null +++ b/internal/metadata/chain_builtin_db_test.go @@ -0,0 +1,314 @@ +package metadata + +import ( + "context" + "fmt" + "os" + "reflect" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func chainBuiltinTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func insertTestInstallation(t *testing.T, pool *pgxpool.Pool, kind string, enabled bool) int { + t.Helper() + var id int + pluginID := fmt.Sprintf("test.chain-%s-%d", kind, time.Now().UnixNano()) + err := pool.QueryRow(context.Background(), + `INSERT INTO plugin_installations (plugin_id, version, install_path, enabled, update_policy, kind) + VALUES ($1, '0', '/nonexistent/test-chain-builtin', $2, 'manual', $3) + RETURNING id`, pluginID, enabled, kind).Scan(&id) + if err != nil { + t.Fatalf("seed %s installation: %v", kind, err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM plugin_installations WHERE id = $1`, id) + }) + return id +} + +func insertTestCapability(t *testing.T, pool *pgxpool.Pool, installationID int, capabilityID, metadataJSON string) { + t.Helper() + _, err := pool.Exec(context.Background(), + `INSERT INTO plugin_capabilities (plugin_installation_id, capability_type, capability_id, metadata) + VALUES ($1, 'metadata_provider.v1', $2, $3::jsonb)`, + installationID, capabilityID, metadataJSON) + if err != nil { + t.Fatalf("seed capability %s: %v", capabilityID, err) + } +} + +func insertTestFolder(t *testing.T, pool *pgxpool.Pool, folderType string) int { + t.Helper() + var id int + err := pool.QueryRow(context.Background(), + `INSERT INTO media_folders (type, name, enabled) VALUES ($1, $2, true) RETURNING id`, + folderType, fmt.Sprintf("chain-builtin-test-%d", time.Now().UnixNano())).Scan(&id) + if err != nil { + t.Fatalf("seed folder: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM media_folders WHERE id = $1`, id) + }) + return id +} + +// The migration must seed the reserved builtin installation and the NFO +// capability exactly as specified. +func TestMigrationSeedsReservedBuiltinInstallation(t *testing.T) { + pool := chainBuiltinTestPool(t) + ctx := context.Background() + + var ( + id int + kind string + enabled bool + installPath string + updatePolicy string + ) + err := pool.QueryRow(ctx, + `SELECT id, kind, enabled, install_path, update_policy + FROM plugin_installations WHERE plugin_id = 'silo.builtin'`). + Scan(&id, &kind, &enabled, &installPath, &updatePolicy) + if err != nil { + t.Fatalf("reserved silo.builtin installation missing: %v", err) + } + if kind != "builtin" || !enabled || installPath != "/nonexistent/silo-builtin" || updatePolicy != "manual" { + t.Errorf("reserved row = kind=%q enabled=%v install_path=%q update_policy=%q", kind, enabled, installPath, updatePolicy) + } + + md := lookupCapabilityMetadata(ctx, pool, id, "nfo") + if md == nil { + t.Fatal("silo.builtin nfo capability missing") + } + if extractDefaultEnabled(md) { + t.Error("nfo capability default_enabled must be false") + } + if got := extractDefaultPriority(md, "movie"); got != 1 { + t.Errorf("nfo movie default_priority = %d, want 1", got) + } + if got := extractDefaultPriority(md, "series"); got != 1 { + t.Errorf("nfo series default_priority = %d, want 1", got) + } + // Phase D declares season/episode support so the builtin chain sync + // appends the NFO provider to those chains. + if got := extractDefaultPriority(md, "season"); got != 1 { + t.Errorf("nfo season default_priority = %d, want 1 (Phase D migration)", got) + } + if got := extractDefaultPriority(md, "episode"); got != 1 { + t.Errorf("nfo episode default_priority = %d, want 1 (Phase D migration)", got) + } +} + +// An enabled chain entry pointing at a builtin capability must resolve to the +// in-process provider through the real enabled-check (generic installation +// reads must NOT filter builtins). +func TestResolveChainWithChecker_BuiltinEntryResolvesInProcessProvider(t *testing.T) { + pool := chainBuiltinTestPool(t) + ctx := context.Background() + + capID := fmt.Sprintf("test-builtin-%d", time.Now().UnixNano()) + RegisterBuiltinProvider(capID, func() Provider { return &builtinStubProvider{slug: capID} }) + + installationID := insertTestInstallation(t, pool, "builtin", true) + insertTestCapability(t, pool, installationID, capID, + `{"display_name":"Test Builtin","default_priority":{"movie":1},"default_enabled":false}`) + folderID := insertTestFolder(t, pool, "movies") + + chainRepo := NewChainRepository(pool) + if err := chainRepo.SetChain(ctx, folderID, []ChainEntry{ + {PluginInstallationID: installationID, CapabilityID: capID, ContentLevel: "movie", Priority: 0, Enabled: true}, + }); err != nil { + t.Fatalf("set chain: %v", err) + } + + providers, err := ResolveChainWithChecker(ctx, folderID, "movie", chainRepo, nil, nil) + if err != nil { + t.Fatalf("resolve chain: %v", err) + } + if len(providers) != 1 { + t.Fatalf("providers = %d, want 1", len(providers)) + } + if _, ok := providers[0].(*builtinStubProvider); !ok { + t.Fatalf("provider type = %T, want in-process *builtinStubProvider", providers[0]) + } + + // A disabled builtin installation must be skipped like any plugin. + if _, err := pool.Exec(ctx, `UPDATE plugin_installations SET enabled = false WHERE id = $1`, installationID); err != nil { + t.Fatalf("disable installation: %v", err) + } + providers, err = ResolveChainWithChecker(ctx, folderID, "movie", chainRepo, nil, nil) + if err != nil { + t.Fatalf("resolve chain: %v", err) + } + // The chain entry is still enabled, so the fallback does not run; the + // disabled installation is filtered by buildProviders. + if len(providers) != 0 { + t.Fatalf("providers after disabling installation = %d, want 0", len(providers)) + } +} + +// The chain-less fallback must skip default_enabled=false capabilities: a +// chain-less library, and a library whose only chain row is the disabled +// seeded NFO entry, must not activate the provider. +func TestChainlessFallback_SkipsDefaultDisabledCapability(t *testing.T) { + pool := chainBuiltinTestPool(t) + ctx := context.Background() + + capID := fmt.Sprintf("test-fallback-%d", time.Now().UnixNano()) + RegisterBuiltinProvider(capID, func() Provider { return &builtinStubProvider{slug: capID} }) + + installationID := insertTestInstallation(t, pool, "builtin", true) + insertTestCapability(t, pool, installationID, capID, + `{"display_name":"Test Fallback","default_priority":{"movie":1},"default_enabled":false}`) + chainRepo := NewChainRepository(pool) + + assertNotResolved := func(folderID int, scenario string) { + t.Helper() + providers, err := ResolveChainWithChecker(ctx, folderID, "movie", chainRepo, nil, nil) + if err != nil { + t.Fatalf("%s: resolve chain: %v", scenario, err) + } + for _, p := range providers { + if p.Slug() == capID { + t.Errorf("%s: default_enabled=false capability leaked into the fallback", scenario) + } + } + } + + // Chain-less library. + assertNotResolved(insertTestFolder(t, pool, "movies"), "chain-less library") + + // Library whose only row is the disabled seeded entry (all-disabled chain). + folderID := insertTestFolder(t, pool, "movies") + if err := chainRepo.SetChain(ctx, folderID, []ChainEntry{ + {PluginInstallationID: installationID, CapabilityID: capID, ContentLevel: "movie", Priority: 0, Enabled: false}, + }); err != nil { + t.Fatalf("set chain: %v", err) + } + assertNotResolved(folderID, "all-disabled chain") +} + +// Startup sync: a library whose chain rows are only legacy content_level=” +// must be materialized per level (same order/enabled, ” rows left in place), +// then the builtin appended disabled at MAX(priority)+1. The resolved +// (enabled-filtered) chain must be identical before and after. +func TestSyncBuiltinProviderChains_LegacyMaterializationFixture(t *testing.T) { + pool := chainBuiltinTestPool(t) + ctx := context.Background() + + pluginInstallID := insertTestInstallation(t, pool, "plugin", true) + capID := fmt.Sprintf("test-legacy-%d", time.Now().UnixNano()) + insertTestCapability(t, pool, pluginInstallID, capID, + `{"display_name":"Legacy Plugin","metadata":{"default_priority":{"movie":2}}}`) + + builtinInstallID := insertTestInstallation(t, pool, "builtin", true) + builtinCapID := fmt.Sprintf("test-legacy-builtin-%d", time.Now().UnixNano()) + insertTestCapability(t, pool, builtinInstallID, builtinCapID, + `{"display_name":"Legacy Builtin","default_priority":{"movie":1,"series":1},"default_enabled":false}`) + + folderID := insertTestFolder(t, pool, "movies") + chainRepo := NewChainRepository(pool) + if err := chainRepo.SetChain(ctx, folderID, []ChainEntry{ + {PluginInstallationID: pluginInstallID, CapabilityID: capID, ContentLevel: "", Priority: 0, Enabled: true}, + }); err != nil { + t.Fatalf("set legacy chain: %v", err) + } + + resolvedBefore, err := chainRepo.GetChain(ctx, folderID, "movie") + if err != nil { + t.Fatalf("get chain before sync: %v", err) + } + + if err := SyncBuiltinProviderChains(ctx, chainRepo); err != nil { + t.Fatalf("sync builtin provider chains: %v", err) + } + + after, err := chainRepo.GetChain(ctx, folderID, "movie") + if err != nil { + t.Fatalf("get chain after sync: %v", err) + } + + // Enabled-filtered resolution must be identical (modulo the now-real level). + var enabledAfter []ChainEntry + for _, e := range after { + if e.Enabled { + e.ContentLevel = "" + enabledAfter = append(enabledAfter, e) + } + } + var enabledBefore []ChainEntry + for _, e := range resolvedBefore { + if e.Enabled { + e.ContentLevel = "" + enabledBefore = append(enabledBefore, e) + } + } + if !reflect.DeepEqual(enabledBefore, enabledAfter) { + t.Errorf("resolved chain changed across sync:\nbefore=%+v\nafter=%+v", enabledBefore, enabledAfter) + } + + // The editor now sees real per-level rows: the materialized plugin row plus + // the appended disabled builtin. + foundPlugin, foundBuiltin := false, false + maxPluginPriority := -1 + for _, e := range after { + switch { + case e.PluginInstallationID == pluginInstallID && e.CapabilityID == capID: + foundPlugin = true + if !e.Enabled { + t.Error("materialized plugin row lost enabled state") + } + if e.Priority > maxPluginPriority { + maxPluginPriority = e.Priority + } + case e.PluginInstallationID == builtinInstallID && e.CapabilityID == builtinCapID: + foundBuiltin = true + if e.Enabled { + t.Error("builtin must be appended disabled") + } + } + } + if !foundPlugin { + t.Error("legacy '' plugin row was not materialized at the movie level") + } + if !foundBuiltin { + t.Error("builtin capability was not appended to the materialized movie level") + } + + // The legacy '' rows stay in place for old binaries. + legacy, err := chainRepo.GetChain(ctx, folderID, "") + if err != nil { + t.Fatalf("get legacy chain: %v", err) + } + if len(legacy) != 1 { + t.Errorf("legacy '' rows = %d, want 1 (left in place)", len(legacy)) + } + + // Idempotency: running the sync again changes nothing. + if err := SyncBuiltinProviderChains(ctx, chainRepo); err != nil { + t.Fatalf("second sync: %v", err) + } + again, err := chainRepo.GetChain(ctx, folderID, "movie") + if err != nil { + t.Fatalf("get chain after second sync: %v", err) + } + if !reflect.DeepEqual(after, again) { + t.Errorf("sync is not idempotent:\nfirst=%+v\nsecond=%+v", after, again) + } +} diff --git a/internal/metadata/export_nfo_test.go b/internal/metadata/export_nfo_test.go new file mode 100644 index 00000000..c10165a0 --- /dev/null +++ b/internal/metadata/export_nfo_test.go @@ -0,0 +1,147 @@ +package metadata + +import ( + "context" + "fmt" + "sort" + "sync" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// NFOSeriesHarness bridges the in-package fakes to the external NFO +// integration tests (package metadata_test). Those tests must live outside +// the package so they can import internal/metadata/nfo (which itself imports +// this package) without an import cycle. +type NFOSeriesHarness struct { + h *testHarness + seasons *fakeSeasonRepo + episodes *fakeEpisodeRepo + jobs *capturingImageJobEnqueuer +} + +type capturingImageJobEnqueuer struct { + mu sync.Mutex + inputs []EnqueueImageCacheJobInput +} + +func (c *capturingImageJobEnqueuer) Enqueue(_ context.Context, in EnqueueImageCacheJobInput) error { + c.mu.Lock() + defer c.mu.Unlock() + c.inputs = append(c.inputs, in) + return nil +} + +func (c *capturingImageJobEnqueuer) EnqueueBatch(_ context.Context, inputs []EnqueueImageCacheJobInput) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.inputs = append(c.inputs, inputs...) + return len(inputs), nil +} + +func NewNFOSeriesHarness() *NFOSeriesHarness { + h := newTestHarness() + x := &NFOSeriesHarness{ + h: h, + seasons: newFakeSeasonRepo(), + episodes: newFakeEpisodeRepo(), + jobs: &capturingImageJobEnqueuer{}, + } + h.service.seasonRepo = x.seasons + h.service.episodeRepo = x.episodes + h.service.imageCacheJobs = x.jobs + h.service.autoCacheImages.Store(true) + return x +} + +func (x *NFOSeriesHarness) Service() *MetadataService { return x.h.service } + +// SeedSeriesSkeleton seeds a pending series item, mirroring the scanner's +// skeleton for an unmatched series root. +func (x *NFOSeriesHarness) SeedSeriesSkeleton(contentID, title string) error { + return x.seedSkeleton(contentID, title, "series") +} + +// SeedMovieSkeleton seeds a pending movie item, mirroring the scanner's +// skeleton for an unmatched movie group (e.g. the movie lane of a mixed +// library). +func (x *NFOSeriesHarness) SeedMovieSkeleton(contentID, title string) error { + return x.seedSkeleton(contentID, title, "movie") +} + +func (x *NFOSeriesHarness) seedSkeleton(contentID, title, contentType string) error { + return x.h.itemRepo.Upsert(context.Background(), &models.MediaItem{ + ContentID: contentID, + Type: contentType, + Title: title, + Status: "pending_match", + Studios: []string{}, + Networks: []string{}, + Countries: []string{}, + Genres: []string{}, + }) +} + +func (x *NFOSeriesHarness) Item(contentID string) (*models.MediaItem, error) { + return x.h.itemRepo.GetByID(context.Background(), contentID) +} + +// SeedSeriesFile registers a persisted media file linked to a content id, so +// refresh flows (which reconstruct local context from the file repo instead +// of scan hints) can find the series' episode files. observedRootPath is the +// series root eligible for directory-level sidecars. +func (x *NFOSeriesHarness) SeedSeriesFile(fileID, folderID int, filePath, observedRootPath, contentID string) { + x.h.fileRepo.setGroupFiles(folderID, 1, fmt.Sprintf("group-%s-%d", contentID, fileID), &models.MediaFile{ + ID: fileID, + MediaFolderID: folderID, + FilePath: filePath, + ObservedRootPath: observedRootPath, + }) + x.h.fileRepo.mu.Lock() + x.h.fileRepo.contentIDs[fileID] = contentID + x.h.fileRepo.mu.Unlock() +} + +// Seasons returns the persisted seasons for a series sorted by season number. +func (x *NFOSeriesHarness) Seasons(seriesID string) []*models.Season { + x.seasons.mu.Lock() + defer x.seasons.mu.Unlock() + var out []*models.Season + for _, s := range x.seasons.seasons { + if s.SeriesID == seriesID { + cp := *s + out = append(out, &cp) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].SeasonNumber < out[j].SeasonNumber }) + return out +} + +// Episodes returns the persisted episodes for a series sorted by (season, episode). +func (x *NFOSeriesHarness) Episodes(seriesID string) []*models.Episode { + x.episodes.mu.Lock() + defer x.episodes.mu.Unlock() + var out []*models.Episode + for _, e := range x.episodes.episodes { + if e.SeriesID == seriesID { + cp := *e + out = append(out, &cp) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].SeasonNumber != out[j].SeasonNumber { + return out[i].SeasonNumber < out[j].SeasonNumber + } + return out[i].EpisodeNumber < out[j].EpisodeNumber + }) + return out +} + +// EnqueuedImageJobs returns every image cache job the pipeline enqueued. +func (x *NFOSeriesHarness) EnqueuedImageJobs() []EnqueueImageCacheJobInput { + x.jobs.mu.Lock() + defer x.jobs.mu.Unlock() + out := make([]EnqueueImageCacheJobInput, len(x.jobs.inputs)) + copy(out, x.jobs.inputs) + return out +} diff --git a/internal/metadata/image_cache_job_repo.go b/internal/metadata/image_cache_job_repo.go index 19b8662b..0239b0c3 100644 --- a/internal/metadata/image_cache_job_repo.go +++ b/internal/metadata/image_cache_job_repo.go @@ -81,7 +81,19 @@ func isStableProviderImageFailure(errText string) bool { return strings.Contains(errText, "unexpected status 403") || strings.Contains(errText, "unexpected status 404") || strings.Contains(errText, "unexpected status 410") || - strings.Contains(errText, "unexpected status 418") + strings.Contains(errText, "unexpected status 418") || + // Local sidecar failures that will not heal on the normal backoff: + // deleted files (ENOENT), unreadable files (EPERM), paths outside the + // owning library's roots, and files that are structurally unusable + // (non-regular or over the size cap) — these do not self-heal without an + // on-disk change, so a hot retry loop is pointless; recovery is + // refresh-driven. Other local read errors keep the standard retry + // schedule. Texts match the processor's local branch. + strings.Contains(errText, "local image missing") || + strings.Contains(errText, "local image forbidden") || + strings.Contains(errText, "local image path outside library roots") || + strings.Contains(errText, "local image is not a regular file") || + strings.Contains(errText, "local image exceeds") } func (r *ImageCacheJobRepository) Enqueue(ctx context.Context, in EnqueueImageCacheJobInput) error { @@ -127,7 +139,7 @@ func (r *ImageCacheJobRepository) enqueueBatch(ctx context.Context, inputs []Enq func normalizeImageCacheJobInput(in EnqueueImageCacheJobInput) (EnqueueImageCacheJobInput, bool) { in.SourcePath = strings.TrimSpace(in.SourcePath) in.TargetLanguage = strings.TrimSpace(in.TargetLanguage) - if in.SourcePath == "" || !strings.Contains(in.SourcePath, "://") || isNonProviderImageScheme(strings.ToLower(in.SourcePath)) { + if !isCacheableImageSourcePath(in.SourcePath) { return EnqueueImageCacheJobInput{}, false } if in.ContentType == "" { @@ -468,6 +480,31 @@ func (r *ImageCacheJobRepository) CurrentTargetSourcePath(ctx context.Context, j return current, nil } +// CurrentTargetCachedPath reports the cached image path currently stored on +// the job's target row. The processor's local-artwork branch uses it to skip +// unchanged art and to delete the stale hashed local/ prefix after a +// successful re-cache. Returns ("", nil) when the row no longer exists or the +// target type is unknown. +func (r *ImageCacheJobRepository) CurrentTargetCachedPath(ctx context.Context, job *models.MetadataImageCacheJob) (string, error) { + if r == nil || r.pool == nil || job == nil { + return "", nil + } + query, args, ok := currentTargetSourceQuery(job) + if !ok { + return "", nil + } + query = strings.Replace(query, "_source_path", "_path", 1) + var current string + err := r.pool.QueryRow(ctx, query, args...).Scan(¤t) + if errors.Is(err, pgx.ErrNoRows) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("reading current target cached path: %w", err) + } + return current, nil +} + func currentTargetSourceQuery(job *models.MetadataImageCacheJob) (string, []any, bool) { switch job.TargetType { case ImageCacheTargetItem: @@ -542,7 +579,11 @@ func (r *ImageCacheJobRepository) EnqueueExistingProviderArtwork(ctx context.Con return 0, nil } // Each branch is restricted to provider-origin sources (LIKE '%://%' minus - // cached/system schemes) AND to targets whose stored *_path is not already a + // cached/system schemes). Local file:// sidecar sources are deliberately + // excluded from this sweep: their recovery is refresh-driven (a metadata + // refresh re-discovers the sidecar and enqueues a fresh job), so a stable + // local failure cannot be resurrected here every cycle. + // Each branch is also restricted to targets whose stored *_path is not already a // cached relative path. The destination check makes the cached row itself the // durable dedup marker, so pruning succeeded job rows does not cause the whole // catalog to be re-downloaded once the rows age out. @@ -798,7 +839,14 @@ func (r *ImageCacheJobRepository) EnqueueExistingProviderArtwork(ctx context.Con return r.enqueueBatch(ctx, inputs, true) } +// imageCacheLocalProviderID is the synthetic provider slug for local sidecar +// artwork; cached keys live under "local/..." like audiobook/ebook covers. +const imageCacheLocalProviderID = "local" + func imageCacheProviderIDFromSource(sourcePath, fallback string) string { + if isLocalImageSourcePath(sourcePath) { + return imageCacheLocalProviderID + } if provider := providerIDFromPluginURL(sourcePath); provider != "" { return provider } diff --git a/internal/metadata/image_cache_job_repo_test.go b/internal/metadata/image_cache_job_repo_test.go index ffca1334..56d1d21c 100644 --- a/internal/metadata/image_cache_job_repo_test.go +++ b/internal/metadata/image_cache_job_repo_test.go @@ -57,7 +57,6 @@ func TestNormalizeImageCacheJobInputSkipsNonProviderArtwork(t *testing.T) { "", "tmdb/series/1396/poster/original.webp", "s3://media/tmdb/series/1396/poster/original.webp", - "file:///media/poster.jpg", "local://poster.jpg", "generated://collections/1/poster.jpg", } { diff --git a/internal/metadata/image_cache_processor.go b/internal/metadata/image_cache_processor.go index d5db69fe..a2956f4e 100644 --- a/internal/metadata/image_cache_processor.go +++ b/internal/metadata/image_cache_processor.go @@ -2,9 +2,17 @@ package metadata import ( "context" + "crypto/sha256" + "encoding/hex" + "errors" "fmt" + "io" + "io/fs" "log/slog" + "os" + "path/filepath" "strconv" + "strings" "sync" "sync/atomic" "time" @@ -12,6 +20,10 @@ import ( "github.com/Silo-Server/silo-server/internal/models" ) +// maxLocalImageSourceBytes caps local sidecar artwork reads at 8 MiB, +// matching the NFO provider's discovery guard. +const maxLocalImageSourceBytes = 8 << 20 + // imageCacheDiscoveryInterval throttles the full-catalog backfill sweep so an // idle installation does not re-scan every entity table on every task tick. // Draining of already-queued jobs is unaffected and stays responsive. @@ -27,6 +39,27 @@ type ImageCacheJobClaimer interface { DeleteSucceededBefore(ctx context.Context, before time.Time, limit int) (int, error) } +// imageCacheTargetCachedPathReader is optionally implemented by the job store +// (ImageCacheJobRepository does) to expose the target's currently stored +// cached path for the local-artwork unchanged-skip and stale-prefix cleanup. +type imageCacheTargetCachedPathReader interface { + CurrentTargetCachedPath(ctx context.Context, job *models.MetadataImageCacheJob) (string, error) +} + +// LibraryRootResolver reports the media folder root paths a piece of content +// belongs to (media_folders.Paths via media_item_libraries). The processor +// confines local file:// sources to these roots before reading them. +type LibraryRootResolver interface { + LibraryRootsForContent(ctx context.Context, contentID string) ([]string, error) +} + +// ImagePrefixDeleter deletes cached image objects under a key prefix; used to +// sweep the previous hashed local/ prefix after a successful re-cache. +type ImagePrefixDeleter interface { + DeletePrefix(ctx context.Context, bucket, prefix string) (int, error) + Bucket() string +} + type SeasonArtworkUpdater interface { UpdateArtworkIfSourceMatches(ctx context.Context, contentID, sourcePath, cachedPath, thumbhash string) (bool, error) } @@ -69,6 +102,12 @@ type ImageCacheProcessor struct { targets ImageCacheProcessorTargets logger *slog.Logger + // Local file:// artwork support. libraryRoots confines reads to the + // owning library's roots; prefixDeleter sweeps stale hashed prefixes. + // The processor host must mount the libraries, like the metadata worker. + libraryRoots LibraryRootResolver + prefixDeleter ImagePrefixDeleter + enabled atomic.Bool discoveryInterval time.Duration @@ -76,6 +115,26 @@ type ImageCacheProcessor struct { lastDiscovery time.Time } +// SetLibraryRootResolver wires the folder repository used to confine local +// file:// sources to their owning library's roots. Without it, local jobs +// fail (and retry on the normal backoff) rather than reading arbitrary paths. +func (p *ImageCacheProcessor) SetLibraryRootResolver(resolver LibraryRootResolver) { + if p == nil { + return + } + p.libraryRoots = resolver +} + +// SetImagePrefixDeleter wires the object store used to delete the previous +// hashed local/ prefix after a successful local re-cache. Optional: without +// it, stale prefixes are left behind (item deletion still sweeps them). +func (p *ImageCacheProcessor) SetImagePrefixDeleter(deleter ImagePrefixDeleter) { + if p == nil { + return + } + p.prefixDeleter = deleter +} + // SetEnabled toggles background caching. When disabled the processor performs // no discovery, claiming, or uploading, honoring metadata.cache_images so that // merely configuring object storage does not download the whole catalog. @@ -357,11 +416,9 @@ func (p *ImageCacheProcessor) processOne(ctx context.Context, job *models.Metada return imageCacheProcessResult{outcome: "failed"} } - // Confirm the target still references this job's source before uploading. - // CacheImage writes to a deterministic, source-independent storage key, so a - // stale job whose source an admin or newer refresh has already replaced would - // otherwise overwrite the live artwork object even though the conditional DB - // update later no-ops. Dropping the obsolete job here avoids that. + // Confirm the target still references this job's source before doing the + // download and immutable upload. The conditional update below remains the + // final concurrency guard; this early check avoids work for an obsolete job. current, err := p.jobs.CurrentTargetSourcePath(ctx, job) if err != nil { p.markFailed(ctx, job, err.Error()) @@ -372,6 +429,10 @@ func (p *ImageCacheProcessor) processOne(ctx context.Context, job *models.Metada return imageCacheProcessResult{outcome: "skipped"} } + if isLocalImageSourcePath(job.SourcePath) { + return p.processLocalOne(ctx, job, imageType) + } + downloadURL := job.SourcePath if isProviderImagePath(downloadURL) { if p.resolver == nil { @@ -408,79 +469,306 @@ func (p *ImageCacheProcessor) processOne(ctx context.Context, job *models.Metada uploadedVariants: result.UploadedVariants, existingVariants: result.ExistingVariants, } - cachedPath := cachedOriginalImagePath(result.BasePath, result.Ext) + cachedPath := CachedImageOriginalPath(result) if cachedPath == "" { p.markFailed(ctx, job, "image cache returned empty stored path") processResult.outcome = "failed" return processResult } - var updated bool - switch job.TargetType { - case ImageCacheTargetItem: - if p.targets.Items == nil { - p.markFailed(ctx, job, "missing item updater") - processResult.outcome = "failed" - return processResult - } - updated, err = p.targets.Items.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.ImageType, job.SourcePath, cachedPath, result.Thumbhash) - case ImageCacheTargetItemLocalization: - if p.targets.ItemLocalizations == nil { - p.markFailed(ctx, job, "missing item localization updater") - processResult.outcome = "failed" - return processResult - } - updated, err = p.targets.ItemLocalizations.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.TargetLanguage, job.ImageType, job.SourcePath, cachedPath, result.Thumbhash) - case ImageCacheTargetSeason: - if p.targets.Seasons == nil { - p.markFailed(ctx, job, "missing season updater") - processResult.outcome = "failed" - return processResult - } - updated, err = p.targets.Seasons.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.SourcePath, cachedPath, result.Thumbhash) - case ImageCacheTargetSeasonLocalization: - if p.targets.SeasonLocalizations == nil { - p.markFailed(ctx, job, "missing season localization updater") - processResult.outcome = "failed" - return processResult - } - updated, err = p.targets.SeasonLocalizations.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.TargetLanguage, job.SourcePath, cachedPath, result.Thumbhash) - case ImageCacheTargetEpisode: - if p.targets.Episodes == nil { - p.markFailed(ctx, job, "missing episode updater") - processResult.outcome = "failed" - return processResult - } - updated, err = p.targets.Episodes.UpdateStillIfSourceMatches(ctx, job.TargetContentID, job.SourcePath, cachedPath, result.Thumbhash) - case ImageCacheTargetPerson: - if p.targets.People == nil { - p.markFailed(ctx, job, "missing person updater") - processResult.outcome = "failed" - return processResult - } - personID, parseErr := strconv.ParseInt(job.TargetContentID, 10, 64) - if parseErr != nil { - err = fmt.Errorf("invalid person image cache target %q: %w", job.TargetContentID, parseErr) - break - } - updated, err = p.targets.People.UpdatePhotoIfSourceMatches(ctx, personID, job.SourcePath, cachedPath, result.Thumbhash) - default: - err = fmt.Errorf("unknown image cache target type %q", job.TargetType) - } + p.finishJobWithTargetUpdate(ctx, job, cachedPath, result.Thumbhash, &processResult) + return processResult +} + +// finishJobWithTargetUpdate persists the cached path/thumbhash to the job's +// target row and marks the job terminal, setting processResult.outcome. +func (p *ImageCacheProcessor) finishJobWithTargetUpdate(ctx context.Context, job *models.MetadataImageCacheJob, cachedPath, thumbhash string, processResult *imageCacheProcessResult) { + updated, err := p.updateTargetArtwork(ctx, job, cachedPath, thumbhash) if err != nil { p.markFailed(ctx, job, err.Error()) processResult.outcome = "failed" - return processResult - } - if !updated { - p.markSucceeded(ctx, job) - processResult.outcome = "skipped" - return processResult + return } p.markSucceeded(ctx, job) - processResult.outcome = "succeeded" + if updated { + processResult.outcome = "succeeded" + } else { + processResult.outcome = "skipped" + } +} + +func (p *ImageCacheProcessor) updateTargetArtwork(ctx context.Context, job *models.MetadataImageCacheJob, cachedPath, thumbhash string) (bool, error) { + switch job.TargetType { + case ImageCacheTargetItem: + if p.targets.Items == nil { + return false, errors.New("missing item updater") + } + return p.targets.Items.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.ImageType, job.SourcePath, cachedPath, thumbhash) + case ImageCacheTargetItemLocalization: + if p.targets.ItemLocalizations == nil { + return false, errors.New("missing item localization updater") + } + return p.targets.ItemLocalizations.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.TargetLanguage, job.ImageType, job.SourcePath, cachedPath, thumbhash) + case ImageCacheTargetSeason: + if p.targets.Seasons == nil { + return false, errors.New("missing season updater") + } + return p.targets.Seasons.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.SourcePath, cachedPath, thumbhash) + case ImageCacheTargetSeasonLocalization: + if p.targets.SeasonLocalizations == nil { + return false, errors.New("missing season localization updater") + } + return p.targets.SeasonLocalizations.UpdateArtworkIfSourceMatches(ctx, job.TargetContentID, job.TargetLanguage, job.SourcePath, cachedPath, thumbhash) + case ImageCacheTargetEpisode: + if p.targets.Episodes == nil { + return false, errors.New("missing episode updater") + } + return p.targets.Episodes.UpdateStillIfSourceMatches(ctx, job.TargetContentID, job.SourcePath, cachedPath, thumbhash) + case ImageCacheTargetPerson: + if p.targets.People == nil { + return false, errors.New("missing person updater") + } + personID, parseErr := strconv.ParseInt(job.TargetContentID, 10, 64) + if parseErr != nil { + return false, fmt.Errorf("invalid person image cache target %q: %w", job.TargetContentID, parseErr) + } + return p.targets.People.UpdatePhotoIfSourceMatches(ctx, personID, job.SourcePath, cachedPath, thumbhash) + default: + return false, fmt.Errorf("unknown image cache target type %q", job.TargetType) + } +} + +// processLocalOne caches a local sidecar image (file:// source). The file is +// confined to the owning library's roots by a lexical check on the logical path +// followed by a symlink-resolving re-check (both path and roots are resolved, so +// scanner-recorded logical paths under symlinked roots stay valid while an +// intermediate directory symlink escaping a root is rejected), then read through +// an opened handle with fstat re-checks and pushed to S3 under a content-hashed +// local/ key. +func (p *ImageCacheProcessor) processLocalOne(ctx context.Context, job *models.MetadataImageCacheJob, imageType ImageType) imageCacheProcessResult { + byteCacher, ok := p.cacher.(ImageByteCacher) + if !ok { + p.markFailed(ctx, job, "image cacher does not support local artwork") + return imageCacheProcessResult{outcome: "failed"} + } + if p.libraryRoots == nil { + p.markFailed(ctx, job, "missing library root resolver for local artwork") + return imageCacheProcessResult{outcome: "failed"} + } + + localPath := filepath.Clean(strings.TrimSpace(job.SourcePath)[len("file://"):]) + rootsContentID := firstNonEmpty(job.SeriesID, job.TargetContentID) + roots, err := p.libraryRoots.LibraryRootsForContent(ctx, rootsContentID) + if err != nil { + p.markFailed(ctx, job, fmt.Sprintf("resolving library roots: %v", err)) + return imageCacheProcessResult{outcome: "failed"} + } + if !localImagePathWithinRoots(localPath, roots) { + p.markFailed(ctx, job, "local image path outside library roots: "+localPath) + return imageCacheProcessResult{outcome: "failed"} + } + // The lexical check above cannot see through symlinks. Resolve the path and + // roots and re-confine so an intermediate directory symlink planted inside a + // root (e.g. a link pointing out of the library) cannot pull an out-of-root + // file into the cache. EvalSymlinks resolves both sides, so a legitimately + // symlinked root still matches. This is a confinement GATE only: the read + // below still uses the logical path so readLocalImageFile's Lstat keeps + // rejecting a symlinked leaf. A not-yet-existent path (ErrNotExist) falls + // through to the reader, which classifies it as the stable "missing" failure. + if _, err := localImagePathResolvedWithinRoots(localPath, roots); err != nil && !errors.Is(err, fs.ErrNotExist) { + p.markFailed(ctx, job, "local image path outside library roots: "+localPath) + return imageCacheProcessResult{outcome: "failed"} + } + + data, err := readLocalImageFile(localPath) + if err != nil { + p.markFailed(ctx, job, err.Error()) + return imageCacheProcessResult{outcome: "failed"} + } + + // The target's current cached path drives the unchanged-skip (same hash → + // same key → nothing to sweep) and the stale-prefix cleanup below. + previousCachedPath := "" + if reader, ok := p.jobs.(imageCacheTargetCachedPathReader); ok { + previousCachedPath, err = reader.CurrentTargetCachedPath(ctx, job) + if err != nil { + p.logger.WarnContext(ctx, "metadata image cache: failed to read current cached path", "job_id", job.ID, "error", err) + previousCachedPath = "" + } + } + + digest := sha256.Sum256(data) + result, err := byteCacher.CacheImageBytes(ctx, data, CacheImageRequest{ + SourceURL: job.SourcePath, + ProviderID: imageCacheLocalProviderID, + ContentType: job.ContentType, + ContentID: job.ProviderContentID, + ImageType: imageType, + SeasonNumber: job.SeasonNumber, + EpisodeNumber: job.EpisodeNumber, + Language: job.TargetLanguage, + KeyDiscriminator: hex.EncodeToString(digest[:4]), + }) + if err != nil { + p.markFailed(ctx, job, err.Error()) + return imageCacheProcessResult{outcome: "failed"} + } + if result == nil { + p.markFailed(ctx, job, "image cache returned no result") + return imageCacheProcessResult{outcome: "failed"} + } + processResult := imageCacheProcessResult{ + uploadedVariants: result.UploadedVariants, + existingVariants: result.ExistingVariants, + } + cachedPath := CachedImageOriginalPath(result) + if cachedPath == "" { + p.markFailed(ctx, job, "image cache returned empty stored path") + processResult.outcome = "failed" + return processResult + } + p.finishJobWithTargetUpdate(ctx, job, cachedPath, result.Thumbhash, &processResult) + if processResult.outcome == "succeeded" { + p.deleteStaleLocalPrefix(ctx, previousCachedPath, cachedPath) + } return processResult } +// deleteStaleLocalPrefix removes the previous hashed local/ image prefix +// after a re-cache stored the artwork under a different key. +func (p *ImageCacheProcessor) deleteStaleLocalPrefix(ctx context.Context, previousCachedPath, cachedPath string) { + previousCachedPath = strings.TrimSpace(previousCachedPath) + if p.prefixDeleter == nil || + previousCachedPath == "" || + previousCachedPath == cachedPath || + !strings.HasPrefix(previousCachedPath, "local/") { + return + } + lastSlash := strings.LastIndex(previousCachedPath, "/") + if lastSlash <= 0 { + return + } + prefix := previousCachedPath[:lastSlash+1] + if strings.HasPrefix(cachedPath, prefix) { + return + } + if _, err := p.prefixDeleter.DeletePrefix(ctx, p.prefixDeleter.Bucket(), prefix); err != nil { + p.logger.WarnContext(ctx, "metadata image cache: failed to delete stale local image prefix", "prefix", prefix, "error", err) + } +} + +// errLocalImageOutsideRoots is returned by localImagePathResolvedWithinRoots +// when a fully symlink-resolved path escapes every resolved library root. +var errLocalImageOutsideRoots = errors.New("local image path outside library roots") + +// localImagePathResolvedWithinRoots resolves symlinks on both the path and the +// library roots and returns the real path when it stays within a real root. +// This closes the gap the lexical check leaves open — an intermediate directory +// symlink planted inside a root that points outside it. Resolving both sides +// keeps a legitimately symlinked root valid. A path that does not exist yet +// surfaces as fs.ErrNotExist for the caller to treat as a stable "missing" +// failure via the reader; any other resolution failure fails closed. +func localImagePathResolvedWithinRoots(path string, roots []string) (string, error) { + resolvedPath, err := filepath.EvalSymlinks(path) + if err != nil { + return "", err + } + for _, root := range roots { + resolvedRoot, err := filepath.EvalSymlinks(strings.TrimSpace(root)) + if err != nil { + continue + } + if localImagePathWithinRoots(resolvedPath, []string{resolvedRoot}) { + return resolvedPath, nil + } + } + return "", errLocalImageOutsideRoots +} + +// localImagePathWithinRoots confines path (already cleaned, logical) to the +// given library roots with a separator-aware lexical prefix check. +func localImagePathWithinRoots(path string, roots []string) bool { + if !filepath.IsAbs(path) { + return false + } + for _, root := range roots { + root = filepath.Clean(strings.TrimSpace(root)) + if root == "" || root == "." || !filepath.IsAbs(root) { + continue + } + if path == root { + return true + } + prefix := root + if prefix != string(filepath.Separator) { + prefix += string(filepath.Separator) + } + if strings.HasPrefix(path, prefix) { + return true + } + } + return false +} + +// readLocalImageFile reads a sidecar image with the same guards as discovery: +// Lstat rejects symlinked leaves and non-regular files, the opened handle is +// fstat-re-checked, and reads cap at maxLocalImageSourceBytes. ENOENT/EPERM +// map to the stable-failure texts matched by isStableProviderImageFailure. +func readLocalImageFile(path string) ([]byte, error) { + classify := func(err error) error { + switch { + case errors.Is(err, fs.ErrNotExist): + return fmt.Errorf("local image missing: %s", path) + case errors.Is(err, fs.ErrPermission): + return fmt.Errorf("local image forbidden: %s", path) + default: + return fmt.Errorf("local image read failed: %w", err) + } + } + info, err := os.Lstat(path) + if err != nil { + return nil, classify(err) + } + if !info.Mode().IsRegular() { + return nil, fmt.Errorf("local image is not a regular file: %s", path) + } + if info.Size() > maxLocalImageSourceBytes { + return nil, fmt.Errorf("local image exceeds %d byte limit: %s", maxLocalImageSourceBytes, path) + } + file, err := os.Open(path) + if err != nil { + return nil, classify(err) + } + defer func() { _ = file.Close() }() + stat, err := file.Stat() + if err != nil { + return nil, classify(err) + } + // Close the symlink-swap window: os.Open follows symlinks, so a leaf swapped + // to a symlink between the Lstat above and this Open would be followed to its + // target. Reject unless the opened handle is the exact file Lstat inspected, + // so an out-of-root target can never be pulled into the public cache. + if !os.SameFile(info, stat) { + return nil, fmt.Errorf("local image is not a regular file: %s", path) + } + if !stat.Mode().IsRegular() { + return nil, fmt.Errorf("local image is not a regular file: %s", path) + } + if stat.Size() > maxLocalImageSourceBytes { + return nil, fmt.Errorf("local image exceeds %d byte limit: %s", maxLocalImageSourceBytes, path) + } + data, err := io.ReadAll(io.LimitReader(file, maxLocalImageSourceBytes+1)) + if err != nil { + return nil, classify(err) + } + if len(data) > maxLocalImageSourceBytes { + return nil, fmt.Errorf("local image exceeds %d byte limit: %s", maxLocalImageSourceBytes, path) + } + if len(data) == 0 { + return nil, fmt.Errorf("local image read failed: empty file %s", path) + } + return data, nil +} + func imageCacheJobImageType(value string) (ImageType, error) { switch value { case ImageCacheImagePoster: diff --git a/internal/metadata/image_cache_processor_local_test.go b/internal/metadata/image_cache_processor_local_test.go new file mode 100644 index 00000000..40d31244 --- /dev/null +++ b/internal/metadata/image_cache_processor_local_test.go @@ -0,0 +1,322 @@ +package metadata + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// fakeLocalImageCacheJobs is a fakeImageCacheJobs that also reports the +// target's currently stored cached path, like the real repository. +type fakeLocalImageCacheJobs struct { + fakeImageCacheJobs + currentCached string +} + +func (f *fakeLocalImageCacheJobs) CurrentTargetCachedPath(context.Context, *models.MetadataImageCacheJob) (string, error) { + return f.currentCached, nil +} + +// fakeByteImageCacher supports both remote and byte caching. CacheImageBytes +// builds the base path the way the real cacher does so key assertions hold. +type fakeByteImageCacher struct { + fakeImageCacher + byteReqs [][]byte + bytesReq []CacheImageRequest + thumbhash string +} + +func (f *fakeByteImageCacher) CacheImageBytes(_ context.Context, data []byte, req CacheImageRequest) (*CacheImageResult, error) { + f.byteReqs = append(f.byteReqs, data) + f.bytesReq = append(f.bytesReq, req) + if f.err != nil { + return nil, f.err + } + return &CacheImageResult{ + BasePath: fmt.Sprintf("%s/%s/%s/%s/%s", req.ProviderID, req.ContentType, req.ContentID, req.KeyDiscriminator, ImageTypeToString(req.ImageType)), + Thumbhash: f.thumbhash, + Ext: ".webp", + }, nil +} + +type fakeLibraryRootResolver struct { + roots []string + err error +} + +func (f *fakeLibraryRootResolver) LibraryRootsForContent(context.Context, string) ([]string, error) { + return f.roots, f.err +} + +type fakePrefixDeleter struct { + bucket string + prefixes []string +} + +func (f *fakePrefixDeleter) DeletePrefix(_ context.Context, _ string, prefix string) (int, error) { + f.prefixes = append(f.prefixes, prefix) + return 1, nil +} + +func (f *fakePrefixDeleter) Bucket() string { return f.bucket } + +func localArtworkJob(sourcePath string) *models.MetadataImageCacheJob { + return &models.MetadataImageCacheJob{ + ID: 7, + TargetType: ImageCacheTargetItem, + TargetContentID: "movie-1", + SeriesID: "movie-1", + SourcePath: sourcePath, + ProviderID: "local", + ProviderContentID: "movie-1", + ContentType: "movies", + ImageType: ImageCacheImagePoster, + } +} + +func newLocalProcessorForTest( + job *models.MetadataImageCacheJob, + roots []string, + currentCached string, +) (*ImageCacheProcessor, *fakeLocalImageCacheJobs, *fakeByteImageCacher, *fakeItemArtworkUpdater, *fakePrefixDeleter) { + jobs := &fakeLocalImageCacheJobs{ + fakeImageCacheJobs: fakeImageCacheJobs{claimed: []*models.MetadataImageCacheJob{job}}, + currentCached: currentCached, + } + cacher := &fakeByteImageCacher{thumbhash: "th-new"} + items := &fakeItemArtworkUpdater{updated: true} + deleter := &fakePrefixDeleter{bucket: "images"} + p := NewImageCacheProcessorWithTargets(jobs, cacher, nil, ImageCacheProcessorTargets{Items: items}) + p.SetLibraryRootResolver(&fakeLibraryRootResolver{roots: roots}) + p.SetImagePrefixDeleter(deleter) + return p, jobs, cacher, items, deleter +} + +func writeLocalPoster(t *testing.T, dir string) string { + t.Helper() + path := filepath.Join(dir, "poster.jpg") + if err := os.WriteFile(path, []byte("poster-bytes"), 0o644); err != nil { + t.Fatalf("writing poster: %v", err) + } + return path +} + +func TestProcessLocalImageCachesWithinLibraryRoots(t *testing.T) { + root := t.TempDir() + posterPath := writeLocalPoster(t, root) + job := localArtworkJob("file://" + posterPath) + p, jobs, cacher, items, deleter := newLocalProcessorForTest(job, []string{root}, "") + + stats, err := p.RunOnce(context.Background(), "w1", 10, 1) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Succeeded != 1 { + t.Fatalf("stats = %+v, want 1 succeeded (failed text %q)", stats, jobs.failedText) + } + if len(cacher.bytesReq) != 1 { + t.Fatalf("CacheImageBytes calls = %d, want 1", len(cacher.bytesReq)) + } + req := cacher.bytesReq[0] + if req.ProviderID != "local" || req.ContentType != "movies" || req.ContentID != "movie-1" { + t.Fatalf("cache request = %+v", req) + } + if len(req.KeyDiscriminator) != 8 { + t.Fatalf("KeyDiscriminator = %q, want 8-char content hash", req.KeyDiscriminator) + } + wantCached := "local/movies/movie-1/" + req.KeyDiscriminator + "/poster/original.webp" + if items.cachedPath != wantCached { + t.Fatalf("cachedPath = %q, want %q", items.cachedPath, wantCached) + } + if items.thumbhash != "th-new" { + t.Fatalf("thumbhash = %q", items.thumbhash) + } + if len(deleter.prefixes) != 0 { + t.Fatalf("no stale prefix to delete on first cache, got %v", deleter.prefixes) + } +} + +func TestProcessLocalImageRejectsPathOutsideRoots(t *testing.T) { + root := t.TempDir() + other := t.TempDir() + posterPath := writeLocalPoster(t, other) + job := localArtworkJob("file://" + posterPath) + p, jobs, cacher, _, _ := newLocalProcessorForTest(job, []string{root}, "") + + stats, err := p.RunOnce(context.Background(), "w1", 10, 1) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Failed != 1 { + t.Fatalf("stats = %+v, want 1 failed", stats) + } + if !isStableProviderImageFailure(jobs.failedText) { + t.Fatalf("confinement failure %q must be a stable failure", jobs.failedText) + } + if len(cacher.bytesReq) != 0 { + t.Fatal("must not read or cache a file outside library roots") + } +} + +func TestProcessLocalImageRejectsDotDotTraversal(t *testing.T) { + root := t.TempDir() + outside := t.TempDir() + posterPath := writeLocalPoster(t, outside) + traversal := root + "/../" + filepath.Base(outside) + "/" + filepath.Base(posterPath) + job := localArtworkJob("file://" + traversal) + p, jobs, cacher, _, _ := newLocalProcessorForTest(job, []string{root}, "") + + stats, err := p.RunOnce(context.Background(), "w1", 10, 1) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Failed != 1 { + t.Fatalf("stats = %+v, want 1 failed", stats) + } + if !isStableProviderImageFailure(jobs.failedText) { + t.Fatalf("traversal failure %q must be a stable failure", jobs.failedText) + } + if len(cacher.bytesReq) != 0 { + t.Fatal("must not cache a traversal path") + } +} + +func TestProcessLocalImageAcceptsLogicalPathUnderSymlinkedRoot(t *testing.T) { + // The scanner records logical paths under symlinked roots; confinement is + // deliberately lexical-on-logical (no EvalSymlinks). + realRoot := t.TempDir() + writeLocalPoster(t, realRoot) + linkParent := t.TempDir() + linkedRoot := filepath.Join(linkParent, "library") + if err := os.Symlink(realRoot, linkedRoot); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + job := localArtworkJob("file://" + filepath.Join(linkedRoot, "poster.jpg")) + p, jobs, cacher, _, _ := newLocalProcessorForTest(job, []string{linkedRoot}, "") + + stats, err := p.RunOnce(context.Background(), "w1", 10, 1) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Succeeded != 1 { + t.Fatalf("stats = %+v (failed text %q), want success under symlinked root", stats, jobs.failedText) + } + if len(cacher.bytesReq) != 1 { + t.Fatal("expected the logical path to be read and cached") + } +} + +func TestProcessLocalImageMissingFileIsStableFailure(t *testing.T) { + root := t.TempDir() + job := localArtworkJob("file://" + filepath.Join(root, "poster.jpg")) + p, jobs, _, _, _ := newLocalProcessorForTest(job, []string{root}, "") + + stats, err := p.RunOnce(context.Background(), "w1", 10, 1) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Failed != 1 { + t.Fatalf("stats = %+v, want 1 failed", stats) + } + if !isStableProviderImageFailure(jobs.failedText) { + t.Fatalf("ENOENT failure %q must be a stable failure", jobs.failedText) + } +} + +func TestProcessLocalImageRejectsSymlinkedLeaf(t *testing.T) { + root := t.TempDir() + target := writeLocalPoster(t, root) + link := filepath.Join(root, "linked.jpg") + if err := os.Symlink(target, link); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + job := localArtworkJob("file://" + link) + p, _, cacher, _, _ := newLocalProcessorForTest(job, []string{root}, "") + + stats, err := p.RunOnce(context.Background(), "w1", 10, 1) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Failed != 1 { + t.Fatalf("stats = %+v, want 1 failed", stats) + } + if len(cacher.bytesReq) != 0 { + t.Fatal("must not cache a symlinked leaf") + } +} + +func TestProcessLocalImageDeletesStalePrefixOnRecache(t *testing.T) { + root := t.TempDir() + posterPath := writeLocalPoster(t, root) + job := localArtworkJob("file://" + posterPath) + stale := "local/movies/movie-1/00000000/poster/original.webp" + p, _, cacher, _, deleter := newLocalProcessorForTest(job, []string{root}, stale) + + stats, err := p.RunOnce(context.Background(), "w1", 10, 1) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Succeeded != 1 { + t.Fatalf("stats = %+v, want success", stats) + } + if len(cacher.bytesReq) != 1 { + t.Fatal("expected one cache call") + } + if len(deleter.prefixes) != 1 || deleter.prefixes[0] != "local/movies/movie-1/00000000/poster/" { + t.Fatalf("stale prefixes deleted = %v", deleter.prefixes) + } +} + +func TestProcessLocalImageSkipsPrefixDeleteWhenUnchanged(t *testing.T) { + root := t.TempDir() + posterPath := writeLocalPoster(t, root) + job := localArtworkJob("file://" + posterPath) + + // First pass discovers the hash for these bytes. + probe, _, probeCacher, probeItems, _ := newLocalProcessorForTest(job, []string{root}, "") + if _, err := probe.RunOnce(context.Background(), "w1", 10, 1); err != nil { + t.Fatalf("probe RunOnce: %v", err) + } + if len(probeCacher.bytesReq) != 1 { + t.Fatal("probe cache call missing") + } + + // Second pass with the same stored cached path must not delete anything. + p, _, _, _, deleter := newLocalProcessorForTest(job, []string{root}, probeItems.cachedPath) + stats, err := p.RunOnce(context.Background(), "w1", 10, 1) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Succeeded != 1 { + t.Fatalf("stats = %+v, want success", stats) + } + if len(deleter.prefixes) != 0 { + t.Fatalf("unchanged art must not delete prefixes, got %v", deleter.prefixes) + } +} + +func TestProcessLocalImageFailsWithoutRootResolver(t *testing.T) { + root := t.TempDir() + posterPath := writeLocalPoster(t, root) + job := localArtworkJob("file://" + posterPath) + jobs := &fakeLocalImageCacheJobs{ + fakeImageCacheJobs: fakeImageCacheJobs{claimed: []*models.MetadataImageCacheJob{job}}, + } + cacher := &fakeByteImageCacher{thumbhash: "th"} + p := NewImageCacheProcessorWithTargets(jobs, cacher, nil, ImageCacheProcessorTargets{Items: &fakeItemArtworkUpdater{updated: true}}) + + stats, err := p.RunOnce(context.Background(), "w1", 10, 1) + if err != nil { + t.Fatalf("RunOnce: %v", err) + } + if stats.Failed != 1 { + t.Fatalf("stats = %+v, want failure without a library root resolver", stats) + } + if len(cacher.bytesReq) != 0 { + t.Fatal("must not cache without confinement roots") + } +} diff --git a/internal/metadata/local_artwork_db_test.go b/internal/metadata/local_artwork_db_test.go new file mode 100644 index 00000000..c3d8802a --- /dev/null +++ b/internal/metadata/local_artwork_db_test.go @@ -0,0 +1,107 @@ +package metadata + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func localArtworkTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func TestCurrentTargetCachedPathItem(t *testing.T) { + pool := localArtworkTestPool(t) + ctx := context.Background() + contentID := fmt.Sprintf("local-art-%d", time.Now().UnixNano()) + if _, err := pool.Exec(ctx, ` + INSERT INTO media_items (content_id, type, title, status, genres, poster_path, poster_source_path) + VALUES ($1, 'movie', 'Local Art Test', 'matched', '{}'::text[], $2, $3) + `, contentID, "local/movies/"+contentID+"/deadbeef/poster/original.webp", "file:///media/movies/Film/poster.jpg"); err != nil { + t.Fatalf("seed item: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_items WHERE content_id = $1`, contentID) + }) + + repo := NewImageCacheJobRepository(pool) + job := &models.MetadataImageCacheJob{ + TargetType: ImageCacheTargetItem, + TargetContentID: contentID, + ImageType: ImageCacheImagePoster, + } + cached, err := repo.CurrentTargetCachedPath(ctx, job) + if err != nil { + t.Fatalf("CurrentTargetCachedPath: %v", err) + } + if want := "local/movies/" + contentID + "/deadbeef/poster/original.webp"; cached != want { + t.Fatalf("cached = %q, want %q", cached, want) + } + source, err := repo.CurrentTargetSourcePath(ctx, job) + if err != nil { + t.Fatalf("CurrentTargetSourcePath: %v", err) + } + if source != "file:///media/movies/Film/poster.jpg" { + t.Fatalf("source = %q", source) + } + + // Missing rows report empty, not an error. + missing, err := repo.CurrentTargetCachedPath(ctx, &models.MetadataImageCacheJob{ + TargetType: ImageCacheTargetItem, + TargetContentID: contentID + "-missing", + ImageType: ImageCacheImagePoster, + }) + if err != nil || missing != "" { + t.Fatalf("missing row: cached=%q err=%v", missing, err) + } +} + +func TestEnqueueBatchAcceptsLocalSourceDB(t *testing.T) { + pool := localArtworkTestPool(t) + ctx := context.Background() + contentID := fmt.Sprintf("local-art-enq-%d", time.Now().UnixNano()) + repo := NewImageCacheJobRepository(pool) + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM metadata_image_cache_jobs WHERE target_content_id = $1`, contentID) + }) + + n, err := repo.EnqueueBatch(ctx, []EnqueueImageCacheJobInput{{ + TargetType: ImageCacheTargetItem, + TargetContentID: contentID, + SeriesID: contentID, + SourcePath: "file:///media/movies/Film/poster.jpg", + ContentType: "movies", + ImageType: ImageCacheImagePoster, + }}) + if err != nil { + t.Fatalf("EnqueueBatch: %v", err) + } + if n != 1 { + t.Fatalf("enqueued %d, want 1", n) + } + var providerID string + if err := pool.QueryRow(ctx, + `SELECT provider_id FROM metadata_image_cache_jobs WHERE target_content_id = $1`, contentID, + ).Scan(&providerID); err != nil { + t.Fatalf("read job: %v", err) + } + if providerID != "local" { + t.Fatalf("provider_id = %q, want local", providerID) + } +} diff --git a/internal/metadata/local_image_source_test.go b/internal/metadata/local_image_source_test.go new file mode 100644 index 00000000..2b7ed1a7 --- /dev/null +++ b/internal/metadata/local_image_source_test.go @@ -0,0 +1,231 @@ +package metadata + +import ( + "context" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func TestIsLocalImageSourcePath(t *testing.T) { + cases := []struct { + path string + want bool + }{ + {"file:///media/movies/Film/poster.jpg", true}, + {"FILE:///media/movies/Film/poster.jpg", true}, + {" file:///media/poster.jpg ", true}, + {"https://image.tmdb.org/t/p/original/a.jpg", false}, + {"tvdb://banners/a.jpg", false}, + {"local/movies/abc/poster/original.webp", false}, + {"tmdb/movies/550/poster/original.webp", false}, + {"", false}, + } + for _, tc := range cases { + if got := isLocalImageSourcePath(tc.path); got != tc.want { + t.Errorf("isLocalImageSourcePath(%q) = %v, want %v", tc.path, got, tc.want) + } + } +} + +func TestIsCachedImagePathExcludesLocalSources(t *testing.T) { + if isCachedImagePath("file:///media/movies/Film/poster.jpg") { + t.Fatal("file:// source must not be treated as a cached image path") + } + if !isCachedImagePath("local/movies/abc/deadbeef/poster/original.webp") { + t.Fatal("local/ cached key must remain a cached image path") + } + if !isCachedImagePath("tmdb/movies/550/poster/original.webp") { + t.Fatal("remote cached key must remain a cached image path") + } +} + +func TestPreserveCachedArtworkRoutesLocalSource(t *testing.T) { + // No prior cached art: the file:// source must never land in *_path; the + // pre-cache window keeps the path empty until the cache job completes. + path, thumb, source := preserveCachedArtwork( + "file:///media/movies/Film/poster.jpg", "", "", "", "", + ) + if path != "" { + t.Fatalf("path = %q, want empty during pre-cache window", path) + } + if thumb != "" { + t.Fatalf("thumb = %q, want empty", thumb) + } + if source != "file:///media/movies/Film/poster.jpg" { + t.Fatalf("source = %q", source) + } + + // Prior cached art keeps serving while the local source waits to cache. + path, thumb, source = preserveCachedArtwork( + "file:///media/movies/Film/poster.jpg", "", + "tmdb/movies/550/poster/original.webp", "https://image.tmdb.org/a.jpg", "th", + ) + if path != "tmdb/movies/550/poster/original.webp" { + t.Fatalf("path = %q, want prior cached key", path) + } + if thumb != "th" { + t.Fatalf("thumb = %q", thumb) + } + if source != "file:///media/movies/Film/poster.jpg" { + t.Fatalf("source = %q", source) + } +} + +func TestPrepareItemImagesForQueueRoutesLocalSource(t *testing.T) { + item := &models.MediaItem{ + ContentID: "movie-1", + Type: "movie", + PosterPath: "file:///media/movies/Film/poster.jpg", + } + prepareItemImagesForQueue(item, nil) + if item.PosterPath != "" { + t.Fatalf("PosterPath = %q, want empty during pre-cache window", item.PosterPath) + } + if item.PosterSourcePath != "file:///media/movies/Film/poster.jpg" { + t.Fatalf("PosterSourcePath = %q", item.PosterSourcePath) + } +} + +func TestApplyBestImagesLocalCandidateAlwaysApplies(t *testing.T) { + item := &models.MediaItem{PosterPath: "tmdb/movies/550/poster/original.webp"} + applyBestImages(item, []RemoteImage{ + {ProviderID: "nfo", URL: "file:///media/movies/Film/poster.jpg", Type: ImagePoster, Rating: 0}, + }, MergeFillEmpty, "en") + if item.PosterPath != "file:///media/movies/Film/poster.jpg" { + t.Fatalf("PosterPath = %q, want local candidate to apply over existing art", item.PosterPath) + } +} + +func TestApplyBestImagesRemoteSelectionUnchanged(t *testing.T) { + // Regression pins: a rating-0 remote candidate still cannot displace + // existing art in fill mode, while a rated one still can. + item := &models.MediaItem{PosterPath: "tmdb/movies/550/poster/original.webp"} + applyBestImages(item, []RemoteImage{ + {ProviderID: "tmdb", URL: "https://image.tmdb.org/zero.jpg", Type: ImagePoster, Rating: 0}, + }, MergeFillEmpty, "en") + if item.PosterPath != "tmdb/movies/550/poster/original.webp" { + t.Fatalf("PosterPath = %q, rating-0 remote must not displace existing art", item.PosterPath) + } + + applyBestImages(item, []RemoteImage{ + {ProviderID: "tmdb", URL: "https://image.tmdb.org/rated.jpg", Type: ImagePoster, Rating: 7.5}, + }, MergeFillEmpty, "en") + if item.PosterPath != "https://image.tmdb.org/rated.jpg" { + t.Fatalf("PosterPath = %q, rated remote candidate must still apply", item.PosterPath) + } + + empty := &models.MediaItem{} + applyBestImages(empty, []RemoteImage{ + {ProviderID: "tmdb", URL: "https://image.tmdb.org/zero.jpg", Type: ImagePoster, Rating: 0}, + }, MergeFillEmpty, "en") + if empty.PosterPath != "https://image.tmdb.org/zero.jpg" { + t.Fatalf("PosterPath = %q, rating-0 remote must still fill empty art", empty.PosterPath) + } +} + +func TestBuildItemLocalizationRecordSkipsLocalCandidates(t *testing.T) { + // Language-neutral local files must not duplicate into per-language rows. + loc := buildItemLocalizationRecord( + nil, + "movie-1", + "de", + "movie", + &MetadataResult{}, + []RemoteImage{ + {ProviderID: "nfo", URL: "file:///media/movies/Film/poster.jpg", Type: ImagePoster}, + }, + MergeFillEmpty, + "de", + false, + ) + if loc.PosterPath != "" || loc.PosterSourcePath != "" { + t.Fatalf("localization picked up local candidate: path=%q source=%q", loc.PosterPath, loc.PosterSourcePath) + } +} + +func TestItemImageCacheAttributionLocalSource(t *testing.T) { + item := &models.MediaItem{ContentID: "movie-1", TmdbID: "550"} + providerID, providerContentID := itemImageCacheAttribution( + item, + map[string]string{"tmdb": "550"}, + nil, + "file:///media/movies/Film/poster.jpg", + ) + if providerID != "local" { + t.Fatalf("providerID = %q, want local", providerID) + } + if providerContentID != "movie-1" { + t.Fatalf("providerContentID = %q, want the item content ID", providerContentID) + } +} + +func TestEnqueueItemImagesAcceptsLocalSource(t *testing.T) { + service := &MetadataService{} + enqueuer := &recordingImageCacheJobEnqueuer{} + service.SetAutoCacheImages(true) + service.SetImageCacheJobEnqueuer(enqueuer) + + item := &models.MediaItem{ + ContentID: "movie-1", + Type: "movie", + PosterSourcePath: "file:///media/movies/Film/poster.jpg", + } + service.enqueueItemImages(context.Background(), item, nil, nil) + if len(enqueuer.inputs) != 1 { + t.Fatalf("enqueued %d jobs, want 1", len(enqueuer.inputs)) + } + in := enqueuer.inputs[0] + if in.SourcePath != "file:///media/movies/Film/poster.jpg" { + t.Fatalf("SourcePath = %q", in.SourcePath) + } + if in.ProviderID != "local" { + t.Fatalf("ProviderID = %q, want local", in.ProviderID) + } + if in.ProviderContentID != "movie-1" { + t.Fatalf("ProviderContentID = %q", in.ProviderContentID) + } +} + +func TestNormalizeImageCacheJobInputAcceptsLocalSource(t *testing.T) { + in, ok := normalizeImageCacheJobInput(EnqueueImageCacheJobInput{ + TargetType: ImageCacheTargetItem, + TargetContentID: "movie-1", + SourcePath: "file:///media/movies/Film/poster.jpg", + ContentType: "movies", + }) + if !ok { + t.Fatal("local file:// source must be accepted") + } + if in.ProviderID != "local" { + t.Fatalf("ProviderID = %q, want local", in.ProviderID) + } + // The other system schemes stay rejected. + for _, rejected := range []string{ + "s3://bucket/key.jpg", + "local://x.jpg", + "upload://x.jpg", + "generated://x.jpg", + "tmdb/movies/550/poster/original.webp", + } { + if _, ok := normalizeImageCacheJobInput(EnqueueImageCacheJobInput{SourcePath: rejected}); ok { + t.Errorf("source %q must stay rejected", rejected) + } + } +} + +func TestIsStableProviderImageFailureLocalClasses(t *testing.T) { + for _, text := range []string{ + "local image missing: /media/movies/Film/poster.jpg", + "local image forbidden: /media/movies/Film/poster.jpg", + "local image path outside library roots: /etc/passwd", + "unexpected status 404", + } { + if !isStableProviderImageFailure(text) { + t.Errorf("expected stable failure classification for %q", text) + } + } + if isStableProviderImageFailure("local image read failed: io timeout") { + t.Error("transient local read errors must keep the normal backoff") + } +} diff --git a/internal/metadata/match_candidates.go b/internal/metadata/match_candidates.go index 345b2c0c..82e93e51 100644 --- a/internal/metadata/match_candidates.go +++ b/internal/metadata/match_candidates.go @@ -256,10 +256,17 @@ func NormalizeCandidates(results []SearchResult, contentType string) []MatchCand sort.Strings(sources) b.candidate.Sources = sources - // Compute agreement hints. - if len(sources) > 1 { + // Compute agreement hints from corroborating sources only: a local + // sidecar (nfo) echoing a remote result is not provider agreement. + corroborating := make([]string, 0, len(sources)) + for _, s := range sources { + if !nonCorroboratingSources[strings.ToLower(s)] { + corroborating = append(corroborating, s) + } + } + if len(corroborating) > 1 { b.candidate.AgreementHints = append(b.candidate.AgreementHints, - "agreed_by_"+strings.Join(sources, "_and_")) + "agreed_by_"+strings.Join(corroborating, "_and_")) } candidates = append(candidates, b.candidate) @@ -413,32 +420,85 @@ func topTieGroup(scored []scoredMatchCandidate) []scoredMatchCandidate { return group } +// nonCorroboratingSources are provider slugs whose search results must never +// count as independent corroboration: a local sidecar (NFO) echoing a title +// is not a second provider agreeing on identity. +var nonCorroboratingSources = map[string]bool{"nfo": true} + // pickByProviderPriority returns the group candidate whose Sources include the // highest-priority provider (providerPriority is ordered highest-first, e.g. the // library's chain order). Falls back to the top-scored candidate when there is no // priority info or no source matches. +// +// The group has already been proven to be one distinct show, so two defenses +// against ID-less local candidates apply: when any group member carries +// provider IDs, ID-less members are excluded from the priority pick (a +// title-only NFO first in the chain must not downgrade a remotely-found movie +// to an unenriched local item), and the winner adopts the union of the +// group's provider IDs (sound because candidatesAreSingleDistinctShow already +// verified they cannot conflict). func pickByProviderPriority(group []scoredMatchCandidate, providerPriority []string) *MatchCandidate { + eligible := group + if anyCandidateHasProviderIDs(group) { + eligible = make([]scoredMatchCandidate, 0, len(group)) + for _, c := range group { + if providerIDRichness(c.candidate.ProviderIDs) > 0 { + eligible = append(eligible, c) + } + } + } + + winner := &eligible[0].candidate for _, prov := range providerPriority { - for i := range group { - for _, s := range group[i].candidate.Sources { + for i := range eligible { + for _, s := range eligible[i].candidate.Sources { if strings.EqualFold(s, prov) { - return &group[i].candidate + winner = &eligible[i].candidate + return adoptGroupProviderIDs(winner, group) } } } } - return &group[0].candidate + return adoptGroupProviderIDs(winner, group) +} + +func anyCandidateHasProviderIDs(group []scoredMatchCandidate) bool { + for _, c := range group { + if providerIDRichness(c.candidate.ProviderIDs) > 0 { + return true + } + } + return false +} + +// adoptGroupProviderIDs returns a copy of winner carrying the union of the +// group's canonical provider IDs. +func adoptGroupProviderIDs(winner *MatchCandidate, group []scoredMatchCandidate) *MatchCandidate { + adopted := *winner + adopted.ProviderIDs = make(map[string]string, len(winner.ProviderIDs)) + for k, v := range winner.ProviderIDs { + adopted.ProviderIDs[k] = v + } + for _, c := range group { + for _, key := range canonicalCandidateIDKeys { + if v := strings.TrimSpace(c.candidate.ProviderIDs[key]); v != "" && adopted.ProviderIDs[key] == "" { + adopted.ProviderIDs[key] = v + } + } + } + return &adopted } // distinctSourceCount returns how many distinct providers (case-insensitive // Sources values) appear across the candidate group — i.e. how many independent -// providers returned this show. +// providers returned this show. Non-corroborating local sources (nfo) never +// count. func distinctSourceCount(group []scoredMatchCandidate) int { seen := make(map[string]struct{}) for _, c := range group { for _, s := range c.candidate.Sources { s = strings.ToLower(strings.TrimSpace(s)) - if s != "" { + if s != "" && !nonCorroboratingSources[s] { seen[s] = struct{}{} } } @@ -625,7 +685,11 @@ func candidatePrimaryProvider(candidate MatchCandidate) string { return "" } -func selectRefreshMatchCandidate(existing *models.MediaItem, candidates []MatchCandidate) (*MatchCandidate, bool) { +// selectRefreshMatchCandidate picks the refresh winner anchored on the +// existing item's identity. idOverrides carries identity-hint values that won +// the pre-search conflict policy (e.g. NFO <uniqueid> on a manual refresh) +// and takes precedence over the stored ids for anchoring. +func selectRefreshMatchCandidate(existing *models.MediaItem, idOverrides map[string]string, candidates []MatchCandidate) (*MatchCandidate, bool) { if existing == nil || len(candidates) == 0 { return nil, false } @@ -638,6 +702,7 @@ func selectRefreshMatchCandidate(existing *models.MediaItem, candidates []MatchC TvdbID: existing.TvdbID, ImdbID: existing.ImdbID, } + overrideHintIDs(hints, idOverrides) return selectInitialMatchCandidate(hints, candidates, nil) } diff --git a/internal/metadata/match_candidates_defang_test.go b/internal/metadata/match_candidates_defang_test.go new file mode 100644 index 00000000..0e71cec7 --- /dev/null +++ b/internal/metadata/match_candidates_defang_test.go @@ -0,0 +1,87 @@ +package metadata + +import ( + "testing" +) + +// An ID-less candidate (e.g. the NFO title-only search candidate) must never +// win the proven-single-show tie-break on chain priority when an ID-bearing +// member exists in the group — and the winner must adopt the union of the +// group's provider IDs. +func TestSelectInitialMatch_IDlessCandidateExcludedWhenIDBearingExists(t *testing.T) { + hints := &MatchHints{Title: "Inception", Year: 2010, Type: "movie"} + candidates := []MatchCandidate{ + {Title: "Inception", Year: 2010, ContentType: "movie", ProviderIDs: map[string]string{}, Sources: []string{"nfo"}}, + {Title: "Inception", Year: 2010, ContentType: "movie", ProviderIDs: map[string]string{"tmdb": "27205"}, Sources: []string{"tmdb"}}, + {Title: "Inception", Year: 2010, ContentType: "movie", ProviderIDs: map[string]string{"tvdb": "83898"}, Sources: []string{"tvdb"}}, + } + + // nfo sits first in the library chain priority. + winner, ok := selectInitialMatchCandidate(hints, candidates, []string{"nfo", "tmdb", "tvdb"}) + if !ok || winner == nil { + t.Fatalf("winner = %v ok = %v, want accepted candidate", winner, ok) + } + if winner.ProviderIDs["tmdb"] != "27205" { + t.Errorf("winner tmdb id = %q, want 27205 (ID-less candidate must not win; union adopted)", winner.ProviderIDs["tmdb"]) + } + if winner.ProviderIDs["tvdb"] != "83898" { + t.Errorf("winner tvdb id = %q, want 83898 (union of the proven-single-show group's IDs)", winner.ProviderIDs["tvdb"]) + } +} + +// The nfo source must not count as independent corroboration: a local file +// echoing a title is not a second provider agreeing. With no hint year, a +// remote candidate corroborated only by the NFO must not be auto-accepted via +// the multi-source arm. +func TestSelectInitialMatch_NFOSourceDoesNotCorroborate(t *testing.T) { + hints := &MatchHints{Title: "Some Show", Type: "movie"} // no year hint + candidates := []MatchCandidate{ + {Title: "Some Show", Year: 2015, ContentType: "movie", ProviderIDs: map[string]string{"tmdb": "1"}, Sources: []string{"tmdb"}}, + {Title: "Some Show", Year: 2015, ContentType: "movie", ProviderIDs: map[string]string{}, Sources: []string{"nfo"}}, + } + + winner, ok := selectInitialMatchCandidate(hints, candidates, nil) + if ok { + t.Fatalf("winner = %+v, want rejection (nfo echo must not stand in for a missing year)", winner) + } +} + +// A genuinely multi-source group (two remote providers) still corroborates. +func TestSelectInitialMatch_RemoteMultiSourceStillCorroborates(t *testing.T) { + hints := &MatchHints{Title: "Some Show", Type: "movie"} // no year hint + candidates := []MatchCandidate{ + {Title: "Some Show", Year: 2015, ContentType: "movie", ProviderIDs: map[string]string{"tmdb": "1"}, Sources: []string{"tmdb"}}, + {Title: "Some Show", Year: 2015, ContentType: "movie", ProviderIDs: map[string]string{"tvdb": "2"}, Sources: []string{"tvdb"}}, + } + + winner, ok := selectInitialMatchCandidate(hints, candidates, nil) + if !ok || winner == nil { + t.Fatalf("winner = %v ok = %v, want accepted candidate", winner, ok) + } +} + +// The nfo source must not manufacture an "agreed_by" hint with a remote +// provider when their results merge into one candidate. +func TestNormalizeCandidates_NFOExcludedFromAgreementHints(t *testing.T) { + results := []SearchResult{ + {Name: "Movie", Year: 2020, ProviderIDs: map[string]string{"tmdb": "7"}, Provider: "tmdb"}, + {Name: "Movie", Year: 2020, ProviderIDs: map[string]string{"tmdb": "7"}, Provider: "nfo"}, + } + candidates := NormalizeCandidates(results, "movie") + if len(candidates) != 1 { + t.Fatalf("candidates = %d, want 1 merged candidate", len(candidates)) + } + if len(candidates[0].AgreementHints) != 0 { + t.Errorf("agreement hints = %v, want none (nfo does not corroborate)", candidates[0].AgreementHints) + } + + // Two genuine remote sources still agree. + remote := []SearchResult{ + {Name: "Movie", Year: 2020, ProviderIDs: map[string]string{"tmdb": "7"}, Provider: "tmdb"}, + {Name: "Movie", Year: 2020, ProviderIDs: map[string]string{"tmdb": "7"}, Provider: "trakt"}, + } + candidates = NormalizeCandidates(remote, "movie") + if len(candidates) != 1 || len(candidates[0].AgreementHints) != 1 { + t.Fatalf("remote agreement hints = %+v, want one", candidates) + } +} diff --git a/internal/metadata/match_candidates_test.go b/internal/metadata/match_candidates_test.go index 2e206dfa..f714b44b 100644 --- a/internal/metadata/match_candidates_test.go +++ b/internal/metadata/match_candidates_test.go @@ -751,6 +751,7 @@ func TestSelectRefreshMatchCandidate_AcceptsCandidateWithPartialTrustedIDCoverag TmdbID: "603", ImdbID: "tt0133093", }, + nil, []MatchCandidate{ { Title: "The Matrix", @@ -775,6 +776,7 @@ func TestSelectRefreshMatchCandidate_RejectsCandidateWithoutTrustedIDMatches(t * TmdbID: "603", ImdbID: "tt0133093", }, + nil, []MatchCandidate{ { Title: "The Matrix", @@ -799,6 +801,7 @@ func TestSelectRefreshMatchCandidate_RejectsConflictingTrustedIDCandidate(t *tes TmdbID: "603", ImdbID: "tt0133093", }, + nil, []MatchCandidate{ { Title: "The Matrix", diff --git a/internal/metadata/merge.go b/internal/metadata/merge.go index 2e74beda..2a7a5ccd 100644 --- a/internal/metadata/merge.go +++ b/internal/metadata/merge.go @@ -40,15 +40,17 @@ func MergeMetadata(source, target *MetadataResult, locked []MetadataField, mode mergeFloat(&target.Ratings.RTAudience, source.Ratings.RTAudience, mode) } - // Always fill year (not lockable — identity field) - mergeInt(&target.Year, source.Year, mode) + // Year and release/air dates lock together under FieldReleaseDates. + if !isLocked(FieldReleaseDates) { + mergeInt(&target.Year, source.Year, mode) + mergeScalar(&target.ReleaseDate, source.ReleaseDate, mode) + mergeScalar(&target.FirstAirDate, source.FirstAirDate, mode) + mergeScalar(&target.LastAirDate, source.LastAirDate, mode) + } mergeScalar(&target.OriginalLanguage, source.OriginalLanguage, mode) - mergeScalar(&target.ReleaseDate, source.ReleaseDate, mode) // Series fields (not lockable — structural) mergeInt(&target.SeasonCount, source.SeasonCount, mode) - mergeScalar(&target.FirstAirDate, source.FirstAirDate, mode) - mergeScalar(&target.LastAirDate, source.LastAirDate, mode) mergeScalar(&target.ShowStatus, source.ShowStatus, mode) if !isLocked(FieldAirSchedule) { mergeScalar(&target.AirTime, source.AirTime, mode) @@ -117,13 +119,15 @@ func MergeGlobalMetadata(source, target *MetadataResult, locked []MetadataField, mergeFloat(&target.Ratings.RTAudience, source.Ratings.RTAudience, mode) } - mergeInt(&target.Year, source.Year, mode) + if !isLocked(FieldReleaseDates) { + mergeInt(&target.Year, source.Year, mode) + mergeScalar(&target.ReleaseDate, source.ReleaseDate, mode) + mergeScalar(&target.FirstAirDate, source.FirstAirDate, mode) + mergeScalar(&target.LastAirDate, source.LastAirDate, mode) + } mergeScalar(&target.OriginalTitle, source.OriginalTitle, mode) mergeScalar(&target.OriginalLanguage, source.OriginalLanguage, mode) - mergeScalar(&target.ReleaseDate, source.ReleaseDate, mode) mergeInt(&target.SeasonCount, source.SeasonCount, mode) - mergeScalar(&target.FirstAirDate, source.FirstAirDate, mode) - mergeScalar(&target.LastAirDate, source.LastAirDate, mode) mergeScalar(&target.ShowStatus, source.ShowStatus, mode) if !isLocked(FieldAirSchedule) { mergeScalar(&target.AirTime, source.AirTime, mode) diff --git a/internal/metadata/nfo/builtin_registration_test.go b/internal/metadata/nfo/builtin_registration_test.go new file mode 100644 index 00000000..3f6ed3cc --- /dev/null +++ b/internal/metadata/nfo/builtin_registration_test.go @@ -0,0 +1,64 @@ +package nfo + +import ( + "context" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/Silo-Server/silo-server/internal/metadata" +) + +// Importing the nfo package must register the provider in the builtin +// registry under its capability id — buildProviders resolves the seeded +// 'nfo' chain entry through this single registration point. +func TestPackageImportRegistersBuiltinProvider(t *testing.T) { + if !slices.Contains(metadata.BuiltinCapabilityIDs(), "nfo") { + t.Fatalf("BuiltinCapabilityIDs() = %v, want to contain nfo", metadata.BuiltinCapabilityIDs()) + } +} + +// The provider implements the narrow IdentityHintProvider contract: a curated +// <uniqueid> becomes a trusted identity hint before Phase-1 search. +func TestProviderIdentityHintsFromNFOFile(t *testing.T) { + dir := t.TempDir() + moviePath := filepath.Join(dir, "Obscure Film (2019).mkv") + nfoBody := `<?xml version="1.0" encoding="UTF-8"?> +<movie> + <title>Obscure Film + 2019 + 424242 + tt0424242 +` + if err := os.WriteFile(filepath.Join(dir, "movie.nfo"), []byte(nfoBody), 0o644); err != nil { + t.Fatalf("write nfo: %v", err) + } + + var hinter metadata.IdentityHintProvider = NewProvider() + hints := hinter.IdentityHints(context.Background(), metadata.SearchQuery{ + Title: "Obscure Film", + ContentType: "movie", + FilePath: moviePath, + }) + if hints["tmdb"] != "424242" { + t.Errorf("tmdb hint = %q, want 424242", hints["tmdb"]) + } + if hints["imdb"] != "tt0424242" { + t.Errorf("imdb hint = %q, want tt0424242", hints["imdb"]) + } + + // A title-only NFO yields no identity hints (the title flows through the + // defanged search candidate instead). + titleOnly := t.TempDir() + if err := os.WriteFile(filepath.Join(titleOnly, "movie.nfo"), []byte(`Home Video`), 0o644); err != nil { + t.Fatalf("write nfo: %v", err) + } + hints = hinter.IdentityHints(context.Background(), metadata.SearchQuery{ + ContentType: "movie", + FilePath: filepath.Join(titleOnly, "Home Video.mkv"), + }) + if len(hints) != 0 { + t.Errorf("title-only NFO hints = %v, want none", hints) + } +} diff --git a/internal/metadata/nfo/images.go b/internal/metadata/nfo/images.go new file mode 100644 index 00000000..3faed1b2 --- /dev/null +++ b/internal/metadata/nfo/images.go @@ -0,0 +1,96 @@ +package nfo + +import ( + "context" + "os" + "path/filepath" + "strings" + + "github.com/Silo-Server/silo-server/internal/metadata" +) + +// maxLocalArtworkBytes caps sidecar artwork reads at 8 MiB (mirrors the +// audiobook sidecar cover guard). Discovery rejects larger files up front so +// the cache processor never queues an oversized read. +const maxLocalArtworkBytes = 8 << 20 + +var localArtworkExtensions = []string{".jpg", ".jpeg", ".png", ".webp"} + +// localArtworkNames maps each image type to its sidecar base names in +// precedence order. "{basename}" expands to the media file's basename and is +// searched next to the media files; plain names are searched only in the +// directory-level sidecar search paths, which upstream code restricts to +// single-content observed roots (canUseObservedRootForDirectorySidecars), so +// one folder.jpg in a flat multi-movie folder attaches to none of them. +var localArtworkNames = map[metadata.ImageType][]string{ + metadata.ImagePoster: {"poster", "folder", "cover", "{basename}-poster"}, + metadata.ImageBackdrop: {"fanart", "backdrop", "background", "{basename}-fanart"}, + metadata.ImageLogo: {"logo", "clearlogo", "{basename}-logo"}, +} + +// GetImages implements metadata.ImageProvider by discovering sidecar artwork +// (poster.jpg, fanart.jpg, clearlogo.png, -poster.jpg, ...) next to +// the item's media files. Returned URLs use the file:// scheme with the +// absolute logical path; the image cache processor copies them into S3 so +// library files are never served directly. Local art applies even when no +// .nfo file is present. +func (p *Provider) GetImages(_ context.Context, req metadata.ImageRequest) ([]metadata.RemoteImage, error) { + searchDirs := compactNFOPaths(req.PrimarySidecarSearchPaths) + mediaFiles := compactNFOPaths(append([]string{req.RepresentativeFilePath}, req.AllGroupFilePaths...)) + if len(searchDirs) == 0 && len(mediaFiles) == 0 { + return nil, nil + } + + var images []metadata.RemoteImage + for _, imageType := range []metadata.ImageType{metadata.ImagePoster, metadata.ImageBackdrop, metadata.ImageLogo} { + if path := findLocalArtwork(localArtworkNames[imageType], searchDirs, mediaFiles); path != "" { + images = append(images, metadata.RemoteImage{ + ProviderID: p.Slug(), + URL: "file://" + path, + Type: imageType, + Rating: 0, + }) + } + } + return images, nil +} + +// findLocalArtwork returns the absolute path of the first matching sidecar +// image, trying names in precedence order across every candidate location. +func findLocalArtwork(names []string, searchDirs []string, mediaFiles []string) string { + for _, name := range names { + if suffix, ok := strings.CutPrefix(name, "{basename}"); ok { + for _, mediaFile := range mediaFiles { + dir := filepath.Dir(mediaFile) + base := strings.TrimSuffix(filepath.Base(mediaFile), filepath.Ext(mediaFile)) + if path := findLocalArtworkFile(dir, base+suffix); path != "" { + return path + } + } + continue + } + for _, dir := range searchDirs { + if path := findLocalArtworkFile(dir, name); path != "" { + return path + } + } + } + return "" +} + +func findLocalArtworkFile(dir, name string) string { + for _, ext := range localArtworkExtensions { + path := filepath.Join(dir, name+ext) + info, err := os.Lstat(path) + if err != nil { + continue + } + // Reject symlinked leaves and non-regular files, and cap the size so + // the cache processor's re-checks agree with discovery. + if !info.Mode().IsRegular() || info.Size() == 0 || info.Size() > maxLocalArtworkBytes { + continue + } + return path + } + return "" +} diff --git a/internal/metadata/nfo/images_test.go b/internal/metadata/nfo/images_test.go new file mode 100644 index 00000000..f31359e7 --- /dev/null +++ b/internal/metadata/nfo/images_test.go @@ -0,0 +1,258 @@ +package nfo + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/Silo-Server/silo-server/internal/metadata" +) + +func writeImageFile(t *testing.T, path string) { + t.Helper() + if err := os.WriteFile(path, []byte("fake-image-bytes"), 0o644); err != nil { + t.Fatalf("writing %s: %v", path, err) + } +} + +func imagesByType(images []metadata.RemoteImage) map[metadata.ImageType]metadata.RemoteImage { + byType := make(map[metadata.ImageType]metadata.RemoteImage, len(images)) + for _, img := range images { + if _, ok := byType[img.Type]; !ok { + byType[img.Type] = img + } + } + return byType +} + +func TestGetImagesFindsSidecarArtwork(t *testing.T) { + dir := t.TempDir() + moviePath := filepath.Join(dir, "Film (2020).mkv") + writeImageFile(t, moviePath) + writeImageFile(t, filepath.Join(dir, "poster.jpg")) + writeImageFile(t, filepath.Join(dir, "fanart.png")) + writeImageFile(t, filepath.Join(dir, "clearlogo.webp")) + + p := NewProvider() + images, err := p.GetImages(context.Background(), metadata.ImageRequest{ + ContentType: "movie", + RepresentativeFilePath: moviePath, + AllGroupFilePaths: []string{moviePath}, + PrimarySidecarSearchPaths: []string{dir}, + }) + if err != nil { + t.Fatalf("GetImages: %v", err) + } + byType := imagesByType(images) + if len(byType) != 3 { + t.Fatalf("expected poster/backdrop/logo, got %d images: %+v", len(images), images) + } + poster := byType[metadata.ImagePoster] + if poster.URL != "file://"+filepath.Join(dir, "poster.jpg") { + t.Fatalf("poster URL = %q", poster.URL) + } + if poster.Rating != 0 { + t.Fatalf("poster rating = %v, want 0", poster.Rating) + } + if poster.ProviderID != "nfo" { + t.Fatalf("poster provider = %q", poster.ProviderID) + } + if got := byType[metadata.ImageBackdrop].URL; got != "file://"+filepath.Join(dir, "fanart.png") { + t.Fatalf("backdrop URL = %q", got) + } + if got := byType[metadata.ImageLogo].URL; got != "file://"+filepath.Join(dir, "clearlogo.webp") { + t.Fatalf("logo URL = %q", got) + } +} + +func TestGetImagesFilenamePrecedence(t *testing.T) { + dir := t.TempDir() + moviePath := filepath.Join(dir, "Film.mkv") + writeImageFile(t, moviePath) + // poster beats folder and cover; folder beats cover. + writeImageFile(t, filepath.Join(dir, "poster.png")) + writeImageFile(t, filepath.Join(dir, "folder.jpg")) + writeImageFile(t, filepath.Join(dir, "cover.jpg")) + // backdrop names: fanart beats backdrop/background. + writeImageFile(t, filepath.Join(dir, "background.jpg")) + writeImageFile(t, filepath.Join(dir, "fanart.jpg")) + // logo: logo beats clearlogo. + writeImageFile(t, filepath.Join(dir, "clearlogo.png")) + writeImageFile(t, filepath.Join(dir, "logo.png")) + + p := NewProvider() + images, err := p.GetImages(context.Background(), metadata.ImageRequest{ + ContentType: "movie", + RepresentativeFilePath: moviePath, + PrimarySidecarSearchPaths: []string{dir}, + }) + if err != nil { + t.Fatalf("GetImages: %v", err) + } + byType := imagesByType(images) + if got := byType[metadata.ImagePoster].URL; got != "file://"+filepath.Join(dir, "poster.png") { + t.Fatalf("poster URL = %q, want poster.png", got) + } + if got := byType[metadata.ImageBackdrop].URL; got != "file://"+filepath.Join(dir, "fanart.jpg") { + t.Fatalf("backdrop URL = %q, want fanart.jpg", got) + } + if got := byType[metadata.ImageLogo].URL; got != "file://"+filepath.Join(dir, "logo.png") { + t.Fatalf("logo URL = %q, want logo.png", got) + } +} + +func TestGetImagesBasenameScopedArtwork(t *testing.T) { + dir := t.TempDir() + moviePath := filepath.Join(dir, "Film (2020).mkv") + writeImageFile(t, moviePath) + writeImageFile(t, filepath.Join(dir, "Film (2020)-poster.jpg")) + writeImageFile(t, filepath.Join(dir, "Film (2020)-fanart.jpg")) + writeImageFile(t, filepath.Join(dir, "Film (2020)-logo.png")) + + p := NewProvider() + // No directory search paths: basename-scoped art still applies. + images, err := p.GetImages(context.Background(), metadata.ImageRequest{ + ContentType: "movie", + RepresentativeFilePath: moviePath, + }) + if err != nil { + t.Fatalf("GetImages: %v", err) + } + byType := imagesByType(images) + if got := byType[metadata.ImagePoster].URL; got != "file://"+filepath.Join(dir, "Film (2020)-poster.jpg") { + t.Fatalf("poster URL = %q", got) + } + if got := byType[metadata.ImageBackdrop].URL; got != "file://"+filepath.Join(dir, "Film (2020)-fanart.jpg") { + t.Fatalf("backdrop URL = %q", got) + } + if got := byType[metadata.ImageLogo].URL; got != "file://"+filepath.Join(dir, "Film (2020)-logo.png") { + t.Fatalf("logo URL = %q", got) + } +} + +func TestGetImagesFlatFolderGenericArtSuppressed(t *testing.T) { + // Multi-movie flat folder: the directory is not a safe sidecar search + // path (canUseObservedRootForDirectorySidecars suppression upstream), so + // generic names like folder.jpg must not attach to any movie in it. + dir := t.TempDir() + movieA := filepath.Join(dir, "Movie A.mkv") + writeImageFile(t, movieA) + writeImageFile(t, filepath.Join(dir, "Movie B.mkv")) + writeImageFile(t, filepath.Join(dir, "folder.jpg")) + + p := NewProvider() + images, err := p.GetImages(context.Background(), metadata.ImageRequest{ + ContentType: "movie", + RepresentativeFilePath: movieA, + AllGroupFilePaths: []string{movieA}, + // PrimarySidecarSearchPaths deliberately empty. + }) + if err != nil { + t.Fatalf("GetImages: %v", err) + } + if len(images) != 0 { + t.Fatalf("expected no images for flat multi-movie folder, got %+v", images) + } +} + +func TestGetImagesWorksWithoutNFO(t *testing.T) { + // Documented bonus: local art applies even with no .nfo present. + dir := t.TempDir() + moviePath := filepath.Join(dir, "Film.mkv") + writeImageFile(t, moviePath) + writeImageFile(t, filepath.Join(dir, "poster.jpg")) + + p := NewProvider() + images, err := p.GetImages(context.Background(), metadata.ImageRequest{ + ContentType: "movie", + RepresentativeFilePath: moviePath, + PrimarySidecarSearchPaths: []string{dir}, + }) + if err != nil { + t.Fatalf("GetImages: %v", err) + } + if len(images) != 1 || images[0].Type != metadata.ImagePoster { + t.Fatalf("expected exactly the poster, got %+v", images) + } +} + +func TestGetImagesExtensionSet(t *testing.T) { + dir := t.TempDir() + moviePath := filepath.Join(dir, "Film.mkv") + writeImageFile(t, moviePath) + // .gif and .bmp are not in the accepted extension set. + writeImageFile(t, filepath.Join(dir, "poster.gif")) + writeImageFile(t, filepath.Join(dir, "fanart.bmp")) + writeImageFile(t, filepath.Join(dir, "logo.jpeg")) + + p := NewProvider() + images, err := p.GetImages(context.Background(), metadata.ImageRequest{ + ContentType: "movie", + RepresentativeFilePath: moviePath, + PrimarySidecarSearchPaths: []string{dir}, + }) + if err != nil { + t.Fatalf("GetImages: %v", err) + } + if len(images) != 1 || images[0].Type != metadata.ImageLogo { + t.Fatalf("expected only logo.jpeg, got %+v", images) + } +} + +func TestGetImagesRejectsSymlinksAndNonRegularFiles(t *testing.T) { + dir := t.TempDir() + moviePath := filepath.Join(dir, "Film.mkv") + writeImageFile(t, moviePath) + target := filepath.Join(dir, "elsewhere.jpg") + writeImageFile(t, target) + if err := os.Symlink(target, filepath.Join(dir, "poster.jpg")); err != nil { + t.Skipf("symlinks unavailable: %v", err) + } + if err := os.Mkdir(filepath.Join(dir, "fanart.jpg"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + + p := NewProvider() + images, err := p.GetImages(context.Background(), metadata.ImageRequest{ + ContentType: "movie", + RepresentativeFilePath: moviePath, + PrimarySidecarSearchPaths: []string{dir}, + }) + if err != nil { + t.Fatalf("GetImages: %v", err) + } + if len(images) != 0 { + t.Fatalf("expected symlinked/non-regular sidecars rejected, got %+v", images) + } +} + +func TestGetImagesRejectsOversizedFiles(t *testing.T) { + dir := t.TempDir() + moviePath := filepath.Join(dir, "Film.mkv") + writeImageFile(t, moviePath) + oversized := filepath.Join(dir, "poster.jpg") + f, err := os.Create(oversized) + if err != nil { + t.Fatalf("create: %v", err) + } + if err := f.Truncate(maxLocalArtworkBytes + 1); err != nil { + t.Fatalf("truncate: %v", err) + } + if err := f.Close(); err != nil { + t.Fatalf("close: %v", err) + } + + p := NewProvider() + images, err := p.GetImages(context.Background(), metadata.ImageRequest{ + ContentType: "movie", + RepresentativeFilePath: moviePath, + PrimarySidecarSearchPaths: []string{dir}, + }) + if err != nil { + t.Fatalf("GetImages: %v", err) + } + if len(images) != 0 { + t.Fatalf("expected oversized sidecar rejected, got %+v", images) + } +} diff --git a/internal/metadata/nfo/nfo.go b/internal/metadata/nfo/nfo.go index f2009977..1eafd0c1 100644 --- a/internal/metadata/nfo/nfo.go +++ b/internal/metadata/nfo/nfo.go @@ -4,7 +4,20 @@ import ( "bytes" "encoding/xml" "fmt" + "regexp" + "strconv" "strings" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// Content types emitted in parsedNFO.Type, matching the metadata package's +// content-type/level vocabulary. +const ( + typeMovie = "movie" + typeSeries = "series" + typeSeason = "season" + typeEpisode = "episode" ) // nfoUniqueID represents a element inside an NFO XML file. @@ -14,45 +27,135 @@ type nfoUniqueID struct { Value string `xml:",chardata"` } +// nfoActor represents an element (Kodi/Jellyfin cast entry). +// URLs are intentionally ignored in v1. +type nfoActor struct { + Name string `xml:"name"` + Role string `xml:"role"` + Order *int `xml:"order"` +} + +// nfoNamedRating represents a entry inside a block. +type nfoNamedRating struct { + Name string `xml:"name,attr"` + Max string `xml:"max,attr"` + Value string `xml:"value"` +} + +// nfoRatingsBlock represents the modern multi-source container. +type nfoRatingsBlock struct { + Ratings []nfoNamedRating `xml:"rating"` +} + +// nfoCommon holds the fields shared by and roots. +// encoding/xml flattens the embedded struct so both roots decode it in place. +// +// (movie collections) is deliberately not mapped: MetadataResult +// has no collection field yet — see +// docs/superpowers/plans/2026-07-09-local-nfo-metadata-and-artwork.md §9. +type nfoCommon struct { + Title string `xml:"title"` + OriginalTitle string `xml:"originaltitle"` + Tagline string `xml:"tagline"` + Year int `xml:"year"` + Plot string `xml:"plot"` + Runtime string `xml:"runtime"` + Premiered string `xml:"premiered"` + MPAA string `xml:"mpaa"` + Genres []string `xml:"genre"` + Studios []string `xml:"studio"` + Countries []string `xml:"country"` + Tags []string `xml:"tag"` + LegacyRating string `xml:"rating"` + Ratings nfoRatingsBlock `xml:"ratings"` + UserRating string `xml:"userrating"` + Actors []nfoActor `xml:"actor"` + Directors []string `xml:"director"` + Credits []string `xml:"credits"` + UniqueIDs []nfoUniqueID `xml:"uniqueid"` +} + // nfoMovie represents the root element. type nfoMovie struct { - XMLName xml.Name `xml:"movie"` - Title string `xml:"title"` - Year int `xml:"year"` - Plot string `xml:"plot"` - UniqueIDs []nfoUniqueID `xml:"uniqueid"` + XMLName xml.Name `xml:"movie"` + nfoCommon + ReleaseDate string `xml:"releasedate"` } // nfoTVShow represents the root element. type nfoTVShow struct { - XMLName xml.Name `xml:"tvshow"` - Title string `xml:"title"` - Year int `xml:"year"` - Plot string `xml:"plot"` - UniqueIDs []nfoUniqueID `xml:"uniqueid"` + XMLName xml.Name `xml:"tvshow"` + nfoCommon + Aired string `xml:"aired"` } -// nfoEpisode represents the root element. +// nfoEpisode represents the root element. Season/Episode are +// pointers so "declared as 0" (specials) is distinguishable from "absent" — +// declared numbers are advisory and checked against the filename-derived +// structure by the provider. type nfoEpisode struct { - XMLName xml.Name `xml:"episodedetails"` - Title string `xml:"title"` - Season int `xml:"season"` - Episode int `xml:"episode"` - Plot string `xml:"plot"` - UniqueIDs []nfoUniqueID `xml:"uniqueid"` + XMLName xml.Name `xml:"episodedetails"` + Title string `xml:"title"` + Season *int `xml:"season"` + Episode *int `xml:"episode"` + Plot string `xml:"plot"` + Aired string `xml:"aired"` + Runtime string `xml:"runtime"` + LegacyRating string `xml:"rating"` + Ratings nfoRatingsBlock `xml:"ratings"` + UniqueIDs []nfoUniqueID `xml:"uniqueid"` } -// parsedNFO holds extracted data from an NFO file. +// nfoSeason represents the root element (Kodi/Jellyfin season +// sidecar). SeasonNumber is a pointer for the same absent-vs-zero reason as +// nfoEpisode. +type nfoSeason struct { + XMLName xml.Name `xml:"season"` + Title string `xml:"title"` + Plot string `xml:"plot"` + SeasonNumber *int `xml:"seasonnumber"` +} + +// parsedNFO holds extracted data from an NFO file. Collections stay nil when +// the NFO declares no entries so downstream MergeFillEmpty early-returns and +// remote providers can fill them (no placeholder emissions). type parsedNFO struct { - Title string - Year int - Overview string - TmdbID string - ImdbID string - TvdbID string - Type string // movie, series, episode - Season int - Episode int + Title string + OriginalTitle string + Tagline string + Year int + Overview string + Runtime int // minutes + ReleaseDate string // movies: then + FirstAirDate string // series: then + ContentRating string + Genres []string + Studios []string + Countries []string + Keywords []string + // Ratings normalized to their MetadataResult scales: IMDB/TMDB out of + // 10, Rotten Tomatoes out of 100. is parsed but never + // emitted — MetadataResult has no per-user rating slot. + RatingIMDB float64 + RatingTMDB float64 + RatingRTCritic float64 + RatingRTAudience float64 + People []models.ItemPerson + TmdbID string + ImdbID string + TvdbID string + Type string // movie, series, season, episode + // Season/Episode carry the NFO's declared numbers when the matching Set + // flag is true. They are advisory only: naming owns structure, so the + // directory/filename-derived numbers win on conflict. + Season int + SeasonSet bool + Episode int + EpisodeSet bool + // MultiEpisode marks a document with more than one + // root (multi-episode file). Out of scope in v1: the first block is + // parsed and the caller warns. + MultiEpisode bool } // parseNFOData parses NFO XML data and returns structured results. @@ -66,10 +169,12 @@ func parseNFOData(data []byte) (*parsedNFO, error) { } switch rootTag { - case "movie": + case typeMovie: return parseMovieNFO(data) case "tvshow": return parseTVShowNFO(data) + case typeSeason: + return parseSeasonNFO(data) case "episodedetails": return parseEpisodeNFO(data) default: @@ -95,13 +200,9 @@ func parseMovieNFO(data []byte) (*parsedNFO, error) { if err := xmlUnmarshal(data, &m); err != nil { return nil, fmt.Errorf("nfo: failed to parse movie: %w", err) } - p := &parsedNFO{ - Title: strings.TrimSpace(m.Title), - Year: m.Year, - Overview: strings.TrimSpace(m.Plot), - Type: "movie", - } - applyUniqueIDs(p, m.UniqueIDs) + p := parsedFromCommon(&m.nfoCommon, typeMovie) + p.ReleaseDate = firstDate(m.Premiered, m.ReleaseDate) + deriveYear(p, m.Premiered, m.ReleaseDate) return p, nil } @@ -110,32 +211,254 @@ func parseTVShowNFO(data []byte) (*parsedNFO, error) { if err := xmlUnmarshal(data, &s); err != nil { return nil, fmt.Errorf("nfo: failed to parse tvshow: %w", err) } + p := parsedFromCommon(&s.nfoCommon, typeSeries) + p.FirstAirDate = firstDate(s.Premiered, s.Aired) + deriveYear(p, s.Premiered, s.Aired) + return p, nil +} + +func parseSeasonNFO(data []byte) (*parsedNFO, error) { + var s nfoSeason + if err := xmlUnmarshal(data, &s); err != nil { + return nil, fmt.Errorf("nfo: failed to parse season: %w", err) + } p := &parsedNFO{ Title: strings.TrimSpace(s.Title), - Year: s.Year, Overview: strings.TrimSpace(s.Plot), - Type: "series", + Type: typeSeason, + } + if s.SeasonNumber != nil { + p.Season = *s.SeasonNumber + p.SeasonSet = true } - applyUniqueIDs(p, s.UniqueIDs) return p, nil } func parseEpisodeNFO(data []byte) (*parsedNFO, error) { + decoder := xml.NewDecoder(bytes.NewReader(data)) var e nfoEpisode - if err := xmlUnmarshal(data, &e); err != nil { + if err := decoder.Decode(&e); err != nil { return nil, fmt.Errorf("nfo: failed to parse episodedetails: %w", err) } p := &parsedNFO{ - Title: strings.TrimSpace(e.Title), - Overview: strings.TrimSpace(e.Plot), - Type: "episode", - Season: e.Season, - Episode: e.Episode, + Title: strings.TrimSpace(e.Title), + Overview: strings.TrimSpace(e.Plot), + FirstAirDate: firstDate(e.Aired), + Type: typeEpisode, + MultiEpisode: hasMoreEpisodeDetails(decoder), } + if e.Season != nil { + p.Season = *e.Season + p.SeasonSet = true + } + if e.Episode != nil { + p.Episode = *e.Episode + p.EpisodeSet = true + } + if minutes, err := strconv.Atoi(strings.TrimSpace(e.Runtime)); err == nil && minutes > 0 { + p.Runtime = minutes + } + applyRatingValues(p, e.Ratings, e.LegacyRating) applyUniqueIDs(p, e.UniqueIDs) return p, nil } +// hasMoreEpisodeDetails reports whether the decoder (positioned after the +// first decoded root) still holds another element. Kodi +// multi-episode files concatenate several roots into one .nfo. +func hasMoreEpisodeDetails(decoder *xml.Decoder) bool { + for { + tok, err := decoder.Token() + if err != nil { + return false + } + if se, ok := tok.(xml.StartElement); ok { + return se.Name.Local == "episodedetails" + } + } +} + +// parsedFromCommon maps the shared / field set. +func parsedFromCommon(c *nfoCommon, contentType string) *parsedNFO { + p := &parsedNFO{ + Title: strings.TrimSpace(c.Title), + OriginalTitle: strings.TrimSpace(c.OriginalTitle), + Tagline: strings.TrimSpace(c.Tagline), + Year: c.Year, + Overview: strings.TrimSpace(c.Plot), + ContentRating: strings.TrimSpace(c.MPAA), + Genres: cleanStrings(c.Genres), + Studios: cleanStrings(c.Studios), + Countries: cleanStrings(c.Countries), + Keywords: cleanStrings(c.Tags), + Type: contentType, + } + if minutes, err := strconv.Atoi(strings.TrimSpace(c.Runtime)); err == nil && minutes > 0 { + p.Runtime = minutes + } + applyRatings(p, c) + applyPeople(p, c) + applyUniqueIDs(p, c.UniqueIDs) + return p +} + +var nfoDatePattern = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`) + +// firstDate returns the first candidate that looks like an ISO date +// (YYYY-MM-DD); anything else is ignored rather than persisted. +func firstDate(candidates ...string) string { + for _, c := range candidates { + c = strings.TrimSpace(c) + if nfoDatePattern.MatchString(c) { + return c + } + } + return "" +} + +// deriveYear fills Year when the NFO omits , from the first candidate +// carrying a leading 4-digit year. Kodi exports commonly ship only , +// sometimes as a bare year (2021) or a datetime (2021-05-01T20:00:00) that +// firstDate's strict ISO match rejects for storage; the year is still +// recoverable here for display. +func deriveYear(p *parsedNFO, candidates ...string) { + if p.Year != 0 { + return + } + for _, c := range candidates { + c = strings.TrimSpace(c) + if len(c) < 4 { + continue + } + if year, err := strconv.Atoi(c[:4]); err == nil && year > 0 { + p.Year = year + return + } + } +} + +// applyRatings maps the modern multi-source block with the legacy +// bare as an IMDB-slot fallback (Kodi's legacy field historically +// held the scraper's IMDB rating). +func applyRatings(p *parsedNFO, c *nfoCommon) { + applyRatingValues(p, c.Ratings, c.LegacyRating) +} + +// applyRatingValues is the shared ratings mapper for / +// (via nfoCommon) and roots. +func applyRatingValues(p *parsedNFO, block nfoRatingsBlock, legacyRating string) { + for _, r := range block.Ratings { + value, ok := parseRatingValue(r.Value) + if !ok { + continue + } + name := strings.ToLower(strings.TrimSpace(r.Name)) + // Default the source scale when omits max: Rotten Tomatoes + // sources are 0-100, everything else 0-10. A blanket default of 10 for a + // bare RT score (e.g. tomatometer 60) would scale ×10 and clamp to 100. + // An explicit max attribute always wins. + max := 10.0 + if name == "tomatometerallcritics" || name == "tomatometerallaudience" || name == "rottentomatoes" { + max = 100.0 + } + if parsed, parsedOK := parseRatingValue(r.Max); parsedOK { + max = parsed + } + switch name { + case "imdb": + p.RatingIMDB = scaleRating(value, max, 10) + case "tmdb", "themoviedb": + p.RatingTMDB = scaleRating(value, max, 10) + case "tomatometerallcritics", "rottentomatoes": + p.RatingRTCritic = scaleRating(value, max, 100) + case "tomatometerallaudience": + p.RatingRTAudience = scaleRating(value, max, 100) + } + } + if p.RatingIMDB == 0 { + if value, ok := parseRatingValue(legacyRating); ok { + p.RatingIMDB = scaleRating(value, 10, 10) + } + } + // is intentionally not emitted; see parsedNFO. +} + +func parseRatingValue(raw string) (float64, bool) { + value, err := strconv.ParseFloat(strings.TrimSpace(raw), 64) + if err != nil || value <= 0 { + return 0, false + } + return value, true +} + +func scaleRating(value, max, targetMax float64) float64 { + if max <= 0 { + max = targetMax + } + scaled := value * targetMax / max + if scaled > targetMax { + return targetMax + } + return scaled +} + +// applyPeople maps entries to cast and / to crew. +func applyPeople(p *parsedNFO, c *nfoCommon) { + people := make([]models.ItemPerson, 0, len(c.Actors)+len(c.Directors)+len(c.Credits)) + for i, actor := range c.Actors { + name := strings.TrimSpace(actor.Name) + if name == "" { + continue + } + order := i + if actor.Order != nil { + order = *actor.Order + } + people = append(people, models.ItemPerson{ + Person: models.Person{Name: name}, + Kind: models.PersonKindActor, + Character: strings.TrimSpace(actor.Role), + SortOrder: order, + }) + } + people = append(people, crewPeople(c.Directors, models.PersonKindDirector)...) + people = append(people, crewPeople(c.Credits, models.PersonKindWriter)...) + if len(people) > 0 { + p.People = people + } +} + +func crewPeople(names []string, kind models.PersonKind) []models.ItemPerson { + people := make([]models.ItemPerson, 0, len(names)) + for _, raw := range names { + name := strings.TrimSpace(raw) + if name == "" { + continue + } + people = append(people, models.ItemPerson{ + Person: models.Person{Name: name}, + Kind: kind, + SortOrder: len(people), + }) + } + return people +} + +// cleanStrings trims entries and drops empties; returns nil (not an empty +// slice) when nothing remains so merge early-returns apply. +func cleanStrings(values []string) []string { + out := make([]string, 0, len(values)) + for _, v := range values { + if trimmed := strings.TrimSpace(v); trimmed != "" { + out = append(out, trimmed) + } + } + if len(out) == 0 { + return nil + } + return out +} + func xmlUnmarshal(data []byte, v any) error { decoder := xml.NewDecoder(bytes.NewReader(data)) return decoder.Decode(v) @@ -144,10 +467,10 @@ func xmlUnmarshal(data []byte, v any) error { func applyUniqueIDs(p *parsedNFO, ids []nfoUniqueID) { for _, uid := range ids { val := strings.TrimSpace(uid.Value) - switch strings.ToLower(uid.Type) { + switch strings.ToLower(strings.TrimSpace(uid.Type)) { case "imdb": p.ImdbID = val - case "tmdb": + case "tmdb", "themoviedb": p.TmdbID = val case "tvdb": p.TvdbID = val diff --git a/internal/metadata/nfo/nfo_test.go b/internal/metadata/nfo/nfo_test.go new file mode 100644 index 00000000..8ea8f352 --- /dev/null +++ b/internal/metadata/nfo/nfo_test.go @@ -0,0 +1,413 @@ +package nfo + +import ( + "reflect" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// fullMovieNFO is a realistic Kodi/Jellyfin-style export exercising the whole +// Phase-B field set: repeated collections, multi-source ratings, cast with +// role/order, directors and writers, and legacy + modern rating forms. +const fullMovieNFO = ` + + Blade Runner + Blade Runner: The Original + Blade Runner 1 + Man has made his match + A blade runner must pursue replicants. + 117 + 1982-06-25 + R + Science Fiction + Thriller + Warner Bros. + The Ladd Company + United States + Hong Kong + dystopia + neo-noir + 9 + + + 8.1 + 700000 + + + 7.9 + + + 89 + + + 91 + + + + Harrison Ford + Rick Deckard + 0 + https://example.com/ford.jpg + + + Rutger Hauer + Roy Batty + 1 + + Ridley Scott + Hampton Fancher + David Peoples + tt0083658 + 78 + + Blade Runner Collection + + + + + + +` + +// fullTVShowNFO exercises the series root, including as the +// FirstAirDate fallback and series-shaped collections. +const fullTVShowNFO = ` + + Mr. Robot + Mr. Robot (Original) + A cybersecurity engineer by day, hacker by night. + 49 + 2015-06-24 + TV-MA + Drama + Crime + USA Network + hacking + + + 8.5 + + + + Rami Malek + Elliot Alderson + 0 + + 289590 +` + +func TestParseNFOData_FullMovie(t *testing.T) { + t.Parallel() + + p, err := parseNFOData([]byte(fullMovieNFO)) + if err != nil { + t.Fatalf("parseNFOData() error = %v", err) + } + if p.Type != "movie" { + t.Errorf("Type = %q, want movie", p.Type) + } + if p.Title != "Blade Runner" { + t.Errorf("Title = %q", p.Title) + } + if p.OriginalTitle != "Blade Runner: The Original" { + t.Errorf("OriginalTitle = %q", p.OriginalTitle) + } + if p.Tagline != "Man has made his match" { + t.Errorf("Tagline = %q", p.Tagline) + } + if p.Runtime != 117 { + t.Errorf("Runtime = %d, want 117", p.Runtime) + } + if p.ReleaseDate != "1982-06-25" { + t.Errorf("ReleaseDate = %q, want 1982-06-25", p.ReleaseDate) + } + if p.FirstAirDate != "" { + t.Errorf("FirstAirDate = %q, want empty on a movie", p.FirstAirDate) + } + if p.Year != 1982 { + t.Errorf("Year = %d, want 1982 (derived from premiered)", p.Year) + } + if p.ContentRating != "R" { + t.Errorf("ContentRating = %q, want R", p.ContentRating) + } + if want := []string{"Science Fiction", "Thriller"}; !reflect.DeepEqual(p.Genres, want) { + t.Errorf("Genres = %#v, want %#v", p.Genres, want) + } + if want := []string{"Warner Bros.", "The Ladd Company"}; !reflect.DeepEqual(p.Studios, want) { + t.Errorf("Studios = %#v, want %#v", p.Studios, want) + } + if want := []string{"United States", "Hong Kong"}; !reflect.DeepEqual(p.Countries, want) { + t.Errorf("Countries = %#v, want %#v", p.Countries, want) + } + if want := []string{"dystopia", "neo-noir"}; !reflect.DeepEqual(p.Keywords, want) { + t.Errorf("Keywords = %#v, want %#v", p.Keywords, want) + } + if p.RatingIMDB != 8.1 { + t.Errorf("RatingIMDB = %v, want 8.1", p.RatingIMDB) + } + if p.RatingTMDB != 7.9 { + t.Errorf("RatingTMDB = %v, want 7.9", p.RatingTMDB) + } + if p.RatingRTCritic != 89 { + t.Errorf("RatingRTCritic = %v, want 89", p.RatingRTCritic) + } + if p.RatingRTAudience != 91 { + t.Errorf("RatingRTAudience = %v, want 91", p.RatingRTAudience) + } + wantPeople := []models.ItemPerson{ + {Person: models.Person{Name: "Harrison Ford"}, Kind: models.PersonKindActor, Character: "Rick Deckard", SortOrder: 0}, + {Person: models.Person{Name: "Rutger Hauer"}, Kind: models.PersonKindActor, Character: "Roy Batty", SortOrder: 1}, + {Person: models.Person{Name: "Ridley Scott"}, Kind: models.PersonKindDirector, SortOrder: 0}, + {Person: models.Person{Name: "Hampton Fancher"}, Kind: models.PersonKindWriter, SortOrder: 0}, + {Person: models.Person{Name: "David Peoples"}, Kind: models.PersonKindWriter, SortOrder: 1}, + } + if !reflect.DeepEqual(p.People, wantPeople) { + t.Errorf("People = %#v, want %#v", p.People, wantPeople) + } + if p.ImdbID != "tt0083658" || p.TmdbID != "78" { + t.Errorf("ids = imdb %q tmdb %q", p.ImdbID, p.TmdbID) + } +} + +func TestParseNFOData_FullTVShow(t *testing.T) { + t.Parallel() + + p, err := parseNFOData([]byte(fullTVShowNFO)) + if err != nil { + t.Fatalf("parseNFOData() error = %v", err) + } + if p.Type != "series" { + t.Errorf("Type = %q, want series", p.Type) + } + if p.FirstAirDate != "2015-06-24" { + t.Errorf("FirstAirDate = %q, want 2015-06-24", p.FirstAirDate) + } + if p.ReleaseDate != "" { + t.Errorf("ReleaseDate = %q, want empty on a series", p.ReleaseDate) + } + if p.Year != 2015 { + t.Errorf("Year = %d, want 2015 (derived from premiered)", p.Year) + } + if p.ContentRating != "TV-MA" { + t.Errorf("ContentRating = %q, want TV-MA", p.ContentRating) + } + if want := []string{"Drama", "Crime"}; !reflect.DeepEqual(p.Genres, want) { + t.Errorf("Genres = %#v, want %#v", p.Genres, want) + } + if p.RatingIMDB != 8.5 { + t.Errorf("RatingIMDB = %v, want 8.5", p.RatingIMDB) + } + if len(p.People) != 1 || p.People[0].Name != "Rami Malek" || p.People[0].Character != "Elliot Alderson" { + t.Errorf("People = %#v", p.People) + } + if p.TvdbID != "289590" { + t.Errorf("TvdbID = %q", p.TvdbID) + } +} + +func TestParseNFOData_TVShowAiredFallsBackToFirstAirDate(t *testing.T) { + t.Parallel() + + p, err := parseNFOData([]byte(`Show2010-09-01`)) + if err != nil { + t.Fatalf("parseNFOData() error = %v", err) + } + if p.FirstAirDate != "2010-09-01" { + t.Errorf("FirstAirDate = %q, want 2010-09-01 (from )", p.FirstAirDate) + } +} + +func TestParseNFOData_MovieReleaseDateFallback(t *testing.T) { + t.Parallel() + + p, err := parseNFOData([]byte(`M1999-03-31`)) + if err != nil { + t.Fatalf("parseNFOData() error = %v", err) + } + if p.ReleaseDate != "1999-03-31" { + t.Errorf("ReleaseDate = %q, want 1999-03-31 (from )", p.ReleaseDate) + } +} + +func TestParseNFOData_TableCases(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + data string + wantErr bool + check func(t *testing.T, p *parsedNFO) + }{ + { + name: "BOM prefix retained", + data: "\xEF\xBB\xBFBOM Movie", + check: func(t *testing.T, p *parsedNFO) { + if p.Title != "BOM Movie" { + t.Errorf("Title = %q", p.Title) + } + }, + }, + { + name: "malformed XML is an error (no NFO)", + data: `Broken`, + wantErr: true, + }, + { + name: "unsupported root element", + data: `<musicvideo><title>x`, + wantErr: true, + }, + { + name: "unknown elements ignored", + data: `Xzzz`, + check: func(t *testing.T, p *parsedNFO) { + if p.Title != "X" { + t.Errorf("Title = %q", p.Title) + } + }, + }, + { + name: "empty collections stay nil (no placeholder emissions)", + data: `Bare `, + check: func(t *testing.T, p *parsedNFO) { + if p.Genres != nil { + t.Errorf("Genres = %#v, want nil", p.Genres) + } + if p.Studios != nil { + t.Errorf("Studios = %#v, want nil", p.Studios) + } + if p.Countries != nil { + t.Errorf("Countries = %#v, want nil", p.Countries) + } + if p.Keywords != nil { + t.Errorf("Keywords = %#v, want nil", p.Keywords) + } + if p.People != nil { + t.Errorf("People = %#v, want nil", p.People) + } + }, + }, + { + name: "non-numeric runtime ignored", + data: `Xtwo hours`, + check: func(t *testing.T, p *parsedNFO) { + if p.Runtime != 0 { + t.Errorf("Runtime = %d, want 0", p.Runtime) + } + }, + }, + { + name: "runtime with surrounding whitespace", + data: `X 95 `, + check: func(t *testing.T, p *parsedNFO) { + if p.Runtime != 95 { + t.Errorf("Runtime = %d, want 95", p.Runtime) + } + }, + }, + { + name: "non-date premiered ignored", + data: `Xnext summer`, + check: func(t *testing.T, p *parsedNFO) { + if p.ReleaseDate != "" { + t.Errorf("ReleaseDate = %q, want empty", p.ReleaseDate) + } + }, + }, + { + name: "explicit year wins over premiered-derived year", + data: `X19811982-06-25`, + check: func(t *testing.T, p *parsedNFO) { + if p.Year != 1981 { + t.Errorf("Year = %d, want 1981", p.Year) + } + }, + }, + { + name: "legacy bare rating fills the imdb slot", + data: `X7.4`, + check: func(t *testing.T, p *parsedNFO) { + if p.RatingIMDB != 7.4 { + t.Errorf("RatingIMDB = %v, want 7.4", p.RatingIMDB) + } + }, + }, + { + name: "named imdb rating wins over legacy bare rating", + data: `X6.08.2`, + check: func(t *testing.T, p *parsedNFO) { + if p.RatingIMDB != 8.2 { + t.Errorf("RatingIMDB = %v, want 8.2", p.RatingIMDB) + } + }, + }, + { + name: "ratings normalized to their slot scale", + data: `X` + + `4.0` + + `9` + + ``, + check: func(t *testing.T, p *parsedNFO) { + if p.RatingIMDB != 8.0 { + t.Errorf("RatingIMDB = %v, want 8.0 (scaled from max=5)", p.RatingIMDB) + } + if p.RatingRTCritic != 90 { + t.Errorf("RatingRTCritic = %v, want 90 (scaled from max=10)", p.RatingRTCritic) + } + }, + }, + { + name: "userrating alone emits no ratings", + data: `X9`, + check: func(t *testing.T, p *parsedNFO) { + if p.RatingIMDB != 0 || p.RatingTMDB != 0 || p.RatingRTCritic != 0 || p.RatingRTAudience != 0 { + t.Errorf("ratings = %v/%v/%v/%v, want all zero", p.RatingIMDB, p.RatingTMDB, p.RatingRTCritic, p.RatingRTAudience) + } + }, + }, + { + name: "actor without order gets file order", + data: `X` + + `A OneR1` + + `B Two`, + check: func(t *testing.T, p *parsedNFO) { + want := []models.ItemPerson{ + {Person: models.Person{Name: "A One"}, Kind: models.PersonKindActor, Character: "R1", SortOrder: 0}, + {Person: models.Person{Name: "B Two"}, Kind: models.PersonKindActor, SortOrder: 1}, + } + if !reflect.DeepEqual(p.People, want) { + t.Errorf("People = %#v, want %#v", p.People, want) + } + }, + }, + { + name: "actor without a name is dropped", + data: `XGhost`, + check: func(t *testing.T, p *parsedNFO) { + if p.People != nil { + t.Errorf("People = %#v, want nil", p.People) + } + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + p, err := parseNFOData([]byte(tc.data)) + if tc.wantErr { + if err == nil { + t.Fatalf("parseNFOData() error = nil, want error") + } + return + } + if err != nil { + t.Fatalf("parseNFOData() error = %v", err) + } + if tc.check != nil { + tc.check(t, p) + } + }) + } +} diff --git a/internal/metadata/nfo/provider.go b/internal/metadata/nfo/provider.go index 0540b257..eeaafd8a 100644 --- a/internal/metadata/nfo/provider.go +++ b/internal/metadata/nfo/provider.go @@ -16,24 +16,38 @@ func NewProvider() *Provider { return &Provider{} } func (p *Provider) Slug() string { return "nfo" } func (p *Provider) Name() string { return "NFO Files" } -func (p *Provider) ForTypes() []string { return []string{"movie", "series"} } +func (p *Provider) ForTypes() []string { return []string{typeMovie, typeSeries} } + +// IdentityHints implements metadata.IdentityHintProvider: the external IDs a +// curated NFO declares ( tmdb/imdb/tvdb) are trusted identity hints +// that anchor Phase-1 candidate selection. A title-only NFO contributes no +// hints; its title participates only as a defanged search candidate. +func (p *Provider) IdentityHints(_ context.Context, query metadata.SearchQuery) map[string]string { + parsed := findNFOForQuery(query) + if parsed == nil { + return nil + } + hints := make(map[string]string) + if parsed.TmdbID != "" { + hints["tmdb"] = parsed.TmdbID + } + if parsed.ImdbID != "" { + hints["imdb"] = parsed.ImdbID + } + if parsed.TvdbID != "" { + hints["tvdb"] = parsed.TvdbID + } + if len(hints) == 0 { + return nil + } + return hints +} // Search extracts external IDs from NFO for matching. func (p *Provider) Search(ctx context.Context, query metadata.SearchQuery) ([]metadata.SearchResult, error) { - nfoPath := findNFOForQuery(query) - if nfoPath == "" { - return nil, nil - } - data, err := os.ReadFile(nfoPath) - if err != nil { - return nil, nil // NFO read errors are not fatal - } - parsed, err := parseNFOData(data) - if err != nil { - return nil, nil // NFO parse errors are not fatal - } - if query.ContentType != "" && parsed.Type != "" && parsed.Type != query.ContentType { - return nil, nil + parsed := findNFOForQuery(query) + if parsed == nil { + return nil, nil // missing/unreadable/mismatched NFOs are not fatal } ids := make(map[string]string) if parsed.TmdbID != "" { @@ -56,25 +70,36 @@ func (p *Provider) Search(ctx context.Context, query metadata.SearchQuery) ([]me }}, nil } -// GetMetadata parses full metadata from NFO file. +// GetMetadata parses full metadata from an NFO file. Like Search, it carries +// the ContentType guard (enforced inside findNFO): a tvshow.nfo next to a +// movie file must not inject series data into a movie item at top priority. func (p *Provider) GetMetadata(ctx context.Context, req metadata.MetadataRequest) (*metadata.MetadataResult, error) { - nfoPath := findNFOForRequest(req) - if nfoPath == "" { - return &metadata.MetadataResult{}, nil - } - data, err := os.ReadFile(nfoPath) - if err != nil { - return &metadata.MetadataResult{}, nil - } - parsed, err := parseNFOData(data) - if err != nil { + parsed := findNFOForRequest(req) + if parsed == nil { return &metadata.MetadataResult{}, nil } result := &metadata.MetadataResult{ - HasMetadata: true, - Title: parsed.Title, - Year: parsed.Year, - Overview: parsed.Overview, + HasMetadata: true, + Title: parsed.Title, + OriginalTitle: parsed.OriginalTitle, + Tagline: parsed.Tagline, + Year: parsed.Year, + Overview: parsed.Overview, + Runtime: parsed.Runtime, + ReleaseDate: parsed.ReleaseDate, + FirstAirDate: parsed.FirstAirDate, + ContentRating: parsed.ContentRating, + Genres: parsed.Genres, + Studios: parsed.Studios, + Countries: parsed.Countries, + Keywords: parsed.Keywords, + Ratings: metadata.Ratings{ + IMDB: parsed.RatingIMDB, + TMDB: parsed.RatingTMDB, + RTCritic: parsed.RatingRTCritic, + RTAudience: parsed.RatingRTAudience, + }, + People: parsed.People, ProviderIDs: make(map[string]string), } if parsed.TmdbID != "" { @@ -89,7 +114,7 @@ func (p *Provider) GetMetadata(ctx context.Context, req metadata.MetadataRequest return result, nil } -func findNFOForQuery(query metadata.SearchQuery) string { +func findNFOForQuery(query metadata.SearchQuery) *parsedNFO { candidatePaths := candidateSidecarPaths( query.FilePath, query.RepresentativeFilePath, @@ -97,17 +122,19 @@ func findNFOForQuery(query metadata.SearchQuery) string { query.PrimarySidecarSearchPaths, query.ProviderIDs["_filepath"], ) - return findNFO(candidatePaths) + _, parsed := findNFO(candidatePaths, query.ContentType) + return parsed } -func findNFOForRequest(req metadata.MetadataRequest) string { +func findNFOForRequest(req metadata.MetadataRequest) *parsedNFO { candidatePaths := candidateSidecarPaths( req.FilePath, req.RepresentativeFilePath, req.AllGroupFilePaths, req.PrimarySidecarSearchPaths, ) - return findNFO(candidatePaths) + _, parsed := findNFO(candidatePaths, req.ContentType) + return parsed } func candidateSidecarPaths(primary string, representative string, groupFiles []string, searchPaths []string, extras ...string) []string { @@ -142,19 +169,32 @@ func compactNFOPaths(paths []string) []string { // findNFO locates the best candidate NFO file across media-file and directory // search paths. File paths check legacy directory-level sidecars first, then a -// basename-matched sidecar in the same directory. -func findNFO(paths []string) string { +// basename-matched sidecar in the same directory. A candidate that exists but +// fails to parse, or whose root type mismatches contentType, falls through to +// the next candidate — a stray movie.nfo in a series root must not shadow +// tvshow.nfo, and a tvshow.nfo beside a movie file must not inject series +// data. Returns the winning path and its parsed contents, or ("", nil). +func findNFO(paths []string, contentType string) (string, *parsedNFO) { candidates := make([]string, 0, len(paths)*3) for _, path := range paths { candidates = append(candidates, nfoCandidatesForPath(path)...) } candidates = compactNFOPaths(candidates) for _, c := range candidates { - if _, err := os.Stat(c); err == nil { - return c + data, err := os.ReadFile(c) + if err != nil { + continue } + parsed, err := parseNFOData(data) + if err != nil { + continue + } + if contentType != "" && parsed.Type != "" && parsed.Type != contentType { + continue + } + return c, parsed } - return "" + return "", nil } func nfoCandidatesForPath(path string) []string { diff --git a/internal/metadata/nfo/provider_test.go b/internal/metadata/nfo/provider_test.go index 2c4a7cdc..1be99418 100644 --- a/internal/metadata/nfo/provider_test.go +++ b/internal/metadata/nfo/provider_test.go @@ -1,10 +1,13 @@ package nfo import ( + "context" "os" "path/filepath" "slices" "testing" + + "github.com/Silo-Server/silo-server/internal/metadata" ) func TestFindNFO_FilePathFallsBackToDirectoryLevelSidecar(t *testing.T) { @@ -15,12 +18,12 @@ func TestFindNFO_FilePathFallsBackToDirectoryLevelSidecar(t *testing.T) { if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { t.Fatalf("WriteFile(file) error = %v", err) } - if err := os.WriteFile(filepath.Join(dir, "movie.nfo"), []byte("dir"), 0o644); err != nil { + if err := os.WriteFile(filepath.Join(dir, "movie.nfo"), []byte("Dir"), 0o644); err != nil { t.Fatalf("WriteFile(movie.nfo) error = %v", err) } want := filepath.Join(dir, "movie.nfo") - if got := findNFO([]string{filePath}); got != want { + if got, _ := findNFO([]string{filePath}, ""); got != want { t.Fatalf("findNFO(file path) = %q, want %q", got, want) } } @@ -30,11 +33,11 @@ func TestFindNFO_DirectoryPathUsesDirectoryLevelSidecar(t *testing.T) { dir := t.TempDir() want := filepath.Join(dir, "movie.nfo") - if err := os.WriteFile(want, []byte("dir"), 0o644); err != nil { + if err := os.WriteFile(want, []byte("Dir"), 0o644); err != nil { t.Fatalf("WriteFile(movie.nfo) error = %v", err) } - if got := findNFO([]string{dir}); got != want { + if got, _ := findNFO([]string{dir}, ""); got != want { t.Fatalf("findNFO(directory path) = %q, want %q", got, want) } } @@ -48,11 +51,11 @@ func TestFindNFO_FilePathUsesBasenameMatchedNFO(t *testing.T) { if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { t.Fatalf("WriteFile(file) error = %v", err) } - if err := os.WriteFile(want, []byte("file"), 0o644); err != nil { + if err := os.WriteFile(want, []byte("File"), 0o644); err != nil { t.Fatalf("WriteFile(file.nfo) error = %v", err) } - if got := findNFO([]string{filePath}); got != want { + if got, _ := findNFO([]string{filePath}, ""); got != want { t.Fatalf("findNFO(file path) = %q, want %q", got, want) } } @@ -66,11 +69,11 @@ func TestFindNFO_ExtensionlessFilePathUsesDirectoryLevelSidecar(t *testing.T) { if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { t.Fatalf("WriteFile(file) error = %v", err) } - if err := os.WriteFile(want, []byte("dir"), 0o644); err != nil { + if err := os.WriteFile(want, []byte("Dir"), 0o644); err != nil { t.Fatalf("WriteFile(movie.nfo) error = %v", err) } - if got := findNFO([]string{filePath}); got != want { + if got, _ := findNFO([]string{filePath}, ""); got != want { t.Fatalf("findNFO(extensionless file path) = %q, want %q", got, want) } } @@ -84,11 +87,11 @@ func TestFindNFO_ExtensionlessFilePathUsesBasenameMatchedNFO(t *testing.T) { if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { t.Fatalf("WriteFile(file) error = %v", err) } - if err := os.WriteFile(want, []byte("file"), 0o644); err != nil { + if err := os.WriteFile(want, []byte("File"), 0o644); err != nil { t.Fatalf("WriteFile(file.nfo) error = %v", err) } - if got := findNFO([]string{filePath}); got != want { + if got, _ := findNFO([]string{filePath}, ""); got != want { t.Fatalf("findNFO(extensionless file path) = %q, want %q", got, want) } } @@ -108,3 +111,194 @@ func TestNFOCandidatesForMissingDottedDirectoryPathIncludeDirectoryLevelSidecars t.Fatalf("nfoCandidatesForPath(%q) missing %q in %#v", path, wantTV, got) } } + +// A tvshow.nfo sitting next to a movie file must not inject series data into +// a movie item at top priority: GetMetadata carries the same ContentType +// guard Search has. +func TestGetMetadata_TVShowNFOBesideMovieFileDoesNotInject(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + filePath := filepath.Join(dir, "Home Movie (2021).mkv") + if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { + t.Fatalf("WriteFile(file) error = %v", err) + } + nfoXML := `Some Series123` + if err := os.WriteFile(filepath.Join(dir, "tvshow.nfo"), []byte(nfoXML), 0o644); err != nil { + t.Fatalf("WriteFile(tvshow.nfo) error = %v", err) + } + + p := NewProvider() + result, err := p.GetMetadata(context.Background(), metadata.MetadataRequest{ + ContentType: "movie", + FilePath: filePath, + }) + if err != nil { + t.Fatalf("GetMetadata() error = %v", err) + } + if result.HasMetadata { + t.Fatalf("GetMetadata() = %#v, want no metadata (series NFO must not inject into a movie)", result) + } +} + +// A stray movie.nfo in a series root must not shadow tvshow.nfo: findNFO +// falls through to the next candidate on type mismatch. +func TestFindNFO_StrayMovieNFOInSeriesRootFallsThroughToTVShow(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "movie.nfo"), []byte(`Stray`), 0o644); err != nil { + t.Fatalf("WriteFile(movie.nfo) error = %v", err) + } + want := filepath.Join(dir, "tvshow.nfo") + if err := os.WriteFile(want, []byte(`The Show`), 0o644); err != nil { + t.Fatalf("WriteFile(tvshow.nfo) error = %v", err) + } + + got, parsed := findNFO([]string{dir}, "series") + if got != want { + t.Fatalf("findNFO(series root) = %q, want %q", got, want) + } + if parsed == nil || parsed.Title != "The Show" { + t.Fatalf("parsed = %#v, want tvshow content", parsed) + } +} + +// A candidate that exists but fails to parse falls through to the next +// candidate instead of masking a valid sidecar. +func TestFindNFO_ParseFailureFallsThroughToNextCandidate(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + filePath := filepath.Join(dir, "Blade Runner (1982).mkv") + if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { + t.Fatalf("WriteFile(file) error = %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "movie.nfo"), []byte("not xml at all"), 0o644); err != nil { + t.Fatalf("WriteFile(movie.nfo) error = %v", err) + } + want := filepath.Join(dir, "Blade Runner (1982).nfo") + if err := os.WriteFile(want, []byte(`Blade Runner`), 0o644); err != nil { + t.Fatalf("WriteFile(basename nfo) error = %v", err) + } + + got, parsed := findNFO([]string{filePath}, "movie") + if got != want { + t.Fatalf("findNFO(parse-failure fallthrough) = %q, want %q", got, want) + } + if parsed == nil || parsed.Title != "Blade Runner" { + t.Fatalf("parsed = %#v, want valid movie content", parsed) + } +} + +// GetMetadata surfaces the full Phase-B field set from a curated movie NFO. +func TestGetMetadata_FullMovieFieldSet(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + filePath := filepath.Join(dir, "Blade Runner (1982).mkv") + if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { + t.Fatalf("WriteFile(file) error = %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "movie.nfo"), []byte(fullMovieNFO), 0o644); err != nil { + t.Fatalf("WriteFile(movie.nfo) error = %v", err) + } + + p := NewProvider() + result, err := p.GetMetadata(context.Background(), metadata.MetadataRequest{ + ContentType: "movie", + FilePath: filePath, + }) + if err != nil { + t.Fatalf("GetMetadata() error = %v", err) + } + if !result.HasMetadata { + t.Fatal("HasMetadata = false, want true") + } + if result.OriginalTitle != "Blade Runner: The Original" { + t.Errorf("OriginalTitle = %q", result.OriginalTitle) + } + if result.Tagline != "Man has made his match" { + t.Errorf("Tagline = %q", result.Tagline) + } + if result.Runtime != 117 { + t.Errorf("Runtime = %d", result.Runtime) + } + if result.ReleaseDate != "1982-06-25" { + t.Errorf("ReleaseDate = %q", result.ReleaseDate) + } + if result.ContentRating != "R" { + t.Errorf("ContentRating = %q", result.ContentRating) + } + if len(result.Genres) != 2 || len(result.Studios) != 2 || len(result.Countries) != 2 || len(result.Keywords) != 2 { + t.Errorf("collections = genres %d studios %d countries %d keywords %d, want 2 each", + len(result.Genres), len(result.Studios), len(result.Countries), len(result.Keywords)) + } + if result.Ratings.IMDB != 8.1 || result.Ratings.TMDB != 7.9 || result.Ratings.RTCritic != 89 || result.Ratings.RTAudience != 91 { + t.Errorf("Ratings = %#v", result.Ratings) + } + if len(result.People) != 5 { + t.Errorf("People len = %d, want 5", len(result.People)) + } + if result.FirstAirDate != "" { + t.Errorf("FirstAirDate = %q, want empty for a movie", result.FirstAirDate) + } +} + +// A minimal NFO must not emit placeholder collections: empty slices stay nil +// so MergeFillEmpty's early-return lets remote providers fill them. +func TestGetMetadata_MinimalNFOEmitsNoPlaceholders(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + filePath := filepath.Join(dir, "Home Movie (2021).mkv") + if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { + t.Fatalf("WriteFile(file) error = %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "movie.nfo"), []byte(`Home Movie`), 0o644); err != nil { + t.Fatalf("WriteFile(movie.nfo) error = %v", err) + } + + p := NewProvider() + result, err := p.GetMetadata(context.Background(), metadata.MetadataRequest{ + ContentType: "movie", + FilePath: filePath, + }) + if err != nil { + t.Fatalf("GetMetadata() error = %v", err) + } + if result.Genres != nil || result.Studios != nil || result.Countries != nil || result.Keywords != nil || result.People != nil { + t.Errorf("placeholder collections emitted: %#v", result) + } + if result.Runtime != 0 || result.ReleaseDate != "" || result.ContentRating != "" { + t.Errorf("placeholder scalars emitted: %#v", result) + } +} + +// The series GetMetadata path maps aired/premiered onto FirstAirDate. +func TestGetMetadata_SeriesFirstAirDate(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "tvshow.nfo"), []byte(fullTVShowNFO), 0o644); err != nil { + t.Fatalf("WriteFile(tvshow.nfo) error = %v", err) + } + + p := NewProvider() + result, err := p.GetMetadata(context.Background(), metadata.MetadataRequest{ + ContentType: "series", + PrimarySidecarSearchPaths: []string{dir}, + }) + if err != nil { + t.Fatalf("GetMetadata() error = %v", err) + } + if !result.HasMetadata { + t.Fatal("HasMetadata = false, want true") + } + if result.FirstAirDate != "2015-06-24" { + t.Errorf("FirstAirDate = %q, want 2015-06-24", result.FirstAirDate) + } + if result.ReleaseDate != "" { + t.Errorf("ReleaseDate = %q, want empty for a series", result.ReleaseDate) + } +} diff --git a/internal/metadata/nfo/register.go b/internal/metadata/nfo/register.go new file mode 100644 index 00000000..fa1a242f --- /dev/null +++ b/internal/metadata/nfo/register.go @@ -0,0 +1,14 @@ +package nfo + +import ( + "github.com/Silo-Server/silo-server/internal/metadata" +) + +// The NFO provider is a built-in host provider: it is represented in the +// database by the reserved 'silo.builtin' installation's 'nfo' capability and +// resolved in-process through the builtin registry. Registration lives here +// (not in the metadata package) because this package implements the metadata +// package's interfaces; cmd/silo blank-imports this package to activate it. +func init() { + metadata.RegisterBuiltinProvider("nfo", func() metadata.Provider { return NewProvider() }) +} diff --git a/internal/metadata/nfo/series_depth.go b/internal/metadata/nfo/series_depth.go new file mode 100644 index 00000000..802236b6 --- /dev/null +++ b/internal/metadata/nfo/series_depth.go @@ -0,0 +1,198 @@ +package nfo + +import ( + "context" + "fmt" + "log/slog" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/Silo-Server/silo-server/internal/metadata" +) + +// Provider implements metadata.EpisodeProvider for its content types using +// the request plumbing added in Phase D: per-season directory paths and +// per-episode media file paths. NFO files never create structure — season +// and episode numbers come from directory/filename parsing upstream and the +// numbers declared inside an NFO are advisory only. + +// GetSeasons reads season.nfo sidecars and season posters for each +// directory-derived season. A season with neither a season.nfo nor a poster +// contributes nothing, leaving remote providers (or fallback synthesis) in +// charge. +func (p *Provider) GetSeasons(ctx context.Context, req metadata.SeasonsRequest) ([]metadata.SeasonResult, error) { + if req.ContentType != "" && req.ContentType != typeSeries { + return nil, nil + } + if len(req.SeasonDirectoryPaths) == 0 { + return nil, nil + } + seriesRoots := compactNFOPaths(req.SeriesRootPaths) + seasonNumbers := make([]int, 0, len(req.SeasonDirectoryPaths)) + for number := range req.SeasonDirectoryPaths { + seasonNumbers = append(seasonNumbers, number) + } + sort.Ints(seasonNumbers) + + var seasons []metadata.SeasonResult + for _, number := range seasonNumbers { + dirs := compactNFOPaths(req.SeasonDirectoryPaths[number]) + parsed := parseSeasonSidecar(ctx, dirs, number) + poster := findSeasonPoster(dirs, seriesRoots, number) + if parsed == nil && poster == "" { + continue + } + season := metadata.SeasonResult{SeasonNumber: number} + if parsed != nil { + season.Title = parsed.Title + season.Overview = parsed.Overview + } + if poster != "" { + season.PosterPath = "file://" + poster + } + seasons = append(seasons, season) + } + return seasons, nil +} + +// parseSeasonSidecar loads the first parseable season.nfo across the season's +// candidate directories. A declared that disagrees with the +// directory-derived number is advisory: naming owns structure, so the +// directory number wins with a warning. +func parseSeasonSidecar(ctx context.Context, seasonDirs []string, seasonNumber int) *parsedNFO { + for _, dir := range seasonDirs { + path := filepath.Join(dir, "season.nfo") + data, err := os.ReadFile(path) + if err != nil { + continue + } + parsed, err := parseNFOData(data) + if err != nil || parsed.Type != typeSeason { + continue + } + if parsed.SeasonSet && parsed.Season != seasonNumber { + slog.WarnContext(ctx, "nfo: season.nfo seasonnumber disagrees with directory; preferring directory number", + "component", "metadata", "path", path, + "nfo_season", parsed.Season, "directory_season", seasonNumber) + } + return parsed + } + return nil +} + +// findSeasonPoster locates season artwork: poster/folder/cover inside the +// season directory first, then the Kodi-style seasonNN-poster form in the +// series root. +func findSeasonPoster(seasonDirs []string, seriesRoots []string, seasonNumber int) string { + for _, name := range []string{"poster", "folder", "cover"} { + for _, dir := range seasonDirs { + if path := findLocalArtworkFile(dir, name); path != "" { + return path + } + } + } + rootNames := []string{fmt.Sprintf("season%02d-poster", seasonNumber)} + if seasonNumber == 0 { + rootNames = append(rootNames, "season-specials-poster") + } + for _, name := range rootNames { + for _, dir := range seriesRoots { + if path := findLocalArtworkFile(dir, name); path != "" { + return path + } + } + } + return "" +} + +// GetEpisodes reads .nfo and -thumb sidecars for each +// episode media file of the requested season. Episodes with neither +// contribute nothing (synthesized fallback stays in charge of them). +func (p *Provider) GetEpisodes(ctx context.Context, req metadata.EpisodesRequest) ([]metadata.EpisodeResult, error) { + if len(req.EpisodeFilePaths) == 0 { + return nil, nil + } + episodeNumbers := make([]int, 0, len(req.EpisodeFilePaths)) + for number := range req.EpisodeFilePaths { + episodeNumbers = append(episodeNumbers, number) + } + sort.Ints(episodeNumbers) + + var episodes []metadata.EpisodeResult + for _, number := range episodeNumbers { + paths := compactNFOPaths(req.EpisodeFilePaths[number]) + parsed, thumb := findEpisodeSidecars(ctx, paths, req.SeasonNumber, number) + if parsed == nil && thumb == "" { + continue + } + episode := metadata.EpisodeResult{ + SeasonNumber: req.SeasonNumber, + EpisodeNumber: number, + } + if parsed != nil { + episode.Title = parsed.Title + episode.Overview = parsed.Overview + episode.AirDate = parsed.FirstAirDate + episode.Runtime = parsed.Runtime + episode.Ratings = metadata.Ratings{ + IMDB: parsed.RatingIMDB, + TMDB: parsed.RatingTMDB, + RTCritic: parsed.RatingRTCritic, + RTAudience: parsed.RatingRTAudience, + } + } + if thumb != "" { + episode.StillPath = "file://" + thumb + } + episodes = append(episodes, episode) + } + return episodes, nil +} + +// findEpisodeSidecars locates the episode's .nfo and +// -thumb image across its media file paths. Numbers declared in +// the NFO are checked against the filename-derived SxxEyy, which wins on +// conflict (naming owns structure). +func findEpisodeSidecars(ctx context.Context, mediaPaths []string, seasonNumber, episodeNumber int) (*parsedNFO, string) { + var parsed *parsedNFO + thumb := "" + for _, mediaPath := range mediaPaths { + dir := filepath.Dir(mediaPath) + base := strings.TrimSuffix(filepath.Base(mediaPath), filepath.Ext(mediaPath)) + if parsed == nil { + parsed = parseEpisodeSidecar(ctx, filepath.Join(dir, base+".nfo"), seasonNumber, episodeNumber) + } + if thumb == "" { + thumb = findLocalArtworkFile(dir, base+"-thumb") + } + if parsed != nil && thumb != "" { + break + } + } + return parsed, thumb +} + +func parseEpisodeSidecar(ctx context.Context, path string, seasonNumber, episodeNumber int) *parsedNFO { + data, err := os.ReadFile(path) + if err != nil { + return nil + } + parsed, err := parseNFOData(data) + if err != nil || parsed.Type != typeEpisode { + return nil + } + if parsed.MultiEpisode { + slog.WarnContext(ctx, "nfo: multi-episode NFO documents are not supported; using the first block", + "component", "metadata", "path", path) + } + if (parsed.SeasonSet && parsed.Season != seasonNumber) || + (parsed.EpisodeSet && parsed.Episode != episodeNumber) { + slog.WarnContext(ctx, "nfo: episode NFO numbers disagree with filename; preferring filename numbers", + "component", "metadata", "path", path, + "nfo_season", parsed.Season, "nfo_episode", parsed.Episode, + "file_season", seasonNumber, "file_episode", episodeNumber) + } + return parsed +} diff --git a/internal/metadata/nfo/series_depth_test.go b/internal/metadata/nfo/series_depth_test.go new file mode 100644 index 00000000..2ac4d17f --- /dev/null +++ b/internal/metadata/nfo/series_depth_test.go @@ -0,0 +1,376 @@ +package nfo + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/Silo-Server/silo-server/internal/metadata" +) + +// --------------------------------------------------------------------------- +// Parser tables: +// --------------------------------------------------------------------------- + +func TestParseSeasonNFO(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + xml string + wantTitle string + wantOverview string + wantSeason int + wantSeasonSet bool + }{ + { + name: "full season", + xml: `Course A: ClassicThe classic 90-day schedule.1`, + wantTitle: "Course A: Classic", + wantOverview: "The classic 90-day schedule.", + wantSeason: 1, + wantSeasonSet: true, + }, + { + name: "no seasonnumber", + xml: `Course B`, + wantTitle: "Course B", + wantSeasonSet: false, + }, + { + name: "specials season zero", + xml: `Bonus Workouts0`, + wantTitle: "Bonus Workouts", + wantSeason: 0, + wantSeasonSet: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + p, err := parseNFOData([]byte(tc.xml)) + if err != nil { + t.Fatalf("parseNFOData: %v", err) + } + if p.Type != "season" { + t.Errorf("Type = %q, want season", p.Type) + } + if p.Title != tc.wantTitle { + t.Errorf("Title = %q, want %q", p.Title, tc.wantTitle) + } + if p.Overview != tc.wantOverview { + t.Errorf("Overview = %q, want %q", p.Overview, tc.wantOverview) + } + if p.SeasonSet != tc.wantSeasonSet { + t.Errorf("SeasonSet = %v, want %v", p.SeasonSet, tc.wantSeasonSet) + } + if tc.wantSeasonSet && p.Season != tc.wantSeason { + t.Errorf("Season = %d, want %d", p.Season, tc.wantSeason) + } + }) + } +} + +// --------------------------------------------------------------------------- +// Parser tables: extended +// --------------------------------------------------------------------------- + +func TestParseEpisodeNFO_ExtendedFields(t *testing.T) { + t.Parallel() + + xml := ` + Chest and Back + Push-ups and pull-ups. + 2004-01-01 + 1 + 1 + 53 + 8.4 +` + p, err := parseNFOData([]byte(xml)) + if err != nil { + t.Fatalf("parseNFOData: %v", err) + } + if p.Type != "episode" { + t.Errorf("Type = %q, want episode", p.Type) + } + if p.Title != "Chest and Back" || p.Overview != "Push-ups and pull-ups." { + t.Errorf("title/plot = %q/%q", p.Title, p.Overview) + } + if p.FirstAirDate != "2004-01-01" { + t.Errorf("FirstAirDate = %q, want 2004-01-01", p.FirstAirDate) + } + if !p.SeasonSet || p.Season != 1 || !p.EpisodeSet || p.Episode != 1 { + t.Errorf("numbers = season(%v,%d) episode(%v,%d), want set 1/1", p.SeasonSet, p.Season, p.EpisodeSet, p.Episode) + } + if p.Runtime != 53 { + t.Errorf("Runtime = %d, want 53", p.Runtime) + } + if p.RatingIMDB != 8.4 { + t.Errorf("RatingIMDB = %v, want 8.4", p.RatingIMDB) + } + if p.MultiEpisode { + t.Error("MultiEpisode = true for single-episode document") + } +} + +func TestParseEpisodeNFO_LegacyRatingAndNoNumbers(t *testing.T) { + t.Parallel() + + p, err := parseNFOData([]byte(`Core7.5`)) + if err != nil { + t.Fatalf("parseNFOData: %v", err) + } + if p.SeasonSet || p.EpisodeSet { + t.Errorf("SeasonSet/EpisodeSet = %v/%v, want false (numbers absent)", p.SeasonSet, p.EpisodeSet) + } + if p.RatingIMDB != 7.5 { + t.Errorf("legacy RatingIMDB = %v, want 7.5", p.RatingIMDB) + } +} + +// Multi- documents (multi-episode files) are out of scope in +// v1: the first block wins and the parser flags the document so the provider +// can warn. +func TestParseEpisodeNFO_MultiEpisodeDocumentTakesFirst(t *testing.T) { + t.Parallel() + + xml := `First Half1 +Second Half2` + p, err := parseNFOData([]byte(xml)) + if err != nil { + t.Fatalf("parseNFOData: %v", err) + } + if p.Title != "First Half" { + t.Errorf("Title = %q, want First Half (take first)", p.Title) + } + if !p.MultiEpisode { + t.Error("MultiEpisode = false, want true") + } +} + +// --------------------------------------------------------------------------- +// Provider: GetSeasons +// --------------------------------------------------------------------------- + +func writeTestFile(t *testing.T, path, contents string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("MkdirAll(%s): %v", path, err) + } + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + t.Fatalf("WriteFile(%s): %v", path, err) + } +} + +func TestGetSeasons_SeasonNFOAndPoster(t *testing.T) { + t.Parallel() + + root := t.TempDir() + s1 := filepath.Join(root, "Season 01") + s2 := filepath.Join(root, "Season 02") + writeTestFile(t, filepath.Join(s1, "season.nfo"), + `Course A: ClassicClassic plan.1`) + writeTestFile(t, filepath.Join(s1, "poster.jpg"), "img") + writeTestFile(t, filepath.Join(s2, "season.nfo"), `Course B`) + // Season 2 poster lives at the series root using the seasonNN-poster form. + writeTestFile(t, filepath.Join(root, "season02-poster.jpg"), "img") + + p := NewProvider() + seasons, err := p.GetSeasons(context.Background(), metadata.SeasonsRequest{ + ContentType: "series", + SeriesRootPaths: []string{root}, + SeasonDirectoryPaths: map[int][]string{ + 1: {s1}, + 2: {s2}, + }, + }) + if err != nil { + t.Fatalf("GetSeasons: %v", err) + } + if len(seasons) != 2 { + t.Fatalf("len(seasons) = %d, want 2 (%#v)", len(seasons), seasons) + } + if seasons[0].SeasonNumber != 1 || seasons[0].Title != "Course A: Classic" || seasons[0].Overview != "Classic plan." { + t.Errorf("season 1 = %#v", seasons[0]) + } + if want := "file://" + filepath.Join(s1, "poster.jpg"); seasons[0].PosterPath != want { + t.Errorf("season 1 poster = %q, want %q", seasons[0].PosterPath, want) + } + if seasons[1].SeasonNumber != 2 || seasons[1].Title != "Course B" { + t.Errorf("season 2 = %#v", seasons[1]) + } + if want := "file://" + filepath.Join(root, "season02-poster.jpg"); seasons[1].PosterPath != want { + t.Errorf("season 2 poster = %q, want %q", seasons[1].PosterPath, want) + } +} + +// A mismatched is advisory: naming owns structure, so the +// directory-derived season number wins. +func TestGetSeasons_NumberConflictPrefersDirectoryNumber(t *testing.T) { + t.Parallel() + + root := t.TempDir() + s2 := filepath.Join(root, "Season 02") + writeTestFile(t, filepath.Join(s2, "season.nfo"), + `Course B7`) + + p := NewProvider() + seasons, err := p.GetSeasons(context.Background(), metadata.SeasonsRequest{ + ContentType: "series", + SeriesRootPaths: []string{root}, + SeasonDirectoryPaths: map[int][]string{2: {s2}}, + }) + if err != nil { + t.Fatalf("GetSeasons: %v", err) + } + if len(seasons) != 1 || seasons[0].SeasonNumber != 2 || seasons[0].Title != "Course B" { + t.Fatalf("seasons = %#v, want season 2 titled Course B", seasons) + } +} + +func TestGetSeasons_NoSidecarsReturnsNothing(t *testing.T) { + t.Parallel() + + root := t.TempDir() + s1 := filepath.Join(root, "Season 01") + if err := os.MkdirAll(s1, 0o755); err != nil { + t.Fatal(err) + } + p := NewProvider() + seasons, err := p.GetSeasons(context.Background(), metadata.SeasonsRequest{ + ContentType: "series", + SeriesRootPaths: []string{root}, + SeasonDirectoryPaths: map[int][]string{1: {s1}}, + }) + if err != nil { + t.Fatalf("GetSeasons: %v", err) + } + if len(seasons) != 0 { + t.Fatalf("seasons = %#v, want none", seasons) + } +} + +func TestGetSeasons_NonSeriesContentTypeReturnsNothing(t *testing.T) { + t.Parallel() + + root := t.TempDir() + s1 := filepath.Join(root, "Season 01") + writeTestFile(t, filepath.Join(s1, "season.nfo"), `Nope`) + p := NewProvider() + seasons, err := p.GetSeasons(context.Background(), metadata.SeasonsRequest{ + ContentType: "movie", + SeasonDirectoryPaths: map[int][]string{1: {s1}}, + }) + if err != nil { + t.Fatalf("GetSeasons: %v", err) + } + if len(seasons) != 0 { + t.Fatalf("seasons = %#v, want none for non-series content", seasons) + } +} + +// --------------------------------------------------------------------------- +// Provider: GetEpisodes +// --------------------------------------------------------------------------- + +func TestGetEpisodes_EpisodeNFOAndThumb(t *testing.T) { + t.Parallel() + + root := t.TempDir() + s1 := filepath.Join(root, "Season 01") + e1 := filepath.Join(s1, "P90X S01E01 - Chest and Back.mkv") + writeTestFile(t, e1, "video") + writeTestFile(t, filepath.Join(s1, "P90X S01E01 - Chest and Back.nfo"), + `Chest and BackPush and pull.2004-01-011153`) + writeTestFile(t, filepath.Join(s1, "P90X S01E01 - Chest and Back-thumb.jpg"), "img") + // Episode 2 has a media file but no NFO and no thumb: the provider must + // return nothing for it (synthesized fallback stays in charge). + e2 := filepath.Join(s1, "P90X S01E02 - Plyometrics.mkv") + writeTestFile(t, e2, "video") + + p := NewProvider() + episodes, err := p.GetEpisodes(context.Background(), metadata.EpisodesRequest{ + SeasonNumber: 1, + SeriesRootPaths: []string{root}, + EpisodeFilePaths: map[int][]string{ + 1: {e1}, + 2: {e2}, + }, + }) + if err != nil { + t.Fatalf("GetEpisodes: %v", err) + } + if len(episodes) != 1 { + t.Fatalf("len(episodes) = %d, want 1 (%#v)", len(episodes), episodes) + } + ep := episodes[0] + if ep.SeasonNumber != 1 || ep.EpisodeNumber != 1 { + t.Errorf("numbers = S%dE%d, want S1E1", ep.SeasonNumber, ep.EpisodeNumber) + } + if ep.Title != "Chest and Back" || ep.Overview != "Push and pull." { + t.Errorf("title/plot = %q/%q", ep.Title, ep.Overview) + } + if ep.AirDate != "2004-01-01" { + t.Errorf("AirDate = %q, want 2004-01-01", ep.AirDate) + } + if ep.Runtime != 53 { + t.Errorf("Runtime = %d, want 53", ep.Runtime) + } + if want := "file://" + filepath.Join(s1, "P90X S01E01 - Chest and Back-thumb.jpg"); ep.StillPath != want { + t.Errorf("StillPath = %q, want %q", ep.StillPath, want) + } +} + +// NFO numbers are advisory: the filename-parsed SxxEyy (the request key) wins +// on conflict. +func TestGetEpisodes_NumberConflictFilenameWins(t *testing.T) { + t.Parallel() + + root := t.TempDir() + s1 := filepath.Join(root, "Season 01") + e3 := filepath.Join(s1, "Show S01E03.mkv") + writeTestFile(t, e3, "video") + writeTestFile(t, filepath.Join(s1, "Show S01E03.nfo"), + `Third49`) + + p := NewProvider() + episodes, err := p.GetEpisodes(context.Background(), metadata.EpisodesRequest{ + SeasonNumber: 1, + EpisodeFilePaths: map[int][]string{3: {e3}}, + }) + if err != nil { + t.Fatalf("GetEpisodes: %v", err) + } + if len(episodes) != 1 || episodes[0].SeasonNumber != 1 || episodes[0].EpisodeNumber != 3 { + t.Fatalf("episodes = %#v, want S1E3 from filename", episodes) + } + if episodes[0].Title != "Third" { + t.Errorf("Title = %q, want Third", episodes[0].Title) + } +} + +// A non-episode NFO next to the media file (e.g. a stray movie.nfo-shaped +// basename sidecar) must not inject data into an episode. +func TestGetEpisodes_RootTypeMismatchIgnored(t *testing.T) { + t.Parallel() + + root := t.TempDir() + s1 := filepath.Join(root, "Season 01") + e1 := filepath.Join(s1, "Show S01E01.mkv") + writeTestFile(t, e1, "video") + writeTestFile(t, filepath.Join(s1, "Show S01E01.nfo"), `Not an episode`) + + p := NewProvider() + episodes, err := p.GetEpisodes(context.Background(), metadata.EpisodesRequest{ + SeasonNumber: 1, + EpisodeFilePaths: map[int][]string{1: {e1}}, + }) + if err != nil { + t.Fatalf("GetEpisodes: %v", err) + } + if len(episodes) != 0 { + t.Fatalf("episodes = %#v, want none for mismatched root type", episodes) + } +} diff --git a/internal/metadata/nfo_identity_test.go b/internal/metadata/nfo_identity_test.go new file mode 100644 index 00000000..62ae0ffa --- /dev/null +++ b/internal/metadata/nfo_identity_test.go @@ -0,0 +1,494 @@ +package metadata + +import ( + "context" + "strings" + "sync" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// localHintStubProvider mimics the built-in NFO provider's shape in the chain: +// it implements SearchProvider + MetadataProvider + IdentityHintProvider with +// canned data, exactly like a parsed sidecar NFO would supply. +type localHintStubProvider struct { + mu sync.Mutex + hints map[string]string + searchResults []SearchResult + metadata *MetadataResult + + hintCalls int + searchCalls int + metadataCalls int +} + +func (p *localHintStubProvider) Slug() string { return "nfo" } +func (p *localHintStubProvider) Name() string { return "NFO Files" } +func (p *localHintStubProvider) ForTypes() []string { return []string{"movie", "series"} } + +func (p *localHintStubProvider) IdentityHints(_ context.Context, _ SearchQuery) map[string]string { + p.mu.Lock() + defer p.mu.Unlock() + p.hintCalls++ + return copyMap(p.hints) +} + +func (p *localHintStubProvider) Search(_ context.Context, _ SearchQuery) ([]SearchResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.searchCalls++ + out := make([]SearchResult, len(p.searchResults)) + copy(out, p.searchResults) + return out, nil +} + +func (p *localHintStubProvider) GetMetadata(_ context.Context, _ MetadataRequest) (*MetadataResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.metadataCalls++ + if p.metadata != nil { + cp := *p.metadata + cp.ProviderIDs = copyMap(p.metadata.ProviderIDs) + return &cp, nil + } + return &MetadataResult{}, nil +} + +// remoteStubProvider is a canned remote search+metadata provider that records +// the provider IDs it was asked to fetch metadata for. +type remoteStubProvider struct { + mu sync.Mutex + slug string + searchResults []SearchResult + metadata *MetadataResult + lastSearchIDs map[string]string + lastMetaIDs map[string]string + metadataCalls int +} + +func (p *remoteStubProvider) Slug() string { return p.slug } +func (p *remoteStubProvider) Name() string { return p.slug } +func (p *remoteStubProvider) ForTypes() []string { return []string{"movie", "series"} } + +func (p *remoteStubProvider) Search(_ context.Context, query SearchQuery) ([]SearchResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.lastSearchIDs = copyMap(query.ProviderIDs) + out := make([]SearchResult, len(p.searchResults)) + copy(out, p.searchResults) + return out, nil +} + +func (p *remoteStubProvider) GetMetadata(_ context.Context, req MetadataRequest) (*MetadataResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.metadataCalls++ + p.lastMetaIDs = copyMap(req.ProviderIDs) + if p.metadata != nil { + cp := *p.metadata + cp.ProviderIDs = copyMap(p.metadata.ProviderIDs) + return &cp, nil + } + return &MetadataResult{}, nil +} + +func (p *remoteStubProvider) lastMetadataIDs() map[string]string { + p.mu.Lock() + defer p.mu.Unlock() + return copyMap(p.lastMetaIDs) +} + +func seedMovieItem(t *testing.T, h *testHarness, contentID, title string, year int) { + t.Helper() + if err := h.itemRepo.Upsert(context.Background(), &models.MediaItem{ + ContentID: contentID, + Type: "movie", + Title: title, + Year: year, + Status: "matched", + Studios: []string{}, + Networks: []string{}, + Countries: []string{}, + Genres: []string{}, + }); err != nil { + t.Fatalf("seed item: %v", err) + } +} + +// A title-only NFO with no remote search results must still produce an +// accepted match: the item stays on its path-deterministic local: content id, +// is persisted as matched, and gains no durable provider IDs (#216). +func TestInitialMatch_TitleOnlyNFO_NoRemoteResults_MatchesUnderLocalID(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + + if err := h.itemRepo.Upsert(ctx, &models.MediaItem{ + ContentID: "local-abc123", + Type: "movie", + Title: "My Home Movie", + Status: "pending_match", + Studios: []string{}, + Networks: []string{}, + Countries: []string{}, + Genres: []string{}, + }); err != nil { + t.Fatalf("seed skeleton: %v", err) + } + + nfo := &localHintStubProvider{ + searchResults: []SearchResult{{Name: "My Home Movie", Year: 2021, Provider: "nfo"}}, + metadata: &MetadataResult{HasMetadata: true, Title: "My Home Movie", Year: 2021, Overview: "Curated."}, + } + remote := &remoteStubProvider{slug: "tmdb"} + + result, err := h.service.ProcessWithProviders(ctx, ProcessRequest{ + ContentID: "local-abc123", + Hints: &MatchHints{ + Title: "My Home Movie", + Year: 2021, + Type: "movie", + }, + Language: "en", + Mode: ModeInitialMatch, + }, []Provider{nfo, remote}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want Updated=true", result) + } + item, err := h.itemRepo.GetByID(ctx, "local-abc123") + if err != nil { + t.Fatalf("item not found under local id after title-only NFO match: %v", err) + } + if item.Status != "matched" { + t.Errorf("item status = %q, want matched", item.Status) + } + if item.Title != "My Home Movie" { + t.Errorf("item title = %q, want My Home Movie", item.Title) + } + if item.TmdbID != "" || item.TvdbID != "" || item.ImdbID != "" { + t.Errorf("title-only NFO match must carry no provider ids, got tmdb=%q tvdb=%q imdb=%q", + item.TmdbID, item.TvdbID, item.ImdbID) + } +} + +// The same title-only flow at the worker level: the queued movie file must +// leave the match queue (row deleted) with the item matched under local:. +func TestWorker_TitleOnlyNFO_MovieLeavesMatchQueueMatched(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + h.service.folderRepo = &fakeWorkerFolderRepo{ + folders: map[int]*models.MediaFolder{ + 10: {ID: 10, Type: "movies", Enabled: true}, + }, + } + + nfo := &localHintStubProvider{ + searchResults: []SearchResult{{Name: "My Home Movie", Year: 2021, Provider: "nfo"}}, + metadata: &MetadataResult{HasMetadata: true, Title: "My Home Movie", Year: 2021}, + } + remote := &remoteStubProvider{slug: "tmdb"} + h.service.hooks.process = func(ctx context.Context, req ProcessRequest) (*ProcessResult, error) { + return h.service.ProcessWithProviders(ctx, req, []Provider{nfo, remote}) + } + + file := &models.MediaFile{ + ID: 1, + MediaFolderID: 10, + FilePath: "/media/movies/My Home Movie (2021)/My Home Movie (2021).mkv", + } + movieRepo := newFakeMovieQueueRepo(file) + + worker := NewMatchWorker(h.service, h.fileRepo, 1, 10, 0) + worker.movieClaimer = movieRepo + worker.processUnmatched(ctx) + + if _, deleted := movieRepo.deleted[file.ID]; !deleted { + t.Fatalf("movie queue row not deleted; errors=%v", movieRepo.errors) + } + contentID := h.fileRepo.contentIDs[file.ID] + if contentID == "" { + t.Fatal("file was not linked to a content id") + } + if !strings.HasPrefix(contentID, "local-") { + t.Errorf("content id = %q, want local- prefix", contentID) + } + item, err := h.itemRepo.GetByID(ctx, contentID) + if err != nil { + t.Fatalf("load matched item: %v", err) + } + if item.Status != "matched" { + t.Errorf("item status = %q, want matched", item.Status) + } +} + +// A curated must anchor identity even when remote search-by-title +// returns nothing: the NFO's tmdb id is seeded as a trusted hint and the +// remote provider's GetMetadata runs with it (full enrichment by ID). +func TestInitialMatch_NFOUniqueIDAnchorsWhenRemoteSearchEmpty(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + + nfo := &localHintStubProvider{ + hints: map[string]string{"tmdb": "424242"}, + searchResults: []SearchResult{{Name: "Obscure Film", Year: 2019, ProviderIDs: map[string]string{"tmdb": "424242"}, Provider: "nfo"}}, + metadata: &MetadataResult{HasMetadata: true, Title: "Obscure Film", Year: 2019}, + } + remote := &remoteStubProvider{ + slug: "tmdb", + metadata: &MetadataResult{HasMetadata: true, Title: "Obscure Film: Remote Cut", Year: 2019, Overview: "From TMDB.", ProviderIDs: map[string]string{"tmdb": "424242", "imdb": "tt0424242"}}, + } + + result, err := h.service.ProcessWithProviders(ctx, ProcessRequest{ + Hints: &MatchHints{ + Title: "Obscure Film", + Year: 2019, + Type: "movie", + }, + Language: "en", + Mode: ModeInitialMatch, + }, []Provider{remote, nfo}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want Updated=true", result) + } + if got := remote.lastMetadataIDs()["tmdb"]; got != "424242" { + t.Errorf("remote GetMetadata tmdb id = %q, want 424242 (NFO hint must anchor identity)", got) + } + item, err := h.itemRepo.GetByID(ctx, result.ContentID) + if err != nil { + t.Fatalf("load matched item: %v", err) + } + if item.TmdbID != "424242" { + t.Errorf("item tmdb id = %q, want 424242", item.TmdbID) + } + if item.Overview != "From TMDB." { + t.Errorf("item overview = %q, want remote enrichment applied", item.Overview) + } +} + +// When a title-only NFO coexists with a successful remote search, the remote +// match must win: no downgrade of a found movie to an unenriched local: item, +// and the winning candidate keeps the remote provider IDs. +func TestInitialMatch_TitleOnlyNFO_RemoteSearchWins(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + + nfo := &localHintStubProvider{ + searchResults: []SearchResult{{Name: "Inception", Year: 2010, Provider: "nfo"}}, + metadata: &MetadataResult{HasMetadata: true, Title: "Inception", Year: 2010}, + } + remote := &remoteStubProvider{ + slug: "tmdb", + searchResults: []SearchResult{{Name: "Inception", Year: 2010, ProviderIDs: map[string]string{"tmdb": "27205"}, Provider: "tmdb"}}, + metadata: &MetadataResult{HasMetadata: true, Title: "Inception", Year: 2010, ProviderIDs: map[string]string{"tmdb": "27205"}}, + } + + // NFO deliberately first in the chain: chain priority must not let the + // ID-less local candidate beat the ID-bearing remote candidate. + result, err := h.service.ProcessWithProviders(ctx, ProcessRequest{ + Hints: &MatchHints{ + Title: "Inception", + Year: 2010, + Type: "movie", + }, + Language: "en", + Mode: ModeInitialMatch, + }, []Provider{nfo, remote}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want Updated=true", result) + } + item, err := h.itemRepo.GetByID(ctx, result.ContentID) + if err != nil { + t.Fatalf("load matched item: %v", err) + } + if item.TmdbID != "27205" { + t.Errorf("item tmdb id = %q, want 27205 (remote match must not be downgraded to local:)", item.TmdbID) + } +} + +// Initial match: NFO hints beat folder-name-derived hints on conflict. +func TestInitialMatch_NFOHintBeatsFolderNameHint(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + + nfo := &localHintStubProvider{ + hints: map[string]string{"tmdb": "222"}, + metadata: &MetadataResult{HasMetadata: true, Title: "Right Movie"}, + } + remote := &remoteStubProvider{ + slug: "tmdb", + metadata: &MetadataResult{HasMetadata: true, Title: "Right Movie", ProviderIDs: map[string]string{"tmdb": "222"}}, + } + + _, err := h.service.ProcessWithProviders(ctx, ProcessRequest{ + Hints: &MatchHints{ + Title: "Right Movie", + Year: 2020, + Type: "movie", + TmdbID: "111", // folder-name-derived hint + }, + Language: "en", + Mode: ModeInitialMatch, + }, []Provider{nfo, remote}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if got := remote.lastMetadataIDs()["tmdb"]; got != "222" { + t.Errorf("remote GetMetadata tmdb id = %q, want 222 (NFO hint beats folder-name hint)", got) + } +} + +// Scheduled refresh: stored durable IDs beat NFO hints (no background +// identity flips). +func TestScheduledRefresh_StoredIDBeatsNFOHint(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + seedMovieItem(t, h, "movie:tmdb:100", "Stable Movie", 2018) + h.itemRepo.items["movie:tmdb:100"].TmdbID = "100" + + nfo := &localHintStubProvider{ + hints: map[string]string{"tmdb": "999"}, + metadata: &MetadataResult{HasMetadata: true, Title: "Stable Movie"}, + } + remote := &remoteStubProvider{ + slug: "tmdb", + metadata: &MetadataResult{HasMetadata: true, Title: "Stable Movie", ProviderIDs: map[string]string{"tmdb": "100"}}, + } + + _, err := h.service.ProcessWithProviders(ctx, ProcessRequest{ + ContentID: "movie:tmdb:100", + Language: "en", + Mode: ModeScheduledRefresh, + }, []Provider{nfo, remote}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if got := remote.lastMetadataIDs()["tmdb"]; got != "100" { + t.Errorf("scheduled refresh tmdb id = %q, want 100 (stored id beats NFO hint)", got) + } +} + +// Manual refresh: NFO hints beat stored IDs — fixing a wrong NFO id and +// refreshing re-anchors the item (recovery path). +func TestManualRefresh_NFOHintBeatsStoredID(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + seedMovieItem(t, h, "movie:tmdb:100", "Misidentified Movie", 2018) + h.itemRepo.items["movie:tmdb:100"].TmdbID = "100" + + nfo := &localHintStubProvider{ + hints: map[string]string{"tmdb": "200"}, + metadata: &MetadataResult{HasMetadata: true, Title: "Corrected Movie"}, + } + remote := &remoteStubProvider{ + slug: "tmdb", + metadata: &MetadataResult{HasMetadata: true, Title: "Corrected Movie", ProviderIDs: map[string]string{"tmdb": "200"}}, + } + + _, err := h.service.ProcessWithProviders(ctx, ProcessRequest{ + ContentID: "movie:tmdb:100", + Language: "en", + Mode: ModeManualRefresh, + }, []Provider{nfo, remote}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if got := remote.lastMetadataIDs()["tmdb"]; got != "200" { + t.Errorf("manual refresh tmdb id = %q, want 200 (NFO hint beats stored id)", got) + } +} + +// Manual Identify: the user's explicit identification wins and the NFO +// provider is skipped entirely — no hints, no metadata overlay. +func TestIdentify_SkipsNFOProviderEntirely(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + seedMovieItem(t, h, "movie:tmdb:100", "Old Title", 2018) + + nfo := &localHintStubProvider{ + hints: map[string]string{"tmdb": "999"}, + metadata: &MetadataResult{HasMetadata: true, Title: "Stale NFO Title"}, + } + remote := &remoteStubProvider{ + slug: "tmdb", + metadata: &MetadataResult{HasMetadata: true, Title: "User Chosen Title", ProviderIDs: map[string]string{"tmdb": "555"}}, + } + + result, err := h.service.ProcessWithProviders(ctx, ProcessRequest{ + ContentID: "movie:tmdb:100", + ProviderIDs: map[string]string{"tmdb": "555"}, + Language: "en", + Mode: ModeIdentify, + }, []Provider{nfo, remote}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want Updated=true", result) + } + if nfo.hintCalls != 0 || nfo.searchCalls != 0 || nfo.metadataCalls != 0 { + t.Errorf("NFO provider consulted during identify: hints=%d search=%d metadata=%d, want all 0", + nfo.hintCalls, nfo.searchCalls, nfo.metadataCalls) + } + if got := remote.lastMetadataIDs()["tmdb"]; got != "555" { + t.Errorf("identify tmdb id = %q, want 555 (user identification wins)", got) + } + item, err := h.itemRepo.GetByID(ctx, "movie:tmdb:100") + if err != nil { + t.Fatalf("load item: %v", err) + } + if item.Title != "User Chosen Title" { + t.Errorf("item title = %q, want User Chosen Title (stale NFO title must not overlay identify)", item.Title) + } +} + +// Phase-2: NFO GetMetadata results merge fields but never inject provider-id +// keys that conflict with or extend a remotely-anchored identity. +func TestPhase2_NFOMetadataDoesNotInjectProviderIDs(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + + nfo := &localHintStubProvider{ + metadata: &MetadataResult{ + HasMetadata: true, + Title: "Anchored Movie", + ProviderIDs: map[string]string{"tmdb": "999", "imdb": "tt0000001"}, + }, + } + remote := &remoteStubProvider{ + slug: "tmdb", + searchResults: []SearchResult{{Name: "Anchored Movie", Year: 2015, ProviderIDs: map[string]string{"tmdb": "42"}, Provider: "tmdb"}}, + metadata: &MetadataResult{HasMetadata: true, Title: "Anchored Movie", Year: 2015, ProviderIDs: map[string]string{"tmdb": "42"}}, + } + + result, err := h.service.ProcessWithProviders(ctx, ProcessRequest{ + Hints: &MatchHints{ + Title: "Anchored Movie", + Year: 2015, + Type: "movie", + }, + Language: "en", + Mode: ModeInitialMatch, + }, []Provider{remote, nfo}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + item, err := h.itemRepo.GetByID(ctx, result.ContentID) + if err != nil { + t.Fatalf("load item: %v", err) + } + if item.TmdbID != "42" { + t.Errorf("item tmdb id = %q, want 42 (NFO metadata ids must not overwrite the anchor)", item.TmdbID) + } + if item.ImdbID != "" { + t.Errorf("item imdb id = %q, want empty (NFO must not extend a remotely-anchored identity)", item.ImdbID) + } +} diff --git a/internal/metadata/nfo_merge_semantics_test.go b/internal/metadata/nfo_merge_semantics_test.go new file mode 100644 index 00000000..6078b88a --- /dev/null +++ b/internal/metadata/nfo_merge_semantics_test.go @@ -0,0 +1,188 @@ +package metadata + +import ( + "context" + "reflect" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +// Phase-2 accumulation runs NFO (priority 1) first, then remote providers, +// always with MergeFillEmpty: the NFO wins every field it populates and the +// remote result backfills the rest. +func TestMerge_NFOFirstFillEmpty_NFOWinsRemoteBackfills(t *testing.T) { + t.Parallel() + + nfoResult := &MetadataResult{ + HasMetadata: true, + Title: "Curated Title", + Overview: "Curated overview.", + Genres: []string{"Documentary"}, + Ratings: Ratings{IMDB: 8.1}, + } + remoteResult := &MetadataResult{ + HasMetadata: true, + Title: "Remote Title", + Overview: "Remote overview.", + Tagline: "Remote tagline", + Runtime: 117, + Genres: []string{"Action", "Thriller"}, + Studios: []string{"Remote Studio"}, + Ratings: Ratings{IMDB: 7.0, TMDB: 7.5}, + People: []models.ItemPerson{{Person: models.Person{Name: "Remote Actor"}, Kind: models.PersonKindActor}}, + ContentRating: "R", + } + + accumulator := &MetadataResult{} + MergeMetadata(nfoResult, accumulator, nil, MergeFillEmpty) + MergeMetadata(remoteResult, accumulator, nil, MergeFillEmpty) + + // NFO wins the fields it populated. + if accumulator.Title != "Curated Title" { + t.Errorf("Title = %q, want Curated Title", accumulator.Title) + } + if accumulator.Overview != "Curated overview." { + t.Errorf("Overview = %q, want the NFO overview", accumulator.Overview) + } + if accumulator.Ratings.IMDB != 8.1 { + t.Errorf("Ratings.IMDB = %v, want 8.1 (NFO wins)", accumulator.Ratings.IMDB) + } + // Genres are first-provider-wins as a whole list: no remote union. + if want := []string{"Documentary"}; !reflect.DeepEqual(accumulator.Genres, want) { + t.Errorf("Genres = %#v, want %#v (whole-list first-provider-wins)", accumulator.Genres, want) + } + // Remote backfills what the NFO left empty. + if accumulator.Tagline != "Remote tagline" || accumulator.Runtime != 117 || accumulator.ContentRating != "R" { + t.Errorf("backfill = tagline %q runtime %d rating %q", accumulator.Tagline, accumulator.Runtime, accumulator.ContentRating) + } + if accumulator.Ratings.TMDB != 7.5 { + t.Errorf("Ratings.TMDB = %v, want 7.5 (remote backfills)", accumulator.Ratings.TMDB) + } + if len(accumulator.Studios) != 1 || len(accumulator.People) != 1 { + t.Errorf("Studios = %#v People = %#v, want remote backfill", accumulator.Studios, accumulator.People) + } +} + +// An NFO that declares no must leave Genres untouched so the remote +// provider's list applies (the provider emits nil, never an empty slice). +func TestMerge_NFOWithoutGenres_RemoteGenresApply(t *testing.T) { + t.Parallel() + + nfoResult := &MetadataResult{HasMetadata: true, Title: "Curated Title"} + remoteResult := &MetadataResult{HasMetadata: true, Genres: []string{"Action", "Thriller"}} + + accumulator := &MetadataResult{} + MergeMetadata(nfoResult, accumulator, nil, MergeFillEmpty) + MergeMetadata(remoteResult, accumulator, nil, MergeFillEmpty) + + if want := []string{"Action", "Thriller"}; !reflect.DeepEqual(accumulator.Genres, want) { + t.Errorf("Genres = %#v, want %#v", accumulator.Genres, want) + } +} + +// FieldReleaseDates locks Year and the release/air dates against refreshes. +func TestMerge_ReleaseDatesLockBlocksReplace(t *testing.T) { + t.Parallel() + + source := &MetadataResult{ + HasMetadata: true, + Year: 1982, + ReleaseDate: "1982-06-25", + FirstAirDate: "1982-06-25", + LastAirDate: "1983-01-01", + } + target := &MetadataResult{ + Year: 1990, + ReleaseDate: "1990-01-01", + FirstAirDate: "1990-01-01", + LastAirDate: "1991-01-01", + } + + MergeMetadata(source, target, []MetadataField{FieldReleaseDates}, MergeReplaceUnlocked) + + if target.Year != 1990 || target.ReleaseDate != "1990-01-01" || + target.FirstAirDate != "1990-01-01" || target.LastAirDate != "1991-01-01" { + t.Errorf("locked dates overwritten: %+v", target) + } + + // Unlocked, replace mode applies the source values. + MergeMetadata(source, target, nil, MergeReplaceUnlocked) + if target.Year != 1982 || target.ReleaseDate != "1982-06-25" { + t.Errorf("unlocked replace did not apply: %+v", target) + } +} + +// Refresh contract, scheduled side: MergeFillEmpty seeds the accumulator from +// the existing item, so an edited NFO title does NOT propagate on the 6h cycle. +func TestScheduledRefresh_DoesNotPropagateNFOEdits(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + seedMovieItem(t, h, "movie:tmdb:100", "Original Title", 2018) + h.itemRepo.items["movie:tmdb:100"].TmdbID = "100" + h.itemRepo.items["movie:tmdb:100"].Overview = "Original overview." + + nfo := &localHintStubProvider{ + metadata: &MetadataResult{HasMetadata: true, Title: "Edited via NFO", Overview: "Edited overview."}, + } + remote := &remoteStubProvider{ + slug: "tmdb", + metadata: &MetadataResult{HasMetadata: true, Title: "Original Title", Year: 2018, ProviderIDs: map[string]string{"tmdb": "100"}}, + } + + _, err := h.service.ProcessWithProviders(ctx, ProcessRequest{ + ContentID: "movie:tmdb:100", + Language: "en", + Mode: ModeScheduledRefresh, + }, []Provider{nfo, remote}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + item, err := h.itemRepo.GetByID(ctx, "movie:tmdb:100") + if err != nil { + t.Fatalf("load item: %v", err) + } + if item.Title != "Original Title" { + t.Errorf("scheduled refresh title = %q, want Original Title (NFO edits must not propagate)", item.Title) + } + if item.Overview != "Original overview." { + t.Errorf("scheduled refresh overview = %q, want the original", item.Overview) + } +} + +// Refresh contract, manual side: MergeReplaceUnlocked propagates NFO edits +// ("edit the NFO, then Refresh metadata"). +func TestManualRefresh_PropagatesNFOEdits(t *testing.T) { + h := newTestHarness() + ctx := context.Background() + seedMovieItem(t, h, "movie:tmdb:100", "Original Title", 2018) + h.itemRepo.items["movie:tmdb:100"].TmdbID = "100" + h.itemRepo.items["movie:tmdb:100"].Overview = "Original overview." + + nfo := &localHintStubProvider{ + metadata: &MetadataResult{HasMetadata: true, Title: "Edited via NFO", Overview: "Edited overview."}, + } + remote := &remoteStubProvider{ + slug: "tmdb", + metadata: &MetadataResult{HasMetadata: true, Title: "Original Title", Year: 2018, ProviderIDs: map[string]string{"tmdb": "100"}}, + } + + _, err := h.service.ProcessWithProviders(ctx, ProcessRequest{ + ContentID: "movie:tmdb:100", + Language: "en", + Mode: ModeManualRefresh, + }, []Provider{nfo, remote}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + item, err := h.itemRepo.GetByID(ctx, "movie:tmdb:100") + if err != nil { + t.Fatalf("load item: %v", err) + } + if item.Title != "Edited via NFO" { + t.Errorf("manual refresh title = %q, want Edited via NFO", item.Title) + } + if item.Overview != "Edited overview." { + t.Errorf("manual refresh overview = %q, want the NFO edit", item.Overview) + } +} diff --git a/internal/metadata/nfo_mixed_library_test.go b/internal/metadata/nfo_mixed_library_test.go new file mode 100644 index 00000000..90b9be16 --- /dev/null +++ b/internal/metadata/nfo_mixed_library_test.go @@ -0,0 +1,394 @@ +package metadata_test + +// Mixed sports library tests (plan §2a win 3): one library of type "mixed" +// holding WWE pay-per-view events as movies alongside "WWE SmackDown" as a +// show, metadata driven mainly by NFO (sports content often has no TVDB/TMDB +// presence). The classification contract: naming decides movie-vs-series per +// file before any metadata provider runs; NFOs supply metadata and identity, +// never type (the provider's ContentType guard ignores a tvshow.nfo next to +// a movie-classified file). These tests reuse the NFOSeriesHarness bridge +// and the real built-in NFO provider, mirroring the fitness-library tests. + +import ( + "context" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/Silo-Server/silo-server/internal/metadata" + "github.com/Silo-Server/silo-server/internal/metadata/nfo" + "github.com/Silo-Server/silo-server/internal/naming" +) + +// --------------------------------------------------------------------------- +// (a) Classification pin: naming decides type in a mixed library. +// --------------------------------------------------------------------------- + +// Sports-flavored classification cases for the mixed default branch +// (internal/naming/filename.go): SxxEyy/season-folder routes to the series +// lane, everything else to the movie lane. NFO contents never participate. +func TestMixedLibrary_NamingDecidesType(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + path string + wantType string + wantSeason int + wantEpisode int + }{ + { + name: "PPV event in its own movie folder", + path: "/sports/WWE/WrestleMania 41 (2025)/WrestleMania 41 (2025).mkv", + wantType: "movie", + }, + { + name: "bare PPV event file defaults to the movie lane", + path: "/sports/WWE/Royal Rumble 2025.mkv", + wantType: "movie", + }, + { + name: "weekly show with season folder and SxxEyy", + path: "/sports/WWE/WWE SmackDown/Season 27/WWE SmackDown S27E15.mkv", + wantType: "series", + wantSeason: 27, + wantEpisode: 15, + }, + { + name: "SxxEyy without a season folder still routes to series", + path: "/sports/WWE/WWE SmackDown/WWE SmackDown S27E15.mkv", + wantType: "series", + wantSeason: 27, + wantEpisode: 15, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + ctx := naming.ResolvePathContext(tc.path, "mixed") + if ctx.Type != tc.wantType { + t.Fatalf("ResolvePathContext(%q, mixed).Type = %q, want %q", tc.path, ctx.Type, tc.wantType) + } + if ctx.SeasonNum != tc.wantSeason || ctx.EpisodeNum != tc.wantEpisode { + t.Errorf("season/episode = S%dE%d, want S%dE%d", + ctx.SeasonNum, ctx.EpisodeNum, tc.wantSeason, tc.wantEpisode) + } + }) + } +} + +// --------------------------------------------------------------------------- +// (b) Movie lane: an NFO-only PPV event. +// --------------------------------------------------------------------------- + +func TestMixedLibrary_MovieLane_NFOOnlyPPV(t *testing.T) { + root := filepath.Join(t.TempDir(), "WWE", "WrestleMania 41 (2025)") + movieFile := filepath.Join(root, "WrestleMania 41 (2025).mkv") + writeFixtureFile(t, movieFile, "video") + writeFixtureFile(t, filepath.Join(root, "movie.nfo"), + `WrestleMania 41Two nights from Las Vegas.2025`) + writeFixtureFile(t, filepath.Join(root, "poster.jpg"), "ppv-poster") + + x := metadata.NewNFOSeriesHarness() + ctx := context.Background() + const movieID = "local-wm41" + if err := x.SeedMovieSkeleton(movieID, "WrestleMania 41"); err != nil { + t.Fatalf("seed skeleton: %v", err) + } + + result, err := x.Service().ProcessWithProviders(ctx, metadata.ProcessRequest{ + ContentID: movieID, + Hints: &metadata.MatchHints{ + Title: "WrestleMania 41", + Year: 2025, + Type: "movie", + FilePath: movieFile, + RepresentativeFilePath: movieFile, + AllGroupFilePaths: []string{movieFile}, + PrimarySidecarSearchPaths: []string{root}, + }, + Language: "en", + Mode: metadata.ModeInitialMatch, + }, []metadata.Provider{nfo.NewProvider()}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want Updated=true", result) + } + + item, err := x.Item(movieID) + if err != nil { + t.Fatalf("PPV movie not found under local id: %v", err) + } + if item.Status != "matched" { + t.Errorf("item status = %q, want matched", item.Status) + } + if item.Type != "movie" { + t.Errorf("item type = %q, want movie (NFO must not flip type)", item.Type) + } + if item.Title != "WrestleMania 41" || item.Overview != "Two nights from Las Vegas." { + t.Errorf("item title/overview = %q/%q", item.Title, item.Overview) + } + if item.TmdbID != "" || item.TvdbID != "" || item.ImdbID != "" { + t.Errorf("title-only PPV must carry no remote ids, got tmdb=%q tvdb=%q imdb=%q", + item.TmdbID, item.TvdbID, item.ImdbID) + } + if want := "file://" + filepath.Join(root, "poster.jpg"); item.PosterSourcePath != want { + t.Errorf("item PosterSourcePath = %q, want %q", item.PosterSourcePath, want) + } + + jobs := x.EnqueuedImageJobs() + found := false + for _, job := range jobs { + if job.SourcePath == "file://"+filepath.Join(root, "poster.jpg") { + found = true + if job.TargetType != metadata.ImageCacheTargetItem || job.ProviderID != "local" { + t.Errorf("poster job = target %q provider %q, want item/local", job.TargetType, job.ProviderID) + } + } + if !strings.HasPrefix(job.SourcePath, "file://") { + t.Errorf("enqueued non-local source %q", job.SourcePath) + } + } + if !found { + t.Errorf("PPV poster not enqueued as a file:// local source (jobs: %#v)", jobs) + } +} + +// --------------------------------------------------------------------------- +// (c) Series lane: the weekly show next to the PPVs, full NFO depth. +// --------------------------------------------------------------------------- + +func TestMixedLibrary_SeriesLane_WeeklyShow(t *testing.T) { + root := filepath.Join(t.TempDir(), "WWE", "WWE SmackDown") + writeFixtureFile(t, filepath.Join(root, "tvshow.nfo"), + `WWE SmackDownFriday night wrestling.`) + s27 := filepath.Join(root, "Season 27") + writeFixtureFile(t, filepath.Join(s27, "season.nfo"), + `SmackDown 202527`) + writeFixtureFile(t, filepath.Join(s27, "poster.jpg"), "s27-poster") + episodeFile := filepath.Join(s27, "WWE SmackDown S27E15.mkv") + writeFixtureFile(t, episodeFile, "video") + writeFixtureFile(t, filepath.Join(s27, "WWE SmackDown S27E15.nfo"), + `SmackDown: April 11, 2025Go-home show before Mania.`) + writeFixtureFile(t, filepath.Join(s27, "WWE SmackDown S27E15-thumb.jpg"), "e15-thumb") + + x := metadata.NewNFOSeriesHarness() + ctx := context.Background() + const seriesID = "local-smackdown" + if err := x.SeedSeriesSkeleton(seriesID, "WWE SmackDown"); err != nil { + t.Fatalf("seed skeleton: %v", err) + } + + result, err := x.Service().ProcessWithProviders(ctx, metadata.ProcessRequest{ + ContentID: seriesID, + Hints: &metadata.MatchHints{ + Title: "WWE SmackDown", + Type: "series", + FilePath: episodeFile, + RepresentativeFilePath: episodeFile, + AllGroupFilePaths: []string{episodeFile}, + PrimarySidecarSearchPaths: []string{root}, + }, + Language: "en", + Mode: metadata.ModeInitialMatch, + }, []metadata.Provider{nfo.NewProvider()}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want Updated=true", result) + } + + item, err := x.Item(seriesID) + if err != nil { + t.Fatalf("show not found under local id: %v", err) + } + if item.Type != "series" || item.Title != "WWE SmackDown" { + t.Errorf("item = type %q title %q, want series / WWE SmackDown", item.Type, item.Title) + } + + seasons := x.Seasons(seriesID) + if len(seasons) != 1 || seasons[0].SeasonNumber != 27 || seasons[0].Title != "SmackDown 2025" { + t.Fatalf("seasons = %#v, want season 27 titled SmackDown 2025", seasons) + } + if want := "file://" + filepath.Join(s27, "poster.jpg"); seasons[0].PosterSourcePath != want { + t.Errorf("season PosterSourcePath = %q, want %q", seasons[0].PosterSourcePath, want) + } + + episodes := x.Episodes(seriesID) + if len(episodes) != 1 || episodes[0].SeasonNumber != 27 || episodes[0].EpisodeNumber != 15 { + t.Fatalf("episodes = %#v, want S27E15", episodes) + } + if episodes[0].Title != "SmackDown: April 11, 2025" || episodes[0].Overview != "Go-home show before Mania." { + t.Errorf("episode title/plot = %q/%q", episodes[0].Title, episodes[0].Overview) + } + if want := "file://" + filepath.Join(s27, "WWE SmackDown S27E15-thumb.jpg"); episodes[0].StillSourcePath != want { + t.Errorf("episode StillSourcePath = %q, want %q", episodes[0].StillSourcePath, want) + } +} + +// --------------------------------------------------------------------------- +// (d) Cross-type guard: a tvshow.nfo next to a movie-classified file. +// --------------------------------------------------------------------------- + +// A tvshow.nfo adjacent to a movie-classified PPV file must be ignored by +// the ContentType guard: no series metadata is injected into the movie, and +// with nothing else to match against the item stays an unmatched skeleton +// (pipeline-level mirror of the Phase B provider guard tests). +func TestMixedLibrary_TVShowNFONextToMovieIgnored(t *testing.T) { + root := filepath.Join(t.TempDir(), "WWE", "WrestleMania 41 (2025)") + movieFile := filepath.Join(root, "WrestleMania 41 (2025).mkv") + writeFixtureFile(t, movieFile, "video") + // Only a series-shaped NFO — e.g. an export tool misplaced it. + writeFixtureFile(t, filepath.Join(root, "tvshow.nfo"), + `WWE SmackDownWrong sidecar.`) + + x := metadata.NewNFOSeriesHarness() + ctx := context.Background() + const movieID = "local-wm41-guard" + if err := x.SeedMovieSkeleton(movieID, "WrestleMania 41"); err != nil { + t.Fatalf("seed skeleton: %v", err) + } + + result, err := x.Service().ProcessWithProviders(ctx, metadata.ProcessRequest{ + ContentID: movieID, + Hints: &metadata.MatchHints{ + Title: "WrestleMania 41", + Year: 2025, + Type: "movie", + FilePath: movieFile, + RepresentativeFilePath: movieFile, + AllGroupFilePaths: []string{movieFile}, + PrimarySidecarSearchPaths: []string{root}, + }, + Language: "en", + Mode: metadata.ModeInitialMatch, + }, []metadata.Provider{nfo.NewProvider()}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if result != nil && result.Updated { + t.Fatalf("result = %#v, want Updated=false (mismatched NFO must not produce a match)", result) + } + + item, err := x.Item(movieID) + if err != nil { + t.Fatalf("load skeleton: %v", err) + } + if item.Type != "movie" { + t.Errorf("item type = %q, want movie (tvshow.nfo must not flip type)", item.Type) + } + if item.Title != "WrestleMania 41" { + t.Errorf("item title = %q, want untouched WrestleMania 41 (no series title injection)", item.Title) + } + if item.Overview != "" { + t.Errorf("item overview = %q, want empty (series plot must not be injected)", item.Overview) + } + if item.Status != "pending_match" { + t.Errorf("item status = %q, want pending_match (skeleton unchanged)", item.Status) + } +} + +// --------------------------------------------------------------------------- +// (e) Mixed identity: an ID-bearing PPV next to NFO-only ones. +// --------------------------------------------------------------------------- + +// remoteMovieStubProvider is a canned remote movie provider that records the +// provider IDs it was asked to fetch metadata for (Phase A harness shape). +type remoteMovieStubProvider struct { + mu sync.Mutex + slug string + metadata *metadata.MetadataResult + lastMetaIDs map[string]string +} + +func (p *remoteMovieStubProvider) Slug() string { return p.slug } +func (p *remoteMovieStubProvider) Name() string { return p.slug } +func (p *remoteMovieStubProvider) ForTypes() []string { return []string{"movie", "series"} } + +func (p *remoteMovieStubProvider) Search(_ context.Context, _ metadata.SearchQuery) ([]metadata.SearchResult, error) { + return nil, nil // remote search knows nothing about obscure sports events +} + +func (p *remoteMovieStubProvider) GetMetadata(_ context.Context, req metadata.MetadataRequest) (*metadata.MetadataResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + p.lastMetaIDs = make(map[string]string, len(req.ProviderIDs)) + for k, v := range req.ProviderIDs { + p.lastMetaIDs[k] = v + } + if p.metadata != nil { + cp := *p.metadata + return &cp, nil + } + return &metadata.MetadataResult{}, nil +} + +func (p *remoteMovieStubProvider) lastMetadataIDs() map[string]string { + p.mu.Lock() + defer p.mu.Unlock() + return p.lastMetaIDs +} + +// A PPV whose movie.nfo carries a tmdb anchors a remote match via +// the trusted-hint phase even when remote search-by-title finds nothing — +// remote-capable events enrich fully while NFO-only events (case b) fall +// back locally, side by side in the same mixed library. +func TestMixedLibrary_PPVWithUniqueIDAnchorsRemoteMatch(t *testing.T) { + root := filepath.Join(t.TempDir(), "WWE", "WrestleMania 41 (2025)") + movieFile := filepath.Join(root, "WrestleMania 41 (2025).mkv") + writeFixtureFile(t, movieFile, "video") + writeFixtureFile(t, filepath.Join(root, "movie.nfo"), + `WrestleMania 412025424242`) + + x := metadata.NewNFOSeriesHarness() + ctx := context.Background() + + remote := &remoteMovieStubProvider{ + slug: "tmdb", + metadata: &metadata.MetadataResult{ + HasMetadata: true, + Title: "WrestleMania 41", + Year: 2025, + Overview: "From TMDB.", + ProviderIDs: map[string]string{"tmdb": "424242"}, + }, + } + + result, err := x.Service().ProcessWithProviders(ctx, metadata.ProcessRequest{ + Hints: &metadata.MatchHints{ + Title: "WrestleMania 41", + Year: 2025, + Type: "movie", + FilePath: movieFile, + RepresentativeFilePath: movieFile, + AllGroupFilePaths: []string{movieFile}, + PrimarySidecarSearchPaths: []string{root}, + }, + Language: "en", + Mode: metadata.ModeInitialMatch, + }, []metadata.Provider{nfo.NewProvider(), remote}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want Updated=true", result) + } + if got := remote.lastMetadataIDs()["tmdb"]; got != "424242" { + t.Errorf("remote GetMetadata tmdb id = %q, want 424242 (NFO uniqueid must anchor identity)", got) + } + + item, err := x.Item(result.ContentID) + if err != nil { + t.Fatalf("load matched item: %v", err) + } + if item.TmdbID != "424242" { + t.Errorf("item tmdb id = %q, want 424242", item.TmdbID) + } + if item.Overview != "From TMDB." { + t.Errorf("item overview = %q, want remote enrichment applied", item.Overview) + } +} diff --git a/internal/metadata/nfo_series_depth_test.go b/internal/metadata/nfo_series_depth_test.go new file mode 100644 index 00000000..01ab254f --- /dev/null +++ b/internal/metadata/nfo_series_depth_test.go @@ -0,0 +1,412 @@ +package metadata_test + +// Phase D integration tests: an NFO-only series library scans into a fully +// presented tree using the real built-in NFO provider — series metadata and +// root art (Phases A+C), season names/posters from season.nfo, and episode +// titles/plots/thumbs from .nfo (Phase D). These live in an +// external test package so they can import internal/metadata/nfo (which +// imports internal/metadata) without an import cycle; the in-package +// fakes are exposed through NFOSeriesHarness (export_nfo_test.go). + +import ( + "context" + "os" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/Silo-Server/silo-server/internal/metadata" + "github.com/Silo-Server/silo-server/internal/metadata/nfo" +) + +func writeFixtureFile(t *testing.T, path, contents string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("MkdirAll(%s): %v", path, err) + } + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + t.Fatalf("WriteFile(%s): %v", path, err) + } +} + +// buildFitnessFixture lays out the P90X fitness library from the Phase D spec +// and returns (seriesRoot, episode file paths). +func buildFitnessFixture(t *testing.T) (string, []string) { + t.Helper() + root := filepath.Join(t.TempDir(), "Fitness", "P90X") + writeFixtureFile(t, filepath.Join(root, "tvshow.nfo"), + `P90X90-day home fitness program.`) + writeFixtureFile(t, filepath.Join(root, "poster.jpg"), "series-poster") + writeFixtureFile(t, filepath.Join(root, "fanart.jpg"), "series-fanart") + + s1 := filepath.Join(root, "Season 01") + writeFixtureFile(t, filepath.Join(s1, "season.nfo"), + `Course A: ClassicThe classic schedule.1`) + writeFixtureFile(t, filepath.Join(s1, "poster.jpg"), "s1-poster") + e1 := filepath.Join(s1, "P90X S01E01 - Chest and Back.mkv") + writeFixtureFile(t, e1, "video") + writeFixtureFile(t, filepath.Join(s1, "P90X S01E01 - Chest and Back.nfo"), + `Chest and BackPush and pull.11`) + writeFixtureFile(t, filepath.Join(s1, "P90X S01E01 - Chest and Back-thumb.jpg"), "e1-thumb") + + s2 := filepath.Join(root, "Season 02") + writeFixtureFile(t, filepath.Join(s2, "season.nfo"), + `Course B2`) + writeFixtureFile(t, filepath.Join(s2, "poster.jpg"), "s2-poster") + e2 := filepath.Join(s2, "P90X S02E01 - Core.mkv") + writeFixtureFile(t, e2, "video") + writeFixtureFile(t, filepath.Join(s2, "P90X S02E01 - Core.nfo"), + `CoreCore synergistics.`) + writeFixtureFile(t, filepath.Join(s2, "P90X S02E01 - Core-thumb.jpg"), "e2-thumb") + + return root, []string{e1, e2} +} + +// The headline Phase D acceptance case: an NFO-only fitness library with zero +// remote IDs anywhere scans into a matched show under a local: content id, +// seasons carrying the NFO names and local posters, episodes carrying NFO +// titles/plots and thumbs, and every image enqueued as a file:// local source. +func TestNFOOnlyFitnessLibrary_FullSeriesDepth(t *testing.T) { + root, episodeFiles := buildFitnessFixture(t) + + x := metadata.NewNFOSeriesHarness() + ctx := context.Background() + const seriesID = "local-p90x" + if err := x.SeedSeriesSkeleton(seriesID, "P90X"); err != nil { + t.Fatalf("seed skeleton: %v", err) + } + + result, err := x.Service().ProcessWithProviders(ctx, metadata.ProcessRequest{ + ContentID: seriesID, + Hints: &metadata.MatchHints{ + Title: "P90X", + Type: "series", + FilePath: episodeFiles[0], + RepresentativeFilePath: episodeFiles[0], + AllGroupFilePaths: episodeFiles, + PrimarySidecarSearchPaths: []string{root}, + }, + Language: "en", + Mode: metadata.ModeInitialMatch, + }, []metadata.Provider{nfo.NewProvider()}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want Updated=true", result) + } + + // Show: matched under the local: content id with NFO metadata, no remote IDs. + item, err := x.Item(seriesID) + if err != nil { + t.Fatalf("item not found under local id: %v", err) + } + if item.Status != "matched" { + t.Errorf("item status = %q, want matched", item.Status) + } + if item.Title != "P90X" || item.Overview != "90-day home fitness program." { + t.Errorf("item title/overview = %q/%q", item.Title, item.Overview) + } + if item.TmdbID != "" || item.TvdbID != "" || item.ImdbID != "" { + t.Errorf("NFO-only series must carry no remote ids, got tmdb=%q tvdb=%q imdb=%q", + item.TmdbID, item.TvdbID, item.ImdbID) + } + if want := "file://" + filepath.Join(root, "poster.jpg"); item.PosterSourcePath != want { + t.Errorf("item PosterSourcePath = %q, want %q", item.PosterSourcePath, want) + } + if want := "file://" + filepath.Join(root, "fanart.jpg"); item.BackdropSourcePath != want { + t.Errorf("item BackdropSourcePath = %q, want %q", item.BackdropSourcePath, want) + } + + // Seasons: NFO names and local posters. + seasons := x.Seasons(seriesID) + if len(seasons) != 2 { + t.Fatalf("len(seasons) = %d, want 2 (%#v)", len(seasons), seasons) + } + if seasons[0].SeasonNumber != 1 || seasons[0].Title != "Course A: Classic" { + t.Errorf("season 1 = number %d title %q, want 1 / Course A: Classic", + seasons[0].SeasonNumber, seasons[0].Title) + } + if seasons[0].Overview != "The classic schedule." { + t.Errorf("season 1 overview = %q", seasons[0].Overview) + } + if want := "file://" + filepath.Join(root, "Season 01", "poster.jpg"); seasons[0].PosterSourcePath != want { + t.Errorf("season 1 PosterSourcePath = %q, want %q", seasons[0].PosterSourcePath, want) + } + if seasons[0].PosterPath != "" { + t.Errorf("season 1 PosterPath = %q, want empty until the cache job lands", seasons[0].PosterPath) + } + if seasons[1].SeasonNumber != 2 || seasons[1].Title != "Course B" { + t.Errorf("season 2 = number %d title %q, want 2 / Course B", + seasons[1].SeasonNumber, seasons[1].Title) + } + if want := "file://" + filepath.Join(root, "Season 02", "poster.jpg"); seasons[1].PosterSourcePath != want { + t.Errorf("season 2 PosterSourcePath = %q, want %q", seasons[1].PosterSourcePath, want) + } + + // Episodes: NFO titles/plots and thumbs. + episodes := x.Episodes(seriesID) + if len(episodes) != 2 { + t.Fatalf("len(episodes) = %d, want 2 (%#v)", len(episodes), episodes) + } + if episodes[0].SeasonNumber != 1 || episodes[0].EpisodeNumber != 1 || + episodes[0].Title != "Chest and Back" || episodes[0].Overview != "Push and pull." { + t.Errorf("episode S1E1 = %#v", episodes[0]) + } + if want := "file://" + filepath.Join(root, "Season 01", "P90X S01E01 - Chest and Back-thumb.jpg"); episodes[0].StillSourcePath != want { + t.Errorf("episode S1E1 StillSourcePath = %q, want %q", episodes[0].StillSourcePath, want) + } + if episodes[1].SeasonNumber != 2 || episodes[1].EpisodeNumber != 1 || episodes[1].Title != "Core" { + t.Errorf("episode S2E1 = %#v", episodes[1]) + } + + // Every image is enqueued as a file:// local source. + jobs := x.EnqueuedImageJobs() + wantSources := map[string]string{ + "file://" + filepath.Join(root, "poster.jpg"): metadata.ImageCacheTargetItem, + "file://" + filepath.Join(root, "fanart.jpg"): metadata.ImageCacheTargetItem, + "file://" + filepath.Join(root, "Season 01", "poster.jpg"): metadata.ImageCacheTargetSeason, + "file://" + filepath.Join(root, "Season 02", "poster.jpg"): metadata.ImageCacheTargetSeason, + "file://" + filepath.Join(root, "Season 01", "P90X S01E01 - Chest and Back-thumb.jpg"): metadata.ImageCacheTargetEpisode, + "file://" + filepath.Join(root, "Season 02", "P90X S02E01 - Core-thumb.jpg"): metadata.ImageCacheTargetEpisode, + } + got := make(map[string]string, len(jobs)) + for _, job := range jobs { + got[job.SourcePath] = job.TargetType + if !strings.HasPrefix(job.SourcePath, "file://") { + t.Errorf("enqueued non-local source %q", job.SourcePath) + } + if job.ProviderID != "local" { + t.Errorf("job for %q attributed to provider %q, want local", job.SourcePath, job.ProviderID) + } + } + for source, target := range wantSources { + if got[source] != target { + t.Errorf("image job for %q = target %q, want %q (all jobs: %#v)", source, got[source], target, got) + } + } +} + +// remoteEpisodeStubProvider is a canned remote series provider with +// season/episode support, standing in for TVDB in the mixed case. +type remoteEpisodeStubProvider struct { + mu sync.Mutex + slug string + metadata *metadata.MetadataResult + seasons []metadata.SeasonResult + episodes map[int][]metadata.EpisodeResult +} + +func (p *remoteEpisodeStubProvider) Slug() string { return p.slug } +func (p *remoteEpisodeStubProvider) Name() string { return p.slug } +func (p *remoteEpisodeStubProvider) ForTypes() []string { return []string{"movie", "series"} } + +func (p *remoteEpisodeStubProvider) GetMetadata(_ context.Context, _ metadata.MetadataRequest) (*metadata.MetadataResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.metadata != nil { + cp := *p.metadata + return &cp, nil + } + return &metadata.MetadataResult{}, nil +} + +func (p *remoteEpisodeStubProvider) GetSeasons(_ context.Context, _ metadata.SeasonsRequest) ([]metadata.SeasonResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + out := make([]metadata.SeasonResult, len(p.seasons)) + copy(out, p.seasons) + return out, nil +} + +func (p *remoteEpisodeStubProvider) GetEpisodes(_ context.Context, req metadata.EpisodesRequest) ([]metadata.EpisodeResult, error) { + p.mu.Lock() + defer p.mu.Unlock() + out := make([]metadata.EpisodeResult, len(p.episodes[req.SeasonNumber])) + copy(out, p.episodes[req.SeasonNumber]) + return out, nil +} + +// Mixed case: a real remote show with a curated season.nfo. With the NFO +// provider at chain priority 1, the local season name wins and the remote +// provider backfills everything the NFO does not supply (air date, episodes). +func TestMixedRemoteShow_CuratedSeasonNFOWins(t *testing.T) { + root := filepath.Join(t.TempDir(), "TV", "Real Show") + s1 := filepath.Join(root, "Season 01") + writeFixtureFile(t, filepath.Join(s1, "season.nfo"), + `The Curated Season`) + e1 := filepath.Join(s1, "Real Show S01E01.mkv") + writeFixtureFile(t, e1, "video") + + x := metadata.NewNFOSeriesHarness() + ctx := context.Background() + const seriesID = "series:tvdb:81189" + if err := x.SeedSeriesSkeleton(seriesID, "Real Show"); err != nil { + t.Fatalf("seed skeleton: %v", err) + } + + remote := &remoteEpisodeStubProvider{ + slug: "tvdb", + metadata: &metadata.MetadataResult{HasMetadata: true, Title: "Real Show", Overview: "From TVDB."}, + seasons: []metadata.SeasonResult{ + {SeasonNumber: 1, Title: "Season 1", Overview: "Remote season overview.", AirDate: "2010-01-01"}, + }, + episodes: map[int][]metadata.EpisodeResult{ + 1: {{SeasonNumber: 1, EpisodeNumber: 1, Title: "Remote Episode", Overview: "From TVDB."}}, + }, + } + + // NFO first in the chain = chain priority 1. + result, err := x.Service().ProcessWithProviders(ctx, metadata.ProcessRequest{ + ContentID: seriesID, + Hints: &metadata.MatchHints{ + Title: "Real Show", + Type: "series", + FilePath: e1, + RepresentativeFilePath: e1, + AllGroupFilePaths: []string{e1}, + PrimarySidecarSearchPaths: []string{root}, + }, + Language: "en", + Mode: metadata.ModeInitialMatch, + }, []metadata.Provider{nfo.NewProvider(), remote}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + if result == nil || !result.Updated { + t.Fatalf("result = %#v, want Updated=true", result) + } + + seasons := x.Seasons(seriesID) + if len(seasons) != 1 { + t.Fatalf("len(seasons) = %d, want 1 (%#v)", len(seasons), seasons) + } + if seasons[0].Title != "The Curated Season" { + t.Errorf("season title = %q, want The Curated Season (NFO at priority 1 wins)", seasons[0].Title) + } + if seasons[0].Overview != "Remote season overview." { + t.Errorf("season overview = %q, want remote backfill", seasons[0].Overview) + } + if seasons[0].AirDate == nil { + t.Error("season air date not backfilled from remote provider") + } + episodes := x.Episodes(seriesID) + if len(episodes) != 1 || episodes[0].Title != "Remote Episode" { + t.Fatalf("episodes = %#v, want the remote episode (no episode NFO present)", episodes) + } +} + +// Season NFO edits propagate on manual refresh only: a scheduled refresh +// fills empty fields but never re-applies an edited season.nfo over an +// existing name; a manual refresh replaces unlocked fields with the merged +// provider results, NFO first (same contract as item fields). +func TestSeasonNFOEdit_ManualRefreshOnlyPropagation(t *testing.T) { + root, episodeFiles := buildFitnessFixture(t) + + x := metadata.NewNFOSeriesHarness() + ctx := context.Background() + const seriesID = "local-p90x" + if err := x.SeedSeriesSkeleton(seriesID, "P90X"); err != nil { + t.Fatalf("seed skeleton: %v", err) + } + for i, path := range episodeFiles { + x.SeedSeriesFile(100+i, 10, path, root, seriesID) + } + + if _, err := x.Service().ProcessWithProviders(ctx, metadata.ProcessRequest{ + ContentID: seriesID, + Hints: &metadata.MatchHints{ + Title: "P90X", + Type: "series", + FilePath: episodeFiles[0], + RepresentativeFilePath: episodeFiles[0], + AllGroupFilePaths: episodeFiles, + PrimarySidecarSearchPaths: []string{root}, + }, + Language: "en", + Mode: metadata.ModeInitialMatch, + }, []metadata.Provider{nfo.NewProvider()}); err != nil { + t.Fatalf("initial match: %v", err) + } + if seasons := x.Seasons(seriesID); len(seasons) == 0 || seasons[0].Title != "Course A: Classic" { + t.Fatalf("initial seasons = %#v, want Course A: Classic first", seasons) + } + + // Edit the season NFO after the initial match. + writeFixtureFile(t, filepath.Join(root, "Season 01", "season.nfo"), + `Course A: Renamed1`) + + // Scheduled refresh: fill-empty only, the edit must NOT propagate. + if _, err := x.Service().ProcessWithProviders(ctx, metadata.ProcessRequest{ + ContentID: seriesID, + Language: "en", + Mode: metadata.ModeScheduledRefresh, + }, []metadata.Provider{nfo.NewProvider()}); err != nil { + t.Fatalf("scheduled refresh: %v", err) + } + if seasons := x.Seasons(seriesID); seasons[0].Title != "Course A: Classic" { + t.Errorf("after scheduled refresh season title = %q, want unchanged Course A: Classic", seasons[0].Title) + } + + // Manual refresh: the edit propagates. + if _, err := x.Service().ProcessWithProviders(ctx, metadata.ProcessRequest{ + ContentID: seriesID, + Language: "en", + Mode: metadata.ModeManualRefresh, + }, []metadata.Provider{nfo.NewProvider()}); err != nil { + t.Fatalf("manual refresh: %v", err) + } + if seasons := x.Seasons(seriesID); seasons[0].Title != "Course A: Renamed" { + t.Errorf("after manual refresh season title = %q, want Course A: Renamed", seasons[0].Title) + } +} + +// Regression: a series with no season/episode NFOs must behave exactly as +// before — the provider contributes nothing at season/episode depth and the +// synthesized fallback path stays in charge. +func TestSeriesWithoutSeasonOrEpisodeNFOs_UnchangedFallback(t *testing.T) { + root := filepath.Join(t.TempDir(), "TV", "Plain Show") + s1 := filepath.Join(root, "Season 01") + e1 := filepath.Join(s1, "Plain Show S01E01.mkv") + writeFixtureFile(t, e1, "video") + + x := metadata.NewNFOSeriesHarness() + ctx := context.Background() + const seriesID = "local-plain" + if err := x.SeedSeriesSkeleton(seriesID, "Plain Show"); err != nil { + t.Fatalf("seed skeleton: %v", err) + } + + result, err := x.Service().ProcessWithProviders(ctx, metadata.ProcessRequest{ + ContentID: seriesID, + Hints: &metadata.MatchHints{ + Title: "Plain Show", + Type: "series", + FilePath: e1, + RepresentativeFilePath: e1, + AllGroupFilePaths: []string{e1}, + PrimarySidecarSearchPaths: []string{root}, + }, + Language: "en", + Mode: metadata.ModeInitialMatch, + }, []metadata.Provider{nfo.NewProvider()}) + if err != nil { + t.Fatalf("ProcessWithProviders: %v", err) + } + // With no NFOs at all the provider finds nothing: the pipeline reports + // no update (the worker then synthesizes fallback structure), exactly as + // before Phase D. + if result == nil || result.Updated { + t.Fatalf("result = %#v, want Updated=false when the provider finds nothing", result) + } + if seasons := x.Seasons(seriesID); len(seasons) != 0 { + t.Errorf("seasons = %#v, want none from the provider path (fallback synthesis owns them)", seasons) + } + if episodes := x.Episodes(seriesID); len(episodes) != 0 { + t.Errorf("episodes = %#v, want none from the provider path", episodes) + } + if jobs := x.EnqueuedImageJobs(); len(jobs) != 0 { + t.Errorf("image jobs = %#v, want none without sidecar art", jobs) + } +} diff --git a/internal/metadata/person_refresh.go b/internal/metadata/person_refresh.go index 24ff9889..99417a0d 100644 --- a/internal/metadata/person_refresh.go +++ b/internal/metadata/person_refresh.go @@ -262,7 +262,7 @@ func (s *PersonRefreshService) cachePersonPhoto( return "", "", err } - return cachedOriginalImagePath(result.BasePath, result.Ext), result.Thumbhash, nil + return CachedImageOriginalPath(result), result.Thumbhash, nil } func personProviderIDs(person models.Person) map[string]string { @@ -370,16 +370,3 @@ func personCacheContentID( } return strconv.FormatInt(person.ID, 10) } - -func cachedOriginalImagePath(basePath, ext string) string { - if basePath == "" { - return "" - } - if strings.Contains(basePath, "/original.") { - return basePath - } - if ext == "" { - ext = ".jpg" - } - return strings.TrimRight(basePath, "/") + "/original" + ext -} diff --git a/internal/metadata/service.go b/internal/metadata/service.go index e81c2502..5cf39c96 100644 --- a/internal/metadata/service.go +++ b/internal/metadata/service.go @@ -7,6 +7,7 @@ import ( "log/slog" "maps" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -893,6 +894,52 @@ func (s *MetadataService) localProviderContextForContent(ctx context.Context, co } } +// seriesChildLocalContext carries the local-sidecar context for season- and +// episode-level provider fetches: the series root sidecar directories plus +// the per-season directories and per-episode media file paths derived from +// filename parsing (naming owns structure; NFOs never create it). +type seriesChildLocalContext struct { + seriesRootPaths []string + seasonDirectoryPaths map[int][]string + episodeFilePaths map[int]map[int][]string // season -> episode -> media files +} + +// buildSeriesChildLocalContext derives the season/episode sidecar context +// from the series' media file paths. Files without a filename-parseable +// episode number contribute nothing — exactly the set fallback synthesis +// also skips. +func buildSeriesChildLocalContext(seriesRootPaths []string, filePaths []string) seriesChildLocalContext { + childCtx := seriesChildLocalContext{ + seriesRootPaths: compactUniqueFilePaths(seriesRootPaths), + seasonDirectoryPaths: make(map[int][]string), + episodeFilePaths: make(map[int]map[int][]string), + } + for _, path := range compactUniqueFilePaths(filePaths) { + hints := naming.ParseFilename(path, "series") + if hints == nil || hints.EpisodeNum == 0 { + continue + } + seasonNum, episodeNum := hints.SeasonNum, hints.EpisodeNum + dir := filepath.Dir(path) + if !slices.Contains(childCtx.seasonDirectoryPaths[seasonNum], dir) { + childCtx.seasonDirectoryPaths[seasonNum] = append(childCtx.seasonDirectoryPaths[seasonNum], dir) + } + if childCtx.episodeFilePaths[seasonNum] == nil { + childCtx.episodeFilePaths[seasonNum] = make(map[int][]string) + } + childCtx.episodeFilePaths[seasonNum][episodeNum] = append(childCtx.episodeFilePaths[seasonNum][episodeNum], path) + } + return childCtx +} + +// seriesChildLocalContextForContent builds the season/episode sidecar context +// from a series' persisted media files (refresh paths, where no scan hints +// are available). +func (s *MetadataService) seriesChildLocalContextForContent(ctx context.Context, contentID string, folderID int) seriesChildLocalContext { + localCtx := s.localProviderContextForContent(ctx, contentID, folderID) + return buildSeriesChildLocalContext(localCtx.primarySidecarSearchPaths, localCtx.allGroupFilePaths) +} + func (s *MetadataService) directorySidecarSearchPathsForFiles(ctx context.Context, files []*models.MediaFile) []string { if len(files) == 0 { return nil @@ -1145,6 +1192,20 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques AllGroupFilePaths: append([]string(nil), req.Hints.AllGroupFilePaths...), PrimarySidecarSearchPaths: append([]string(nil), req.Hints.PrimarySidecarSearchPaths...), } + // Seed curated identity hints (NFO ) before Phase-1 search. + // At initial match NFO hints beat folder-name-derived hints, but stored + // durable IDs (injected into req.ProviderIDs) beat NFO hints. + protectedKeys := make(map[string]bool, len(req.ProviderIDs)) + for key := range req.ProviderIDs { + protectedKeys[key] = true + } + wonHints := applyBuiltinIdentityHints(ctx, itemChain, searchQuery, accumulatedIDs, protectedKeys) + selectionHints := req.Hints + if len(wonHints) > 0 { + hintsCopy := *req.Hints + overrideHintIDs(&hintsCopy, wonHints) + selectionHints = &hintsCopy + } searchQuery = suppressTitleYearFallbackForTrustedIDs(searchQuery) allResults := make([]SearchResult, 0) for _, p := range itemChain { @@ -1196,7 +1257,7 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques for _, p := range itemChain { providerPriority = append(providerPriority, p.Slug()) } - if winner, ok := selectInitialMatchCandidate(req.Hints, candidates, providerPriority); ok && winner != nil { + if winner, ok := selectInitialMatchCandidate(selectionHints, candidates, providerPriority); ok && winner != nil { for k, v := range winner.ProviderIDs { if v != "" { accumulatedIDs[k] = v @@ -1262,6 +1323,18 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques searchQuery.ObservedRootPath = localCtx.observedRootPath searchQuery.AllGroupFilePaths = append([]string(nil), localCtx.allGroupFilePaths...) searchQuery.PrimarySidecarSearchPaths = append([]string(nil), localCtx.primarySidecarSearchPaths...) + // Seed curated identity hints (NFO ) before the refresh + // search. On a scheduled refresh stored durable IDs win over NFO hints + // (no background identity flips); on a manual refresh NFO hints win, + // which is the recovery path for a corrected NFO. + var protectedKeys map[string]bool + if req.Mode == ModeScheduledRefresh { + protectedKeys = make(map[string]bool, len(accumulatedIDs)) + for key := range accumulatedIDs { + protectedKeys[key] = true + } + } + wonHints := applyBuiltinIdentityHints(ctx, itemChain, searchQuery, accumulatedIDs, protectedKeys) searchQuery = suppressTitleYearFallbackForTrustedIDs(searchQuery) allResults := make([]SearchResult, 0) for _, p := range itemChain { @@ -1288,7 +1361,7 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques } } candidates := NormalizeCandidates(allResults, contentType) - if winner, ok := selectRefreshMatchCandidate(existing, candidates); ok && winner != nil { + if winner, ok := selectRefreshMatchCandidate(existing, wonHints, candidates); ok && winner != nil { for k, v := range winner.ProviderIDs { if v != "" { accumulatedIDs[k] = v @@ -1333,6 +1406,13 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques if !ok { continue } + _, isIdentityHinter := p.(IdentityHintProvider) + // A manual Identify is the user's explicit identification: the local + // hint provider (NFO) is skipped entirely so a stale sidecar cannot + // re-apply its title/ids over the user's choice in the same operation. + if isIdentityHinter && req.Mode == ModeIdentify { + continue + } result, err := mp.GetMetadata(ctx, MetadataRequest{ ProviderIDs: accumulatedIDs, ContentType: contentType, @@ -1358,6 +1438,13 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques if result == nil || !result.HasMetadata { continue } + // Identity-hint providers contribute IDs exclusively through the + // trusted-hint phase: their Phase-2 results merge metadata fields but + // never inject provider-id keys that conflict with or extend the + // established identity (kills chimeric ID sets). + if isIdentityHinter { + result.ProviderIDs = nil + } // Bootstrap: feed new IDs to subsequent providers. mergeProviderIDs(accumulator, result) accumulatedIDs = accumulator.ProviderIDs @@ -1371,10 +1458,19 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques if !ok { continue } + // A manual Identify skips the local hint provider (NFO) across every + // phase: its sidecar art is resolved by file path, not by the chosen id, + // so a stale poster/fanart must not be re-applied over the user's choice. + if _, isIdentityHinter := p.(IdentityHintProvider); isIdentityHinter && req.Mode == ModeIdentify { + continue + } images, err := ip.GetImages(ctx, ImageRequest{ - ProviderIDs: accumulatedIDs, - ContentType: contentType, - Language: req.Language, + ProviderIDs: accumulatedIDs, + ContentType: contentType, + Language: req.Language, + RepresentativeFilePath: representativeFilePath, + AllGroupFilePaths: allGroupFilePaths, + PrimarySidecarSearchPaths: primarySidecarSearchPaths, }) if err != nil { if handleProvider404(provider404s, accumulatedIDs, p.Slug(), err, @@ -1393,6 +1489,10 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques var allSeasons []SeasonResult var allEpisodes []EpisodeResult if contentType == "series" { + childCtx := buildSeriesChildLocalContext( + primarySidecarSearchPaths, + append(append([]string(nil), representativeFilePath), allGroupFilePaths...), + ) seasonChain, err := resolveChain("season") if err != nil { return nil, err @@ -1403,10 +1503,17 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques if !ok { continue } + // Identify skips the local hint provider (NFO): a stale season.nfo + // must not overlay season names on the user's chosen identity. + if _, isIdentityHinter := p.(IdentityHintProvider); isIdentityHinter && req.Mode == ModeIdentify { + continue + } seasons, err := ep.GetSeasons(ctx, SeasonsRequest{ - ProviderIDs: accumulatedIDs, - ContentType: contentType, - Language: req.Language, + ProviderIDs: accumulatedIDs, + ContentType: contentType, + Language: req.Language, + SeriesRootPaths: childCtx.seriesRootPaths, + SeasonDirectoryPaths: childCtx.seasonDirectoryPaths, }) if err != nil { // Pass nil for provider404s so this refresh can drop the @@ -1426,33 +1533,54 @@ func (s *MetadataService) processInternal(ctx context.Context, req ProcessReques } allSeasons = flattenSeasonResults(seasonResults) - // Phase 4b: Episodes — resolve with "episode" content level. - if len(allSeasons) > 0 { + // Phase 4b: Episodes — resolve with "episode" content level. The + // season set is the provider-returned seasons plus any filename- + // derived on-disk seasons, so local episode NFOs are read even when + // no provider returned their season (persist then creates the + // implicit "Season N" rows). + episodeSeasonNumbers := make([]int, 0, len(allSeasons)+len(childCtx.episodeFilePaths)) + for _, season := range allSeasons { + episodeSeasonNumbers = append(episodeSeasonNumbers, season.SeasonNumber) + } + for seasonNumber := range childCtx.episodeFilePaths { + if !slices.Contains(episodeSeasonNumbers, seasonNumber) { + episodeSeasonNumbers = append(episodeSeasonNumbers, seasonNumber) + } + } + sort.Ints(episodeSeasonNumbers) + if len(episodeSeasonNumbers) > 0 { episodeChain, err := resolveChain("episode") if err != nil { return nil, err } episodeResults := make(map[episodeResultKey]*EpisodeResult) - for _, season := range allSeasons { + for _, seasonNumber := range episodeSeasonNumbers { for _, p := range episodeChain { ep, ok := p.(EpisodeProvider) if !ok { continue } + // Identify skips the local hint provider (NFO): stale episode + // NFOs/thumbs must not overlay the user's chosen identity. + if _, isIdentityHinter := p.(IdentityHintProvider); isIdentityHinter && req.Mode == ModeIdentify { + continue + } episodes, err := ep.GetEpisodes(ctx, EpisodesRequest{ - ProviderIDs: accumulatedIDs, - SeasonNumber: season.SeasonNumber, - Language: req.Language, + ProviderIDs: accumulatedIDs, + SeasonNumber: seasonNumber, + Language: req.Language, + SeriesRootPaths: childCtx.seriesRootPaths, + EpisodeFilePaths: childCtx.episodeFilePaths[seasonNumber], }) if err != nil { if handleChildProvider404(p.Slug(), accumulatedIDs, err, "content_id", req.ContentID, - "season", season.SeasonNumber, + "season", seasonNumber, ) { continue } slog.WarnContext(ctx, "metadata: episode provider error", "component", "metadata", - "provider", p.Slug(), "season", season.SeasonNumber, "error", err) + "provider", p.Slug(), "season", seasonNumber, "error", err) continue } accumulateEpisodeResults(episodeResults, episodes) @@ -1645,6 +1773,33 @@ func (s *MetadataService) mergeAndPersist( } } + // Re-anchor an already provider-anchored item whose corrected identity now + // derives a different anchor — the recovery path when an admin fixes a wrong + // in an NFO. Manual refresh only: the accumulator's IDs are seeded + // from the item's stored IDs and only a trusted NFO hint (which wins on + // manual refresh) can change them here, so a scheduled/background refresh + // never flips a stored identity. Reuses the local-promotion machinery under + // the provider-dedup lock; a no-op when the derived anchor is unchanged. + if !isNew && req.Mode == ModeManualRefresh && contentid.IsProviderAnchored(contentID) { + reanchored, err := s.reanchorContentID( + ctx, contentID, providerIDsStruct(accumulator.ProviderIDs), contentType) + if err != nil { + return nil, fmt.Errorf("reanchor provider content id: %w", err) + } + if reanchored != contentID { + contentID = reanchored + existingItem, err = s.itemRepo.GetByID(ctx, contentID) + if err != nil { + return nil, fmt.Errorf("loading reanchored item: %w", err) + } + locked = intSliceToFields(existingItem.LockedFields) + durableIDs, err = s.loadDurableProviderIDs(ctx, contentID) + if err != nil { + return nil, err + } + } + } + if len(durableIDs) > 0 { if accumulator.ProviderIDs == nil { accumulator.ProviderIDs = make(map[string]string, len(durableIDs)) @@ -1827,11 +1982,16 @@ func (s *MetadataService) mergeAndPersist( } } - // Persist seasons and episodes for series. + // Persist seasons and episodes for series. Episode-only results (e.g. + // local episode NFOs without a season.nfo) persist too — the persist + // path creates their implicit "Season N" rows. Fallback synthesis then + // covers any files the providers left unlinked (episodes with no NFO + // and no remote row); it is a no-op when every file is linked. if contentType == "series" { - if len(seasons) > 0 { + if len(seasons) > 0 || len(episodes) > 0 { s.persistSeasonsAndEpisodes(ctx, item, accumulator.ProviderIDs, canonicalLanguage, req.Language, seasons, episodes, mergeMode) - } else if err := s.SynthesizeFallbackEpisodes(ctx, contentID); err != nil { + } + if err := s.SynthesizeFallbackEpisodes(ctx, contentID); err != nil { slog.WarnContext(ctx, "metadata: failed to synthesize fallback series structure", "component", "metadata", "content_id", contentID, "error", err) } @@ -2536,6 +2696,7 @@ func (s *MetadataService) refreshSeriesChildTarget( } updated := false + childCtx := s.seriesChildLocalContextForContent(ctx, seriesID, folderID) for _, language := range languages { canonicalLanguage := strings.TrimSpace(series.DefaultMetadataLanguage) if canonicalLanguage == "" { @@ -2549,11 +2710,11 @@ func (s *MetadataService) refreshSeriesChildTarget( if err != nil { return err } - seasons, err := s.fetchTargetSeasonResults(ctx, providerIDs, folderID, language, seasonNumber) + seasons, err := s.fetchTargetSeasonResults(ctx, providerIDs, folderID, language, seasonNumber, childCtx) if err != nil { return err } - episodes, err := s.fetchTargetEpisodeResults(ctx, providerIDs, folderID, language, seasonNumber, episodeNumber) + episodes, err := s.fetchTargetEpisodeResults(ctx, providerIDs, folderID, language, seasonNumber, episodeNumber, childCtx) if err != nil { return err } @@ -2637,7 +2798,7 @@ func (s *MetadataService) resolveSeriesRefreshProviderIDs(ctx context.Context, s } } candidates := NormalizeCandidates(allResults, series.Type) - if winner, ok := selectRefreshMatchCandidate(series, candidates); ok && winner != nil { + if winner, ok := selectRefreshMatchCandidate(series, nil, candidates); ok && winner != nil { for k, v := range winner.ProviderIDs { if v != "" { accumulatedIDs[k] = v @@ -2650,7 +2811,7 @@ func (s *MetadataService) resolveSeriesRefreshProviderIDs(ctx context.Context, s return accumulatedIDs, nil } -func (s *MetadataService) fetchTargetSeasonResults(ctx context.Context, providerIDs map[string]string, folderID int, language string, seasonNumber int) ([]SeasonResult, error) { +func (s *MetadataService) fetchTargetSeasonResults(ctx context.Context, providerIDs map[string]string, folderID int, language string, seasonNumber int, childCtx seriesChildLocalContext) ([]SeasonResult, error) { seasonChain, err := s.resolveChainCached(ctx, folderID, "season") if err != nil { return nil, fmt.Errorf("resolve season provider chain: %w", err) @@ -2662,9 +2823,11 @@ func (s *MetadataService) fetchTargetSeasonResults(ctx context.Context, provider continue } seasons, err := ep.GetSeasons(ctx, SeasonsRequest{ - ProviderIDs: providerIDs, - ContentType: "series", - Language: language, + ProviderIDs: providerIDs, + ContentType: "series", + Language: language, + SeriesRootPaths: childCtx.seriesRootPaths, + SeasonDirectoryPaths: childCtx.seasonDirectoryPaths, }) if err != nil { if handleProvider404(nil, providerIDs, p.Slug(), err, "season", seasonNumber) { @@ -2684,7 +2847,7 @@ func (s *MetadataService) fetchTargetSeasonResults(ctx context.Context, provider return flattenSeasonResults(seasonResults), nil } -func (s *MetadataService) fetchTargetEpisodeResults(ctx context.Context, providerIDs map[string]string, folderID int, language string, seasonNumber int, episodeNumber int) ([]EpisodeResult, error) { +func (s *MetadataService) fetchTargetEpisodeResults(ctx context.Context, providerIDs map[string]string, folderID int, language string, seasonNumber int, episodeNumber int, childCtx seriesChildLocalContext) ([]EpisodeResult, error) { episodeChain, err := s.resolveChainCached(ctx, folderID, "episode") if err != nil { return nil, fmt.Errorf("resolve episode provider chain: %w", err) @@ -2696,9 +2859,11 @@ func (s *MetadataService) fetchTargetEpisodeResults(ctx context.Context, provide continue } episodes, err := ep.GetEpisodes(ctx, EpisodesRequest{ - ProviderIDs: providerIDs, - SeasonNumber: seasonNumber, - Language: language, + ProviderIDs: providerIDs, + SeasonNumber: seasonNumber, + Language: language, + SeriesRootPaths: childCtx.seriesRootPaths, + EpisodeFilePaths: childCtx.episodeFilePaths[seasonNumber], }) if err != nil { if handleChildProvider404(p.Slug(), providerIDs, err, "season", seasonNumber) { @@ -2771,7 +2936,8 @@ func isCachedImagePath(path string) bool { path != "-" && !strings.HasPrefix(path, "http://") && !strings.HasPrefix(path, "https://") && - !isProviderImagePath(path) + !isProviderImagePath(path) && + !isLocalImageSourcePath(path) } func isRemoteImageSourcePath(path string) bool { @@ -2783,6 +2949,20 @@ func isRemoteImageSourcePath(path string) bool { !isNonProviderImageScheme(lower) } +// isLocalImageSourcePath reports whether path is a local sidecar artwork +// source (file:// scheme, produced by the NFO provider). Local sources route +// into *_source_path like remote sources and are copied into the S3 image +// cache by the processor; they are never served directly. +func isLocalImageSourcePath(path string) bool { + return strings.HasPrefix(strings.ToLower(strings.TrimSpace(path)), "file://") +} + +// isCacheableImageSourcePath gates the image-cache enqueue paths: remote +// provider URLs and local file:// sidecars both queue cache jobs. +func isCacheableImageSourcePath(path string) bool { + return isRemoteImageSourcePath(path) || isLocalImageSourcePath(path) +} + func isNonProviderImageScheme(lowerPath string) bool { return strings.HasPrefix(lowerPath, "s3://") || strings.HasPrefix(lowerPath, "file://") || @@ -2799,7 +2979,29 @@ func providerImageSourcePath(path string) string { return "" } +// splitProviderImagePath separates a provider-returned image path into the +// served path and the cacheable source path. A local file:// sidecar must +// never land in the served column: it routes into *_source_path only (the +// cache job fills the served path later). Remote sources keep the existing +// contract: the URL occupies both columns until the cache job lands. +func splitProviderImagePath(path string) (servedPath, sourcePath string) { + if isLocalImageSourcePath(path) { + return "", strings.TrimSpace(path) + } + return path, providerImageSourcePath(path) +} + func preserveCachedArtwork(providerPath, providerThumbhash, existingCachedPath, existingSourcePath, existingThumbhash string) (string, string, string) { + // A local file:// source must never land in the served *_path column: it + // routes into *_source_path and, during the pre-cache window, the path + // keeps the prior cached key (or stays empty) until the job completes. + if isLocalImageSourcePath(providerPath) { + providerPath = strings.TrimSpace(providerPath) + if isCachedImagePath(existingCachedPath) { + return existingCachedPath, existingThumbhash, providerPath + } + return "", "", providerPath + } if !isRemoteImageSourcePath(providerPath) { if strings.TrimSpace(providerPath) == "" && isCachedImagePath(existingCachedPath) { return existingCachedPath, existingThumbhash, existingSourcePath @@ -2961,7 +3163,21 @@ func buildItemLocalizationRecord( LogoPath: loc.LogoPath, LogoSourcePath: loc.LogoSourcePath, } - applyBestImages(locItem, images, mergeMode, preferredLanguage) + // Local sidecar art is language-neutral: it must not duplicate into every + // localization row, so local candidates only compete at the item level. + remoteImages := images + for _, img := range images { + if isLocalImageSourcePath(img.URL) { + remoteImages = make([]RemoteImage, 0, len(images)) + for _, candidate := range images { + if !isLocalImageSourcePath(candidate.URL) { + remoteImages = append(remoteImages, candidate) + } + } + break + } + } + applyBestImages(locItem, remoteImages, mergeMode, preferredLanguage) prepareItemImagesForQueue(locItem, existingLocItem) loc.PosterPath = locItem.PosterPath @@ -3208,6 +3424,11 @@ func (s *MetadataService) persistSeasonsAndEpisodes( imageJobs := make([]EnqueueImageCacheJobInput, 0, len(seasons)+len(episodes)) fallbackProvider := primaryProviderID(providerIDs) keyAttribution := func(sourcePath string) (string, string) { + // Local sidecar sources attribute to the synthetic "local" provider, + // keyed by the series' own content ID (mirrors item attribution). + if isLocalImageSourcePath(sourcePath) { + return imageCacheLocalProviderID, seriesID + } providerID := providerIDFromPluginURL(sourcePath) if providerID == "" { providerID = fallbackProvider @@ -3215,7 +3436,7 @@ func (s *MetadataService) persistSeasonsAndEpisodes( return providerID, findContentID(series, providerID) } addSeasonImageJob := func(season *models.Season) { - if season == nil || !isRemoteImageSourcePath(season.PosterSourcePath) { + if season == nil || !isCacheableImageSourcePath(season.PosterSourcePath) { return } providerID, providerContentID := keyAttribution(season.PosterSourcePath) @@ -3233,7 +3454,7 @@ func (s *MetadataService) persistSeasonsAndEpisodes( }) } addEpisodeImageJob := func(episode *models.Episode) { - if episode == nil || !isRemoteImageSourcePath(episode.StillSourcePath) { + if episode == nil || !isCacheableImageSourcePath(episode.StillSourcePath) { return } providerID, providerContentID := keyAttribution(episode.StillSourcePath) @@ -3253,7 +3474,7 @@ func (s *MetadataService) persistSeasonsAndEpisodes( }) } addSeasonLocalizationImageJob := func(season *models.Season, loc *models.SeasonLocalization) { - if season == nil || loc == nil || !isRemoteImageSourcePath(loc.PosterSourcePath) { + if season == nil || loc == nil || !isCacheableImageSourcePath(loc.PosterSourcePath) { return } providerID, providerContentID := keyAttribution(loc.PosterSourcePath) @@ -3282,13 +3503,15 @@ func (s *MetadataService) persistSeasonsAndEpisodes( continue } providerSeason := season - providerSeason.PosterSourcePath = providerImageSourcePath(providerSeason.PosterPath) + providerSeason.PosterPath, providerSeason.PosterSourcePath = splitProviderImagePath(season.PosterPath) if existingSeason != nil && isCanonicalWrite { mergedSeason := seasonResultFromModel(existingSeason) MergeSeasonResult(&providerSeason, &mergedSeason, mergeMode) + // preserveCachedArtwork sees the raw provider path so a local + // file:// source still routes into *_source_path here. nextPath, nextThumbhash, nextSourcePath := preserveCachedArtwork( - providerSeason.PosterPath, - providerSeason.PosterThumbhash, + season.PosterPath, + season.PosterThumbhash, existingSeason.PosterPath, existingSeason.PosterSourcePath, existingSeason.PosterThumbhash, @@ -3333,6 +3556,13 @@ func (s *MetadataService) persistSeasonsAndEpisodes( dbSeason.AirDate = &t } } + // An artwork-only season (poster but no season.nfo) would otherwise + // persist a blank title and then be skipped by fallback synthesis + // (which only fills not-yet-existing rows), so default it to the same + // "Season N"/"Specials" label synthesis uses. Graceful degradation. + if dbSeason.Title == "" { + dbSeason.Title = fallbackSeasonTitle(dbSeason.SeasonNumber) + } if err := s.seasonRepo.Upsert(ctx, dbSeason); err != nil { slog.WarnContext(ctx, "metadata: failed to upsert season", "component", "metadata", "series_id", seriesID, "season", season.SeasonNumber, "error", err) @@ -3439,13 +3669,15 @@ func (s *MetadataService) persistSeasonsAndEpisodes( continue } providerEpisode := ep - providerEpisode.StillSourcePath = providerImageSourcePath(providerEpisode.StillPath) + providerEpisode.StillPath, providerEpisode.StillSourcePath = splitProviderImagePath(ep.StillPath) if existingEpisode != nil && isCanonicalWrite { mergedEpisode := episodeResultFromModel(existingEpisode) MergeEpisodeResult(&providerEpisode, &mergedEpisode, mergeMode) + // preserveCachedArtwork sees the raw provider path so a local + // file:// source still routes into *_source_path here. nextPath, nextThumbhash, nextSourcePath := preserveCachedArtwork( - providerEpisode.StillPath, - providerEpisode.StillThumbhash, + ep.StillPath, + ep.StillThumbhash, existingEpisode.StillPath, existingEpisode.StillSourcePath, existingEpisode.StillThumbhash, @@ -3505,6 +3737,13 @@ func (s *MetadataService) persistSeasonsAndEpisodes( v := providerEpisode.Ratings.IMDB dbEp.RatingIMDB = &v } + // An artwork-only episode (a -thumb.jpg but no episode .nfo) would + // otherwise persist a blank title and then be skipped by fallback + // synthesis, so default it to the same "Episode N" label synthesis + // uses. Graceful degradation for partially curated libraries. + if dbEp.Title == "" { + dbEp.Title = fallbackEpisodeTitle(dbEp.EpisodeNumber) + } if err := s.episodeRepo.Upsert(ctx, dbEp); err != nil { slog.WarnContext(ctx, "metadata: failed to upsert episode", "component", "metadata", "series_id", seriesID, "season", ep.SeasonNumber, @@ -5593,7 +5832,11 @@ func applyBestImages(item *models.MediaItem, images []RemoteImage, mode MergeMod if b.url == "" { return } - if *current == "" || mode == MergeReplaceUnlocked || b.rating > 0 { + // Local sidecar candidates always apply: they carry rating 0, so an + // already-matched item could otherwise never pick up local art, and a + // rated remote image could stickily displace it after a transient + // local read failure. + if *current == "" || mode == MergeReplaceUnlocked || b.rating > 0 || isLocalImageSourcePath(b.url) { *current = b.url } } @@ -5658,7 +5901,7 @@ func (s *MetadataService) enqueueItemImages(ctx context.Context, item *models.Me inputs := make([]EnqueueImageCacheJobInput, 0, 3) for _, field := range itemArtworkFields(item) { sourcePath := strings.TrimSpace(*field.source) - if !isRemoteImageSourcePath(sourcePath) { + if !isCacheableImageSourcePath(sourcePath) { continue } providerID, providerContentID := itemImageCacheAttribution(item, providerIDs, images, sourcePath) @@ -5690,7 +5933,7 @@ func (s *MetadataService) enqueueItemLocalizationImages(ctx context.Context, ite inputs := make([]EnqueueImageCacheJobInput, 0, 3) for _, field := range itemArtworkFields(locItem) { sourcePath := strings.TrimSpace(*field.source) - if !isRemoteImageSourcePath(sourcePath) { + if !isCacheableImageSourcePath(sourcePath) { continue } providerID, providerContentID := itemImageCacheAttribution(item, providerIDs, images, sourcePath) @@ -5710,6 +5953,16 @@ func (s *MetadataService) enqueueItemLocalizationImages(ctx context.Context, ite } func itemImageCacheAttribution(item *models.MediaItem, providerIDs map[string]string, images []RemoteImage, sourcePath string) (string, string) { + // Local sidecar sources are attributed to the synthetic "local" provider + // (the generic scheme parse below would derive "file") and keyed by the + // item's own content ID, matching the audiobook/ebook local/ precedent. + if isLocalImageSourcePath(sourcePath) { + contentID := "" + if item != nil { + contentID = item.ContentID + } + return imageCacheLocalProviderID, contentID + } providerID := providerIDFromPluginURL(sourcePath) if providerID == "" { providerID = findProviderID(images, sourcePath) @@ -5801,7 +6054,7 @@ func (s *MetadataService) cacheItemImages(ctx context.Context, item *models.Medi // URLs from it (local storage URLs never leave the server). item.PosterSourcePath = j.url } - *j.field.path = cachedOriginalImagePath(cr.result.BasePath, cr.result.Ext) + *j.field.path = CachedImageOriginalPath(cr.result) if j.field.thumbhash != nil && cr.result.Thumbhash != "" { *j.field.thumbhash = cr.result.Thumbhash } @@ -5954,9 +6207,10 @@ func (s *MetadataService) ApplyItemImage(ctx context.Context, req ApplyItemImage return nil, fmt.Errorf("caching image: %w", err) } - storedPath := cachedOriginalImagePath(result.BasePath, result.Ext) + storedPath := CachedImageOriginalPath(result) return &ApplyItemImageResult{ StoredPath: storedPath, + Revision: result.Revision, Thumbhash: result.Thumbhash, }, nil } @@ -5977,6 +6231,7 @@ type ApplyItemImageRequest struct { // ApplyItemImageResult contains the stored S3 path and thumbhash. type ApplyItemImageResult struct { StoredPath string + Revision string Thumbhash string } @@ -6102,6 +6357,68 @@ func searchResultConflictsWithTrustedIDs(hintedIDs, candidateIDs map[string]stri return false } +// applyBuiltinIdentityHints consults the chain's IdentityHintProviders (the +// built-in NFO provider) and folds their curated tmdb/imdb/tvdb ids into +// accumulatedIDs ahead of Phase-1 search. Keys in protected (stored durable +// ids) are never overwritten — a conflict is logged and the stored value +// kept. The returned map holds the hint values that won a slot in +// accumulatedIDs, so candidate selection can anchor on them through the +// trusted-hint machinery. +func applyBuiltinIdentityHints( + ctx context.Context, + chain []Provider, + query SearchQuery, + accumulatedIDs map[string]string, + protected map[string]bool, +) map[string]string { + var won map[string]string + for _, p := range chain { + hinter, ok := p.(IdentityHintProvider) + if !ok { + continue + } + hints := hinter.IdentityHints(ctx, query) + for _, key := range trustedSearchIDKeys { + value := strings.TrimSpace(hints[key]) + if value == "" { + continue + } + current := strings.TrimSpace(accumulatedIDs[key]) + if current != "" && protected[key] { + if current != value { + slog.WarnContext(ctx, "metadata: local identity hint conflicts with stored id; keeping stored id", "component", "metadata", + "provider", p.Slug(), "key", key, "stored", current, "hint", value) + } + continue + } + if current != "" && current != value { + slog.WarnContext(ctx, "metadata: local identity hint overrides conflicting hint id", "component", "metadata", + "provider", p.Slug(), "key", key, "previous", current, "hint", value) + } + accumulatedIDs[key] = value + if won == nil { + won = make(map[string]string, len(trustedSearchIDKeys)) + } + won[key] = value + } + } + return won +} + +// overrideHintIDs applies won identity-hint values onto a MatchHints copy so +// trustedHintIDsPresent/candidateMatchesTrustedIDs anchor selection on them. +func overrideHintIDs(hints *MatchHints, won map[string]string) { + if v := won["tmdb"]; v != "" { + hints.TmdbID = v + } + if v := won["tvdb"]; v != "" { + hints.TvdbID = v + } + if v := won["imdb"]; v != "" { + hints.ImdbID = v + } +} + func suppressTitleYearFallbackForTrustedIDs(query SearchQuery) SearchQuery { for _, key := range trustedSearchIDKeys { if strings.TrimSpace(query.ProviderIDs[key]) != "" { diff --git a/internal/metadata/types.go b/internal/metadata/types.go index 47661854..60fa078e 100644 --- a/internal/metadata/types.go +++ b/internal/metadata/types.go @@ -41,6 +41,9 @@ const ( FieldImages FieldAirSchedule FieldVideos + // FieldReleaseDates locks Year, ReleaseDate, and First/LastAirDate so a + // refresh (or an NFO at priority 1) cannot overwrite manual corrections. + FieldReleaseDates ) // RefreshPriority controls queue ordering. @@ -229,6 +232,14 @@ type ImageRequest struct { ProviderIDs map[string]string ContentType string Language string + // Local sidecar context (additive; empty for purely remote providers). + // RepresentativeFilePath is the group's representative media file, + // AllGroupFilePaths are every file in the content group, and + // PrimarySidecarSearchPaths are directories already validated as safe for + // directory-level sidecars (single-content observed roots). + RepresentativeFilePath string + AllGroupFilePaths []string + PrimarySidecarSearchPaths []string } // RemoteImage describes an available image from a provider. @@ -266,13 +277,19 @@ type CacheImageRequest struct { SeasonNumber *int EpisodeNumber *int Language string + // KeyDiscriminator, when set, is inserted into the S3 key between the + // content ID and the image type (e.g. the 8-hex content hash of a local + // sidecar file) so re-cached art rotates to a fresh key. + KeyDiscriminator string } // CacheImageResult is returned by ImageCacher on success. type CacheImageResult struct { - BasePath string // S3 key prefix - Thumbhash string // base64-encoded - Ext string // file extension including dot (e.g. ".jpg", ".png") + BasePath string // image-type prefix retained for legacy callers + OriginalPath string // exact immutable original-variant object key + Revision string // content revision shared by all generated variants + Thumbhash string // base64-encoded + Ext string // encoded file extension including dot UploadedVariants int ExistingVariants int @@ -283,6 +300,13 @@ type ImageCacher interface { CacheImage(ctx context.Context, req CacheImageRequest) (*CacheImageResult, error) } +// ImageByteCacher caches an image already read into memory (e.g. a local +// sidecar file) to object storage. Implemented by imagecache.Cacher alongside +// ImageCacher; the processor type-asserts it for file:// sources. +type ImageByteCacher interface { + CacheImageBytes(ctx context.Context, data []byte, req CacheImageRequest) (*CacheImageResult, error) +} + type ImageCacheJobEnqueuer interface { Enqueue(ctx context.Context, in EnqueueImageCacheJobInput) error EnqueueBatch(ctx context.Context, inputs []EnqueueImageCacheJobInput) (int, error) @@ -293,6 +317,13 @@ type SeasonsRequest struct { ProviderIDs map[string]string ContentType string Language string + // Local sidecar context (additive, internal — mirrors MetadataRequest). + // SeriesRootPaths are directories already validated as safe for + // directory-level sidecars (the series root); SeasonDirectoryPaths maps + // each directory-derived season number to its candidate directories. + // Both are empty for purely remote providers, which key off ProviderIDs. + SeriesRootPaths []string + SeasonDirectoryPaths map[int][]string } // EpisodesRequest is passed to EpisodeProvider.GetEpisodes(). @@ -300,6 +331,11 @@ type EpisodesRequest struct { ProviderIDs map[string]string SeasonNumber int Language string + // Local sidecar context (additive, internal). SeriesRootPaths as on + // SeasonsRequest; EpisodeFilePaths maps each filename-derived episode + // number within SeasonNumber to that episode's media file paths. + SeriesRootPaths []string + EpisodeFilePaths map[int][]string } // SeasonResult carries season data from a provider. diff --git a/internal/models/media.go b/internal/models/media.go index 1c6caa40..8ee7ae6c 100644 --- a/internal/models/media.go +++ b/internal/models/media.go @@ -153,31 +153,79 @@ func (f *MediaFile) PrimaryDVProfile() int { return f.VideoTracks[0].DVProfile } +// NormalizeVideoBitDepth returns an explicit probe value when available and +// otherwise derives the bit depth from stable ffprobe fields. FFprobe commonly +// omits bits_per_raw_sample for HEVC even though the pixel format and profile +// are conclusive (for example yuv420p10le / Main 10). +func NormalizeVideoBitDepth(explicit int, pixelFormat, profile string) int { + if explicit > 0 { + return explicit + } + + pixelFormat = strings.ToLower(strings.TrimSpace(pixelFormat)) + for _, candidate := range []struct { + markers []string + depth int + }{ + {markers: []string{"p016", "p16", "gray16", "16le", "16be"}, depth: 16}, + {markers: []string{"p014", "p14", "gray14", "14le", "14be"}, depth: 14}, + {markers: []string{"p012", "p12", "gray12", "12le", "12be"}, depth: 12}, + {markers: []string{"p010", "p10", "gray10", "10le", "10be"}, depth: 10}, + {markers: []string{"p009", "p9", "gray9", "9le", "9be"}, depth: 9}, + } { + for _, marker := range candidate.markers { + if strings.Contains(pixelFormat, marker) { + return candidate.depth + } + } + } + + profile = strings.ToLower(strings.TrimSpace(profile)) + for _, marker := range []string{"main 12", "main12", "12-bit", "12 bit"} { + if strings.Contains(profile, marker) { + return 12 + } + } + for _, marker := range []string{"main 10", "main10", "10-bit", "10 bit"} { + if strings.Contains(profile, marker) { + return 10 + } + } + + switch pixelFormat { + case "yuv420p", "yuv422p", "yuv444p", "yuvj420p", "yuvj422p", "yuvj444p", "nv12", "nv21", "rgb24", "bgr24": + return 8 + } + return 0 +} + // VideoTrack represents a probed video stream stored as JSONB. type VideoTrack struct { - Title string `json:"title,omitempty"` - Codec string `json:"codec,omitempty"` - DolbyVision string `json:"dolby_vision,omitempty"` - DVProfile int `json:"dv_profile,omitempty"` - DVBLCompatID int `json:"dv_bl_compat_id,omitempty"` - DVELPresent bool `json:"dv_el_present,omitempty"` - HDR10Plus bool `json:"hdr10_plus,omitempty"` - Profile string `json:"profile,omitempty"` - Level int `json:"level,omitempty"` - Width int `json:"width,omitempty"` - Height int `json:"height,omitempty"` - AspectRatio string `json:"aspect_ratio,omitempty"` - Interlaced bool `json:"interlaced"` - FrameRate string `json:"frame_rate,omitempty"` - Bitrate int `json:"bitrate,omitempty"` - VideoRange string `json:"video_range,omitempty"` - VideoRangeType string `json:"video_range_type,omitempty"` - ColorPrimaries string `json:"color_primaries,omitempty"` - ColorSpace string `json:"color_space,omitempty"` - ColorTransfer string `json:"color_transfer,omitempty"` - BitDepth int `json:"bit_depth,omitempty"` - PixelFormat string `json:"pixel_format,omitempty"` - ReferenceFrames int `json:"reference_frames,omitempty"` + Title string `json:"title,omitempty"` + Codec string `json:"codec,omitempty"` + DolbyVision string `json:"dolby_vision,omitempty"` + DVProfile int `json:"dv_profile,omitempty"` + DVBLCompatID int `json:"dv_bl_compat_id,omitempty"` + DVELPresent bool `json:"dv_el_present,omitempty"` + DVEnhancementLayer string `json:"dv_enhancement_layer,omitempty"` // none, mel, fel, unknown + HDR10Plus bool `json:"hdr10_plus,omitempty"` + Profile string `json:"profile,omitempty"` + Level int `json:"level,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + AspectRatio string `json:"aspect_ratio,omitempty"` + Interlaced bool `json:"interlaced"` + FrameRate string `json:"frame_rate,omitempty"` + Bitrate int `json:"bitrate,omitempty"` + VideoRange string `json:"video_range,omitempty"` + VideoRangeType string `json:"video_range_type,omitempty"` + ColorRange string `json:"color_range,omitempty"` + ColorPrimaries string `json:"color_primaries,omitempty"` + ColorSpace string `json:"color_space,omitempty"` + ColorTransfer string `json:"color_transfer,omitempty"` + BitDepth int `json:"bit_depth,omitempty"` + PixelFormat string `json:"pixel_format,omitempty"` + ReferenceFrames int `json:"reference_frames,omitempty"` } // AudioTrack represents a probed audio stream stored as JSONB. diff --git a/internal/models/media_test.go b/internal/models/media_test.go index e74c529c..21c602bc 100644 --- a/internal/models/media_test.go +++ b/internal/models/media_test.go @@ -1,6 +1,9 @@ package models -import "testing" +import ( + "encoding/json" + "testing" +) func TestPersonKindAudiobookRoles(t *testing.T) { cases := []struct { @@ -16,3 +19,58 @@ func TestPersonKindAudiobookRoles(t *testing.T) { } } } + +func TestNormalizeVideoBitDepth(t *testing.T) { + tests := []struct { + name string + explicit int + pixelFormat string + profile string + want int + }{ + {name: "explicit wins", explicit: 8, pixelFormat: "yuv420p10le", profile: "Main 10", want: 8}, + {name: "hevc ten bit pixel format", pixelFormat: "yuv420p10le", profile: "Main 10", want: 10}, + {name: "p010 pixel format", pixelFormat: "p010le", profile: "Main 10", want: 10}, + {name: "main ten profile fallback", pixelFormat: "", profile: "Main 10", want: 10}, + {name: "ordinary planar eight bit", pixelFormat: "yuv420p", profile: "Main", want: 8}, + {name: "unknown remains unknown", pixelFormat: "unknown", profile: "", want: 0}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := NormalizeVideoBitDepth(test.explicit, test.pixelFormat, test.profile); got != test.want { + t.Fatalf("NormalizeVideoBitDepth(%d, %q, %q) = %d, want %d", test.explicit, test.pixelFormat, test.profile, got, test.want) + } + }) + } +} + +func TestVideoTrackColorRangeJSON(t *testing.T) { + for _, test := range []struct { + name string + value string + present bool + }{ + {name: "limited", value: "tv", present: true}, + {name: "full", value: "pc", present: true}, + {name: "unspecified", value: "unknown", present: true}, + {name: "empty omitted", value: "", present: false}, + } { + t.Run(test.name, func(t *testing.T) { + data, err := json.Marshal(VideoTrack{ColorRange: test.value}) + if err != nil { + t.Fatal(err) + } + var decoded map[string]any + if err := json.Unmarshal(data, &decoded); err != nil { + t.Fatal(err) + } + got, present := decoded["color_range"] + if present != test.present { + t.Fatalf("color_range present = %v, want %v (%s)", present, test.present, data) + } + if test.present && got != test.value { + t.Fatalf("color_range = %#v, want %q", got, test.value) + } + }) + } +} diff --git a/internal/nodepool/planner.go b/internal/nodepool/planner.go index 9f058e56..137b25d2 100644 --- a/internal/nodepool/planner.go +++ b/internal/nodepool/planner.go @@ -88,6 +88,17 @@ func NewPlanner(proxies *ProxyPool, transcodes *TranscodePool) *Planner { // bandwidth-cap admission. Re-planning the same session replaces its previous // reservation, so quality switches don't double-count. func (p *Planner) PlanSession(sessionID, currentTranscodeURL string, needsTranscode bool, estBitrateKbps int) Plan { + return p.PlanSessionWith(sessionID, currentTranscodeURL, needsTranscode, estBitrateKbps, nil) +} + +// PlanSessionWith behaves like PlanSession but restricts transcode-node +// selection to nodes accepted by eligible (nil accepts every node). Capability +// -aware playback planning uses it so a recipe that only some pooled nodes +// can execute is never load-balanced onto a node that cannot. The predicate +// runs under the planner lock and must be cheap and non-blocking (a set +// lookup, never a network call). Group health is still computed over the +// full pool: eligibility narrows selection, not co-location semantics. +func (p *Planner) PlanSessionWith(sessionID, currentTranscodeURL string, needsTranscode bool, estBitrateKbps int, eligible func(*Node) bool) Plan { if p == nil { return Plan{} } @@ -106,6 +117,15 @@ func (p *Planner) PlanSession(sessionID, currentTranscodeURL string, needsTransc proxies := p.proxies.Nodes() transcodes := p.transcodes.Nodes() groupHealthy := groupHealth(proxies, transcodes) + if eligible != nil { + filtered := make([]*Node, 0, len(transcodes)) + for _, node := range transcodes { + if eligible(node) { + filtered = append(filtered, node) + } + } + transcodes = filtered + } var plan Plan if needsTranscode { @@ -131,6 +151,35 @@ func (p *Planner) PlanSession(sessionID, currentTranscodeURL string, needsTransc return plan } +// TranscodeNodeURLs lists the URLs of every enabled pooled transcode node, +// healthy or not: capability planning wants the deployment's toolchain, and +// an unreachable node excludes itself when its capability fetch fails. An +// empty slice means no nodes are pooled. +func (p *Planner) TranscodeNodeURLs() []string { + if p == nil || p.transcodes == nil { + return nil + } + nodes := p.transcodes.Nodes() + urls := make([]string, 0, len(nodes)) + for _, node := range nodes { + if node != nil && node.URL != "" { + urls = append(urls, node.URL) + } + } + return urls +} + +// ReleaseSession removes a provisional node reservation when playback setup +// fails or falls back locally before a node health report can account for it. +func (p *Planner) ReleaseSession(sessionID string) { + if p == nil { + return + } + p.mu.Lock() + delete(p.reserved, sessionID) + p.mu.Unlock() +} + // groupHealth reports, for every group label present in either pool, whether // all of its enabled members are healthy. Pools only hold enabled nodes, so // disabled nodes never count against a group. diff --git a/internal/nodepool/planner_test.go b/internal/nodepool/planner_test.go index f2f29b79..a7ad1ae5 100644 --- a/internal/nodepool/planner_test.go +++ b/internal/nodepool/planner_test.go @@ -59,6 +59,46 @@ func TestPlanTranscodePairsProxyFromSameGroup(t *testing.T) { } } +func TestPlanSessionWithRestrictsEligibleTranscodeNodes(t *testing.T) { + f := newFixture(nil, []*Node{ + transcodeNode(1, "http://tc-a", nil, 0), + transcodeNode(2, "http://tc-b", nil, 5), + }) + eligible := func(n *Node) bool { return n != nil && n.URL == "http://tc-b" } + + plan := f.planner.PlanSessionWith("s1", "", true, 0, eligible) + if plan.TranscodeNode == nil || plan.TranscodeNode.URL != "http://tc-b" { + t.Fatalf("expected the eligible node despite its higher load, got %+v", plan.TranscodeNode) + } + if none := f.planner.PlanSessionWith("s2", "", true, 0, func(*Node) bool { return false }); none.TranscodeNode != nil { + t.Fatalf("no eligible node must select nothing, got %+v", none.TranscodeNode) + } + // Soft affinity to the session's current node must not survive the + // current node becoming ineligible. + if sticky := f.planner.PlanSessionWith("s3", "http://tc-a", true, 0, eligible); sticky.TranscodeNode == nil || sticky.TranscodeNode.URL != "http://tc-b" { + t.Fatalf("affinity to an ineligible node must yield to an eligible one, got %+v", sticky.TranscodeNode) + } + if unrestricted := f.planner.PlanSessionWith("s4", "", true, 0, nil); unrestricted.TranscodeNode == nil || unrestricted.TranscodeNode.URL != "http://tc-a" { + t.Fatalf("nil predicate must behave like PlanSession, got %+v", unrestricted.TranscodeNode) + } +} + +func TestReleaseSessionDropsProvisionalReservation(t *testing.T) { + node := transcodeNode(1, "http://tc-1", nil, 0) + node.MaxJobs = intPtr(1) + f := newFixture(nil, []*Node{node}) + if got := f.planner.PlanSession("s1", "", true, 0).TranscodeNode; got == nil { + t.Fatal("first session was not reserved") + } + if got := f.planner.PlanSession("s2", "", true, 0).TranscodeNode; got != nil { + t.Fatalf("second session bypassed reservation: %+v", got) + } + f.planner.ReleaseSession("s1") + if got := f.planner.PlanSession("s2", "", true, 0).TranscodeNode; got == nil { + t.Fatal("released reservation still blocked the node") + } +} + func TestDegradedGroupExcludesItsTranscodeNodes(t *testing.T) { unhealthyProxy := proxyNode(1, "http://proxy-a", strPtr("rack-a")) unhealthyProxy.Healthy = false diff --git a/internal/notifications/fanout_worker.go b/internal/notifications/fanout_worker.go index 556489ce..b2d87ff9 100644 --- a/internal/notifications/fanout_worker.go +++ b/internal/notifications/fanout_worker.go @@ -354,10 +354,15 @@ type pendingDelivery struct { flags ReasonFlags } -// enqueuePushOutbox inserts pending Apple push attempt rows for each newly -// inserted delivery and enabled private-push device. +// enqueuePushOutbox inserts pending push attempt rows for each newly inserted +// delivery and enabled private-push device on every admin-enabled platform +// (Apple and Android). func (w *FanoutWorker) enqueuePushOutbox(ctx context.Context, tx pgx.Tx, inserted []InsertedDelivery) error { - if w.pushDevices == nil || len(inserted) == 0 || !w.settings.ApplePushDeliveryEnabled(ctx) { + if w.pushDevices == nil || len(inserted) == 0 { + return nil + } + platforms := w.settings.EnabledPushPlatforms(ctx) + if len(platforms) == 0 { return nil } profileSet := make(map[string]struct{}, len(inserted)) @@ -368,7 +373,7 @@ func (w *FanoutWorker) enqueuePushOutbox(ctx context.Context, tx pgx.Tx, inserte profileIDs = append(profileIDs, row.ProfileID) } } - devicesByProfile, err := w.pushDevices.ListEnabledAppleByProfiles(ctx, tx, profileIDs) + devicesByProfile, err := w.pushDevices.ListEnabledPushByProfiles(ctx, tx, profileIDs, platforms) if err != nil { return err } diff --git a/internal/notifications/interest_hooks.go b/internal/notifications/interest_hooks.go index 807bfbb6..48f2c1bc 100644 --- a/internal/notifications/interest_hooks.go +++ b/internal/notifications/interest_hooks.go @@ -107,12 +107,12 @@ func (s *interestTrackingStore) AddFavorite(ctx context.Context, profileID, medi return err } -func (s *interestTrackingStore) AddFavoriteAt(ctx context.Context, profileID, mediaItemID string, addedAt time.Time) error { - err := s.UserStore.AddFavoriteAt(ctx, profileID, mediaItemID, addedAt) - if err == nil { +func (s *interestTrackingStore) AddFavoriteAt(ctx context.Context, profileID, mediaItemID string, addedAt time.Time) (bool, error) { + inserted, err := s.UserStore.AddFavoriteAt(ctx, profileID, mediaItemID, addedAt) + if err == nil && inserted { s.updater.QueueItemMutation(s.userID, profileID, mediaItemID) } - return err + return inserted, err } func (s *interestTrackingStore) RemoveFavorite(ctx context.Context, profileID, mediaItemID string) error { diff --git a/internal/notifications/operational_dispatch.go b/internal/notifications/operational_dispatch.go index 081ebbff..de48d1bd 100644 --- a/internal/notifications/operational_dispatch.go +++ b/internal/notifications/operational_dispatch.go @@ -79,14 +79,16 @@ func (s *System) DispatchOperational(ctx context.Context, delivery Delivery, opt return nil, err } } - if s.pushDeviceRepo != nil && s.Settings.ApplePushDeliveryEnabled(ctx) { - devicesByProfile, err := s.pushDeviceRepo.ListEnabledAppleByProfiles(ctx, tx, []string{delivery.ProfileID}) - if err != nil { - return nil, err - } - attempts := newPushDeliveryAttempts(row.ID, devicesByProfile[delivery.ProfileID]) - if err := s.pushDeviceRepo.EnqueuePushAttempts(ctx, tx, attempts); err != nil { - return nil, err + if s.pushDeviceRepo != nil { + if platforms := s.Settings.EnabledPushPlatforms(ctx); len(platforms) > 0 { + devicesByProfile, err := s.pushDeviceRepo.ListEnabledPushByProfiles(ctx, tx, []string{delivery.ProfileID}, platforms) + if err != nil { + return nil, err + } + attempts := newPushDeliveryAttempts(row.ID, devicesByProfile[delivery.ProfileID]) + if err := s.pushDeviceRepo.EnqueuePushAttempts(ctx, tx, attempts); err != nil { + return nil, err + } } } if err := tx.Commit(ctx); err != nil { diff --git a/internal/notifications/push_delivery.go b/internal/notifications/push_delivery.go index 7955baa1..2659a706 100644 --- a/internal/notifications/push_delivery.go +++ b/internal/notifications/push_delivery.go @@ -58,6 +58,8 @@ func newPushDeliveryAttempts(deliveryID string, devices []PushDevice) []PushDeli NotificationDeliveryID: &deliveryID, PushDeviceID: device.ID, TriggerType: PushTriggerDelivery, + Provider: device.Provider, + Platform: device.Platform, }) } return attempts @@ -98,20 +100,21 @@ func scanPushDeliveryAttempts(rows pgx.Rows) ([]PushDeliveryAttempt, error) { return attempts, rows.Err() } -// ListEnabledAppleByProfiles loads delivery-eligible APNs devices keyed by profile. -func (r *PushDeviceRepository) ListEnabledAppleByProfiles(ctx context.Context, tx pgx.Tx, profileIDs []string) (map[string][]PushDevice, error) { +// ListEnabledPushByProfiles loads delivery-eligible push devices for the +// admin-enabled platforms, keyed by profile. +func (r *PushDeviceRepository) ListEnabledPushByProfiles(ctx context.Context, tx pgx.Tx, profileIDs, platforms []string) (map[string][]PushDevice, error) { out := make(map[string][]PushDevice, len(profileIDs)) - if len(profileIDs) == 0 { + if len(profileIDs) == 0 || len(platforms) == 0 { return out, nil } rows, err := tx.Query(ctx, `SELECT `+pushDeviceColumns+` FROM push_devices WHERE profile_id = ANY($1) - AND platform = $2 + AND platform = ANY($2) AND provider = $3 AND push_mode = $4 AND enabled`, - profileIDs, PushPlatformApple, PushProviderSiloRelay, PushModePrivatePush) + profileIDs, platforms, PushProviderSiloRelay, PushModePrivatePush) if err != nil { return nil, fmt.Errorf("list enabled push devices: %w", err) } @@ -149,8 +152,8 @@ func (r *PushDeviceRepository) EnqueuePushAttempts(ctx context.Context, tx pgx.T attempt.NotificationDeliveryID, attempt.PushDeviceID, defaultString(attempt.TriggerType, PushTriggerDelivery), - PushProviderSiloRelay, - PushPlatformApple, + defaultString(attempt.Provider, PushProviderSiloRelay), + defaultString(attempt.Platform, PushPlatformApple), 0, PushOutcomePending, ) @@ -162,8 +165,9 @@ func (r *PushDeviceRepository) EnqueuePushAttempts(ctx context.Context, tx pgx.T return nil } -// EnqueueAppleTestAttempt creates a pending diagnostic attempt for one enabled device. -func (r *PushDeviceRepository) EnqueueAppleTestAttempt(ctx context.Context, profileID, serverDeviceID string) (*PushDeliveryAttempt, *PushDevice, error) { +// EnqueueTestAttempt creates a pending diagnostic attempt for one enabled +// device on the given platform. +func (r *PushDeviceRepository) EnqueueTestAttempt(ctx context.Context, platform, profileID, serverDeviceID string) (*PushDeliveryAttempt, *PushDevice, error) { if r == nil || r.pool == nil { return nil, nil, ErrPushDeliveryUnavailable } @@ -185,7 +189,7 @@ func (r *PushDeviceRepository) EnqueueAppleTestAttempt(ctx context.Context, prof AND provider = $3 AND push_mode = $4 AND enabled` - args := []any{profileID, PushPlatformApple, PushProviderSiloRelay, PushModePrivatePush} + args := []any{profileID, platform, PushProviderSiloRelay, PushModePrivatePush} if serverDeviceID != "" { args = append(args, serverDeviceID) query += fmt.Sprintf(" AND server_device_id = $%d", len(args)) @@ -205,7 +209,7 @@ func (r *PushDeviceRepository) EnqueueAppleTestAttempt(ctx context.Context, prof INSERT INTO push_delivery_attempts (id, notification_delivery_id, push_device_id, trigger_type, provider, platform, attempt_number, outcome) VALUES ($1, NULL, $2, $3, $4, $5, 0, $6)`+pushAttemptReturning, - attemptID, device.ID, PushTriggerTest, PushProviderSiloRelay, PushPlatformApple, PushOutcomePending) + attemptID, device.ID, PushTriggerTest, PushProviderSiloRelay, platform, PushOutcomePending) if err != nil { return nil, nil, fmt.Errorf("insert push test attempt: %w", err) } diff --git a/internal/notifications/push_devices.go b/internal/notifications/push_devices.go index d9595caf..8ea546ae 100644 --- a/internal/notifications/push_devices.go +++ b/internal/notifications/push_devices.go @@ -18,6 +18,7 @@ import ( const ( PushPlatformApple = "apple" + PushPlatformAndroid = "android" PushProviderSiloRelay = "silo_relay" PushModeOff = "off" PushModeInAppOnly = "in_app_only" @@ -33,6 +34,8 @@ var ( ErrPushDeviceUnsupported = errors.New("unsupported push device registration") apnsTokenHexPattern = regexp.MustCompile(`^[0-9a-f]+$`) + // The relay validates FCM registration tokens against the same shape. + fcmTokenPattern = regexp.MustCompile(`^[A-Za-z0-9_:-]{64,512}$`) ) // PushDevice represents one profile-scoped notification endpoint. @@ -47,6 +50,8 @@ type PushDevice struct { APNsTopic string APNsTokenCiphertext string APNsTokenHash string + FCMTokenCiphertext string + FCMTokenHash string ServerDeviceID string PushMode string Enabled bool @@ -76,8 +81,24 @@ type ApplePushDeviceRegistration struct { PushMode string } +type FCMPushRegistrationInput struct { + DeviceID string + FCMToken string + PushMode string +} + +type FCMPushDeviceRegistration struct { + UserID int + ProfileID string + DeviceID string + FCMToken string + PushMode string +} + type PushDeviceStore interface { UpsertApple(ctx context.Context, registration ApplePushDeviceRegistration, cipher *secret.Cipher) (*PushDevice, error) + UpsertFCM(ctx context.Context, registration FCMPushDeviceRegistration, cipher *secret.Cipher) (*PushDevice, error) + DeleteByProfileDevice(ctx context.Context, profileID, deviceID string) error } type PushDeviceRepository struct { @@ -121,6 +142,40 @@ func (s *PushDeviceService) RegisterApple(ctx context.Context, userID int, profi return s.store.UpsertApple(ctx, registration, s.cipher) } +func (s *PushDeviceService) RegisterFCM(ctx context.Context, userID int, profileID string, input FCMPushRegistrationInput) (*PushDevice, error) { + if !s.Available() { + return nil, ErrPushDeviceUnavailable + } + if userID <= 0 { + return nil, fmt.Errorf("%w: user_id is required", ErrPushDeviceInvalid) + } + profileID = strings.TrimSpace(profileID) + if profileID == "" { + return nil, fmt.Errorf("%w: profile_id is required", ErrPushDeviceInvalid) + } + registration, err := normalizeFCMPushRegistration(input) + if err != nil { + return nil, err + } + registration.UserID = userID + registration.ProfileID = profileID + return s.store.UpsertFCM(ctx, registration, s.cipher) +} + +// Unregister removes every registration this device install holds for the +// profile, regardless of platform. +func (s *PushDeviceService) Unregister(ctx context.Context, profileID, deviceID string) error { + if !s.Available() { + return ErrPushDeviceUnavailable + } + profileID = strings.TrimSpace(profileID) + deviceID = strings.TrimSpace(deviceID) + if profileID == "" || deviceID == "" { + return fmt.Errorf("%w: profile_id and device_id are required", ErrPushDeviceInvalid) + } + return s.store.DeleteByProfileDevice(ctx, profileID, deviceID) +} + func normalizeApplePushRegistration(input ApplePushRegistrationInput) (ApplePushDeviceRegistration, error) { deviceID := strings.TrimSpace(input.DeviceID) if deviceID == "" { @@ -170,15 +225,61 @@ func normalizeApplePushRegistration(input ApplePushRegistrationInput) (ApplePush }, nil } +func normalizeFCMPushRegistration(input FCMPushRegistrationInput) (FCMPushDeviceRegistration, error) { + deviceID := strings.TrimSpace(input.DeviceID) + if deviceID == "" { + return FCMPushDeviceRegistration{}, fmt.Errorf("%w: device_id is required", ErrPushDeviceInvalid) + } + if len(deviceID) > 128 { + return FCMPushDeviceRegistration{}, fmt.Errorf("%w: device_id is too long", ErrPushDeviceInvalid) + } + + // FCM registration tokens are case-sensitive; only trim surrounding space. + token := strings.TrimSpace(input.FCMToken) + if token == "" { + return FCMPushDeviceRegistration{}, fmt.Errorf("%w: token is required", ErrPushDeviceInvalid) + } + if !fcmTokenPattern.MatchString(token) { + return FCMPushDeviceRegistration{}, fmt.Errorf("%w: token is not a plausible FCM registration token", ErrPushDeviceInvalid) + } + + pushMode := strings.TrimSpace(input.PushMode) + if pushMode == "" { + pushMode = PushModePrivatePush + } + switch pushMode { + case PushModeOff, PushModeInAppOnly, PushModePrivatePush: + default: + return FCMPushDeviceRegistration{}, fmt.Errorf("%w: push_mode is not supported", ErrPushDeviceUnsupported) + } + + return FCMPushDeviceRegistration{ + DeviceID: deviceID, + FCMToken: token, + PushMode: pushMode, + }, nil +} + func apnsTokenHash(token string) string { sum := sha256.Sum256([]byte(strings.ToLower(strings.TrimSpace(token)))) return hex.EncodeToString(sum[:]) } +func fcmTokenHash(token string) string { + sum := sha256.Sum256([]byte(strings.TrimSpace(token))) + return hex.EncodeToString(sum[:]) +} + func pushDeviceAPNsTokenAAD(id string) string { return secret.RowAAD("push_devices", "apns_token", id) } +func pushDeviceFCMTokenAAD(id string) string { + return secret.RowAAD("push_devices", "fcm_token", id) +} + +// Platform-specific token columns are NULL for the other platform's rows; +// COALESCE keeps the scan targets plain strings. const pushDeviceColumns = ` id, user_id, @@ -186,10 +287,12 @@ const pushDeviceColumns = ` device_id, platform, provider, - apns_environment, - apns_topic, - apns_token_ciphertext, - apns_token_hash, + COALESCE(apns_environment, ''), + COALESCE(apns_topic, ''), + COALESCE(apns_token_ciphertext, ''), + COALESCE(apns_token_hash, ''), + COALESCE(fcm_token_ciphertext, ''), + COALESCE(fcm_token_hash, ''), server_device_id, push_mode, enabled, @@ -221,7 +324,7 @@ func (r *PushDeviceRepository) UpsertApple(ctx context.Context, registration App return nil, fmt.Errorf("purge reassigned push device: %w", err) } - device, err := r.selectAppleForUpdate(ctx, tx, registration.ProfileID, registration.DeviceID) + device, err := r.selectForUpdate(ctx, tx, registration.ProfileID, registration.DeviceID, PushPlatformApple) if err != nil { return nil, err } @@ -238,7 +341,7 @@ func (r *PushDeviceRepository) UpsertApple(ctx context.Context, registration App return device, nil } - device, err = r.selectAppleForUpdate(ctx, tx, registration.ProfileID, registration.DeviceID) + device, err = r.selectForUpdate(ctx, tx, registration.ProfileID, registration.DeviceID, PushPlatformApple) if err != nil { return nil, err } @@ -257,6 +360,61 @@ func (r *PushDeviceRepository) UpsertApple(ctx context.Context, registration App return device, nil } +func (r *PushDeviceRepository) UpsertFCM(ctx context.Context, registration FCMPushDeviceRegistration, cipher *secret.Cipher) (*PushDevice, error) { + if r == nil || r.pool == nil || cipher == nil { + return nil, ErrPushDeviceUnavailable + } + + tx, err := r.pool.Begin(ctx) + if err != nil { + return nil, fmt.Errorf("begin push device upsert: %w", err) + } + defer func() { _ = tx.Rollback(ctx) }() + + // Same profile-switch cleanup as UpsertApple: one install notifies one + // profile at a time. + if _, err := tx.Exec(ctx, + `DELETE FROM push_devices WHERE device_id = $1 AND platform = $2 AND profile_id <> $3`, + registration.DeviceID, PushPlatformAndroid, registration.ProfileID); err != nil { + return nil, fmt.Errorf("purge reassigned push device: %w", err) + } + + device, err := r.selectForUpdate(ctx, tx, registration.ProfileID, registration.DeviceID, PushPlatformAndroid) + if err != nil { + return nil, err + } + + if device == nil { + device, err = r.insertFCM(ctx, tx, registration, cipher) + if err != nil && !errors.Is(err, pgx.ErrNoRows) { + return nil, err + } + if device != nil { + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit push device insert: %w", err) + } + return device, nil + } + + device, err = r.selectForUpdate(ctx, tx, registration.ProfileID, registration.DeviceID, PushPlatformAndroid) + if err != nil { + return nil, err + } + if device == nil { + return nil, fmt.Errorf("push device upsert conflict row missing") + } + } + + device, err = r.updateFCM(ctx, tx, registration, cipher, device) + if err != nil { + return nil, err + } + if err := tx.Commit(ctx); err != nil { + return nil, fmt.Errorf("commit push device update: %w", err) + } + return device, nil +} + // DeleteAllForProfile removes push registrations for a deleted profile. func (r *PushDeviceRepository) DeleteAllForProfile(ctx context.Context, profileID string) error { if r == nil || r.pool == nil { @@ -266,8 +424,19 @@ func (r *PushDeviceRepository) DeleteAllForProfile(ctx context.Context, profileI return err } -func (r *PushDeviceRepository) selectAppleForUpdate(ctx context.Context, tx pgx.Tx, profileID, deviceID string) (*PushDevice, error) { - row := tx.QueryRow(ctx, `SELECT `+pushDeviceColumns+` FROM push_devices WHERE profile_id = $1 AND device_id = $2 AND platform = $3 FOR UPDATE`, profileID, deviceID, PushPlatformApple) +// DeleteByProfileDevice removes one install's registrations for a profile +// across every platform (attempts cascade with the device rows). +func (r *PushDeviceRepository) DeleteByProfileDevice(ctx context.Context, profileID, deviceID string) error { + if r == nil || r.pool == nil { + return ErrPushDeviceUnavailable + } + _, err := r.pool.Exec(ctx, + `DELETE FROM push_devices WHERE profile_id = $1 AND device_id = $2`, profileID, deviceID) + return err +} + +func (r *PushDeviceRepository) selectForUpdate(ctx context.Context, tx pgx.Tx, profileID, deviceID, platform string) (*PushDevice, error) { + row := tx.QueryRow(ctx, `SELECT `+pushDeviceColumns+` FROM push_devices WHERE profile_id = $1 AND device_id = $2 AND platform = $3 FOR UPDATE`, profileID, deviceID, platform) device, err := scanPushDevice(row) if errors.Is(err, pgx.ErrNoRows) { return nil, nil @@ -364,6 +533,84 @@ func (r *PushDeviceRepository) updateApple(ctx context.Context, tx pgx.Tx, regis return device, nil } +func (r *PushDeviceRepository) insertFCM(ctx context.Context, tx pgx.Tx, registration FCMPushDeviceRegistration, cipher *secret.Cipher) (*PushDevice, error) { + id := ulid.Make().String() + serverDeviceID := ulid.Make().String() + ciphertext, err := cipher.Encrypt(registration.FCMToken, pushDeviceFCMTokenAAD(id)) + if err != nil { + return nil, fmt.Errorf("encrypt fcm token: %w", err) + } + + row := tx.QueryRow(ctx, ` + INSERT INTO push_devices ( + id, + user_id, + profile_id, + device_id, + platform, + provider, + fcm_token_ciphertext, + fcm_token_hash, + server_device_id, + push_mode, + enabled, + last_seen_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, true, now()) + ON CONFLICT (profile_id, device_id, platform) DO NOTHING + RETURNING `+pushDeviceColumns, + id, + registration.UserID, + registration.ProfileID, + registration.DeviceID, + PushPlatformAndroid, + PushProviderSiloRelay, + ciphertext, + fcmTokenHash(registration.FCMToken), + serverDeviceID, + registration.PushMode, + ) + device, err := scanPushDevice(row) + if err != nil { + return nil, err + } + return device, nil +} + +func (r *PushDeviceRepository) updateFCM(ctx context.Context, tx pgx.Tx, registration FCMPushDeviceRegistration, cipher *secret.Cipher, existing *PushDevice) (*PushDevice, error) { + ciphertext, err := cipher.Encrypt(registration.FCMToken, pushDeviceFCMTokenAAD(existing.ID)) + if err != nil { + return nil, fmt.Errorf("encrypt fcm token: %w", err) + } + + row := tx.QueryRow(ctx, ` + UPDATE push_devices + SET user_id = $1, + provider = $2, + fcm_token_ciphertext = $3, + fcm_token_hash = $4, + push_mode = $5, + enabled = true, + last_seen_at = now(), + last_failure_at = NULL, + last_failure_code = NULL, + updated_at = now() + WHERE id = $6 + RETURNING `+pushDeviceColumns, + registration.UserID, + PushProviderSiloRelay, + ciphertext, + fcmTokenHash(registration.FCMToken), + registration.PushMode, + existing.ID, + ) + device, err := scanPushDevice(row) + if err != nil { + return nil, fmt.Errorf("update push device: %w", err) + } + return device, nil +} + func scanPushDevice(row pgx.Row) (*PushDevice, error) { var device PushDevice if err := row.Scan( @@ -377,6 +624,8 @@ func scanPushDevice(row pgx.Row) (*PushDevice, error) { &device.APNsTopic, &device.APNsTokenCiphertext, &device.APNsTokenHash, + &device.FCMTokenCiphertext, + &device.FCMTokenHash, &device.ServerDeviceID, &device.PushMode, &device.Enabled, diff --git a/internal/notifications/push_devices_test.go b/internal/notifications/push_devices_test.go index 754fed8a..fa1573d9 100644 --- a/internal/notifications/push_devices_test.go +++ b/internal/notifications/push_devices_test.go @@ -12,10 +12,12 @@ import ( ) type fakePushDeviceStore struct { - got ApplePushDeviceRegistration - calls int - device *PushDevice - err error + got ApplePushDeviceRegistration + gotFCM FCMPushDeviceRegistration + calls int + device *PushDevice + err error + deleted []string } func (f *fakePushDeviceStore) UpsertApple(ctx context.Context, registration ApplePushDeviceRegistration, cipher *secret.Cipher) (*PushDevice, error) { @@ -35,6 +37,29 @@ func (f *fakePushDeviceStore) UpsertApple(ctx context.Context, registration Appl }, nil } +func (f *fakePushDeviceStore) UpsertFCM(ctx context.Context, registration FCMPushDeviceRegistration, cipher *secret.Cipher) (*PushDevice, error) { + f.calls++ + f.gotFCM = registration + if f.err != nil { + return nil, f.err + } + if f.device != nil { + return f.device, nil + } + return &PushDevice{ + ID: "device-row", + Platform: PushPlatformAndroid, + ServerDeviceID: "server-device", + Enabled: true, + PushMode: registration.PushMode, + }, nil +} + +func (f *fakePushDeviceStore) DeleteByProfileDevice(ctx context.Context, profileID, deviceID string) error { + f.deleted = append(f.deleted, profileID+"/"+deviceID) + return f.err +} + func testPushCipher(t *testing.T) *secret.Cipher { t.Helper() cipher, err := secret.New([]byte("01234567890123456789012345678901")) @@ -147,6 +172,87 @@ func TestPushDeviceServiceRegisterAppleRejectsInvalidOrUnsupported(t *testing.T) } } +func TestPushDeviceServiceRegisterFCMNormalizesAndStores(t *testing.T) { + store := &fakePushDeviceStore{} + service := NewPushDeviceService(store, testPushCipher(t)) + + token := strings.Repeat("F", 100) + ":APA91b-" + strings.Repeat("x", 40) + device, err := service.RegisterFCM(context.Background(), 42, " profile-1 ", FCMPushRegistrationInput{ + DeviceID: " local-device ", + FCMToken: " " + token + " ", + PushMode: "", + }) + if err != nil { + t.Fatalf("register fcm: %v", err) + } + if device.ServerDeviceID != "server-device" || !device.Enabled || device.PushMode != PushModePrivatePush { + t.Fatalf("unexpected device response: %+v", device) + } + if store.gotFCM.UserID != 42 || store.gotFCM.ProfileID != "profile-1" || store.gotFCM.DeviceID != "local-device" { + t.Fatalf("unexpected owner scope: %+v", store.gotFCM) + } + if store.gotFCM.FCMToken != token { + t.Fatalf("fcm token was not trimmed exactly: %q", store.gotFCM.FCMToken) + } + if store.gotFCM.PushMode != PushModePrivatePush { + t.Fatalf("push mode = %q", store.gotFCM.PushMode) + } +} + +func TestPushDeviceServiceRegisterFCMRejectsInvalidOrUnsupported(t *testing.T) { + validToken := strings.Repeat("F", 140) + tests := []struct { + name string + input FCMPushRegistrationInput + wantErr error + }{ + { + name: "missing device id", + input: FCMPushRegistrationInput{DeviceID: " ", FCMToken: validToken}, + wantErr: ErrPushDeviceInvalid, + }, + { + name: "short token", + input: FCMPushRegistrationInput{DeviceID: "local-device", FCMToken: "abc"}, + wantErr: ErrPushDeviceInvalid, + }, + { + name: "token with invalid characters", + input: FCMPushRegistrationInput{DeviceID: "local-device", FCMToken: strings.Repeat("!", 140)}, + wantErr: ErrPushDeviceInvalid, + }, + { + name: "unsupported push mode", + input: FCMPushRegistrationInput{DeviceID: "local-device", FCMToken: validToken, PushMode: "custom_fcm"}, + wantErr: ErrPushDeviceUnsupported, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + store := &fakePushDeviceStore{} + service := NewPushDeviceService(store, testPushCipher(t)) + _, err := service.RegisterFCM(context.Background(), 42, "profile-1", tt.input) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("error = %v, want %v", err, tt.wantErr) + } + if store.calls != 0 { + t.Fatalf("store was called for rejected input") + } + }) + } +} + +func TestPushDeviceFCMTokenHashIsCaseSensitive(t *testing.T) { + token := strings.Repeat("f", 140) + if fcmTokenHash(token) == fcmTokenHash(strings.ToUpper(token)) { + t.Fatal("fcm token hash must be case-sensitive") + } + if fcmTokenHash(" "+token+" ") != fcmTokenHash(token) { + t.Fatal("fcm token hash must ignore surrounding whitespace") + } +} + func TestPushDeviceAPNsTokenEncryptionUsesRowAAD(t *testing.T) { cipher := testPushCipher(t) token := strings.Repeat("a", 64) diff --git a/internal/notifications/push_sender.go b/internal/notifications/push_sender.go index 70067065..27368517 100644 --- a/internal/notifications/push_sender.go +++ b/internal/notifications/push_sender.go @@ -31,6 +31,7 @@ const ( pushRelayRequestTimeout = 15 * time.Second pushRelayMaxRetryAfter = 23 * time.Hour relayAppleSendPath = "/v1/apple/send" + relayFcmSendPath = "/v1/fcm/send" ) func pushRetryDelay(completedAttempt int) (time.Duration, bool) { @@ -64,6 +65,26 @@ func terminalAPNsDeviceRejection(status int, code, message string) bool { } } +// terminalFCMDeviceRejection recognizes the relay's fcm_rejected responses +// whose FCM error code means the registration token is permanently gone. +// Request-level rejections (e.g. INVALID_ARGUMENT payload complaints) keep the +// device enabled, matching the conservative APNs list above. +func terminalFCMDeviceRejection(status int, code, message string) bool { + if status != http.StatusUnprocessableEntity || code != "fcm_rejected" { + return false + } + const prefix = "FCM rejected the notification:" + reason := strings.TrimSpace(strings.TrimPrefix(message, prefix)) + return reason == "UNREGISTERED" +} + +func terminalDeviceRejection(platform string, status int, code, message string) bool { + if platform == PushPlatformAndroid { + return terminalFCMDeviceRejection(status, code, message) + } + return terminalAPNsDeviceRejection(status, code, message) +} + type pushRelayAppleRequest struct { Token string `json:"token"` Environment string `json:"environment"` @@ -80,6 +101,16 @@ type pushRelayAppleResponse struct { Status string `json:"status"` } +// pushRelayFcmRequest matches the relay's /v1/fcm/send contract: no +// environment or topic — the relay's Firebase project is the boundary. +type pushRelayFcmRequest struct { + Token string `json:"token"` + Mode string `json:"mode"` + ServerDeviceID string `json:"server_device_id"` + DeliveryID string `json:"delivery_id"` + CollapseID *string `json:"collapse_id,omitempty"` +} + type pushRelayErrorResponse struct { Error struct { Code string `json:"code"` @@ -138,8 +169,12 @@ func (s *pushSender) processAttempt(ctx context.Context, attempt PushDeliveryAtt } return nil } - if !device.Enabled || device.PushMode != PushModePrivatePush || !s.settings.ApplePushDeliveryEnabled(ctx) { - return s.finalize(ctx, attempt, PushOutcomeFailed, "delivery_disabled", "apple push delivery disabled", nil, "", nil) + deliveryEnabled := s.settings.ApplePushDeliveryEnabled(ctx) + if device.Platform == PushPlatformAndroid { + deliveryEnabled = s.settings.AndroidPushDeliveryEnabled(ctx) + } + if !device.Enabled || device.PushMode != PushModePrivatePush || !deliveryEnabled { + return s.finalize(ctx, attempt, PushOutcomeFailed, "delivery_disabled", "push delivery disabled", nil, "", nil) } if attempt.NotificationDeliveryID != nil { row, err := s.deliveries.GetRowByID(ctx, *attempt.NotificationDeliveryID) @@ -160,9 +195,13 @@ func (s *pushSender) processAttempt(ctx context.Context, attempt PushDeliveryAtt } } - token, err := s.cipher.Decrypt(device.APNsTokenCiphertext, pushDeviceAPNsTokenAAD(device.ID)) + ciphertext, aad := device.APNsTokenCiphertext, pushDeviceAPNsTokenAAD(device.ID) + if device.Platform == PushPlatformAndroid { + ciphertext, aad = device.FCMTokenCiphertext, pushDeviceFCMTokenAAD(device.ID) + } + token, err := s.cipher.Decrypt(ciphertext, aad) if err != nil { - return s.finalize(ctx, attempt, PushOutcomeFailed, "decrypt_failed", "APNs token decrypt failed", nil, "", nil) + return s.finalize(ctx, attempt, PushOutcomeFailed, "decrypt_failed", "push token decrypt failed", nil, "", nil) } result := s.send(ctx, attempt, device, token) attemptNumber := attempt.AttemptNumber + 1 @@ -265,7 +304,8 @@ func (s *pushSender) sendWithCapability(ctx context.Context, attempt PushDeliver deliveryID = *attempt.NotificationDeliveryID } collapseID := deliveryID - body, err := json.Marshal(pushRelayAppleRequest{ + sendPath := relayAppleSendPath + var payload any = pushRelayAppleRequest{ Token: token, Environment: device.APNsEnvironment, Topic: device.APNsTopic, @@ -273,11 +313,22 @@ func (s *pushSender) sendWithCapability(ctx context.Context, attempt PushDeliver ServerDeviceID: device.ServerDeviceID, DeliveryID: deliveryID, CollapseID: &collapseID, - }) + } + if device.Platform == PushPlatformAndroid { + sendPath = relayFcmSendPath + payload = pushRelayFcmRequest{ + Token: token, + Mode: "private_alert", + ServerDeviceID: device.ServerDeviceID, + DeliveryID: deliveryID, + CollapseID: &collapseID, + } + } + body, err := json.Marshal(payload) if err != nil { return pushSendResult{Message: "relay payload build failed", UpstreamReason: "payload_build_failed"} } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, relayURL+relayAppleSendPath, bytes.NewReader(body)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, relayURL+sendPath, bytes.NewReader(body)) if err != nil { return pushSendResult{Message: "invalid push relay URL", UpstreamReason: "invalid_relay_url"} } @@ -331,7 +382,7 @@ func (s *pushSender) sendWithCapability(ctx context.Context, attempt PushDeliver RelayRequestID: parsed.Error.RequestID, UpstreamReason: code, Message: strings.TrimSpace(message), - TerminalDevice: terminalAPNsDeviceRejection(resp.StatusCode, code, message), + TerminalDevice: terminalDeviceRejection(device.Platform, resp.StatusCode, code, message), } } @@ -379,7 +430,9 @@ func newPushDispatcher(sender *pushSender) *PushDispatcher { process: func(ctx context.Context, attempt PushDeliveryAttempt) { sender.processAttempt(ctx, attempt) }, - enabled: sender.settings.ApplePushDeliveryEnabled, + // The dispatcher claims attempts for every platform; processAttempt + // applies the per-platform delivery toggle to each device. + enabled: sender.settings.PushDeliveryEnabled, claimDue: sender.devices.ClaimDuePushAttempts, claimLimit: pushRetryClaimLimit, }} @@ -412,16 +465,28 @@ type ApplePushTestResult struct { } func (s *System) SendApplePushTest(ctx context.Context, profileID, serverDeviceID string) (*ApplePushTestResult, error) { + return s.sendPushTest(ctx, PushPlatformApple, profileID, serverDeviceID) +} + +func (s *System) SendAndroidPushTest(ctx context.Context, profileID, serverDeviceID string) (*ApplePushTestResult, error) { + return s.sendPushTest(ctx, PushPlatformAndroid, profileID, serverDeviceID) +} + +func (s *System) sendPushTest(ctx context.Context, platform, profileID, serverDeviceID string) (*ApplePushTestResult, error) { if s == nil || s.pushDeviceRepo == nil || s.pushSender == nil { return nil, ErrPushDeliveryUnavailable } - if !s.Settings.ApplePushDeliveryEnabled(ctx) { + deliveryEnabled := s.Settings.ApplePushDeliveryEnabled(ctx) + if platform == PushPlatformAndroid { + deliveryEnabled = s.Settings.AndroidPushDeliveryEnabled(ctx) + } + if !deliveryEnabled { return nil, ErrPushDeliveryUnavailable } if s.Settings.PushRelayAPIKey(ctx) == "" { return nil, ErrPushDeliveryUnavailable } - attempt, device, err := s.pushDeviceRepo.EnqueueAppleTestAttempt(ctx, profileID, serverDeviceID) + attempt, device, err := s.pushDeviceRepo.EnqueueTestAttempt(ctx, platform, profileID, serverDeviceID) if err != nil { return nil, err } diff --git a/internal/notifications/push_sender_test.go b/internal/notifications/push_sender_test.go index 65745d08..e0864216 100644 --- a/internal/notifications/push_sender_test.go +++ b/internal/notifications/push_sender_test.go @@ -153,6 +153,143 @@ func TestPushSenderSendMapsRelayTerminalAPNsRejection(t *testing.T) { } } +func TestPushSenderSendBuildsFcmRelayRequest(t *testing.T) { + token := strings.Repeat("F", 140) + var got struct { + auth string + idempotencyKey string + body pushRelayFcmRequest + } + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got.auth = r.Header.Get("Authorization") + got.idempotencyKey = r.Header.Get("Idempotency-Key") + if r.URL.Path != relayFcmSendPath { + t.Fatalf("path = %q, want %q", r.URL.Path, relayFcmSendPath) + } + if err := json.NewDecoder(r.Body).Decode(&got.body); err != nil { + t.Fatalf("decode request body: %v", err) + } + _ = json.NewEncoder(w).Encode(map[string]string{ + "request_id": "relay-request-fcm-1", + "fcm_message_id": "fcm-1", + "status": "accepted", + }) + })) + defer server.Close() + + settings := NewSettings(mapSettingReader{ + SettingPushRelayURL: server.URL, + SettingPushRelayAPIKey: "relay-key", + }) + sender := newPushSender(nil, nil, nil, settings) + sender.client = server.Client() + sender.developmentRelayURL = server.URL + + deliveryID := "delivery-fcm-1" + result := sender.send(context.Background(), PushDeliveryAttempt{ + ID: "attempt-fcm-1", + NotificationDeliveryID: &deliveryID, + }, &PushDevice{ + Platform: PushPlatformAndroid, + ServerDeviceID: "server-device-fcm", + }, token) + + if !result.OK || result.RelayRequestID != "relay-request-fcm-1" { + t.Fatalf("result = %+v", result) + } + if got.auth != "Bearer relay-key" || got.idempotencyKey != "attempt-fcm-1" { + t.Fatalf("headers = %+v", got) + } + if got.body.Token != token || got.body.Mode != "private_alert" || got.body.DeliveryID != deliveryID { + t.Fatalf("relay body = %+v", got.body) + } + if got.body.CollapseID == nil || *got.body.CollapseID != deliveryID { + t.Fatalf("collapse_id = %+v, want delivery id", got.body.CollapseID) + } +} + +func TestPushSenderSendMapsRelayTerminalFCMRejection(t *testing.T) { + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusUnprocessableEntity) + _ = json.NewEncoder(w).Encode(pushRelayErrorResponse{ + Error: struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id"` + }{ + Code: "fcm_rejected", + Message: "FCM rejected the notification: UNREGISTERED", + RequestID: "relay-request-fcm-2", + }, + }) + })) + defer server.Close() + + sender := newPushSender(nil, nil, nil, NewSettings(mapSettingReader{ + SettingPushRelayURL: server.URL, + SettingPushRelayAPIKey: "relay-key", + })) + sender.client = server.Client() + sender.developmentRelayURL = server.URL + + result := sender.send(context.Background(), PushDeliveryAttempt{ID: "attempt-fcm-2"}, &PushDevice{ + Platform: PushPlatformAndroid, + ServerDeviceID: "server-device-fcm", + }, strings.Repeat("F", 140)) + + if result.OK || !result.TerminalDevice || result.HTTPStatus != http.StatusUnprocessableEntity { + t.Fatalf("terminal result = %+v", result) + } + if result.UpstreamReason != "fcm_rejected" || result.RelayRequestID != "relay-request-fcm-2" { + t.Fatalf("terminal diagnostic = %+v", result) + } +} + +func TestTerminalFCMDeviceRejectionReasons(t *testing.T) { + if !terminalFCMDeviceRejection(http.StatusUnprocessableEntity, "fcm_rejected", + "FCM rejected the notification: UNREGISTERED") { + t.Fatal("UNREGISTERED was not terminal for the device") + } + if terminalFCMDeviceRejection(http.StatusUnprocessableEntity, "fcm_rejected", + "FCM rejected the notification: INVALID_ARGUMENT") { + t.Fatal("request-level FCM rejection was terminal for the device") + } + if terminalFCMDeviceRejection(http.StatusUnprocessableEntity, "apns_rejected", + "APNs rejected the notification: Unregistered") { + t.Fatal("APNs rejection satisfied the FCM matcher") + } + if terminalDeviceRejection(PushPlatformAndroid, http.StatusUnprocessableEntity, "fcm_rejected", + "FCM rejected the notification: UNREGISTERED") != + terminalFCMDeviceRejection(http.StatusUnprocessableEntity, "fcm_rejected", + "FCM rejected the notification: UNREGISTERED") { + t.Fatal("platform dispatch mismatch") + } +} + +func TestAndroidPushDeliverySettings(t *testing.T) { + ctx := context.Background() + settings := NewSettings(nil) + if settings.AndroidPushDeliveryEnabled(ctx) { + t.Fatal("AndroidPushDeliveryEnabled must default to false") + } + if settings.PushDeliveryEnabled(ctx) { + t.Fatal("PushDeliveryEnabled must default to false") + } + if got := settings.EnabledPushPlatforms(ctx); len(got) != 0 { + t.Fatalf("EnabledPushPlatforms = %v, want empty", got) + } + + settings = NewSettings(mapSettingReader{ + SettingAndroidPushDeliveryEnabled: "true", + }) + if !settings.AndroidPushDeliveryEnabled(ctx) || !settings.PushDeliveryEnabled(ctx) { + t.Fatal("android delivery setting did not enable push delivery") + } + if got := settings.EnabledPushPlatforms(ctx); len(got) != 1 || got[0] != PushPlatformAndroid { + t.Fatalf("EnabledPushPlatforms = %v, want [android]", got) + } +} + func TestTerminalAPNsDeviceRejectionReasons(t *testing.T) { for _, reason := range []string{ "BadDeviceToken", diff --git a/internal/notifications/settings.go b/internal/notifications/settings.go index b87a7658..21d9de05 100644 --- a/internal/notifications/settings.go +++ b/internal/notifications/settings.go @@ -56,13 +56,14 @@ const ( SettingServerChannelsBatchSeconds = "notifications.server_channels.batch_seconds" SettingServerChannelMentionRequesters = "notifications.server_channels.mention_requesters" - SettingApplePushDeliveryEnabled = "notifications.apple_push_delivery_enabled" - SettingPushRelayURL = "notifications.push_relay_url" - SettingPushRelayDeploymentID = "notifications.push_relay_deployment_id" - SettingPushRelayAPIKey = "notifications.push_relay_api_key" - SettingPushRelayExpiresAt = "notifications.push_relay_expires_at" - SettingPushRelayKeyPrefix = "notifications.push_relay_key_prefix" - SettingPushRelayReregister = "notifications.push_relay_reregistration_required" + SettingApplePushDeliveryEnabled = "notifications.apple_push_delivery_enabled" + SettingAndroidPushDeliveryEnabled = "notifications.android_push_delivery_enabled" + SettingPushRelayURL = "notifications.push_relay_url" + SettingPushRelayDeploymentID = "notifications.push_relay_deployment_id" + SettingPushRelayAPIKey = "notifications.push_relay_api_key" + SettingPushRelayExpiresAt = "notifications.push_relay_expires_at" + SettingPushRelayKeyPrefix = "notifications.push_relay_key_prefix" + SettingPushRelayReregister = "notifications.push_relay_reregistration_required" // Discord application credentials live under the discord.* namespace // (admin-configured, alongside email.smtp_*). The secret and bot token @@ -416,6 +417,32 @@ func (s *Settings) ApplePushDeliveryEnabled(ctx context.Context) bool { return s.boolSetting(ctx, SettingApplePushDeliveryEnabled, false) } +// AndroidPushDeliveryEnabled is the Android counterpart of +// ApplePushDeliveryEnabled: it gates relay FCM sends and the capability +// endpoint's android_push availability. +func (s *Settings) AndroidPushDeliveryEnabled(ctx context.Context) bool { + return s.boolSetting(ctx, SettingAndroidPushDeliveryEnabled, false) +} + +// PushDeliveryEnabled reports whether any push platform may deliver; the +// shared dispatcher uses it as its wake-up gate. +func (s *Settings) PushDeliveryEnabled(ctx context.Context) bool { + return s.ApplePushDeliveryEnabled(ctx) || s.AndroidPushDeliveryEnabled(ctx) +} + +// EnabledPushPlatforms lists the platforms whose admin delivery toggle is on, +// in the shape ListEnabledPushByProfiles expects. +func (s *Settings) EnabledPushPlatforms(ctx context.Context) []string { + platforms := make([]string, 0, 2) + if s.ApplePushDeliveryEnabled(ctx) { + platforms = append(platforms, PushPlatformApple) + } + if s.AndroidPushDeliveryEnabled(ctx) { + platforms = append(platforms, PushPlatformAndroid) + } + return platforms +} + // PushRelayURL is the public Silo relay origin. func (s *Settings) PushRelayURL(ctx context.Context) string { value := strings.TrimRight(strings.TrimSpace(s.raw(ctx, SettingPushRelayURL)), "/") diff --git a/internal/overlays/summary.go b/internal/overlays/summary.go index b068b55e..e90cb3b9 100644 --- a/internal/overlays/summary.go +++ b/internal/overlays/summary.go @@ -51,20 +51,53 @@ func BuildSummary(files []*models.MediaFile) *Summary { func bestFile(files []*models.MediaFile) *models.MediaFile { var best *models.MediaFile - bestRank := -1 + bestRes := -1 + bestRange := -1 for _, file := range files { if file == nil { continue } - rank := resolutionRank(file.Resolution) - if best == nil || rank > bestRank { + res := resolutionRank(file.Resolution) + rng := rangeRank(file) + if best == nil || res > bestRes || (res == bestRes && rng > bestRange) { best = file - bestRank = rank + bestRes = res + bestRange = rng } } return best } +// rangeRank orders files with equal resolution by the richness of their +// dynamic-range metadata so a Dolby Vision version is not masked by a +// first-scanned file that only carries the bare HDR boolean (e.g. a stale +// pre-DV probe row). Ties keep the earliest file in the slice. +func rangeRank(file *models.MediaFile) int { + if hasDolbyVision(file.VideoTracks) { + return 3 + } + if hdrTypeFromTracks(file.VideoTracks) != "" { + return 2 + } + if file.HDR { + return 1 + } + return 0 +} + +// hasDolbyVision reports whether any track carries Dolby Vision metadata. +// Probed rows set DolbyVision and DVProfile together, but seeded/imported +// rows may carry only dv_profile or a DOVI* video_range_type. +func hasDolbyVision(tracks []models.VideoTrack) bool { + for _, track := range tracks { + if track.DolbyVision != "" || track.DVProfile > 0 || + strings.HasPrefix(track.VideoRangeType, "DOVI") { + return true + } + } + return false +} + func normalizeResolution(value string) string { cleaned := strings.ToLower(strings.TrimSpace(value)) switch cleaned { @@ -92,13 +125,7 @@ func resolutionRank(value string) int { } func normalizeHDR(file *models.MediaFile) string { - hasDV := false - for _, track := range file.VideoTracks { - if track.DolbyVision != "" { - hasDV = true - break - } - } + hasDV := hasDolbyVision(file.VideoTracks) hdrType := hdrTypeFromTracks(file.VideoTracks) switch { @@ -115,9 +142,22 @@ func normalizeHDR(file *models.MediaFile) string { } } -// hdrTypeFromTracks inspects video track color transfer to distinguish HDR variants. +// hdrTypeFromTracks distinguishes HDR variants from the scanner-derived +// video_range_type enum (HDR10, HDR10Plus, HLG, DOVIWith*) with color +// transfer as a fallback, so rows without probed color metadata (e.g. +// catalog-seeded imports) still resolve. Keep in sync with trackHdrType in +// web/src/lib/videoRange.ts. func hdrTypeFromTracks(tracks []models.VideoTrack) string { for _, track := range tracks { + rangeType := strings.TrimSpace(track.VideoRangeType) + switch { + case track.HDR10Plus || strings.Contains(rangeType, "HDR10Plus"): + return "HDR10+" + case rangeType == "HDR10" || strings.HasSuffix(rangeType, "WithHDR10"): + return "HDR10" + case rangeType == "HLG" || strings.HasSuffix(rangeType, "WithHLG"): + return "HLG" + } ct := strings.ToLower(track.ColorTransfer) switch { case strings.Contains(ct, "smpte2084"): diff --git a/internal/overlays/summary_test.go b/internal/overlays/summary_test.go index a0de3b2a..47cc79f8 100644 --- a/internal/overlays/summary_test.go +++ b/internal/overlays/summary_test.go @@ -212,6 +212,165 @@ func TestNormalizeReleaseType(t *testing.T) { } } +func TestNormalizeHDR(t *testing.T) { + cases := []struct { + name string + file *models.MediaFile + want string + }{ + {"sdr", &models.MediaFile{}, ""}, + {"bare boolean", &models.MediaFile{HDR: true}, "HDR"}, + {"hdr10 via color transfer", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{ColorTransfer: "smpte2084"}}, + }, "HDR10"}, + {"hlg via color transfer", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{ColorTransfer: "arib-std-b67"}}, + }, "HLG"}, + {"dv only", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{DolbyVision: "Profile 5"}}, + }, "DV"}, + {"dv with hdr10 base layer", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{ + DolbyVision: "Profile 8", + ColorTransfer: "smpte2084", + }}, + }, "DV HDR10"}, + {"dv via profile number only", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{DVProfile: 5}}, + }, "DV"}, + {"dv via DOVI range type only", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{ + VideoRangeType: "DOVIWithHDR10", + ColorTransfer: "smpte2084", + }}, + }, "DV HDR10"}, + {"hdr10 via range type without color transfer", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{VideoRangeType: "HDR10"}}, + }, "HDR10"}, + {"hlg via range type without color transfer", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{VideoRangeType: "HLG"}}, + }, "HLG"}, + {"hdr10+ via flag", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{ + HDR10Plus: true, + ColorTransfer: "smpte2084", + }}, + }, "HDR10+"}, + {"hdr10+ via range type", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{VideoRangeType: "HDR10Plus"}}, + }, "HDR10+"}, + {"dv with hdr10+ base layer", &models.MediaFile{ + HDR: true, + VideoTracks: []models.VideoTrack{{VideoRangeType: "DOVIWithELHDR10Plus"}}, + }, "DV HDR10+"}, + {"sdr range type stays sdr", &models.MediaFile{ + VideoTracks: []models.VideoTrack{{VideoRangeType: "SDR"}}, + }, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := normalizeHDR(tc.file); got != tc.want { + t.Errorf("normalizeHDR = %q, want %q", got, tc.want) + } + }) + } +} + +func TestBestFileRangeTieBreaking(t *testing.T) { + genericHDR := &models.MediaFile{Resolution: "2160p", HDR: true} + explicitHDR10 := &models.MediaFile{ + Resolution: "2160p", + HDR: true, + VideoTracks: []models.VideoTrack{{ColorTransfer: "smpte2084"}}, + } + dv := &models.MediaFile{ + Resolution: "2160p", + HDR: true, + VideoTracks: []models.VideoTrack{{ + DolbyVision: "Profile 8", + ColorTransfer: "smpte2084", + }}, + } + sdr := &models.MediaFile{Resolution: "2160p"} + dvProfileOnly := &models.MediaFile{ + Resolution: "2160p", + HDR: true, + VideoTracks: []models.VideoTrack{{DVProfile: 5}}, + } + lowResDV := &models.MediaFile{ + Resolution: "1080p", + HDR: true, + VideoTracks: []models.VideoTrack{{DolbyVision: "Profile 5"}}, + } + rangeTypeHDR10 := &models.MediaFile{ + Resolution: "2160p", + HDR: true, + VideoTracks: []models.VideoTrack{{VideoRangeType: "HDR10"}}, + } + + cases := []struct { + name string + files []*models.MediaFile + want *models.MediaFile + }{ + {"nil files ignored", []*models.MediaFile{nil, genericHDR}, genericHDR}, + {"dv beats first-scanned generic hdr at same resolution", + []*models.MediaFile{genericHDR, dv}, dv}, + {"dv beats explicit hdr10 at same resolution", + []*models.MediaFile{explicitHDR10, dv}, dv}, + {"dv via profile number only still outranks generic hdr", + []*models.MediaFile{genericHDR, dvProfileOnly}, dvProfileOnly}, + {"range-type-only hdr10 outranks bare boolean", + []*models.MediaFile{genericHDR, rangeTypeHDR10}, rangeTypeHDR10}, + {"explicit hdr10 beats bare boolean at same resolution", + []*models.MediaFile{genericHDR, explicitHDR10}, explicitHDR10}, + {"bare boolean beats sdr at same resolution", + []*models.MediaFile{sdr, genericHDR}, genericHDR}, + {"higher resolution still wins over lower-res dv", + []*models.MediaFile{lowResDV, genericHDR}, genericHDR}, + {"full tie keeps earliest file", + []*models.MediaFile{genericHDR, {Resolution: "2160p", HDR: true}}, genericHDR}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := bestFile(tc.files); got != tc.want { + t.Errorf("bestFile picked %+v, want %+v", got, tc.want) + } + }) + } +} + +func TestBuildSummaryPrefersDolbyVisionSibling(t *testing.T) { + genericHDR := &models.MediaFile{Resolution: "2160p", HDR: true, CodecVideo: "hevc"} + dv := &models.MediaFile{ + Resolution: "2160p", + HDR: true, + CodecVideo: "hevc", + VideoTracks: []models.VideoTrack{{ + Codec: "hevc", + DolbyVision: "Profile 8", + ColorTransfer: "smpte2084", + }}, + } + got := BuildSummary([]*models.MediaFile{genericHDR, dv}) + if got == nil { + t.Fatal("expected non-nil summary") + } + if got.HDR != "DV HDR10" { + t.Errorf("HDR = %q, want %q", got.HDR, "DV HDR10") + } +} + func TestBuildSummaryAggregatesNewFields(t *testing.T) { file := &models.MediaFile{ Resolution: "1080p", diff --git a/internal/pathscope/pathscope.go b/internal/pathscope/pathscope.go index 05933b89..5fbd3558 100644 --- a/internal/pathscope/pathscope.go +++ b/internal/pathscope/pathscope.go @@ -1,6 +1,7 @@ package pathscope import ( + "fmt" "path/filepath" "strings" ) @@ -24,3 +25,21 @@ func PrefixLike(pathPrefix string) string { } return likeEscaper.Replace(clean) + string(filepath.Separator) + "%" } + +// CoverageClauses builds one SQL predicate per root matching rows whose +// column value lies at or under that root, using the exact-match + escaped +// prefix-LIKE shape (`column = $n OR column LIKE $n+1 ESCAPE '\'`). The +// prefix pattern ends with a path separator, so a sibling root that merely +// shares a string prefix (/mnt/movies2 vs /mnt/movies) never matches. +// Placeholder numbering starts at firstArg; the returned args bind pairwise +// (root, prefix pattern) in clause order. +func CoverageClauses(column string, roots []string, firstArg int) ([]string, []any) { + clauses := make([]string, 0, len(roots)) + args := make([]any, 0, len(roots)*2) + for i, root := range roots { + pathArg := firstArg + i*2 + clauses = append(clauses, fmt.Sprintf(`(%s = $%d OR %s LIKE $%d ESCAPE '\')`, column, pathArg, column, pathArg+1)) + args = append(args, root, PrefixLike(root)) + } + return clauses, args +} diff --git a/internal/playback/capabilities_v3.go b/internal/playback/capabilities_v3.go new file mode 100644 index 00000000..59b34c3a --- /dev/null +++ b/internal/playback/capabilities_v3.go @@ -0,0 +1,315 @@ +package playback + +import ( + "strconv" + "strings" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func SourceDescriptorFromFileV3(file *models.MediaFile, audioIndex int) SourceDescriptorV3 { + if file == nil { + return SourceDescriptorV3{DVEnhancementLayer: EnhancementUnknownV3} + } + source := SourceDescriptorV3{ + MediaFileID: file.ID, + Container: normalizeCodecV3(file.Container), + VideoCodec: normalizeCodecV3(file.CodecVideo), + AudioCodec: normalizeCodecV3(file.CodecAudio), + AudioChannels: file.AudioChannels, + BitrateKbps: normalizeBitrateKbpsV3(file.Bitrate), + DVEnhancementLayer: EnhancementNoneV3, + } + if len(file.VideoTracks) > 0 { + track := file.VideoTracks[0] + source.VideoCodec = firstNonEmptyV3(normalizeCodecV3(track.Codec), source.VideoCodec) + source.VideoProfile = strings.ToLower(strings.TrimSpace(track.Profile)) + source.VideoLevel = track.Level + source.BitDepth = models.NormalizeVideoBitDepth(track.BitDepth, track.PixelFormat, track.Profile) + source.ColorRange = normalizeColorRangeV3(track.ColorRange) + source.Width = track.Width + source.Height = track.Height + source.FrameRate = parseFrameRateV3(track.FrameRate) + if track.Bitrate > 0 { + source.BitrateKbps = normalizeBitrateKbpsV3(track.Bitrate) + } + source.DynamicRange = normalizeDynamicRangeV3(track) + source.HDR10Plus = track.HDR10Plus || strings.Contains(strings.ToLower(track.VideoRangeType), "hdr10+") + source.DVProfile = track.DVProfile + source.DVBLCompatID = track.DVBLCompatID + switch EnhancementLayerV3(strings.ToLower(track.DVEnhancementLayer)) { + case EnhancementNoneV3, EnhancementMELV3, EnhancementFELV3, EnhancementUnknownV3: + source.DVEnhancementLayer = EnhancementLayerV3(strings.ToLower(track.DVEnhancementLayer)) + case "": + // Legacy rows predate the explicit enhancement-layer fields. A + // Profile 7 DOVIWithEL label proves an EL exists but cannot prove + // MEL versus FEL, so keep it unknown rather than misclassifying it + // as a safe single-layer stream. + legacyProfile7EL := track.DVProfile == 7 && strings.Contains(strings.ToLower(track.VideoRangeType), "withel") + if track.DVELPresent || legacyProfile7EL { + source.DVEnhancementLayer = EnhancementUnknownV3 + } else { + source.DVEnhancementLayer = EnhancementNoneV3 + } + default: + source.DVEnhancementLayer = EnhancementUnknownV3 + } + } + if source.Width == 0 || source.Height == 0 { + source.Width, source.Height = dimensionsFromResolutionV3(file.Resolution) + } + if audioIndex >= 0 && audioIndex < len(file.AudioTracks) { + track := file.AudioTracks[audioIndex] + source.AudioCodec = firstNonEmptyV3(normalizeCodecV3(track.Codec), source.AudioCodec) + source.AudioChannels = track.Channels + source.AudioLayout = normalizeLayoutV3(track.Layout) + } + if source.DynamicRange == "" { + if file.HDR { + source.DynamicRange = "hdr_unknown" + } else { + source.DynamicRange = "sdr" + } + } + return source +} + +func normalizeColorRangeV3(value string) string { + switch normalized := strings.ToLower(strings.TrimSpace(value)); normalized { + case "tv", "pc", "unknown": + return normalized + default: + return "" + } +} + +func detailedVideoEligibleV3(source SourceDescriptorV3, request StartRequestV3) bool { + if !HasFeatureV3(request.ClientFeatures, FeatureDetailedDecodeV3) && !HasFeatureV3(request.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) { + return false + } + if !detailedVideoEvidenceCompleteV3(source) { + return false + } + for _, capability := range request.Capabilities.VideoDecode { + if !strings.EqualFold(capability.Codec, source.VideoCodec) || !capability.Hardware { + continue + } + if len(capability.Profiles) > 0 && !containsFoldV3(capability.Profiles, source.VideoProfile) { + continue + } + if len(capability.Levels) > 0 && !containsAtLeastV3(capability.Levels, source.VideoLevel) { + continue + } + if len(capability.BitDepths) > 0 && !containsIntV3(capability.BitDepths, source.BitDepth) { + continue + } + if capability.MaxWidth > 0 && source.Width > capability.MaxWidth || capability.MaxHeight > 0 && source.Height > capability.MaxHeight || capability.MaxFrameRate > 0 && source.FrameRate > capability.MaxFrameRate || capability.MaxBitrateKbps > 0 && source.BitrateKbps > capability.MaxBitrateKbps { + continue + } + return true + } + return false +} + +func detailedVideoEvidenceCompleteV3(source SourceDescriptorV3) bool { + return source.VideoCodec != "" && + source.VideoProfile != "" && + source.VideoLevel > 0 && + source.BitDepth > 0 && + source.Width > 0 && + source.Height > 0 && + source.FrameRate > 0 && + source.BitrateKbps > 0 +} + +func outputRangeEligibleV3(source SourceDescriptorV3, request StartRequestV3) (bool, VideoClaimsV3) { + hdr := request.ClientPlaybackContext.Output.HDRDetails + if hdr == nil { + hdr = request.Capabilities.HDRDetails + } + claims := VideoClaimsV3{} + switch source.DynamicRange { + case "", "sdr": + return true, claims + case "hdr10": + claims.HDR10 = hdr != nil && hdr.HDR10 + return claims.HDR10, claims + case "hdr_unknown": + // Legacy rows only recorded a file-level HDR flag without per-track + // range metadata. HDR10 is by far the most common static-HDR range, so + // an HDR10-capable output treats the source as HDR10 instead of + // refusing playback outright; the planner attaches a degradation + // warning for these assumed-range plans. + claims.HDR10 = hdr != nil && hdr.HDR10 + return claims.HDR10, claims + case "hdr10_plus": + claims.HDR10Plus = hdr != nil && hdr.HDR10Plus + return claims.HDR10Plus, claims + case "hlg": + claims.HLG = hdr != nil && hdr.HLG + return claims.HLG, claims + case "dolby_vision": + if source.DVProfile == 7 && source.DVEnhancementLayer == EnhancementUnknownV3 { + claims.DolbyVisionReason = "profile_7_enhancement_layer_unknown" + return false, claims + } + if hdr != nil && containsIntV3(hdr.DolbyVisionProfiles, source.DVProfile) { + claims.DolbyVision = true + claims.DolbyVisionReason = "native_profile_supported" + return true, claims + } + claims.DolbyVisionReason = "native_profile_not_supported" + return false, claims + default: + return false, claims + } +} + +func clientSupportsHDR10V3(request StartRequestV3) bool { + hdr := request.ClientPlaybackContext.Output.HDRDetails + if hdr == nil { + hdr = request.Capabilities.HDRDetails + } + return hdr != nil && hdr.HDR10 +} + +func audioEligibilityV3(source SourceDescriptorV3, request StartRequestV3) (copyOK, passthrough bool, claim AudioClaimsV3) { + claim.Codec = source.AudioCodec + passthroughCaps := request.ClientPlaybackContext.Output.AudioPassthrough + if passthroughCaps == nil { + passthroughCaps = request.Capabilities.AudioPassthrough + } + if passthroughCaps != nil && containsFoldV3(passthroughCaps.PassthroughCodecs, source.AudioCodec) && + (HasFeatureV3(request.ClientFeatures, FeatureLayoutPassthrough) || HasFeatureV3(request.ClientPlaybackContext.Features, FeatureLayoutPassthrough)) { + for _, entry := range passthroughCaps.Entries { + if !strings.EqualFold(entry.Codec, source.AudioCodec) || len(entry.ChannelCounts) == 0 || len(entry.Layouts) == 0 || + !containsIntV3(entry.ChannelCounts, source.AudioChannels) || !containsFoldV3(entry.Layouts, source.AudioLayout) { + continue + } + claim.Passthrough = true + claim.AtmosPreserved = strings.Contains(strings.ToLower(source.AudioLayout), "joc") || strings.Contains(strings.ToLower(source.AudioLayout), "atmos") + claim.Reason = "sink_passthrough_validated" + return true, true, claim + } + } + if containsFoldV3(request.Capabilities.CodecsAudio, source.AudioCodec) { + claim.Reason = "client_decode_supported" + return true, false, claim + } + if passthroughCaps != nil && containsFoldV3(passthroughCaps.PassthroughCodecs, source.AudioCodec) { + claim.Reason = "passthrough_layout_unsupported" + } else { + claim.Reason = "audio_codec_unsupported" + } + return false, false, claim +} + +func normalizeDynamicRangeV3(track models.VideoTrack) string { + if track.DVProfile > 0 || strings.Contains(strings.ToLower(track.VideoRangeType), "dovi") || strings.Contains(strings.ToLower(track.DolbyVision), "dolby") { + return "dolby_vision" + } + if track.HDR10Plus || strings.Contains(strings.ToLower(track.VideoRangeType), "hdr10+") { + return "hdr10_plus" + } + joined := strings.ToLower(strings.Join([]string{track.VideoRange, track.VideoRangeType, track.ColorTransfer}, " ")) + if strings.Contains(joined, "hlg") || strings.Contains(joined, "arib-std-b67") { + return "hlg" + } + if strings.Contains(joined, "hdr") || strings.Contains(joined, "smpte2084") || strings.Contains(joined, "pq") { + return "hdr10" + } + if joined == " " || strings.TrimSpace(joined) == "" { + return "" + } + return "sdr" +} + +func parseFrameRateV3(value string) float64 { + value = strings.TrimSpace(value) + if value == "" { + return 0 + } + if parts := strings.Split(value, "/"); len(parts) == 2 { + n, nErr := strconv.ParseFloat(parts[0], 64) + d, dErr := strconv.ParseFloat(parts[1], 64) + if nErr == nil && dErr == nil && d != 0 { + return n / d + } + } + v, _ := strconv.ParseFloat(value, 64) + return v +} + +func normalizeBitrateKbpsV3(value int) int { + if value > 10_000_000 { + return value / 1000 + } + return value +} + +func dimensionsFromResolutionV3(value string) (int, int) { + switch strings.ToLower(strings.TrimSpace(value)) { + case "4320p", "8k": + return 7680, 4320 + case "2160p", "4k", "uhd": + return 3840, 2160 + case "1080p", "fhd": + return 1920, 1080 + case "720p", "hd": + return 1280, 720 + case "480p", "sd": + return 854, 480 + default: + return 0, 0 + } +} + +func normalizeCodecV3(value string) string { + v := strings.ToLower(strings.TrimSpace(value)) + switch v { + case "h265", "h.265", "x265": + return "hevc" + case "h264", "h.264", "avc", "x264": + return "h264" + case "eac3", "e-ac-3", "ec-3": + return "eac3" + case "truehd", "mlp fba": + return "truehd" + default: + return v + } +} + +func normalizeLayoutV3(value string) string { return strings.ToLower(strings.TrimSpace(value)) } +func firstNonEmptyV3(values ...string) string { + for _, v := range values { + if v != "" { + return v + } + } + return "" +} +func containsFoldV3(values []string, wanted string) bool { + for _, v := range values { + if strings.EqualFold(strings.TrimSpace(v), strings.TrimSpace(wanted)) { + return true + } + } + return false +} +func containsIntV3(values []int, wanted int) bool { + for _, v := range values { + if v == wanted { + return true + } + } + return false +} +func containsAtLeastV3(values []int, wanted int) bool { + for _, v := range values { + if v >= wanted { + return true + } + } + return false +} diff --git a/internal/playback/copy_seek_anchor.go b/internal/playback/copy_seek_anchor.go new file mode 100644 index 00000000..051aabfb --- /dev/null +++ b/internal/playback/copy_seek_anchor.go @@ -0,0 +1,149 @@ +package playback + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "math" + "os/exec" + "strconv" + "strings" + "time" + + "golang.org/x/sync/singleflight" +) + +const ( + maxConcurrentCopySeekProbes = 4 + copySeekProbeTimeout = 15 * time.Second +) + +var ( + copySeekProbeGroup singleflight.Group + copySeekProbeSlots = make(chan struct{}, maxConcurrentCopySeekProbes) +) + +type copySeekProbePacket struct { + PTSSeconds string `json:"pts_time"` + DTSSeconds string `json:"dts_time"` + Flags string `json:"flags"` +} + +type copySeekProbeOutput struct { + Packets []copySeekProbePacket `json:"packets"` +} + +type copySeekAnchor struct { + seconds float64 + segment int +} + +// ResolveCopySeekAnchor returns the keyframe timestamp FFmpeg's input seek will +// actually use for a copy-video restart. FFmpeg cannot discard the pre-roll +// between that keyframe and requestedSeekSeconds while preserving -c:v copy, +// so callers need both timestamps: requestedSeekSeconds remains the -ss input, +// while the returned anchor defines the stream's real timeline origin. +// +// The tiny read interval is intentional. ffprobe asks the demuxer to seek to +// requestedSeekSeconds, which lands on the same preceding key packet FFmpeg +// uses, then emits only the packets at that seek point instead of scanning the +// media from the beginning. +func ResolveCopySeekAnchor( + ctx context.Context, + ffmpegPath string, + inputPath string, + requestedSeekSeconds float64, + segmentDuration int, +) (float64, int, error) { + if requestedSeekSeconds <= 0 { + return 0, 0, nil + } + if strings.TrimSpace(inputPath) == "" { + return 0, 0, fmt.Errorf("resolve copy seek anchor: empty input path") + } + if segmentDuration <= 0 { + segmentDuration = DefaultSegmentDuration + } + + key := strings.Join([]string{ + ffprobePathFromFFmpeg(ffmpegPath), + inputPath, + strconv.FormatFloat(requestedSeekSeconds, 'f', 6, 64), + strconv.Itoa(segmentDuration), + }, "\x00") + resultCh := copySeekProbeGroup.DoChan(key, func() (any, error) { + probeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), copySeekProbeTimeout) + defer cancel() + select { + case copySeekProbeSlots <- struct{}{}: + defer func() { <-copySeekProbeSlots }() + case <-probeCtx.Done(): + return copySeekAnchor{}, probeCtx.Err() + } + seconds, segment, err := resolveCopySeekAnchor(probeCtx, ffmpegPath, inputPath, requestedSeekSeconds, segmentDuration) + return copySeekAnchor{seconds: seconds, segment: segment}, err + }) + + select { + case <-ctx.Done(): + return 0, 0, ctx.Err() + case result := <-resultCh: + if result.Err != nil { + return 0, 0, result.Err + } + anchor := result.Val.(copySeekAnchor) + return anchor.seconds, anchor.segment, nil + } +} + +func resolveCopySeekAnchor( + ctx context.Context, + ffmpegPath string, + inputPath string, + requestedSeekSeconds float64, + segmentDuration int, +) (float64, int, error) { + + interval := strconv.FormatFloat(requestedSeekSeconds, 'f', 6, 64) + "%+0.001" + cmd := exec.CommandContext(ctx, ffprobePathFromFFmpeg(ffmpegPath), + "-v", "error", + "-select_streams", "v:0", + "-read_intervals", interval, + "-show_entries", "packet=pts_time,dts_time,flags", + "-of", "json", + inputPath, + ) + var stdout bytes.Buffer + stderr := newBoundedTailBuffer(stderrTailMaxBytes) + cmd.Stdout = &stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + if tail := truncateStderr(stderr.String()); tail != "" { + return 0, 0, fmt.Errorf("resolve copy seek anchor: ffprobe failed: %w (stderr: %s)", err, tail) + } + return 0, 0, fmt.Errorf("resolve copy seek anchor: ffprobe failed: %w", err) + } + + var output copySeekProbeOutput + if err := json.Unmarshal(stdout.Bytes(), &output); err != nil { + return 0, 0, fmt.Errorf("resolve copy seek anchor: decode ffprobe output: %w", err) + } + for _, packet := range output.Packets { + if !strings.Contains(packet.Flags, "K") { + continue + } + timestamp := packet.PTSSeconds + if timestamp == "" || strings.EqualFold(timestamp, "N/A") { + timestamp = packet.DTSSeconds + } + anchor, err := strconv.ParseFloat(timestamp, 64) + if err != nil || math.IsNaN(anchor) || math.IsInf(anchor, 0) { + continue + } + anchor = math.Max(0, anchor) + return anchor, int(anchor / float64(segmentDuration)), nil + } + + return 0, 0, fmt.Errorf("resolve copy seek anchor: ffprobe returned no key packet") +} diff --git a/internal/playback/copy_seek_anchor_test.go b/internal/playback/copy_seek_anchor_test.go new file mode 100644 index 00000000..e6313d2e --- /dev/null +++ b/internal/playback/copy_seek_anchor_test.go @@ -0,0 +1,174 @@ +package playback + +import ( + "context" + "fmt" + "math" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +func TestResolveCopySeekAnchorUsesKeyPacketTimestamp(t *testing.T) { + dir := t.TempDir() + ffmpegPath := filepath.Join(dir, "ffmpeg") + ffprobePath := filepath.Join(dir, "ffprobe") + if err := os.WriteFile(ffmpegPath, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatalf("write fake ffmpeg: %v", err) + } + probe := `#!/bin/sh +printf '%s' '{"packets":[{"pts_time":"N/A","dts_time":"14.500000","flags":"K__"}]}' +` + if err := os.WriteFile(ffprobePath, []byte(probe), 0o755); err != nil { + t.Fatalf("write fake ffprobe: %v", err) + } + + anchor, segment, err := ResolveCopySeekAnchor(context.Background(), ffmpegPath, "/media/movie.mkv", 18.261, 2) + if err != nil { + t.Fatalf("ResolveCopySeekAnchor: %v", err) + } + if anchor != 14.5 || segment != 7 { + t.Fatalf("resolved anchor = %v, segment = %d; want 14.5, 7", anchor, segment) + } +} + +func TestResolveCopySeekAnchorCoalescesMatchingConcurrentProbes(t *testing.T) { + dir := t.TempDir() + ffmpegPath := filepath.Join(dir, "ffmpeg") + ffprobePath := filepath.Join(dir, "ffprobe") + countPath := filepath.Join(dir, "probe-count") + if err := os.WriteFile(ffmpegPath, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatalf("write fake ffmpeg: %v", err) + } + probe := "#!/bin/sh\n" + + "printf x >> \"" + countPath + "\"\n" + + "sleep 0.1\n" + + "printf '%s' '{\"packets\":[{\"pts_time\":\"16.000000\",\"flags\":\"K__\"}]}'\n" + if err := os.WriteFile(ffprobePath, []byte(probe), 0o755); err != nil { + t.Fatalf("write fake ffprobe: %v", err) + } + + start := make(chan struct{}) + errs := make(chan error, 2) + var ready sync.WaitGroup + ready.Add(2) + for range 2 { + go func() { + ready.Done() + <-start + anchor, segment, err := ResolveCopySeekAnchor(context.Background(), ffmpegPath, "/media/movie.mkv", 18, 2) + if err == nil && (anchor != 16 || segment != 8) { + err = fmt.Errorf("resolved anchor = %v, segment = %d", anchor, segment) + } + errs <- err + }() + } + ready.Wait() + close(start) + for range 2 { + if err := <-errs; err != nil { + t.Fatal(err) + } + } + count, err := os.ReadFile(countPath) + if err != nil { + t.Fatalf("read probe count: %v", err) + } + if string(count) != "x" { + t.Fatalf("ffprobe executions = %d, want 1", len(count)) + } +} + +func TestResolveCopySeekAnchorMatchesRealLongGOPHEVC(t *testing.T) { + if testing.Short() { + t.Skip("real FFmpeg integration test") + } + ffmpegPath, err := exec.LookPath("ffmpeg") + if err != nil { + t.Skip("ffmpeg is not installed") + } + if _, err := exec.LookPath(ffprobePathFromFFmpeg(ffmpegPath)); err != nil { + t.Skip("ffprobe is not installed beside ffmpeg") + } + encoders, err := exec.Command(ffmpegPath, "-hide_banner", "-encoders").CombinedOutput() + if err != nil || !strings.Contains(string(encoders), "libx265") { + t.Skip("ffmpeg does not provide libx265") + } + + sourcePath := filepath.Join(t.TempDir(), "long-gop-hevc.mkv") + encodeCtx, cancelEncode := context.WithTimeout(context.Background(), 20*time.Second) + defer cancelEncode() + encode := exec.CommandContext(encodeCtx, ffmpegPath, + "-hide_banner", "-loglevel", "error", + "-f", "lavfi", "-i", "testsrc2=size=320x180:rate=24", + "-t", "22", + "-c:v", "libx265", "-preset", "ultrafast", + "-x265-params", "keyint=240:min-keyint=240:scenecut=0:log-level=error:pools=1:frame-threads=1", + "-an", "-y", sourcePath, + ) + if output, err := encode.CombinedOutput(); err != nil { + t.Fatalf("generate long-GOP HEVC fixture: %v\n%s", err, output) + } + + probeCtx, cancelProbe := context.WithTimeout(context.Background(), 5*time.Second) + defer cancelProbe() + anchor, segment, err := ResolveCopySeekAnchor(probeCtx, ffmpegPath, sourcePath, 18.261, 2) + if err != nil { + t.Fatalf("ResolveCopySeekAnchor: %v", err) + } + if math.Abs(anchor-10) > 0.001 || segment != 5 { + t.Fatalf("resolved anchor = %v, segment = %d; want 10, 5", anchor, segment) + } + + outputDir := filepath.Join(t.TempDir(), "hls") + if err := os.MkdirAll(outputDir, 0o755); err != nil { + t.Fatalf("create HLS output: %v", err) + } + opts := TranscodeOpts{ + InputPath: sourcePath, + OutputDir: outputDir, + SessionID: "copy-anchor-integration", + SourceVideoCodec: "hevc", + SeekSeconds: 18.261, + StreamOriginSeconds: anchor, + CopySeekAnchorResolved: true, + TargetCodecVideo: "copy", + TargetCodecAudio: "copy", + SegmentDuration: 2, + StartSegmentNumber: segment, + FFmpegPath: ffmpegPath, + } + packageCtx, cancelPackage := context.WithTimeout(context.Background(), 10*time.Second) + defer cancelPackage() + packageHLS := exec.CommandContext(packageCtx, ffmpegPath, buildFFmpegArgs(opts)...) + if output, err := packageHLS.CombinedOutput(); err != nil { + t.Fatalf("package copy HLS: %v\n%s", err, output) + } + + manifestPath := filepath.Join(outputDir, "stream.m3u8") + manifest, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatalf("read copy HLS manifest: %v", err) + } + if !strings.Contains(string(manifest), "#EXT-X-MEDIA-SEQUENCE:5") { + t.Fatalf("copy HLS media sequence does not use resolved anchor:\n%s", manifest) + } + firstFrame, err := exec.Command(ffprobePathFromFFmpeg(ffmpegPath), + "-v", "error", + "-select_streams", "v:0", + "-read_intervals", "%+0.1", + "-show_entries", "frame=best_effort_timestamp_time,key_frame", + "-of", "csv=p=0", + manifestPath, + ).CombinedOutput() + if err != nil { + t.Fatalf("probe copy HLS first frame: %v\n%s", err, firstFrame) + } + if !strings.Contains(string(firstFrame), "10.000000") { + t.Fatalf("copy HLS first frame = %q, want resolved 10-second keyframe", firstFrame) + } +} diff --git a/internal/playback/device_quirks_v3.go b/internal/playback/device_quirks_v3.go new file mode 100644 index 00000000..8f3545eb --- /dev/null +++ b/internal/playback/device_quirks_v3.go @@ -0,0 +1,113 @@ +package playback + +import "strings" + +const ( + QuirkFireTVAFTKRTHigh10V3 = "android.fire_tv.aftkrt.h264_high10_l52_v1" + QuirkFireTVAFTKRTEAC3HLSV3 = "android.fire_tv.aftkrt.eac3_7_1_hls_audio_adapt_v1" + QuirkFireTVDV8HDR10PlusV3 = "android.fire_tv.dv8_hdr10plus_sei_v1" +) + +func high10DecodeOverrideV3(source SourceDescriptorV3, request StartRequestV3) (*AppliedQuirkV3, bool) { + if !deviceQuirkProtocolAvailableV3(request) || !isAmazonModelV3(request, "AFTKRT") || + !strings.EqualFold(source.VideoCodec, "h264") || !isHigh10ProfileV3(source.VideoProfile) || + source.BitDepth != 10 || source.VideoLevel <= 0 || source.VideoLevel > 52 || + source.Width <= 0 || source.Height <= 0 || source.Width > 1920 || source.Height > 1080 { + return nil, false + } + for _, capability := range request.Capabilities.VideoDecode { + if !capability.Hardware || !strings.EqualFold(capability.Codec, "h264") || + capability.MaxWidth > 0 && source.Width > capability.MaxWidth || + capability.MaxHeight > 0 && source.Height > capability.MaxHeight || + capability.MaxFrameRate > 0 && source.FrameRate > capability.MaxFrameRate || + capability.MaxBitrateKbps > 0 && source.BitrateKbps > capability.MaxBitrateKbps { + continue + } + quirk := AppliedQuirkV3{ + ID: QuirkFireTVAFTKRTHigh10V3, + RegistryRevision: DeviceQuirkRegistryRevisionV3, + Action: "positive_decode_override", + Reason: "Exact AFTKRT evidence supports hardware H.264 High 10 through level 5.2 at 1080p.", + } + return &quirk, true + } + return nil, false +} + +func hlsEAC3AudioCorrectionV3(source SourceDescriptorV3, request StartRequestV3) (*AppliedQuirkV3, bool) { + if !deviceQuirkProtocolAvailableV3(request) || !isAmazonModelV3(request, "AFTKRT") || + !strings.EqualFold(source.AudioCodec, "eac3") || source.AudioChannels != 8 { + return nil, false + } + quirk := AppliedQuirkV3{ + ID: QuirkFireTVAFTKRTEAC3HLSV3, + RegistryRevision: DeviceQuirkRegistryRevisionV3, + Action: "audio_only_transcode", + Reason: "AFTKRT cannot reliably consume eight-channel E-AC-3 from an HLS MPEG-TS route.", + } + return &quirk, true +} + +func dv8HDR10PlusRuntimeCorrectionV3(source SourceDescriptorV3, request StartRequestV3, engine EngineV3) (*AppliedQuirkV3, bool) { + if !deviceQuirkProtocolAvailableV3(request) || source.DVProfile != 8 || !source.HDR10Plus || + !isAmazonFireTVV3(request) || !engineSupportsFeatureV3(request, engine, ClientDV8HDR10PlusSanitizerV3) { + return nil, false + } + quirk := AppliedQuirkV3{ + ID: QuirkFireTVDV8HDR10PlusV3, + RegistryRevision: DeviceQuirkRegistryRevisionV3, + Action: "client_runtime_correction", + Reason: "The native Fire TV Dolby Vision path requires HDR10+ dynamic-metadata SEI removal for hybrid Profile 8 samples.", + } + return &quirk, true +} + +func applyCopiedVideoQuirksV3(plan *PlanV3, source SourceDescriptorV3, request StartRequestV3, high10 *AppliedQuirkV3) { + if high10 != nil { + appendAppliedQuirkV3(plan, *high10, "") + } + if quirk, ok := dv8HDR10PlusRuntimeCorrectionV3(source, request, plan.Engine); ok { + appendAppliedQuirkV3(plan, *quirk, ClientDV8HDR10PlusSanitizerV3) + } +} + +func appendAppliedQuirkV3(plan *PlanV3, quirk AppliedQuirkV3, runtimeCorrection string) { + for _, existing := range plan.AppliedQuirks { + if existing.ID == quirk.ID && existing.RegistryRevision == quirk.RegistryRevision { + return + } + } + plan.AppliedQuirks = append(plan.AppliedQuirks, quirk) + if runtimeCorrection != "" && !containsFoldV3(plan.RuntimeCorrections, runtimeCorrection) { + plan.RuntimeCorrections = append(plan.RuntimeCorrections, runtimeCorrection) + } +} + +func deviceQuirkProtocolAvailableV3(request StartRequestV3) bool { + // Like every other dual-location feature check, either advertisement + // location proves the client speaks the quirk protocol. + return HasFeatureV3(request.ClientFeatures, FeatureDeviceQuirksV3) || + HasFeatureV3(request.ClientPlaybackContext.Features, FeatureDeviceQuirksV3) +} + +func engineSupportsFeatureV3(request StartRequestV3, engine EngineV3, feature string) bool { + value, ok := request.ClientPlaybackContext.Engines[string(engine)] + return ok && value.Enabled && value.SupportedOnDevice && HasFeatureV3(value.Features, feature) +} + +func isAmazonModelV3(request StartRequestV3, model string) bool { + device := request.ClientPlaybackContext.Device + return strings.EqualFold(device.Model, model) && + (strings.EqualFold(device.Manufacturer, "Amazon") || strings.EqualFold(device.Brand, "Amazon")) +} + +func isAmazonFireTVV3(request StartRequestV3) bool { + device := request.ClientPlaybackContext.Device + return strings.HasPrefix(strings.ToUpper(strings.TrimSpace(device.Model)), "AFT") && + (strings.EqualFold(device.Manufacturer, "Amazon") || strings.EqualFold(device.Brand, "Amazon")) +} + +func isHigh10ProfileV3(profile string) bool { + normalized := strings.NewReplacer(" ", "", "-", "", "_", "").Replace(strings.ToLower(profile)) + return normalized == "high10" || normalized == "high10intra" +} diff --git a/internal/playback/device_quirks_v3_test.go b/internal/playback/device_quirks_v3_test.go new file mode 100644 index 00000000..6ff6c8fe --- /dev/null +++ b/internal/playback/device_quirks_v3_test.go @@ -0,0 +1,158 @@ +package playback + +import ( + "bytes" + "encoding/json" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func TestAFTKRTHigh10OverrideIsExactAndPreservesVideo(t *testing.T) { + file := &models.MediaFile{ + ID: 42, FilePath: "/media/high10.mkv", Container: "mkv", CodecVideo: "h264", CodecAudio: "aac", + Resolution: "1080p", Bitrate: 12_000, AudioChannels: 2, + VideoTracks: []models.VideoTrack{{Codec: "h264", Profile: "High 10", Level: 52, Width: 1920, Height: 1080, FrameRate: "24000/1001", Bitrate: 12_000, BitDepth: 10, VideoRange: "SDR", VideoRangeType: "SDR"}}, + AudioTracks: []models.AudioTrack{{Codec: "aac", Channels: 2, Layout: "stereo"}}, + } + req := quirkRequestV3() + req.Capabilities.Containers = []string{"mkv"} + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "h264", Profiles: []string{"high"}, Levels: []int{51}, BitDepths: []int{8}, MaxWidth: 1920, MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 20_000, Hardware: true}} + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: false}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || result.PlayMethod != PlayDirect || len(result.Plan.AppliedQuirks) != 1 || result.Plan.AppliedQuirks[0].ID != QuirkFireTVAFTKRTHigh10V3 { + t.Fatalf("result = %#v", result) + } + + req.ClientPlaybackContext.Device.Model = "AFTKA" + withoutExactEvidence := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: false}, Registry: testTransformationRegistryV3()}) + if withoutExactEvidence.Plan != nil && withoutExactEvidence.Plan.Delivery == DeliveryOriginalHTTPV3 { + t.Fatalf("untested model received override: %#v", withoutExactEvidence.Plan) + } +} + +func TestAFTKRTEAC3HLSCorrectionTranscodesAudioOnly(t *testing.T) { + file := &models.MediaFile{ + ID: 42, FilePath: "/media/eac3.avi", Container: "avi", CodecVideo: "h264", CodecAudio: "eac3", + Resolution: "1080p", Bitrate: 12_000, AudioChannels: 8, + VideoTracks: []models.VideoTrack{{Codec: "h264", Profile: "High", Level: 42, Width: 1920, Height: 1080, FrameRate: "24", Bitrate: 12_000, BitDepth: 8, VideoRange: "SDR", VideoRangeType: "SDR"}}, + AudioTracks: []models.AudioTrack{{Codec: "eac3", Channels: 8, Layout: "7.1"}}, + } + req := quirkRequestV3() + req.Capabilities.Containers = []string{"mkv"} + req.Capabilities.CodecsAudio = []string{"aac", "eac3"} + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "h264", Profiles: []string{"high"}, Levels: []int{42}, BitDepths: []int{8}, MaxWidth: 1920, MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 20_000, Hardware: true}} + req.ClientPlaybackContext.Engines[string(EngineMedia3ProgressiveRemuxV3)] = EngineCapabilityV3{} + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: false}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryRemuxHLSV3 || result.PlayMethod != PlayRemux || result.TargetVideoCodec != "copy" || !result.TranscodeAudio || result.TargetAudioCodec != "aac" || result.Plan.EffectiveRecipe.VideoCodec != "h264" { + t.Fatalf("result = %#v", result) + } + if len(result.Plan.AppliedQuirks) != 1 || result.Plan.AppliedQuirks[0].ID != QuirkFireTVAFTKRTEAC3HLSV3 { + t.Fatalf("quirks = %#v", result.Plan.AppliedQuirks) + } + wire, err := json.Marshal(result.Plan) + if err != nil { + t.Fatalf("marshal plan: %v", err) + } + if !bytes.Contains(wire, []byte(`"runtime_corrections":[]`)) { + t.Fatalf("runtime corrections must remain an array: %s", wire) + } + + req.ClientPlaybackContext.Device.Model = "AFTKA" + withoutQuirk := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: false}, Registry: testTransformationRegistryV3()}) + if withoutQuirk.Plan == nil || withoutQuirk.Plan.Delivery != DeliveryRemuxHLSV3 { + t.Fatalf("non-quirk HLS result = %#v", withoutQuirk) + } + wire, err = json.Marshal(withoutQuirk.Plan) + if err != nil { + t.Fatalf("marshal non-quirk plan: %v", err) + } + if !bytes.Contains(wire, []byte(`"applied_quirks":[]`)) || !bytes.Contains(wire, []byte(`"runtime_corrections":[]`)) { + t.Fatalf("quirk fields must remain arrays: %s", wire) + } +} + +func TestFireTVDV8HDR10PlusCorrectionRequiresAdvertisedRuntime(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].DVProfile = 8 + file.VideoTracks[0].DVBLCompatID = 1 + file.VideoTracks[0].HDR10Plus = true + file.VideoTracks[0].VideoRange = "HDR" + file.VideoTracks[0].VideoRangeType = "DOVI HDR10+" + req := quirkRequestV3() + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true, HDR10Plus: true, DolbyVisionProfiles: []int{8}} + req.ClientPlaybackContext.Output.HDRDetails = req.Capabilities.HDRDetails + direct := req.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] + direct.Features = append(direct.Features, ClientDV8HDR10PlusSanitizerV3) + req.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] = direct + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: false}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || len(result.Plan.RuntimeCorrections) != 1 || result.Plan.RuntimeCorrections[0] != ClientDV8HDR10PlusSanitizerV3 || len(result.Plan.AppliedQuirks) != 1 || result.Plan.AppliedQuirks[0].ID != QuirkFireTVDV8HDR10PlusV3 { + t.Fatalf("result = %#v", result) + } + + direct.Features = nil + req.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] = direct + withoutRuntime := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: false}, Registry: testTransformationRegistryV3()}) + if withoutRuntime.Plan == nil || len(withoutRuntime.Plan.AppliedQuirks) != 0 || len(withoutRuntime.Plan.RuntimeCorrections) != 0 { + t.Fatalf("unadvertised correction applied: %#v", withoutRuntime.Plan) + } +} + +func TestDeviceQuirkProtocolAcceptsEitherFeatureLocation(t *testing.T) { + file := &models.MediaFile{ + ID: 42, FilePath: "/media/high10.mkv", Container: "mkv", CodecVideo: "h264", CodecAudio: "aac", + Resolution: "1080p", Bitrate: 12_000, AudioChannels: 2, + VideoTracks: []models.VideoTrack{{Codec: "h264", Profile: "High 10", Level: 52, Width: 1920, Height: 1080, FrameRate: "24000/1001", Bitrate: 12_000, BitDepth: 10, VideoRange: "SDR", VideoRangeType: "SDR"}}, + AudioTracks: []models.AudioTrack{{Codec: "aac", Channels: 2, Layout: "stereo"}}, + } + req := quirkRequestV3() + req.Capabilities.Containers = []string{"mkv"} + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "h264", Profiles: []string{"high"}, Levels: []int{51}, BitDepths: []int{8}, MaxWidth: 1920, MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 20_000, Hardware: true}} + // Advertise the quirk protocol in the top-level client features only. + req.ClientPlaybackContext.Features = []string{FeaturePlaybackPlanV3, FeatureMedia3Only, FeatureDetailedDecodeV3} + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: false}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || len(result.Plan.AppliedQuirks) != 1 || result.Plan.AppliedQuirks[0].ID != QuirkFireTVAFTKRTHigh10V3 { + t.Fatalf("client-features-only advertisement: %#v", result) + } + + // Context-only advertisement is equally sufficient. + contextOnly := quirkRequestV3() + contextOnly.ClientFeatures = []string{FeaturePlaybackPlanV3, FeatureMedia3Only, FeatureDetailedDecodeV3} + if !deviceQuirkProtocolAvailableV3(contextOnly) { + t.Fatal("context-only quirk feature must enable the quirk protocol") + } + + neither := quirkRequestV3() + neither.ClientFeatures = []string{FeaturePlaybackPlanV3, FeatureMedia3Only} + neither.ClientPlaybackContext.Features = neither.ClientFeatures + if deviceQuirkProtocolAvailableV3(neither) { + t.Fatal("quirk protocol enabled without any advertisement") + } +} + +func TestPlanAttemptKeyV3DeviceQuirkMatchesKotlin(t *testing.T) { + width, height, bitrate := 3840, 2160, 60_000 + plan := PlanV3{ + PlanID: "plan:quirk", Delivery: DeliveryOriginalHTTPV3, + Stream: StreamV3{Protocol: StreamHTTPProgressiveV3, Container: "mkv"}, + EffectiveRecipe: EffectiveRecipeV3{VideoCodec: "hevc", AudioCodec: "eac3", Width: &width, Height: &height, BitrateKbps: &bitrate, DynamicRange: "dolby_vision"}, + Subtitle: SubtitleDecisionV3{Mode: SubtitleOffV3}, + AppliedQuirks: []AppliedQuirkV3{{ID: QuirkFireTVDV8HDR10PlusV3, RegistryRevision: DeviceQuirkRegistryRevisionV3, Action: "client_runtime_correction"}}, + RuntimeCorrections: []string{ClientDV8HDR10PlusSanitizerV3}, + } + if got := PlanAttemptKeyV3(plan, 9, nil); got != "v3:8d843bfffeb3adc3" { + t.Fatalf("key = %q", got) + } +} + +func quirkRequestV3() StartRequestV3 { + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3, FeatureDeviceQuirksV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3, FeatureDeviceQuirksV3) + req.ClientPlaybackContext.Device = DeviceContextV3{Manufacturer: "Amazon", Brand: "Amazon", Model: "AFTKRT", SDKInt: 30} + return req +} diff --git a/internal/playback/gpudetect.go b/internal/playback/gpudetect.go index 6d611cd8..468b00fb 100644 --- a/internal/playback/gpudetect.go +++ b/internal/playback/gpudetect.go @@ -36,11 +36,12 @@ var nvencProbeCache = struct { // HWAccelInfo describes the detected hardware acceleration capability. type HWAccelInfo struct { - Resolved string `json:"resolved"` - RenderDevices []string `json:"render_devices"` - IntelDetected bool `json:"intel_detected"` - Source string `json:"source"` - NodeURL string `json:"node_url,omitempty"` + Resolved string `json:"resolved"` + RenderDevices []string `json:"render_devices"` + IntelDetected bool `json:"intel_detected"` + Source string `json:"source"` + NodeURL string `json:"node_url,omitempty"` + Transformations []TransformationV3 `json:"transformations,omitempty"` } // DetectHWAccel probes this host's GPU hardware and returns structured info. diff --git a/internal/playback/plan_key_v3.go b/internal/playback/plan_key_v3.go new file mode 100644 index 00000000..9bb18192 --- /dev/null +++ b/internal/playback/plan_key_v3.go @@ -0,0 +1,101 @@ +package playback + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "hash/fnv" + "sort" + "strconv" + "strings" +) + +func DeterministicPlanIDV3(attemptID string, requestedFileID, effectiveFileID int, plan PlanV3) string { + transformations := make([]string, 0, len(plan.Transformations)) + for _, transformation := range plan.Transformations { + transformations = append(transformations, transformation.Executor+":"+transformation.Name+":"+transformation.RecipeVersion) + } + sort.Strings(transformations) + parts := []string{ + attemptID, + strconv.Itoa(requestedFileID), + strconv.Itoa(effectiveFileID), + string(plan.Delivery), + string(plan.Stream.Protocol), + strings.ToLower(plan.Stream.Container), + strings.ToLower(plan.EffectiveRecipe.VideoCodec), + strings.ToLower(plan.EffectiveRecipe.AudioCodec), + optionalIntV3(plan.EffectiveRecipe.Width), + optionalIntV3(plan.EffectiveRecipe.Height), + optionalIntV3(plan.EffectiveRecipe.BitrateKbps), + strings.ToLower(plan.EffectiveRecipe.DynamicRange), + trackIdentityValueV3(plan.SelectedTracks.Audio), + trackIdentityValueV3(plan.SelectedTracks.Subtitle), + string(plan.Subtitle.Mode), + strings.Join(transformations, ","), + } + parts = appendQuirkIdentityV3(parts, plan) + parts = append(parts, PlanRecipeVersionV3) + sum := sha256.Sum256([]byte(strings.Join(parts, "|"))) + return "plan:" + hex.EncodeToString(sum[:16]) +} + +func PlanAttemptKeyV3(plan PlanV3, outputRouteGeneration int64, localMutations []string) string { + transformations := make([]string, 0, len(plan.Transformations)) + for _, transformation := range plan.Transformations { + transformations = append(transformations, transformation.Executor+":"+transformation.Name+":"+transformation.RecipeVersion) + } + sort.Strings(transformations) + mutations := append([]string(nil), localMutations...) + sort.Strings(mutations) + parts := []string{ + plan.PlanID, + plan.Delivery.KotlinName(), + plan.Stream.Protocol.KotlinName(), + strings.ToLower(plan.Stream.Container), + strings.ToLower(plan.EffectiveRecipe.VideoCodec), + strings.ToLower(plan.EffectiveRecipe.AudioCodec), + optionalIntV3(plan.EffectiveRecipe.Width) + "x" + optionalIntV3(plan.EffectiveRecipe.Height), + optionalIntV3(plan.EffectiveRecipe.BitrateKbps), + strings.ToLower(plan.EffectiveRecipe.DynamicRange), + plan.Subtitle.Mode.KotlinName(), + strings.Join(transformations, ","), + } + parts = appendQuirkIdentityV3(parts, plan) + parts = append(parts, + strconv.FormatInt(outputRouteGeneration, 10), + strings.Join(mutations, ","), + ) + canonical := strings.Join(parts, "|") + h := fnv.New64a() + _, _ = h.Write([]byte(canonical)) + return fmt.Sprintf("v3:%016x", h.Sum64()) +} + +func appendQuirkIdentityV3(parts []string, plan PlanV3) []string { + if len(plan.AppliedQuirks) == 0 && len(plan.RuntimeCorrections) == 0 { + return parts + } + quirks := make([]string, 0, len(plan.AppliedQuirks)) + for _, quirk := range plan.AppliedQuirks { + quirks = append(quirks, quirk.RegistryRevision+":"+quirk.ID) + } + runtimeCorrections := append([]string(nil), plan.RuntimeCorrections...) + sort.Strings(quirks) + sort.Strings(runtimeCorrections) + return append(parts, strings.Join(quirks, ","), strings.Join(runtimeCorrections, ",")) +} + +func optionalIntV3(v *int) string { + if v == nil { + return "0" + } + return strconv.Itoa(*v) +} + +func trackIdentityValueV3(v *TrackIdentityV3) string { + if v == nil { + return "" + } + return v.ID + ":" + optionalIntV3(v.Index) +} diff --git a/internal/playback/plan_v3.go b/internal/playback/plan_v3.go new file mode 100644 index 00000000..f6981d40 --- /dev/null +++ b/internal/playback/plan_v3.go @@ -0,0 +1,817 @@ +package playback + +import ( + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/Silo-Server/silo-server/internal/models" +) + +type PlannerSettingsV3 struct { + TranscodeEnabled bool + Allow4KTranscode bool +} + +type PlannerInputV3 struct { + Request StartRequestV3 + RequestedFile *models.MediaFile + EffectiveFile *models.MediaFile + AudioTrackIndex int + Settings PlannerSettingsV3 + // Registry holds the transformations the local binary can execute. + // Progressive remux routes always gate on it: they run in this process. + Registry *TransformationRegistryV3 + // HLSRegistry optionally widens transformation availability for HLS + // deliveries, which can execute on pooled transcode nodes as well as + // locally. Nil means HLS routes gate on Registry alone. It is a lazy + // producer because building the widened registry can touch the network + // (node capability fetches): the planner only invokes it when a route + // decision genuinely depends on node capabilities, so direct-play and + // other source-preserving starts never pay for it. Producers must + // return a superset of Registry (local ∪ node capabilities) and should + // memoize; the transport layer re-validates whichever executor is + // actually selected. + HLSRegistry func() *TransformationRegistryV3 + Now time.Time + AttemptedKeys []string + AdditionalSubtitles []SubtitleInventoryEntryV3 +} + +// hlsRegistry resolves the registry HLS deliveries gate on: the widened +// local∪node registry when provided, otherwise the local one. Callers must +// keep it behind short-circuits so transformation-free routes never force +// the lazy producer to run. +func (input PlannerInputV3) hlsRegistry() *TransformationRegistryV3 { + if input.HLSRegistry != nil { + if widened := input.HLSRegistry(); widened != nil { + return widened + } + } + return input.Registry +} + +type PlannerResultV3 struct { + Plan *PlanV3 + Terminal *TerminalV3 + PlayMethod PlayMethod + TranscodeAudio bool + TargetVideoCodec string + TargetAudioCodec string + // TargetAudioChannels caps the transcode's re-encoded channel count; + // 0 keeps the historical stereo downmix. + TargetAudioChannels int + TargetResolution string + TargetBitrateKbps int + SubtitleTrackIndex int + SubtitleTransportTrackIndex int + SubtitleBurnIn bool + SubtitleCodec string +} + +func PlanPlaybackV3(input PlannerInputV3) PlannerResultV3 { + if input.RequestedFile == nil { + return terminalPlannerResultV3("source_unavailable", "The requested media source is unavailable.", false) + } + file := input.EffectiveFile + if file == nil { + file = input.RequestedFile + } + if input.Now.IsZero() { + input.Now = time.Now() + } + source := SourceDescriptorFromFileV3(file, input.AudioTrackIndex) + // Subtitle renderability is engine-specific, so every candidate route is + // validated against the capabilities of the engine that would execute it. + // The direct engine remains the canonical policy for source-preserving + // routes and for the up-front terminal decision. + subtitle := ResolveSubtitlePolicyV3(file, input.Request, input.Settings.TranscodeEnabled, EngineMedia3DirectV3, input.AdditionalSubtitles) + if subtitle.Terminal != nil { + return PlannerResultV3{Terminal: subtitle.Terminal, SubtitleTrackIndex: -1, SubtitleTransportTrackIndex: -1} + } + remuxSubtitle := ResolveSubtitlePolicyV3(file, input.Request, input.Settings.TranscodeEnabled, EngineMedia3ProgressiveRemuxV3, input.AdditionalSubtitles) + hlsSubtitle := ResolveSubtitlePolicyV3(file, input.Request, input.Settings.TranscodeEnabled, EngineMedia3HLSV3, input.AdditionalSubtitles) + // A remux route cannot burn subtitles, so it is only viable when its own + // engine can deliver the selected subtitle without one. + remuxSubtitleOK := remuxSubtitle.Terminal == nil && !remuxSubtitle.RequiresBurn + hlsRemuxSubtitleOK := hlsSubtitle.Terminal == nil && !hlsSubtitle.RequiresBurn + quality := ResolveQualityPolicyV3(input.Request, source) + videoOK := detailedVideoEligibleV3(source, input.Request) + var high10Quirk *AppliedQuirkV3 + if !videoOK { + if quirk, ok := high10DecodeOverrideV3(source, input.Request); ok { + videoOK = true + high10Quirk = quirk + } + } + rangeOK, videoClaims := outputRangeEligibleV3(source, input.Request) + audioOK, passthrough, audioClaims := audioEligibilityV3(source, input.Request) + if !audioOK && source.AudioCodec == "" && (file == nil || len(file.AudioTracks) == 0) { + // Video-only media has no audio stream to adapt: treating the absence + // as an unsupported codec would force a pointless AAC conversion — or + // a terminal when conversion is unavailable — on a playable file. An + // audio track whose codec merely failed to probe keeps the codec + // gate: converting unknown audio is safer than copying it. + audioOK = true + audioClaims.Reason = "no_audio_track" + } + containerOK := containsFoldV3(input.Request.Capabilities.Containers, source.Container) + hlsEngineOK := engineAvailableV3(input.Request, EngineMedia3HLSV3) + // DV strip eligibility is split by executor pool: a progressive remux + // executes on this process's ffmpeg, while an HLS remux may run on a + // pooled transcode node advertising the transformation. Node capability + // only counts when the client can actually run an HLS delivery, and the + // widened registry is consulted lazily so non-DV sources never touch it. + dvStripEligibleLocal := canStripDolbyVisionToHDR10V3(source, input.Request, input.Registry) + dvStripEligible := dvStripEligibleLocal + if !dvStripEligible && hlsEngineOK && source.DynamicRange == "dolby_vision" { + dvStripEligible = canStripDolbyVisionToHDR10V3(source, input.Request, input.hlsRegistry()) + } + clientDV81Eligible := canClientTransformDV7ToDV81V3(source, input.Request) + clientHDR10Eligible := canClientTransformDV7ToHDR10V3(source, input.Request) + + base := PlanV3{ + ProtocolVersion: ProtocolV3, + ExpiresAt: NewPlanExpiryV3(input.Now), + SelectedTracks: selectedTracksForPlanV3(file, input.AudioTrackIndex, subtitle), + EffectiveRecipe: recipeFromSourceV3(source), + Claims: ValidationClaimsV3{Video: videoClaims, Audio: audioClaims, Subtitles: subtitle.Claims}, + Subtitle: subtitle.Decision, + Transformations: []TransformationV3{}, + AppliedQuirks: []AppliedQuirkV3{}, + RuntimeCorrections: []string{}, + DegradationWarnings: []DegradationWarningV3{}, + RequestedMediaFileID: input.RequestedFile.ID, + EffectiveMediaFileID: file.ID, + Source: source, + SubtitleFidelityPolicy: subtitlePolicyNameV3(input.Request.SubtitleFidelityPreference), + Timeline: TimelineV3{SourceStartSeconds: floatOrZeroV3(input.Request.StartPosition), PlayerStartSeconds: floatOrZeroV3(input.Request.StartPosition), CanSeekAnywhere: true, SeekRestoration: "player_position"}, + } + base.Claims.Audio.Passthrough = passthrough + if source.DynamicRange == "hdr_unknown" && rangeOK { + base.DegradationWarnings = append(base.DegradationWarnings, DegradationWarningV3{ + Code: "hdr_range_assumed_hdr10", + Message: "The source is flagged HDR without precise range metadata and is delivered as HDR10.", + }) + } + if !detailedVideoEvidenceCompleteV3(source) { + return terminalPlannerResultV3("source_metadata_incomplete", "The source is missing video metadata required for a validated playback route.", true) + } + + // Automatic quality reductions (device resolution limit, bandwidth + // estimate/cap, metered fallback) are best-effort. When the only reason to + // transcode is such a reduction, a validated source-preserving route + // exists, and the transcode itself cannot execute (HDR sources have no + // validated reduced-quality recipe yet, or the client/server lacks the + // transcode route entirely), deliver the source at original quality with a + // degradation warning instead of refusing playback. Explicit user-selected + // rungs keep the existing terminals. + if quality.RequiresTranscode && !quality.ExplicitRung && !subtitle.RequiresBurn && videoOK && + (rangeOK || dvStripEligible || clientDV81Eligible || clientHDR10Eligible) && + !videoTranscodeExecutableV3(input, source) { + warnings := append(quality.Warnings, DegradationWarningV3{ + Code: "quality_reduction_unavailable", + Message: "Reduced-quality transcoding is unavailable for this source; it is delivered at original quality.", + }) + quality = originalQualityResultV3(source) + quality.Warnings = warnings + } + base.DegradationWarnings = append(base.DegradationWarnings, quality.Warnings...) + + if quality.RequiresTranscode || !videoOK || + (!rangeOK && !dvStripEligible && !clientDV81Eligible && !clientHDR10Eligible) || + (subtitle.RequiresBurn && !remuxSubtitleOK && !hlsRemuxSubtitleOK) { + return planVideoTranscodeV3(input, base, source, quality, hlsSubtitle, "") + } + + // Profile 7 is normalized on the client against the original range-capable + // source. A decoder profile/max-instance claim alone is not proof of native + // dual-layer output, so the default Android route mirrors Silo Apple: P8.1 + // base-layer Dolby Vision first, then same-file HDR10. + if source.DVProfile == 7 && quality.PreservesSource && videoOK && containerOK && audioOK && !subtitle.RequiresBurn { + if clientDV81Eligible { + plan := base + plan.Delivery = DeliveryOriginalHTTPV3 + plan.Engine = EngineMedia3DirectV3 + plan.Stream = StreamV3{Protocol: StreamHTTPProgressiveV3, Container: source.Container, MIMEType: MimeFromExtension(file.FilePath), Headers: map[string]string{}, HeaderRefresh: HeaderRefreshSessionV3} + plan.DecisionReason = "client_dv7_to_dv81" + plan.EffectiveRecipe.DynamicRange = "dolby_vision" + plan.Claims.Video = VideoClaimsV3{DolbyVision: true, DolbyVisionReason: "client_profile7_to_profile81"} + plan.Transformations = append(plan.Transformations, TransformationV3{ + Name: ClientDV7ToDV81V3, Executor: "client", RecipeVersion: ClientDVTransformVersionV3, + ValidatedClaims: []string{"profile7_rpu_converted_to_profile81", "hdr10_base_layer_preserved", "enhancement_layer_discarded"}, + }) + plan.DegradationWarnings = append(plan.DegradationWarnings, DegradationWarningV3{ + Code: "dolby_vision_enhancement_layer_discarded", + Message: "Dolby Vision Profile 7 is played as Profile 8.1 base-layer Dolby Vision; enhancement-layer pixel data is discarded.", + }) + finalizePlanIdentityV3(&plan, input.Request.PlaybackAttemptID) + if !planAttemptedV3(plan, input.Request.OutputRouteGeneration, input.AttemptedKeys) { + return PlannerResultV3{Plan: &plan, PlayMethod: PlayDirect, SubtitleTrackIndex: subtitle.SelectedIndex, SubtitleTransportTrackIndex: subtitle.TransportIndex, SubtitleCodec: subtitle.Codec} + } + } + if clientHDR10Eligible { + plan := base + plan.Delivery = DeliveryOriginalHTTPV3 + plan.Engine = EngineMedia3DirectV3 + plan.Stream = StreamV3{Protocol: StreamHTTPProgressiveV3, Container: source.Container, MIMEType: MimeFromExtension(file.FilePath), Headers: map[string]string{}, HeaderRefresh: HeaderRefreshSessionV3} + plan.DecisionReason = "client_dv7_to_hdr10" + plan.EffectiveRecipe.DynamicRange = "hdr10" + plan.Claims.Video = VideoClaimsV3{HDR10: true} + plan.Transformations = append(plan.Transformations, TransformationV3{ + Name: ClientDV7ToHDR10V3, Executor: "client", RecipeVersion: ClientDVTransformVersionV3, + ValidatedClaims: []string{"dolby_vision_metadata_removed", "hdr10_base_layer_preserved", "enhancement_layer_discarded"}, + }) + plan.DegradationWarnings = append(plan.DegradationWarnings, DegradationWarningV3{ + Code: "dolby_vision_removed", + Message: "Dolby Vision Profile 7 is played from the same 4K file as its HDR10 base layer.", + }) + finalizePlanIdentityV3(&plan, input.Request.PlaybackAttemptID) + if !planAttemptedV3(plan, input.Request.OutputRouteGeneration, input.AttemptedKeys) { + return PlannerResultV3{Plan: &plan, PlayMethod: PlayDirect, SubtitleTrackIndex: subtitle.SelectedIndex, SubtitleTransportTrackIndex: subtitle.TransportIndex, SubtitleCodec: subtitle.Codec} + } + } + } + + if source.DVProfile != 7 && engineAvailableV3(input.Request, EngineMedia3DirectV3) && containerOK && videoOK && rangeOK && audioOK && quality.PreservesSource && !subtitle.RequiresBurn { + plan := base + plan.Delivery = DeliveryOriginalHTTPV3 + plan.Engine = EngineMedia3DirectV3 + plan.Stream = StreamV3{Protocol: StreamHTTPProgressiveV3, Container: source.Container, MIMEType: MimeFromExtension(file.FilePath), Headers: map[string]string{}, HeaderRefresh: HeaderRefreshSessionV3} + plan.DecisionReason = "validated_original_playback" + applyCopiedVideoQuirksV3(&plan, source, input.Request, high10Quirk) + finalizePlanIdentityV3(&plan, input.Request.PlaybackAttemptID) + if !planAttemptedV3(plan, input.Request.OutputRouteGeneration, input.AttemptedKeys) { + return PlannerResultV3{Plan: &plan, PlayMethod: PlayDirect, SubtitleTrackIndex: subtitle.SelectedIndex, SubtitleTransportTrackIndex: subtitle.TransportIndex, SubtitleCodec: subtitle.Codec} + } + } + + // A progressive remux maps only the base-layer video stream, so dual-layer + // Profile 7 can never ship as native Dolby Vision here regardless of the + // client's decoder claims; the validated HDR10 strip is the only eligible + // P7 remux recipe. + remuxRangeOK := rangeOK && source.DVProfile != 7 + if videoOK && (remuxRangeOK || dvStripEligible) && (remuxSubtitleOK || hlsRemuxSubtitleOK) { + plan := base + plan.Delivery = DeliveryRemuxProgressiveV3 + plan.Engine = EngineMedia3ProgressiveRemuxV3 + plan.Stream = StreamV3{Protocol: StreamHTTPProgressiveV3, Container: "mp4", MIMEType: "video/mp4", Headers: map[string]string{}, HeaderRefresh: HeaderRefreshSessionV3} + plan.DecisionReason = "container_normalization" + transcodeAudio := !audioOK + localAudioConvertOK := input.Registry != nil && input.Registry.Available("audio_to_aac") + if transcodeAudio { + // The HLS remux branch below can offload the conversion to a + // pooled node, but only for clients that can run an HLS + // delivery: a progressive-only client must keep this terminal + // (its retryable semantics included) rather than fall through + // to a generic adaptation_unavailable for a route it can never + // use. Short-circuit order keeps locally-capable planning from + // consulting node capabilities at all. + audioConvertOK := localAudioConvertOK || + hlsEngineOK && input.hlsRegistry().Available("audio_to_aac") + if !audioConvertOK { + return terminalPlannerResultV3("audio_conversion_unsupported", "The required validated AAC conversion toolchain is unavailable.", true) + } + plan.EffectiveRecipe.AudioCodec = "aac" + plan.EffectiveRecipe.AudioChannels = intPointerV3(2) + plan.EffectiveRecipe.AudioLayout = "stereo" + plan.Claims.Audio = AudioClaimsV3{Codec: "aac", Reason: "server_audio_adaptation"} + plan.Transformations = append(plan.Transformations, TransformationV3{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1", ValidatedClaims: []string{"media3_audio_decode"}}) + plan.DegradationWarnings = append(plan.DegradationWarnings, DegradationWarningV3{Code: "audio_converted", Message: "The selected audio track is converted to AAC stereo."}) + plan.DecisionReason = "audio_adaptation" + } + dvStrip := dvStripEligible && (source.DVProfile == 7 || !rangeOK) + if dvStrip { + plan.Transformations = append(plan.Transformations, TransformationV3{Name: "server_dv7_to_hdr10", Executor: "server", RecipeVersion: "1", ValidatedClaims: []string{"dolby_vision_metadata_removed", "hdr10_base_layer_preserved", "enhancement_layer_discarded"}}) + plan.EffectiveRecipe.DynamicRange = "hdr10" + plan.Claims.Video = VideoClaimsV3{HDR10: true} + plan.DegradationWarnings = append(plan.DegradationWarnings, DegradationWarningV3{Code: "dolby_vision_removed", Message: "Dolby Vision metadata is removed and the validated HDR10 base layer is preserved."}) + } + if !dvStrip { + applyCopiedVideoQuirksV3(&plan, source, input.Request, high10Quirk) + } + // The progressive remux executes on this process's ffmpeg, so its + // server transformations must be locally available; when only pooled + // nodes carry them, the HLS remux below ships the same recipe on a + // node-offloadable delivery instead. + progressiveExecutable := (!transcodeAudio || localAudioConvertOK) && (!dvStrip || dvStripEligibleLocal) + if remuxSubtitleOK && progressiveExecutable { + plan.Subtitle = remuxSubtitle.Decision + plan.Claims.Subtitles = remuxSubtitle.Claims + finalizePlanIdentityV3(&plan, input.Request.PlaybackAttemptID) + if engineAvailableV3(input.Request, EngineMedia3ProgressiveRemuxV3) && !planAttemptedV3(plan, input.Request.OutputRouteGeneration, input.AttemptedKeys) { + return PlannerResultV3{Plan: &plan, PlayMethod: PlayRemux, TranscodeAudio: transcodeAudio, TargetAudioCodec: plan.EffectiveRecipe.AudioCodec, SubtitleTrackIndex: remuxSubtitle.SelectedIndex, SubtitleTransportTrackIndex: remuxSubtitle.TransportIndex, SubtitleCodec: remuxSubtitle.Codec} + } + } + if engineAvailableV3(input.Request, EngineMedia3HLSV3) && hlsRemuxSubtitleOK { + plan.AppliedQuirks = []AppliedQuirkV3{} + plan.RuntimeCorrections = []string{} + plan.Delivery = DeliveryRemuxHLSV3 + plan.Engine = EngineMedia3HLSV3 + plan.Stream = StreamV3{Protocol: StreamHLSV3, Container: "hls", MIMEType: "application/vnd.apple.mpegurl", Headers: map[string]string{}, HeaderRefresh: HeaderRefreshSessionV3} + hlsTranscodeAudio := transcodeAudio + if audioQuirk, ok := hlsEAC3AudioCorrectionV3(source, input.Request); ok && !hlsTranscodeAudio { + if !input.hlsRegistry().Available("audio_to_aac") { + return terminalPlannerResultV3("audio_conversion_unsupported", "The device-specific HLS route requires the validated AAC conversion toolchain.", true) + } + hlsTranscodeAudio = true + plan.EffectiveRecipe.AudioCodec = "aac" + plan.EffectiveRecipe.AudioChannels = intPointerV3(2) + plan.EffectiveRecipe.AudioLayout = "stereo" + plan.Claims.Audio = AudioClaimsV3{Codec: "aac", Reason: "device_hls_audio_adaptation"} + plan.Transformations = append(plan.Transformations, TransformationV3{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1", ValidatedClaims: []string{"media3_audio_decode"}}) + plan.DegradationWarnings = append(plan.DegradationWarnings, DegradationWarningV3{Code: "audio_converted", Message: "The selected audio track is converted to AAC stereo for this device's HLS route."}) + appendAppliedQuirkV3(&plan, *audioQuirk, "") + } + // DTS and TrueHD are not HLS-native audio codecs. A client's + // progressive DTS decode claim does not transfer to segmented + // fMP4/TS: copied DTS in an HLS route drags Media3's audio clock + // (device stall corrections, ~0.3x pacing, frozen position + // reports), so the copy is converted to AAC — surround-preserving + // when the source is multichannel. + hlsAudioChannels := 0 + if !hlsTranscodeAudio && !hlsNativeAudioCodecV3(source.AudioCodec) { + if !input.hlsRegistry().Available("audio_to_aac") { + return terminalPlannerResultV3("audio_conversion_unsupported", "The HLS route requires the validated AAC conversion toolchain.", true) + } + hlsTranscodeAudio = true + hlsAudioChannels = 2 + audioLayout := "stereo" + if source.AudioChannels >= 6 { + hlsAudioChannels = 6 + audioLayout = "5.1" + } + plan.EffectiveRecipe.AudioCodec = "aac" + plan.EffectiveRecipe.AudioChannels = intPointerV3(hlsAudioChannels) + plan.EffectiveRecipe.AudioLayout = audioLayout + plan.Claims.Audio = AudioClaimsV3{Codec: "aac", Reason: "hls_audio_adaptation"} + plan.Transformations = append(plan.Transformations, TransformationV3{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1", ValidatedClaims: []string{"media3_audio_decode"}}) + plan.DegradationWarnings = append(plan.DegradationWarnings, DegradationWarningV3{Code: "audio_converted", Message: "The selected audio track is converted to AAC for HLS delivery."}) + } + if !dvStrip { + applyCopiedVideoQuirksV3(&plan, source, input.Request, high10Quirk) + } + if hlsTranscodeAudio { + plan.DecisionReason = "hls_audio_adaptation" + } else { + plan.DecisionReason = "hls_packaging_required" + } + plan.Subtitle = hlsSubtitle.Decision + plan.Claims.Subtitles = hlsSubtitle.Claims + finalizePlanIdentityV3(&plan, input.Request.PlaybackAttemptID) + if !planAttemptedV3(plan, input.Request.OutputRouteGeneration, input.AttemptedKeys) { + targetAudio := "copy" + if hlsTranscodeAudio { + targetAudio = "aac" + } + return PlannerResultV3{Plan: &plan, PlayMethod: PlayRemux, TranscodeAudio: hlsTranscodeAudio, TargetVideoCodec: "copy", TargetAudioCodec: targetAudio, TargetAudioChannels: hlsAudioChannels, TargetResolution: resolutionLabelV3(source.Height), TargetBitrateKbps: source.BitrateKbps, SubtitleTrackIndex: hlsSubtitle.SelectedIndex, SubtitleTransportTrackIndex: hlsSubtitle.TransportIndex, SubtitleCodec: hlsSubtitle.Codec} + } + } + } + if engineAvailableV3(input.Request, EngineMedia3HLSV3) { + return planVideoTranscodeV3(input, base, source, quality, hlsSubtitle, "copy_routes_exhausted") + } + + return terminalPlannerResultV3("adaptation_unavailable", "No validated playback route is available for this source and output route.", false) +} + +// planVideoTranscodeV3 always executes on the HLS engine, so the caller must +// pass the subtitle policy resolved against EngineMedia3HLSV3. +func planVideoTranscodeV3(input PlannerInputV3, base PlanV3, source SourceDescriptorV3, quality QualityResultV3, subtitle SubtitlePolicyResultV3, reasonOverride string) PlannerResultV3 { + if !engineAvailableV3(input.Request, EngineMedia3HLSV3) { + return terminalPlannerResultV3("client_hls_unsupported", "The client cannot execute the required HLS adaptation route.", false) + } + if subtitle.Terminal != nil { + return PlannerResultV3{Terminal: subtitle.Terminal, SubtitleTrackIndex: -1, SubtitleTransportTrackIndex: -1} + } + if !input.Settings.TranscodeEnabled { + reason := "transcoding_disabled" + if subtitle.RequiresBurn { + reason = "subtitle_conversion_unsupported" + } + return terminalPlannerResultV3(reason, "The source requires video adaptation, but transcoding is unavailable.", false) + } + if is4KSourceV3(input.EffectiveFile, source) && !input.Settings.Allow4KTranscode { + return terminalPlannerResultV3("no_alternate_version", "A lower-resolution source is required because 4K transcoding is disabled.", false) + } + if hdrTranscodeUnavailableV3(source) { + return terminalPlannerResultV3("hdr_transcode_unsupported", "This HDR source requires video encoding, but no validated HDR-preserving or tone-map recipe is installed.", false) + } + if !input.hlsRegistry().Available("video_to_h264") || !input.hlsRegistry().Available("audio_to_aac") { + return terminalPlannerResultV3("conversion_tool_unavailable", "The required validated H.264/AAC conversion toolchain is unavailable.", true) + } + plan := base + plan.Delivery = DeliveryTranscodeHLSV3 + plan.Engine = EngineMedia3HLSV3 + plan.Stream = StreamV3{Protocol: StreamHLSV3, Container: "hls", MIMEType: "application/vnd.apple.mpegurl", Headers: map[string]string{}, HeaderRefresh: HeaderRefreshSessionV3} + plan.EffectiveRecipe.VideoCodec = "h264" + plan.EffectiveRecipe.AudioCodec = "aac" + plan.EffectiveRecipe.Width = intPointerV3(quality.Width) + plan.EffectiveRecipe.Height = intPointerV3(quality.Height) + plan.EffectiveRecipe.BitrateKbps = intPointerV3(quality.BitrateKbps) + // Surround sources keep 5.1 through the AAC re-encode (universal Media3 + // decode); only stereo/mono sources — and unknown layouts — downmix to 2.0. + targetAudioChannels := 2 + audioLayout := "stereo" + if source.AudioChannels >= 6 { + targetAudioChannels = 6 + audioLayout = "5.1" + } + plan.EffectiveRecipe.AudioChannels = intPointerV3(targetAudioChannels) + plan.EffectiveRecipe.AudioLayout = audioLayout + plan.Transformations = append(plan.Transformations, + TransformationV3{Name: "video_to_h264", Executor: "server", RecipeVersion: "1", ValidatedClaims: []string{"media3_h264_decode"}}, + TransformationV3{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1", ValidatedClaims: []string{"media3_audio_decode"}}, + ) + plan.Claims.Audio = AudioClaimsV3{Codec: "aac", Passthrough: false, AtmosPreserved: false, Reason: "server_audio_adaptation"} + plan.Subtitle = subtitle.Decision + plan.Claims.Subtitles = subtitle.Claims + plan.DecisionReason = quality.Reason + if reasonOverride != "" { + plan.DecisionReason = reasonOverride + } + if subtitle.RequiresBurn { + plan.DecisionReason = "subtitle_burn_in_required" + plan.DegradationWarnings = append(plan.DegradationWarnings, DegradationWarningV3{Code: "subtitle_burn_in", Message: "The selected subtitle is rendered into the video."}) + } + plan.EffectiveRecipe.DynamicRange = "sdr" + plan.Claims.Video = VideoClaimsV3{} + finalizePlanIdentityV3(&plan, input.Request.PlaybackAttemptID) + if planAttemptedV3(plan, input.Request.OutputRouteGeneration, input.AttemptedKeys) { + return terminalPlannerResultV3("adaptation_exhausted", "All compatible playback recipes have already failed for this output route.", false) + } + return PlannerResultV3{Plan: &plan, PlayMethod: PlayTranscode, TranscodeAudio: true, TargetVideoCodec: "h264", TargetAudioCodec: "aac", TargetAudioChannels: targetAudioChannels, TargetResolution: quality.Label, TargetBitrateKbps: quality.BitrateKbps, SubtitleTrackIndex: subtitle.SelectedIndex, SubtitleTransportTrackIndex: subtitle.TransportIndex, SubtitleBurnIn: subtitle.RequiresBurn, SubtitleCodec: subtitle.Codec} +} + +func canStripDolbyVisionToHDR10V3(source SourceDescriptorV3, request StartRequestV3, registry *TransformationRegistryV3) bool { + if source.DynamicRange != "dolby_vision" || !clientSupportsHDR10V3(request) || registry == nil || !registry.Available("server_dv7_to_hdr10") { + return false + } + // Profile 7 always carries an HDR10-viewable base layer. Profile 8 is + // safe only when the DOVI compatibility id explicitly identifies HDR10. + return source.DVProfile == 7 || source.DVProfile == 8 && source.DVBLCompatID == 1 +} + +func canClientTransformDV7ToDV81V3(source SourceDescriptorV3, request StartRequestV3) bool { + return source.DynamicRange == "dolby_vision" && source.DVProfile == 7 && + clientSupportsDVProfileV3(request, 8) && + clientTransformationAvailableV3(request, ClientDV7ToDV81V3, ClientDVTransformVersionV3) +} + +func canClientTransformDV7ToHDR10V3(source SourceDescriptorV3, request StartRequestV3) bool { + return source.DynamicRange == "dolby_vision" && source.DVProfile == 7 && clientSupportsHDR10V3(request) && + clientTransformationAvailableV3(request, ClientDV7ToHDR10V3, ClientDVTransformVersionV3) +} + +func clientSupportsDVProfileV3(request StartRequestV3, profile int) bool { + hdr := request.ClientPlaybackContext.Output.HDRDetails + if hdr == nil { + hdr = request.Capabilities.HDRDetails + } + return hdr != nil && containsIntV3(hdr.DolbyVisionProfiles, profile) +} + +func clientTransformationAvailableV3(request StartRequestV3, name, version string) bool { + if !HasFeatureV3(request.ClientFeatures, FeatureClientVideoTransforms) && + !HasFeatureV3(request.ClientPlaybackContext.Features, FeatureClientVideoTransforms) { + return false + } + engine, ok := request.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] + if !ok || !engine.Enabled || !engine.SupportedOnDevice { + return false + } + for _, transformation := range engine.Transformations { + if transformation.Executor == "client" && transformation.Name == name && transformation.RecipeVersion == version { + return true + } + } + return false +} + +func is4KSourceV3(file *models.MediaFile, source SourceDescriptorV3) bool { + resolution := "" + if file != nil { + resolution = strings.ToLower(strings.TrimSpace(file.Resolution)) + } + return resolution == "2160p" || resolution == "4k" || resolution == "uhd" || source.Width >= 3840 || source.Height >= 2160 +} + +type QualityResultV3 struct { + Label string + Width int + Height int + BitrateKbps int + PreservesSource bool + RequiresTranscode bool + // ExplicitRung marks a user-selected fixed rung, as opposed to an + // automatic reduction from device limits, bandwidth evidence, or caps. + ExplicitRung bool + Reason string + Warnings []DegradationWarningV3 +} + +// ResolveQualityPolicyV3 selects the delivery quality for a plan. +// +// bandwidth_cap_kbps is a hard delivery ceiling and is honored in every +// quality mode: "original" delivery is degraded when the source bitrate +// exceeds the cap, fixed rungs are lowered when their ladder bitrate exceeds +// it, and "auto" folds the cap into bandwidth-based rung selection. A metered +// connection with neither a cap nor a bandwidth estimate limits auto +// selection to the conservative 720p rung — the rung auto would pick for a +// mid-range bandwidth estimate — instead of assuming the link can sustain the +// original stream. +func ResolveQualityPolicyV3(request StartRequestV3, source SourceDescriptorV3) QualityResultV3 { + quality, changed := NormalizeQualityV3(request.QualityPreference) + var warnings []DegradationWarningV3 + if changed { + warnings = append(warnings, DegradationWarningV3{Code: "quality_preference_normalized", Message: "Unknown quality preference was normalized to auto."}) + } + capKbps := optionalValueV3(request.BandwidthCapKbps) + capExceededBySource := capKbps > 0 && source.BitrateKbps > capKbps + if quality == "original" && !capExceededBySource { + result := originalQualityResultV3(source) + result.Warnings = warnings + return result + } + targetHeight := source.Height + reason := "quality_auto_source" + explicitRung := false + capApplied := false + switch { + case quality == "original": + // Only reached when the source bitrate exceeds the cap: the cap is a + // hard ceiling and outranks the original preference. + targetHeight = ladderHeightForBandwidthV3(int(float64(capKbps) * 0.8)) + capApplied = true + case quality != "auto": + targetHeight, _ = strconv.Atoi(strings.TrimSuffix(quality, "p")) + reason = "quality_fixed_rung" + explicitRung = true + default: + maxHeight := resolutionHeightV3(request.Capabilities.MaxResolution) + if maxHeight > 0 && (targetHeight == 0 || maxHeight < targetHeight) { + targetHeight = maxHeight + reason = "quality_device_limit" + } + bandwidth := optionalValueV3(request.BandwidthEstimateKbps) + if capKbps > 0 && (bandwidth == 0 || capKbps < bandwidth) { + bandwidth = capKbps + } + if bandwidth > 0 { + targetHeight = minPositiveV3(targetHeight, ladderHeightForBandwidthV3(int(float64(bandwidth)*0.8))) + reason = "quality_bandwidth_limit" + } else if request.Metered { + if capped := minPositiveV3(targetHeight, 720); capped != targetHeight { + targetHeight = capped + reason = "quality_metered_limit" + } + } + } + if targetHeight <= 0 { + targetHeight = 1080 + } + if source.Height > 0 && targetHeight > source.Height { + targetHeight = source.Height + } + // The cap also constrains the rung chosen above: a rung that would + // preserve the source is forced down when the source bitrate exceeds the + // cap, and a transcode rung whose ladder bitrate exceeds the cap drops to + // the cap's rung. + if capKbps > 0 && !capApplied { + wouldPreserve := source.Height > 0 && targetHeight >= source.Height + if (wouldPreserve && capExceededBySource) || (!wouldPreserve && ladderBitrateKbpsV3(targetHeight) > capKbps) { + capApplied = true + if capHeight := ladderHeightForBandwidthV3(int(float64(capKbps) * 0.8)); capHeight < targetHeight { + targetHeight = capHeight + } + } + } + if capApplied { + reason = "quality_bandwidth_cap" + warnings = append(warnings, DegradationWarningV3{Code: "bandwidth_cap_applied", Message: "Delivery quality is limited by the configured bandwidth cap."}) + } + if source.Height > 0 && targetHeight >= source.Height && !capApplied { + return QualityResultV3{ + Label: strconv.Itoa(source.Height) + "p", + Width: source.Width, + Height: source.Height, + BitrateKbps: source.BitrateKbps, + PreservesSource: true, + ExplicitRung: explicitRung, + Reason: reason, + Warnings: warnings, + } + } + label := resolutionLabelV3(targetHeight) + effectiveHeight := resolutionHeightV3(label) + if source.Height > 0 && effectiveHeight > source.Height { + effectiveHeight = source.Height + label = resolutionLabelV3(effectiveHeight) + } + width, bitrate := qualityDimensionsV3(effectiveHeight, source.Width, source.Height) + if capKbps > 0 && bitrate > capKbps { + // The ladder has no rung below 480p, so a cap under the lowest rung's + // bitrate is honored by lowering the encode target directly: the cap + // is a hard delivery ceiling, never advisory. + bitrate = capKbps + } + result := QualityResultV3{Label: label, Width: width, Height: effectiveHeight, BitrateKbps: bitrate, PreservesSource: !capApplied && source.Height > 0 && effectiveHeight >= source.Height, ExplicitRung: explicitRung, Reason: reason, Warnings: warnings} + result.RequiresTranscode = !result.PreservesSource + return result +} + +func originalQualityResultV3(source SourceDescriptorV3) QualityResultV3 { + return QualityResultV3{Label: resolutionLabelV3(source.Height), Width: source.Width, Height: source.Height, BitrateKbps: source.BitrateKbps, PreservesSource: true, Reason: "quality_original"} +} + +// hlsNativeAudioCodecV3 reports whether an audio codec can be stream-copied +// into an HLS delivery. The allowlist follows the HLS authoring spec (AAC, +// AC-3, E-AC-3, MP3); everything else — DTS, TrueHD, PCM, Opus — must be +// converted even when the client can decode it in a progressive container. +func hlsNativeAudioCodecV3(codec string) bool { + switch strings.ToLower(strings.TrimSpace(codec)) { + case "aac", "ac3", "eac3", "mp3": + return true + } + return false +} + +// hdrTranscodeUnavailableV3 mirrors planVideoTranscodeV3's terminal +// condition: no validated HDR-preserving or tone-map transcode recipe exists. +func hdrTranscodeUnavailableV3(source SourceDescriptorV3) bool { + return source.DynamicRange != "" && source.DynamicRange != "sdr" +} + +// videoTranscodeExecutableV3 mirrors planVideoTranscodeV3's terminal +// preconditions: it reports whether a validated video transcode of this +// source could actually run for this client and configuration. +func videoTranscodeExecutableV3(input PlannerInputV3, source SourceDescriptorV3) bool { + if !engineAvailableV3(input.Request, EngineMedia3HLSV3) || !input.Settings.TranscodeEnabled { + return false + } + if is4KSourceV3(input.EffectiveFile, source) && !input.Settings.Allow4KTranscode { + return false + } + if hdrTranscodeUnavailableV3(source) { + return false + } + return input.hlsRegistry().Available("video_to_h264") && input.hlsRegistry().Available("audio_to_aac") +} + +func recipeFromSourceV3(source SourceDescriptorV3) EffectiveRecipeV3 { + return EffectiveRecipeV3{VideoCodec: source.VideoCodec, AudioCodec: source.AudioCodec, Width: intPointerV3(source.Width), Height: intPointerV3(source.Height), FrameRate: floatPointerV3(source.FrameRate), BitrateKbps: intPointerV3(source.BitrateKbps), DynamicRange: source.DynamicRange, AudioChannels: intPointerV3(source.AudioChannels), AudioLayout: source.AudioLayout} +} + +func selectedTracksForPlanV3(file *models.MediaFile, audioIndex int, subtitle SubtitlePolicyResultV3) SelectedTracksV3 { + selected := SelectedTracksV3{} + if file != nil && audioIndex >= 0 && audioIndex < len(file.AudioTracks) { + index := audioIndex + selected.Audio = &TrackIdentityV3{ID: TrackIDV3(file.ID, "audio", audioIndex), Index: &index} + } + if file != nil && subtitle.SelectedIndex >= 0 { + index := subtitle.SelectedIndex + selected.Subtitle = &TrackIdentityV3{ID: TrackIDV3(file.ID, "subtitle", index), Index: &index} + } + return selected +} + +func finalizePlanIdentityV3(plan *PlanV3, attemptID string) { + plan.PlanID = DeterministicPlanIDV3(attemptID, plan.RequestedMediaFileID, plan.EffectiveMediaFileID, *plan) +} + +// planAttemptedV3 compares FNV-hex attempt keys exactly after trimming +// whitespace; the keys are case-sensitive hashes, not free-form labels. +func planAttemptedV3(plan PlanV3, generation int64, attempted []string) bool { + wanted := PlanAttemptKeyV3(plan, generation, nil) + for _, key := range attempted { + if strings.TrimSpace(key) == wanted { + return true + } + } + return false +} + +func terminalPlannerResultV3(reason, message string, retryable bool) PlannerResultV3 { + return PlannerResultV3{Terminal: &TerminalV3{Reason: reason, Message: message, Retryable: retryable}, SubtitleTrackIndex: -1, SubtitleTransportTrackIndex: -1} +} +func subtitlePolicyNameV3(f SubtitleFidelityV3) string { + if f == SubtitleFidelityPreserveV3 { + return "require_authored_fidelity" + } + return "allow_simplified_rendering" +} +func floatOrZeroV3(v *float64) float64 { + if v == nil { + return 0 + } + return *v +} +func intPointerV3(v int) *int { + if v <= 0 { + return nil + } + value := v + return &value +} +func floatPointerV3(v float64) *float64 { + if v <= 0 { + return nil + } + value := v + return &value +} +func optionalValueV3(v *int) int { + if v == nil { + return 0 + } + return *v +} +func resolutionHeightV3(v string) int { + value, _ := strconv.Atoi(strings.TrimSuffix(strings.ToLower(v), "p")) + if strings.EqualFold(v, "4k") { + return 2160 + } + return value +} +func resolutionLabelV3(h int) string { + switch { + case h >= 2160: + return "2160p" + case h >= 1080: + return "1080p" + case h >= 720: + return "720p" + default: + return "480p" + } +} +func ladderHeightForBandwidthV3(kbps int) int { + switch { + case kbps >= 20_000: + return 2160 + case kbps >= 8_000: + return 1080 + case kbps >= 4_000: + return 720 + default: + return 480 + } +} +func minPositiveV3(a, b int) int { + if a <= 0 { + return b + } + if b <= 0 || a < b { + return a + } + return b +} + +// ladderBitrateKbpsV3 matches the established web ladder's standard shared +// rungs; 2160p is the v3-only extension until the web menu exposes a 4K +// transcode tier. +func ladderBitrateKbpsV3(height int) int { + bitrates := map[int]int{480: 1_500, 720: 2_000, 1080: 6_000, 2160: 20_000} + return bitrates[resolutionHeightV3(resolutionLabelV3(height))] +} + +func qualityDimensionsV3(height, sourceWidth, sourceHeight int) (int, int) { + rung := resolutionHeightV3(resolutionLabelV3(height)) + width := 0 + if sourceWidth > 0 && sourceHeight > 0 { + width = sourceWidth * rung / sourceHeight + width -= width % 2 + } + if width == 0 { + width, _ = dimensionsFromResolutionV3(resolutionLabelV3(rung)) + } + return width, ladderBitrateKbpsV3(rung) +} + +func SortedTransformationNamesV3(values []TransformationV3) []string { + result := make([]string, 0, len(values)) + for _, v := range values { + result = append(result, v.Name) + } + sort.Strings(result) + return result +} + +func engineAvailableV3(request StartRequestV3, engine EngineV3) bool { + capability, ok := request.ClientPlaybackContext.Engines[string(engine)] + if !ok { + return false + } + return capability.Enabled && capability.SupportedOnDevice +} +func ExplainPlannerResultV3(result PlannerResultV3) string { + if result.Plan != nil { + return fmt.Sprintf("%s:%s", result.Plan.Delivery, result.Plan.DecisionReason) + } + if result.Terminal != nil { + return "terminal:" + result.Terminal.Reason + } + return "invalid" +} diff --git a/internal/playback/plan_v3_union_test.go b/internal/playback/plan_v3_union_test.go new file mode 100644 index 00000000..f68df978 --- /dev/null +++ b/internal/playback/plan_v3_union_test.go @@ -0,0 +1,204 @@ +package playback + +import ( + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func staticHLSRegistryV3(registry *TransformationRegistryV3) func() *TransformationRegistryV3 { + return func() *TransformationRegistryV3 { return registry } +} + +func TestTransformationRegistryWithAdvertised(t *testing.T) { + registry := NewTransformationRegistryV3([]TransformationSpecV3{ + {Name: "audio_to_aac", RecipeVersion: "1"}, + {Name: "video_to_h264", RecipeVersion: "1"}, + {Name: "server_dv7_to_hdr10", RecipeVersion: "1", Available: true}, + }) + if got := registry.WithAdvertised(nil); got != registry { + t.Fatal("empty advertisement must return the receiver unchanged") + } + widened := registry.WithAdvertised([]TransformationV3{ + {Name: "Audio_To_AAC", Executor: "server", RecipeVersion: "1"}, + {Name: "video_to_h264", Executor: "server", RecipeVersion: "2"}, + {Name: "made_up_transform", Executor: "server", RecipeVersion: "1"}, + }) + if !widened.Available("audio_to_aac") { + t.Fatal("a matching node advertisement must widen availability") + } + if widened.Available("video_to_h264") { + t.Fatal("a recipe-version mismatch must not widen availability") + } + if widened.Available("made_up_transform") { + t.Fatal("advertisements must not introduce specs the server does not define") + } + if !widened.Available("server_dv7_to_hdr10") { + t.Fatal("locally available specs must stay available") + } + if registry.Available("audio_to_aac") { + t.Fatal("widening must not mutate the receiver") + } + clientOnly := registry.WithAdvertised([]TransformationV3{{Name: "audio_to_aac", Executor: "client", RecipeVersion: "1"}}) + if clientOnly.Available("audio_to_aac") { + t.Fatal("client-executor advertisements must not widen server availability") + } +} + +// A deployment whose API host lacks the H.264/AAC toolchain must still plan +// an HLS transcode when pooled transcode nodes advertise it, and must keep +// the terminal when nothing does. +func TestPlanPlaybackV3TranscodeOffloadsToNodeToolchain(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].VideoRange = "SDR" + file.VideoTracks[0].VideoRangeType = "SDR" + file.VideoTracks[0].ColorTransfer = "bt709" + req := validStartRequestV3() + req.QualityPreference = "480p" + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + local := NewTransformationRegistryV3([]TransformationSpecV3{ + {Name: "video_to_h264", RecipeVersion: "1"}, + {Name: "audio_to_aac", RecipeVersion: "1"}, + }) + settings := PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true} + + withoutNodes := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: settings, Registry: local}) + if withoutNodes.Terminal == nil || withoutNodes.Terminal.Reason != "conversion_tool_unavailable" { + t.Fatalf("without nodes = %s", ExplainPlannerResultV3(withoutNodes)) + } + + union := local.WithAdvertised([]TransformationV3{ + {Name: "video_to_h264", Executor: "server", RecipeVersion: "1"}, + {Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}, + }) + withNodes := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: settings, Registry: local, HLSRegistry: staticHLSRegistryV3(union)}) + if withNodes.Plan == nil || withNodes.Plan.Delivery != DeliveryTranscodeHLSV3 { + t.Fatalf("with nodes = %s", ExplainPlannerResultV3(withNodes)) + } +} + +// Audio conversion on the remux family must skip the locally-executed +// progressive remux when only pooled nodes carry the AAC toolchain, shipping +// the same recipe on the node-offloadable HLS remux delivery instead. +func TestPlanPlaybackV3AudioAdaptationOffloadsToHLSRemux(t *testing.T) { + file := detailedFixtureFileV3() + file.AudioTracks[0] = models.AudioTrack{Codec: "truehd", Channels: 8, Layout: "7.1"} + file.CodecAudio = "truehd" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + local := NewTransformationRegistryV3([]TransformationSpecV3{{Name: "audio_to_aac", RecipeVersion: "1"}}) + settings := PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true} + + withoutNodes := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: settings, Registry: local}) + if withoutNodes.Terminal == nil || withoutNodes.Terminal.Reason != "audio_conversion_unsupported" { + t.Fatalf("without nodes = %s", ExplainPlannerResultV3(withoutNodes)) + } + + union := local.WithAdvertised([]TransformationV3{{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}}) + offloaded := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: settings, Registry: local, HLSRegistry: staticHLSRegistryV3(union)}) + if offloaded.Plan == nil || offloaded.Plan.Delivery != DeliveryRemuxHLSV3 || !offloaded.TranscodeAudio || offloaded.TargetAudioCodec != "aac" { + t.Fatalf("with nodes = %s", ExplainPlannerResultV3(offloaded)) + } + + // With the toolchain available locally the progressive remux keeps + // priority — offloadability must never demote a local-capable route. + localCapable := NewTransformationRegistryV3([]TransformationSpecV3{{Name: "audio_to_aac", RecipeVersion: "1", Available: true}}) + preserved := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: settings, Registry: localCapable, HLSRegistry: staticHLSRegistryV3(localCapable)}) + if preserved.Plan == nil || preserved.Plan.Delivery != DeliveryRemuxProgressiveV3 { + t.Fatalf("local capable = %s", ExplainPlannerResultV3(preserved)) + } +} + +// A source that direct-plays must never trigger the lazy node-capability +// producer: building the widened registry can touch the network, and dead +// nodes must not add latency to starts that never use them. +func TestPlanPlaybackV3DirectPlayNeverConsultsNodeCapabilities(t *testing.T) { + file := detailedFixtureFileV3() + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + + result := PlanPlaybackV3(PlannerInputV3{ + Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: testTransformationRegistryV3(), + HLSRegistry: func() *TransformationRegistryV3 { + t.Fatal("direct-play planning must not build the node capability registry") + return nil + }, + }) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } +} + +// A progressive-only client (no HLS engine) cannot consume node-offloaded +// conversions, so node capabilities must not suppress its specific retryable +// audio terminal in favor of a generic non-retryable adaptation_unavailable. +func TestPlanPlaybackV3NodeToolchainDoesNotMaskTerminalForProgressiveOnlyClient(t *testing.T) { + file := detailedFixtureFileV3() + file.AudioTracks[0] = models.AudioTrack{Codec: "truehd", Channels: 8, Layout: "7.1"} + file.CodecAudio = "truehd" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + delete(req.ClientPlaybackContext.Engines, string(EngineMedia3HLSV3)) + + local := NewTransformationRegistryV3([]TransformationSpecV3{{Name: "audio_to_aac", RecipeVersion: "1"}}) + union := local.WithAdvertised([]TransformationV3{{Name: "audio_to_aac", Executor: "server", RecipeVersion: "1"}}) + result := PlanPlaybackV3(PlannerInputV3{ + Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, + Registry: local, HLSRegistry: staticHLSRegistryV3(union), + }) + if result.Terminal == nil || result.Terminal.Reason != "audio_conversion_unsupported" || !result.Terminal.Retryable { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } +} + +// The Profile 7 HDR10 strip must ride the HLS remux when only pooled nodes +// carry the dovi_rpu filter: the progressive remux executes locally and is +// not eligible without the local filter. +func TestPlanPlaybackV3Profile7StripOffloadsToHLSRemux(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].DVProfile = 7 + file.VideoTracks[0].DVBLCompatID = 6 + file.VideoTracks[0].DVELPresent = false + file.VideoTracks[0].DVEnhancementLayer = "" + file.VideoTracks[0].VideoRange = "DolbyVision" + file.VideoTracks[0].VideoRangeType = "DOVIWithEL" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true, DolbyVisionProfiles: []int{5, 8}} + req.ClientPlaybackContext.Output.HDRDetails = req.Capabilities.HDRDetails + local := NewTransformationRegistryV3([]TransformationSpecV3{{Name: "server_dv7_to_hdr10", RecipeVersion: "1"}}) + settings := PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true} + + withoutNodes := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: settings, Registry: local}) + if withoutNodes.Terminal == nil || withoutNodes.Terminal.Reason != "hdr_transcode_unsupported" { + t.Fatalf("without nodes = %s", ExplainPlannerResultV3(withoutNodes)) + } + + union := local.WithAdvertised([]TransformationV3{{Name: "server_dv7_to_hdr10", Executor: "server", RecipeVersion: "1"}}) + offloaded := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: settings, Registry: local, HLSRegistry: staticHLSRegistryV3(union)}) + if offloaded.Plan == nil || offloaded.Plan.Delivery != DeliveryRemuxHLSV3 || offloaded.TargetVideoCodec != "copy" { + t.Fatalf("with nodes = %s", ExplainPlannerResultV3(offloaded)) + } + if len(offloaded.Plan.Transformations) != 1 || offloaded.Plan.Transformations[0].Name != "server_dv7_to_hdr10" { + t.Fatalf("transformations = %#v", offloaded.Plan.Transformations) + } + if offloaded.Plan.EffectiveRecipe.DynamicRange != "hdr10" || !offloaded.Plan.Claims.Video.HDR10 { + t.Fatalf("claims = %#v", offloaded.Plan.Claims) + } +} diff --git a/internal/playback/planstore/postgres.go b/internal/playback/planstore/postgres.go new file mode 100644 index 00000000..921ca3b0 --- /dev/null +++ b/internal/playback/planstore/postgres.go @@ -0,0 +1,339 @@ +package planstore + +import ( + "context" + "encoding/json" + "errors" + "sync" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/playback" +) + +type Postgres struct { + db *pgxpool.Pool +} + +func NewPostgres(db *pgxpool.Pool) *Postgres { return &Postgres{db: db} } + +// SessionLockCapacity reports how many AcquireSessionLock holders the +// underlying pool can sustain concurrently. Each holder pins one pooled +// connection for its advisory-lock transaction while issuing further store +// queries from the same pool, so the bound leaves at least half the pool free +// for those queries and for the rest of the application. +func (s *Postgres) SessionLockCapacity() int { + if s == nil || s.db == nil { + return 0 + } + capacity := int(s.db.Config().MaxConns) / 2 + if capacity < 1 { + capacity = 1 + } + return capacity +} + +func (s *Postgres) AcquireSessionLock(ctx context.Context, sessionID string) (func(), error) { + conn, err := s.db.Acquire(ctx) + if err != nil { + return nil, err + } + tx, err := conn.Begin(ctx) + if err != nil { + conn.Release() + return nil, err + } + release := func() { + rollbackCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + err := tx.Rollback(rollbackCtx) + cancel() + if err != nil && !errors.Is(err, pgx.ErrTxClosed) { + // Closing the physical connection is the fail-safe for an uncertain + // rollback; PostgreSQL releases every transaction advisory lock when + // the backend connection closes. + closeCtx, closeCancel := context.WithTimeout(context.Background(), 2*time.Second) + _ = conn.Conn().Close(closeCtx) + closeCancel() + } + conn.Release() + } + if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock(hashtextextended($1, 0))`, sessionID); err != nil { + release() + return nil, err + } + var once sync.Once + return func() { + once.Do(release) + }, nil +} + +func (s *Postgres) SaveAttempt(ctx context.Context, record playback.AttemptRecordV3) error { + planJSON, err := json.Marshal(record.CurrentPlan) + if err != nil { + return err + } + requestJSON, err := json.Marshal(record.NormalizedRequest) + if err != nil { + return err + } + tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + // Expired rows linger for up to an hour until CleanupExpired runs; they + // must not wedge a legitimate attempt-ID or session reuse into a + // conflict that the recovery lookup (which filters expired rows) can + // never resolve. + if _, err := tx.Exec(ctx, ` + DELETE FROM playback_v3_attempts + WHERE (playback_attempt_id = $1 OR session_id = $2::uuid) AND expires_at <= NOW()`, + record.PlaybackAttemptID, record.SessionID); err != nil { + return err + } + result, err := tx.Exec(ctx, ` + INSERT INTO playback_v3_attempts ( + playback_attempt_id, session_id, user_id, profile_id, + requested_media_file_id, effective_media_file_id, + current_plan_id, current_replan_request_id, current_plan, normalized_request, request_digest, expires_at + ) VALUES ($1, $2::uuid, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + ON CONFLICT DO NOTHING`, + record.PlaybackAttemptID, record.SessionID, record.UserID, record.ProfileID, + record.RequestedMediaFileID, record.EffectiveMediaFileID, + record.CurrentPlanID, record.CurrentReplanRequestID, planJSON, requestJSON, record.RequestDigest, record.ExpiresAt) + if err != nil { + return err + } + if result.RowsAffected() == 0 { + // An attempt-ID reused with different input is an idempotency + // violation, not a replayable duplicate. + var digest string + err := tx.QueryRow(ctx, ` + SELECT request_digest FROM playback_v3_attempts + WHERE playback_attempt_id = $1 AND expires_at > NOW()`, record.PlaybackAttemptID).Scan(&digest) + if err == nil && digest != "" && record.RequestDigest != "" && digest != record.RequestDigest { + return playback.ErrIdempotencyKeyReusedV3 + } + return playback.ErrPlaybackAttemptExistsV3 + } + return tx.Commit(ctx) +} + +func (s *Postgres) GetAttempt(ctx context.Context, sessionID string) (*playback.AttemptRecordV3, error) { + return s.getAttempt(ctx, "session_id = $1::uuid", sessionID) +} + +func (s *Postgres) GetAttemptByPlaybackAttemptID(ctx context.Context, attemptID string) (*playback.AttemptRecordV3, error) { + return s.getAttempt(ctx, "playback_attempt_id = $1", attemptID) +} + +func (s *Postgres) GetAttemptIdentity(ctx context.Context, sessionID string) (*playback.AttemptIdentityV3, error) { + return s.getAttemptIdentity(ctx, "session_id = $1::uuid", sessionID) +} + +func (s *Postgres) GetAttemptIdentityByPlaybackAttemptID(ctx context.Context, attemptID string) (*playback.AttemptIdentityV3, error) { + return s.getAttemptIdentity(ctx, "playback_attempt_id = $1", attemptID) +} + +// getAttemptIdentity fetches only the ownership columns; route-event +// authorization runs per event and must not pay for the plan JSONB decode. +func (s *Postgres) getAttemptIdentity(ctx context.Context, predicate string, value any) (*playback.AttemptIdentityV3, error) { + var identity playback.AttemptIdentityV3 + err := s.db.QueryRow(ctx, ` + SELECT playback_attempt_id, session_id::text, user_id, profile_id + FROM playback_v3_attempts + WHERE `+predicate+` AND expires_at > NOW()`, value).Scan( + &identity.PlaybackAttemptID, &identity.SessionID, &identity.UserID, &identity.ProfileID, + ) + if errors.Is(err, pgx.ErrNoRows) { + return nil, playback.ErrSessionNotFound + } + if err != nil { + return nil, err + } + return &identity, nil +} + +func (s *Postgres) getAttempt(ctx context.Context, predicate string, value any) (*playback.AttemptRecordV3, error) { + var record playback.AttemptRecordV3 + var planJSON, requestJSON []byte + err := s.db.QueryRow(ctx, ` + SELECT playback_attempt_id, session_id::text, user_id, profile_id, + requested_media_file_id, effective_media_file_id, + current_plan_id, current_replan_request_id, current_plan, normalized_request, request_digest, expires_at + FROM playback_v3_attempts + WHERE `+predicate+` AND expires_at > NOW()`, value).Scan( + &record.PlaybackAttemptID, &record.SessionID, &record.UserID, &record.ProfileID, + &record.RequestedMediaFileID, &record.EffectiveMediaFileID, + &record.CurrentPlanID, &record.CurrentReplanRequestID, &planJSON, &requestJSON, &record.RequestDigest, &record.ExpiresAt, + ) + if errors.Is(err, pgx.ErrNoRows) { + return nil, playback.ErrSessionNotFound + } + if err != nil { + return nil, err + } + if err := json.Unmarshal(planJSON, &record.CurrentPlan); err != nil { + return nil, err + } + if err := json.Unmarshal(requestJSON, &record.NormalizedRequest); err != nil { + return nil, err + } + return &record, nil +} + +func (s *Postgres) BeginReplan(ctx context.Context, sessionID, requestID, digest, baseReplanRequestID string, leaseUntil time.Time) (playback.ReplanLeaseV3, error) { + // One retry: if a concurrent writer wins the insert race (possible only + // when a caller skips the advisory session lock), re-read its row and + // resolve to a replay/in-flight lease instead of surfacing a raw 23505. + for attempt := 0; ; attempt++ { + lease, retry, err := s.beginReplanOnce(ctx, sessionID, requestID, digest, baseReplanRequestID, leaseUntil) + if retry && attempt == 0 { + continue + } + return lease, err + } +} + +func (s *Postgres) beginReplanOnce(ctx context.Context, sessionID, requestID, digest, baseReplanRequestID string, leaseUntil time.Time) (playback.ReplanLeaseV3, bool, error) { + tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return playback.ReplanLeaseV3{}, false, err + } + defer func() { _ = tx.Rollback(ctx) }() + var existingDigest, existingBase, state string + var existingLease time.Time + var response []byte + err = tx.QueryRow(ctx, ` + SELECT request_digest, base_replan_request_id, state, lease_expires_at, response + FROM playback_v3_replans + WHERE session_id = $1::uuid AND replan_request_id = $2 + FOR UPDATE`, sessionID, requestID).Scan(&existingDigest, &existingBase, &state, &existingLease, &response) + if errors.Is(err, pgx.ErrNoRows) { + _, err = tx.Exec(ctx, ` + INSERT INTO playback_v3_replans (session_id, replan_request_id, request_digest, base_replan_request_id, lease_expires_at) + VALUES ($1::uuid, $2, $3, $4, $5)`, sessionID, requestID, digest, baseReplanRequestID, leaseUntil) + if err != nil { + var pgErr *pgconn.PgError + if errors.As(err, &pgErr) && pgErr.Code == "23505" { + return playback.ReplanLeaseV3{}, true, nil + } + return playback.ReplanLeaseV3{}, false, err + } + if err := tx.Commit(ctx); err != nil { + return playback.ReplanLeaseV3{}, false, err + } + return playback.ReplanLeaseV3{State: playback.ReplanLeaseOwnedV3}, false, nil + } + if err != nil { + return playback.ReplanLeaseV3{}, false, err + } + if existingDigest != digest { + return playback.ReplanLeaseV3{}, false, playback.ErrIdempotencyKeyReusedV3 + } + if state == "completed" { + if err := tx.Commit(ctx); err != nil { + return playback.ReplanLeaseV3{}, false, err + } + return playback.ReplanLeaseV3{State: playback.ReplanLeaseCompletedV3, Response: response}, false, nil + } + if time.Now().Before(existingLease) { + if err := tx.Commit(ctx); err != nil { + return playback.ReplanLeaseV3{}, false, err + } + return playback.ReplanLeaseV3{State: playback.ReplanLeaseInFlightV3}, false, nil + } + if existingBase != baseReplanRequestID { + return playback.ReplanLeaseV3{}, false, playback.ErrStaleReplanLeaseV3 + } + _, err = tx.Exec(ctx, `UPDATE playback_v3_replans SET lease_expires_at = $3, updated_at = NOW() WHERE session_id = $1::uuid AND replan_request_id = $2`, sessionID, requestID, leaseUntil) + if err != nil { + return playback.ReplanLeaseV3{}, false, err + } + if err := tx.Commit(ctx); err != nil { + return playback.ReplanLeaseV3{}, false, err + } + return playback.ReplanLeaseV3{State: playback.ReplanLeaseOwnedV3}, false, nil +} + +func (s *Postgres) CompleteReplan(ctx context.Context, sessionID, requestID, baseReplanRequestID string, response json.RawMessage, record playback.AttemptRecordV3) error { + tx, err := s.db.BeginTx(ctx, pgx.TxOptions{}) + if err != nil { + return err + } + defer func() { _ = tx.Rollback(ctx) }() + planJSON, err := json.Marshal(record.CurrentPlan) + if err != nil { + return err + } + requestJSON, err := json.Marshal(record.NormalizedRequest) + if err != nil { + return err + } + // The base-revision predicate makes the commit a true compare-and-swap: + // under the advisory session lock it never fails, but a skipped or broken + // lock must surface as a conflict rather than silently last-writer-win + // the durable plan. + attemptResult, err := tx.Exec(ctx, ` + UPDATE playback_v3_attempts SET + effective_media_file_id = $2, current_plan_id = $3, + current_replan_request_id = $4, current_plan = $5, normalized_request = $6, expires_at = $7, updated_at = NOW() + WHERE session_id = $1::uuid AND current_replan_request_id = $8`, + sessionID, record.EffectiveMediaFileID, record.CurrentPlanID, record.CurrentReplanRequestID, planJSON, requestJSON, record.ExpiresAt, baseReplanRequestID) + if err != nil { + return err + } + if attemptResult.RowsAffected() != 1 { + var exists bool + if scanErr := tx.QueryRow(ctx, `SELECT true FROM playback_v3_attempts WHERE session_id = $1::uuid`, sessionID).Scan(&exists); scanErr == nil { + return playback.ErrReplanSupersededV3 + } + return playback.ErrSessionNotFound + } + replanResult, err := tx.Exec(ctx, ` + UPDATE playback_v3_replans SET state = 'completed', response = $3, updated_at = NOW() + WHERE session_id = $1::uuid AND replan_request_id = $2`, sessionID, requestID, response) + if err != nil { + return err + } + if replanResult.RowsAffected() != 1 { + return playback.ErrSessionNotFound + } + return tx.Commit(ctx) +} + +func (s *Postgres) RecordRouteEvent(ctx context.Context, record playback.RouteEventRecordV3) error { + if record.Diagnostics == nil { + record.Diagnostics = map[string]string{} + } + diagnostics, err := json.Marshal(record.Diagnostics) + if err != nil { + return err + } + _, err = s.db.Exec(ctx, ` + INSERT INTO playback_route_events ( + playback_attempt_id, session_id, plan_id, plan_attempt_id, plan_attempt_key, + event, failure_classification, fallback_reason, output_route_generation, + diagnostics, user_id, profile_id, client_name, client_version, client_model + ) VALUES ($1, NULLIF($2, '')::uuid, NULLIF($3, ''), NULLIF($4, ''), NULLIF($5, ''), + $6, NULLIF($7, ''), NULLIF($8, ''), $9, $10, $11, $12, + NULLIF($13, ''), NULLIF($14, ''), NULLIF($15, ''))`, + record.PlaybackAttemptID, record.SessionID, record.PlanID, record.PlanAttemptID, record.PlanAttemptKey, + record.Event, record.FailureClassification, record.FallbackReason, record.OutputRouteGeneration, + diagnostics, record.UserID, record.ProfileID, record.ClientName, record.ClientVersion, record.ClientModel) + return err +} + +func (s *Postgres) CleanupExpired(ctx context.Context, now time.Time) (int64, error) { + if _, err := s.db.Exec(ctx, `DELETE FROM playback_route_events WHERE received_at < $1`, now.Add(-30*24*time.Hour)); err != nil { + return 0, err + } + result, err := s.db.Exec(ctx, `DELETE FROM playback_v3_attempts WHERE expires_at <= $1`, now) + if err != nil { + return 0, err + } + return result.RowsAffected(), nil +} diff --git a/internal/playback/planstore/postgres_test.go b/internal/playback/planstore/postgres_test.go new file mode 100644 index 00000000..a99d6aad --- /dev/null +++ b/internal/playback/planstore/postgres_test.go @@ -0,0 +1,601 @@ +package planstore + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/playback" +) + +// planstoreFixture holds the minimal FK graph (user + media files) that +// playback_v3_attempts and playback_route_events rows require. +type planstoreFixture struct { + pool *pgxpool.Pool + userID int + mediaFileID int + altFileID int +} + +// newPlanstoreFixture connects to SILO_TEST_DATABASE_URL (skipping when +// unset), verifies the v3 migrations are applied, and inserts the fixture +// rows every attempt/event insert depends on. Cleanup deletes everything the +// tests wrote so reruns against the same database stay green. +func newPlanstoreFixture(t *testing.T) *planstoreFixture { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + + var tableName *string + if err := pool.QueryRow(ctx, `SELECT to_regclass('public.playback_v3_attempts')::text`).Scan(&tableName); err != nil { + t.Fatalf("check playback_v3_attempts table: %v", err) + } + if tableName == nil || *tableName == "" { + t.Skip("test database has not applied the playback protocol v3 migration") + } + var hasRevision bool + if err := pool.QueryRow(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'playback_v3_attempts' AND column_name = 'current_replan_request_id' + )`).Scan(&hasRevision); err != nil { + t.Fatalf("check current_replan_request_id column: %v", err) + } + if !hasRevision { + t.Skip("test database has not applied the playback v3 attempt revision migration") + } + + f := &planstoreFixture{pool: pool} + unique := fmt.Sprintf("planstore-test-%d", time.Now().UnixNano()) + + var folderID int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_folders (type, name) VALUES ('movies', $1) RETURNING id`, unique).Scan(&folderID); err != nil { + t.Fatalf("insert fixture media folder: %v", err) + } + if err := pool.QueryRow(ctx, ` + INSERT INTO users (username) VALUES ($1) RETURNING id`, unique).Scan(&f.userID); err != nil { + t.Fatalf("insert fixture user: %v", err) + } + if err := pool.QueryRow(ctx, ` + INSERT INTO media_files (media_folder_id, file_path) VALUES ($1, $2) RETURNING id`, + folderID, "/fixtures/"+unique+"/movie.mkv").Scan(&f.mediaFileID); err != nil { + t.Fatalf("insert fixture media file: %v", err) + } + if err := pool.QueryRow(ctx, ` + INSERT INTO media_files (media_folder_id, file_path) VALUES ($1, $2) RETURNING id`, + folderID, "/fixtures/"+unique+"/movie-alt.mkv").Scan(&f.altFileID); err != nil { + t.Fatalf("insert alternate fixture media file: %v", err) + } + + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + // Attempts cascade to replans; deleting the user cascades any attempt + // or route event a failed subtest left behind; the folder cascades the + // media files. + _, _ = pool.Exec(cleanupCtx, `DELETE FROM playback_route_events WHERE user_id = $1`, f.userID) + _, _ = pool.Exec(cleanupCtx, `DELETE FROM playback_v3_attempts WHERE user_id = $1`, f.userID) + _, _ = pool.Exec(cleanupCtx, `DELETE FROM users WHERE id = $1`, f.userID) + _, _ = pool.Exec(cleanupCtx, `DELETE FROM media_folders WHERE id = $1`, folderID) + }) + return f +} + +func (f *planstoreFixture) attemptRecord(sessionID, attemptID, digest string) playback.AttemptRecordV3 { + return playback.AttemptRecordV3{ + PlaybackAttemptID: attemptID, + SessionID: sessionID, + UserID: f.userID, + ProfileID: "profile-1", + RequestedMediaFileID: f.mediaFileID, + EffectiveMediaFileID: f.mediaFileID, + CurrentPlanID: "plan-1", + CurrentReplanRequestID: "", + CurrentPlan: playback.PlanV3{ + ProtocolVersion: 3, + PlanID: "plan-1", + SessionID: sessionID, + DecisionReason: "direct_play", + RequestedMediaFileID: f.mediaFileID, + EffectiveMediaFileID: f.mediaFileID, + }, + NormalizedRequest: playback.StartRequestV3{ + ProtocolVersion: 3, + FileID: f.mediaFileID, + ProfileID: "profile-1", + PlaybackAttemptID: attemptID, + QualityPreference: "auto", + }, + RequestDigest: digest, + ExpiresAt: time.Now().Add(time.Hour).UTC().Truncate(time.Microsecond), + } +} + +func (f *planstoreFixture) expireAttempt(t *testing.T, attemptID string) { + t.Helper() + tag, err := f.pool.Exec(context.Background(), ` + UPDATE playback_v3_attempts SET expires_at = NOW() - INTERVAL '1 minute' + WHERE playback_attempt_id = $1`, attemptID) + if err != nil { + t.Fatalf("expire attempt %s: %v", attemptID, err) + } + if tag.RowsAffected() != 1 { + t.Fatalf("expire attempt %s: affected %d rows", attemptID, tag.RowsAffected()) + } +} + +// mustJSON canonicalizes a value through encoding/json so structs that +// round-trip via JSONB can be compared without tripping on nil-vs-empty +// map/slice differences. +func mustJSON(t *testing.T, v any) []byte { + t.Helper() + data, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return data +} + +func TestPostgresPlanStore(t *testing.T) { + f := newPlanstoreFixture(t) + store := NewPostgres(f.pool) + ctx := context.Background() + + // Regression test for the CHECK-constraint drift: every event name the + // code emits must be accepted by the playback_route_events CHECK in the + // real schema. + t.Run("RecordRouteEventAcceptsAllEventNames", func(t *testing.T) { + sessionID := uuid.NewString() + names := playback.RouteEventNamesV3() + if len(names) == 0 { + t.Fatal("RouteEventNamesV3 returned no events") + } + for _, name := range names { + err := store.RecordRouteEvent(ctx, playback.RouteEventRecordV3{ + RouteEventV3: playback.RouteEventV3{ + ProtocolVersion: 3, + PlaybackAttemptID: "att-events-" + sessionID, + SessionID: sessionID, + PlanID: "plan-1", + PlanAttemptID: "plan-attempt-1", + PlanAttemptKey: "plan-attempt-key-1", + Event: name, + FailureClassification: "decode_error", + FallbackReason: "test", + OutputRouteGeneration: 1, + Diagnostics: map[string]string{"source": "planstore-test"}, + }, + UserID: f.userID, + ProfileID: "profile-1", + ClientName: "planstore-test", + ClientVersion: "1.0", + ClientModel: "test-model", + }) + if err != nil { + t.Errorf("RecordRouteEvent(%q) rejected by real schema: %v", name, err) + } + } + var count int + if err := f.pool.QueryRow(ctx, ` + SELECT COUNT(*) FROM playback_route_events WHERE session_id = $1::uuid`, sessionID).Scan(&count); err != nil { + t.Fatalf("count route events: %v", err) + } + if count != len(names) { + t.Fatalf("persisted %d route events, want %d", count, len(names)) + } + }) + + t.Run("SaveAttemptIdempotency", func(t *testing.T) { + sessionID := uuid.NewString() + attemptID := "att-save-" + sessionID + record := f.attemptRecord(sessionID, attemptID, "digest-a") + + if err := store.SaveAttempt(ctx, record); err != nil { + t.Fatalf("fresh SaveAttempt: %v", err) + } + + // Exact replay of the same attempt-ID and digest. + if err := store.SaveAttempt(ctx, record); !errors.Is(err, playback.ErrPlaybackAttemptExistsV3) { + t.Fatalf("same-digest replay: got %v, want ErrPlaybackAttemptExistsV3", err) + } + + // Same attempt-ID reused with different input (digest) is an + // idempotency violation, not a replay. + conflicting := f.attemptRecord(uuid.NewString(), attemptID, "digest-b") + if err := store.SaveAttempt(ctx, conflicting); !errors.Is(err, playback.ErrIdempotencyKeyReusedV3) { + t.Fatalf("different-digest reuse: got %v, want ErrIdempotencyKeyReusedV3", err) + } + + // Once the original row expires, the pre-delete path must clear it so + // the attempt-ID becomes reusable. + f.expireAttempt(t, attemptID) + if err := store.SaveAttempt(ctx, record); err != nil { + t.Fatalf("SaveAttempt after expiry should reclaim the attempt-ID: %v", err) + } + }) + + t.Run("GetAttemptRoundTrip", func(t *testing.T) { + sessionID := uuid.NewString() + attemptID := "att-get-" + sessionID + record := f.attemptRecord(sessionID, attemptID, "digest-get") + if err := store.SaveAttempt(ctx, record); err != nil { + t.Fatalf("SaveAttempt: %v", err) + } + + for name, fetch := range map[string]func() (*playback.AttemptRecordV3, error){ + "GetAttempt": func() (*playback.AttemptRecordV3, error) { return store.GetAttempt(ctx, sessionID) }, + "GetAttemptByPlaybackAttemptID": func() (*playback.AttemptRecordV3, error) { return store.GetAttemptByPlaybackAttemptID(ctx, attemptID) }, + } { + got, err := fetch() + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if got.PlaybackAttemptID != attemptID || got.SessionID != sessionID { + t.Fatalf("%s identity mismatch: %+v", name, got) + } + if got.UserID != record.UserID || got.ProfileID != record.ProfileID { + t.Fatalf("%s ownership mismatch: %+v", name, got) + } + if got.RequestedMediaFileID != record.RequestedMediaFileID || got.EffectiveMediaFileID != record.EffectiveMediaFileID { + t.Fatalf("%s media file mismatch: %+v", name, got) + } + if got.CurrentPlanID != record.CurrentPlanID || got.CurrentReplanRequestID != record.CurrentReplanRequestID { + t.Fatalf("%s plan revision mismatch: %+v", name, got) + } + if got.RequestDigest != record.RequestDigest { + t.Fatalf("%s request_digest = %q, want %q", name, got.RequestDigest, record.RequestDigest) + } + if !bytes.Equal(mustJSON(t, got.CurrentPlan), mustJSON(t, record.CurrentPlan)) { + t.Fatalf("%s plan JSON did not round-trip:\n got %s\nwant %s", name, mustJSON(t, got.CurrentPlan), mustJSON(t, record.CurrentPlan)) + } + if !bytes.Equal(mustJSON(t, got.NormalizedRequest), mustJSON(t, record.NormalizedRequest)) { + t.Fatalf("%s normalized request JSON did not round-trip", name) + } + if diff := got.ExpiresAt.Sub(record.ExpiresAt); diff < -time.Millisecond || diff > time.Millisecond { + t.Fatalf("%s expires_at drifted by %v", name, diff) + } + } + + identity, err := store.GetAttemptIdentity(ctx, sessionID) + if err != nil { + t.Fatalf("GetAttemptIdentity: %v", err) + } + byAttempt, err := store.GetAttemptIdentityByPlaybackAttemptID(ctx, attemptID) + if err != nil { + t.Fatalf("GetAttemptIdentityByPlaybackAttemptID: %v", err) + } + for name, got := range map[string]*playback.AttemptIdentityV3{"bySession": identity, "byAttempt": byAttempt} { + if got.PlaybackAttemptID != attemptID || got.SessionID != sessionID || + got.UserID != f.userID || got.ProfileID != "profile-1" { + t.Fatalf("%s identity ownership mismatch: %+v", name, got) + } + } + + // Expired rows must be invisible to every read path. + f.expireAttempt(t, attemptID) + if _, err := store.GetAttempt(ctx, sessionID); !errors.Is(err, playback.ErrSessionNotFound) { + t.Fatalf("GetAttempt on expired row: got %v, want ErrSessionNotFound", err) + } + if _, err := store.GetAttemptByPlaybackAttemptID(ctx, attemptID); !errors.Is(err, playback.ErrSessionNotFound) { + t.Fatalf("GetAttemptByPlaybackAttemptID on expired row: got %v, want ErrSessionNotFound", err) + } + if _, err := store.GetAttemptIdentity(ctx, sessionID); !errors.Is(err, playback.ErrSessionNotFound) { + t.Fatalf("GetAttemptIdentity on expired row: got %v, want ErrSessionNotFound", err) + } + if _, err := store.GetAttemptIdentityByPlaybackAttemptID(ctx, attemptID); !errors.Is(err, playback.ErrSessionNotFound) { + t.Fatalf("GetAttemptIdentityByPlaybackAttemptID on expired row: got %v, want ErrSessionNotFound", err) + } + }) + + t.Run("BeginReplanLifecycle", func(t *testing.T) { + sessionID := uuid.NewString() + attemptID := "att-replan-" + sessionID + if err := store.SaveAttempt(ctx, f.attemptRecord(sessionID, attemptID, "digest-replan")); err != nil { + t.Fatalf("SaveAttempt: %v", err) + } + future := time.Now().Add(time.Minute) + + // New replan request: caller owns the lease. + lease, err := store.BeginReplan(ctx, sessionID, "rq-1", "rq-digest-1", "", future) + if err != nil { + t.Fatalf("BeginReplan new: %v", err) + } + if lease.State != playback.ReplanLeaseOwnedV3 { + t.Fatalf("BeginReplan new state = %q, want owned", lease.State) + } + + // Same request-ID with different input. + if _, err := store.BeginReplan(ctx, sessionID, "rq-1", "rq-digest-other", "", future); !errors.Is(err, playback.ErrIdempotencyKeyReusedV3) { + t.Fatalf("digest mismatch: got %v, want ErrIdempotencyKeyReusedV3", err) + } + + // Active unexpired lease held by someone else. + lease, err = store.BeginReplan(ctx, sessionID, "rq-1", "rq-digest-1", "", future) + if err != nil { + t.Fatalf("BeginReplan in-flight: %v", err) + } + if lease.State != playback.ReplanLeaseInFlightV3 { + t.Fatalf("BeginReplan in-flight state = %q, want in_flight", lease.State) + } + + // Completed replan replays the stored response. + completed := f.attemptRecord(sessionID, attemptID, "digest-replan") + completed.CurrentPlanID = "plan-2" + completed.CurrentReplanRequestID = "rq-1" + completed.CurrentPlan.PlanID = "plan-2" + response := json.RawMessage(`{"plan_id": "plan-2"}`) + if err := store.CompleteReplan(ctx, sessionID, "rq-1", "", response, completed); err != nil { + t.Fatalf("CompleteReplan: %v", err) + } + lease, err = store.BeginReplan(ctx, sessionID, "rq-1", "rq-digest-1", "", future) + if err != nil { + t.Fatalf("BeginReplan completed: %v", err) + } + if lease.State != playback.ReplanLeaseCompletedV3 { + t.Fatalf("BeginReplan completed state = %q, want completed", lease.State) + } + var storedResponse, wantResponse any + if err := json.Unmarshal(lease.Response, &storedResponse); err != nil { + t.Fatalf("unmarshal replayed response: %v", err) + } + if err := json.Unmarshal(response, &wantResponse); err != nil { + t.Fatalf("unmarshal expected response: %v", err) + } + if !bytes.Equal(mustJSON(t, storedResponse), mustJSON(t, wantResponse)) { + t.Fatalf("replayed response = %s, want %s", lease.Response, response) + } + + // Expired lease whose base revision no longer matches the retry. + past := time.Now().Add(-time.Minute) + if _, err := store.BeginReplan(ctx, sessionID, "rq-stale", "rq-digest-stale", "base-x", past); err != nil { + t.Fatalf("BeginReplan seed stale lease: %v", err) + } + if _, err := store.BeginReplan(ctx, sessionID, "rq-stale", "rq-digest-stale", "base-y", future); !errors.Is(err, playback.ErrStaleReplanLeaseV3) { + t.Fatalf("expired lease with stale base: got %v, want ErrStaleReplanLeaseV3", err) + } + + // Expired lease with a matching base is re-owned. + if _, err := store.BeginReplan(ctx, sessionID, "rq-retry", "rq-digest-retry", "rq-1", past); err != nil { + t.Fatalf("BeginReplan seed expired lease: %v", err) + } + lease, err = store.BeginReplan(ctx, sessionID, "rq-retry", "rq-digest-retry", "rq-1", future) + if err != nil { + t.Fatalf("BeginReplan re-own expired lease: %v", err) + } + if lease.State != playback.ReplanLeaseOwnedV3 { + t.Fatalf("re-owned lease state = %q, want owned", lease.State) + } + }) + + t.Run("CompleteReplan", func(t *testing.T) { + sessionID := uuid.NewString() + attemptID := "att-complete-" + sessionID + if err := store.SaveAttempt(ctx, f.attemptRecord(sessionID, attemptID, "digest-complete")); err != nil { + t.Fatalf("SaveAttempt: %v", err) + } + future := time.Now().Add(time.Minute) + if _, err := store.BeginReplan(ctx, sessionID, "rq-1", "rq-digest-1", "", future); err != nil { + t.Fatalf("BeginReplan: %v", err) + } + + updated := f.attemptRecord(sessionID, attemptID, "digest-complete") + updated.EffectiveMediaFileID = f.altFileID + updated.CurrentPlanID = "plan-2" + updated.CurrentReplanRequestID = "rq-1" + updated.CurrentPlan.PlanID = "plan-2" + updated.CurrentPlan.EffectiveMediaFileID = f.altFileID + updated.CurrentPlan.DecisionReason = "transcode_fallback" + updated.ExpiresAt = time.Now().Add(2 * time.Hour).UTC().Truncate(time.Microsecond) + response := json.RawMessage(`{"plan_id": "plan-2", "status": "replanned"}`) + + if err := store.CompleteReplan(ctx, sessionID, "rq-1", "", response, updated); err != nil { + t.Fatalf("CompleteReplan happy path: %v", err) + } + + got, err := store.GetAttempt(ctx, sessionID) + if err != nil { + t.Fatalf("GetAttempt after replan: %v", err) + } + if got.CurrentReplanRequestID != "rq-1" { + t.Fatalf("current_replan_request_id = %q, want rq-1", got.CurrentReplanRequestID) + } + if got.EffectiveMediaFileID != f.altFileID { + t.Fatalf("effective_media_file_id = %d, want %d", got.EffectiveMediaFileID, f.altFileID) + } + if got.CurrentPlanID != "plan-2" || got.CurrentPlan.PlanID != "plan-2" || got.CurrentPlan.DecisionReason != "transcode_fallback" { + t.Fatalf("plan did not round-trip through replan: %+v", got.CurrentPlan) + } + if !bytes.Equal(mustJSON(t, got.CurrentPlan), mustJSON(t, updated.CurrentPlan)) { + t.Fatalf("plan JSON mismatch after replan:\n got %s\nwant %s", mustJSON(t, got.CurrentPlan), mustJSON(t, updated.CurrentPlan)) + } + + // The migration's sync trigger must not fight the in-transaction CAS: + // the raw column must equal the new request ID, with no extra rewrite. + var rawRevision, replanState string + if err := f.pool.QueryRow(ctx, ` + SELECT a.current_replan_request_id, r.state + FROM playback_v3_attempts a + JOIN playback_v3_replans r ON r.session_id = a.session_id AND r.replan_request_id = $2 + WHERE a.session_id = $1::uuid`, sessionID, "rq-1").Scan(&rawRevision, &replanState); err != nil { + t.Fatalf("inspect attempt/replan rows: %v", err) + } + if rawRevision != "rq-1" { + t.Fatalf("raw current_replan_request_id = %q, want rq-1", rawRevision) + } + if replanState != "completed" { + t.Fatalf("replan state = %q, want completed", replanState) + } + + // A second replan whose base does not match the current revision must + // lose the compare-and-swap. + if _, err := store.BeginReplan(ctx, sessionID, "rq-2", "rq-digest-2", "rq-1", future); err != nil { + t.Fatalf("BeginReplan second: %v", err) + } + stale := updated + stale.CurrentReplanRequestID = "rq-2" + if err := store.CompleteReplan(ctx, sessionID, "rq-2", "wrong-base", response, stale); !errors.Is(err, playback.ErrReplanSupersededV3) { + t.Fatalf("CompleteReplan wrong base: got %v, want ErrReplanSupersededV3", err) + } + + // Unknown session. + if err := store.CompleteReplan(ctx, uuid.NewString(), "rq-1", "", response, updated); !errors.Is(err, playback.ErrSessionNotFound) { + t.Fatalf("CompleteReplan missing session: got %v, want ErrSessionNotFound", err) + } + }) + + t.Run("CleanupExpired", func(t *testing.T) { + sessionID := uuid.NewString() + attemptID := "att-cleanup-" + sessionID + if err := store.SaveAttempt(ctx, f.attemptRecord(sessionID, attemptID, "digest-cleanup")); err != nil { + t.Fatalf("SaveAttempt: %v", err) + } + if _, err := store.BeginReplan(ctx, sessionID, "rq-1", "rq-digest-1", "", time.Now().Add(time.Minute)); err != nil { + t.Fatalf("BeginReplan: %v", err) + } + + // A survivor attempt that must not be swept. + keepSession := uuid.NewString() + keepAttempt := "att-keep-" + keepSession + if err := store.SaveAttempt(ctx, f.attemptRecord(keepSession, keepAttempt, "digest-keep")); err != nil { + t.Fatalf("SaveAttempt survivor: %v", err) + } + + event := func(attempt string) playback.RouteEventRecordV3 { + return playback.RouteEventRecordV3{ + RouteEventV3: playback.RouteEventV3{ + ProtocolVersion: 3, + PlaybackAttemptID: attempt, + SessionID: sessionID, + Event: playback.RouteEventNamesV3()[0], + Diagnostics: map[string]string{}, + }, + UserID: f.userID, + ProfileID: "profile-1", + } + } + if err := store.RecordRouteEvent(ctx, event("att-cleanup-old")); err != nil { + t.Fatalf("RecordRouteEvent old: %v", err) + } + if err := store.RecordRouteEvent(ctx, event("att-cleanup-recent")); err != nil { + t.Fatalf("RecordRouteEvent recent: %v", err) + } + if _, err := f.pool.Exec(ctx, ` + UPDATE playback_route_events SET received_at = NOW() - INTERVAL '31 days' + WHERE playback_attempt_id = 'att-cleanup-old'`); err != nil { + t.Fatalf("age route event: %v", err) + } + + f.expireAttempt(t, attemptID) + removed, err := store.CleanupExpired(ctx, time.Now()) + if err != nil { + t.Fatalf("CleanupExpired: %v", err) + } + if removed < 1 { + t.Fatalf("CleanupExpired removed %d attempts, want at least 1", removed) + } + + var attempts, replans, oldEvents, recentEvents int + if err := f.pool.QueryRow(ctx, `SELECT COUNT(*) FROM playback_v3_attempts WHERE playback_attempt_id = $1`, attemptID).Scan(&attempts); err != nil { + t.Fatalf("count attempts: %v", err) + } + if err := f.pool.QueryRow(ctx, `SELECT COUNT(*) FROM playback_v3_replans WHERE session_id = $1::uuid`, sessionID).Scan(&replans); err != nil { + t.Fatalf("count replans: %v", err) + } + if err := f.pool.QueryRow(ctx, `SELECT COUNT(*) FROM playback_route_events WHERE playback_attempt_id = 'att-cleanup-old'`).Scan(&oldEvents); err != nil { + t.Fatalf("count old events: %v", err) + } + if err := f.pool.QueryRow(ctx, `SELECT COUNT(*) FROM playback_route_events WHERE playback_attempt_id = 'att-cleanup-recent'`).Scan(&recentEvents); err != nil { + t.Fatalf("count recent events: %v", err) + } + if attempts != 0 { + t.Fatalf("expired attempt survived cleanup") + } + if replans != 0 { + t.Fatalf("replans did not cascade with the expired attempt") + } + if oldEvents != 0 { + t.Fatalf("31-day-old route event survived cleanup") + } + if recentEvents != 1 { + t.Fatalf("recent route event count = %d, want 1", recentEvents) + } + if _, err := store.GetAttempt(ctx, keepSession); err != nil { + t.Fatalf("unexpired attempt was swept: %v", err) + } + }) + + t.Run("AcquireSessionLock", func(t *testing.T) { + sessionID := uuid.NewString() + + release1, err := store.AcquireSessionLock(ctx, sessionID) + if err != nil { + t.Fatalf("first AcquireSessionLock: %v", err) + } + + // A different session must not be serialized behind the first lock. + otherCtx, cancelOther := context.WithTimeout(ctx, 5*time.Second) + defer cancelOther() + releaseOther, err := store.AcquireSessionLock(otherCtx, uuid.NewString()) + if err != nil { + t.Fatalf("different-session AcquireSessionLock blocked: %v", err) + } + releaseOther() + + // A second acquire on the same session must block until release. + type lockResult struct { + release func() + err error + } + acquired := make(chan lockResult, 1) + go func() { + release2, err := store.AcquireSessionLock(ctx, sessionID) + acquired <- lockResult{release: release2, err: err} + }() + + select { + case result := <-acquired: + if result.err == nil { + result.release() + } + t.Fatalf("second lock acquired while first was held (err=%v)", result.err) + case <-time.After(300 * time.Millisecond): + // Still blocked, as required. + } + + release1() + select { + case result := <-acquired: + if result.err != nil { + t.Fatalf("second AcquireSessionLock after release: %v", result.err) + } + result.release() + case <-time.After(5 * time.Second): + t.Fatal("second lock never acquired after first release") + } + + // Releasing twice is safe (sync.Once) and the lock is free again. + release1() + release3, err := store.AcquireSessionLock(ctx, sessionID) + if err != nil { + t.Fatalf("reacquire after release: %v", err) + } + release3() + }) +} diff --git a/internal/playback/protocol_store_v3.go b/internal/playback/protocol_store_v3.go new file mode 100644 index 00000000..ed567ea5 --- /dev/null +++ b/internal/playback/protocol_store_v3.go @@ -0,0 +1,269 @@ +package playback + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "time" +) + +var ErrIdempotencyKeyReusedV3 = errors.New("idempotency key reused") +var ErrPlaybackAttemptExistsV3 = errors.New("playback attempt already exists") +var ErrStaleReplanLeaseV3 = errors.New("stale replan lease") + +// ErrReplanSupersededV3 means a CompleteReplan lost the revision compare: a +// newer replan already moved the attempt past the caller's base revision. +var ErrReplanSupersededV3 = errors.New("replan superseded") + +type AttemptRecordV3 struct { + PlaybackAttemptID string + SessionID string + UserID int + ProfileID string + RequestedMediaFileID int + EffectiveMediaFileID int + CurrentPlanID string + CurrentReplanRequestID string + CurrentPlan PlanV3 + NormalizedRequest StartRequestV3 + // RequestDigest fingerprints the normalized start request so an attempt-ID + // reused with different input is a detectable idempotency violation rather + // than a silent replay of the old plan. + RequestDigest string + ExpiresAt time.Time +} + +// AttemptIdentityV3 carries only the ownership columns of an attempt so +// per-event authorization checks avoid decoding the plan and request JSONB. +type AttemptIdentityV3 struct { + PlaybackAttemptID string + SessionID string + UserID int + ProfileID string +} + +type RouteEventRecordV3 struct { + RouteEventV3 + UserID int + ProfileID string + ClientName string + ClientVersion string + ClientModel string +} + +type ReplanLeaseStateV3 string + +const ( + ReplanLeaseOwnedV3 ReplanLeaseStateV3 = "owned" + ReplanLeaseInFlightV3 ReplanLeaseStateV3 = "in_flight" + ReplanLeaseCompletedV3 ReplanLeaseStateV3 = "completed" +) + +type ReplanLeaseV3 struct { + State ReplanLeaseStateV3 + Response json.RawMessage +} + +type PlanStoreV3 interface { + AcquireSessionLock(context.Context, string) (func(), error) + SaveAttempt(context.Context, AttemptRecordV3) error + GetAttempt(context.Context, string) (*AttemptRecordV3, error) + GetAttemptByPlaybackAttemptID(context.Context, string) (*AttemptRecordV3, error) + GetAttemptIdentity(context.Context, string) (*AttemptIdentityV3, error) + GetAttemptIdentityByPlaybackAttemptID(context.Context, string) (*AttemptIdentityV3, error) + BeginReplan(context.Context, string, string, string, string, time.Time) (ReplanLeaseV3, error) + // CompleteReplan commits a replan atomically; the attempt row is only + // updated while its current_replan_request_id still equals the caller's + // base revision, otherwise ErrReplanSupersededV3 is returned. + CompleteReplan(ctx context.Context, sessionID, requestID, baseReplanRequestID string, response json.RawMessage, record AttemptRecordV3) error + RecordRouteEvent(context.Context, RouteEventRecordV3) error + CleanupExpired(context.Context, time.Time) (int64, error) +} + +type memoryReplanV3 struct { + digest string + base string + lease time.Time + completed bool + response json.RawMessage +} + +type MemoryPlanStoreV3 struct { + mu sync.Mutex + attempts map[string]AttemptRecordV3 + replans map[string]memoryReplanV3 + events []RouteEventRecordV3 +} + +func NewMemoryPlanStoreV3() *MemoryPlanStoreV3 { + return &MemoryPlanStoreV3{attempts: make(map[string]AttemptRecordV3), replans: make(map[string]memoryReplanV3)} +} + +// AcquireSessionLock is deliberately a no-op. The store lock exists to +// serialize replans across processes sharing one PostgreSQL database; the +// memory store only ever backs a single-process, DB-less deployment, where +// the handler's own per-session replan mutex already provides the same +// serialization before this lock is taken. +func (s *MemoryPlanStoreV3) AcquireSessionLock(context.Context, string) (func(), error) { + return func() {}, nil +} + +func (s *MemoryPlanStoreV3) SaveAttempt(_ context.Context, record AttemptRecordV3) error { + s.mu.Lock() + defer s.mu.Unlock() + now := time.Now() + // Expired rows are replaceable, mirroring the Postgres pre-delete: they + // linger until the hourly cleanup and must not wedge a legitimate retry. + for sessionID, existing := range s.attempts { + if existing.ExpiresAt.After(now) { + continue + } + if existing.PlaybackAttemptID == record.PlaybackAttemptID || sessionID == record.SessionID { + s.deleteAttemptLocked(sessionID) + } + } + for sessionID, existing := range s.attempts { + if existing.PlaybackAttemptID != record.PlaybackAttemptID && sessionID != record.SessionID { + continue + } + if existing.PlaybackAttemptID == record.PlaybackAttemptID && + existing.RequestDigest != "" && record.RequestDigest != "" && existing.RequestDigest != record.RequestDigest { + return ErrIdempotencyKeyReusedV3 + } + return ErrPlaybackAttemptExistsV3 + } + s.attempts[record.SessionID] = record + return nil +} + +// ReplaceAttempt overwrites a session's attempt record unconditionally. It is +// not part of PlanStoreV3: durable stores treat attempts as insert-once and +// replan-updated, so only in-memory test setups may rewrite one in place. +func (s *MemoryPlanStoreV3) ReplaceAttempt(_ context.Context, record AttemptRecordV3) { + s.mu.Lock() + defer s.mu.Unlock() + s.attempts[record.SessionID] = record +} + +func (s *MemoryPlanStoreV3) deleteAttemptLocked(sessionID string) { + delete(s.attempts, sessionID) + for key := range s.replans { + if strings.HasPrefix(key, sessionID+":") { + delete(s.replans, key) + } + } +} + +func (s *MemoryPlanStoreV3) GetAttemptByPlaybackAttemptID(_ context.Context, attemptID string) (*AttemptRecordV3, error) { + s.mu.Lock() + defer s.mu.Unlock() + for _, record := range s.attempts { + if record.PlaybackAttemptID == attemptID && record.ExpiresAt.After(time.Now()) { + copy := record + return ©, nil + } + } + return nil, ErrSessionNotFound +} + +func (s *MemoryPlanStoreV3) GetAttempt(_ context.Context, sessionID string) (*AttemptRecordV3, error) { + s.mu.Lock() + defer s.mu.Unlock() + record, ok := s.attempts[sessionID] + if !ok || !record.ExpiresAt.After(time.Now()) { + return nil, ErrSessionNotFound + } + copy := record + return ©, nil +} + +func (s *MemoryPlanStoreV3) BeginReplan(_ context.Context, sessionID, requestID, digest, baseReplanRequestID string, leaseUntil time.Time) (ReplanLeaseV3, error) { + s.mu.Lock() + defer s.mu.Unlock() + key := sessionID + ":" + requestID + existing, ok := s.replans[key] + if !ok { + s.replans[key] = memoryReplanV3{digest: digest, base: baseReplanRequestID, lease: leaseUntil} + return ReplanLeaseV3{State: ReplanLeaseOwnedV3}, nil + } + if existing.digest != digest { + return ReplanLeaseV3{}, ErrIdempotencyKeyReusedV3 + } + if existing.completed { + return ReplanLeaseV3{State: ReplanLeaseCompletedV3, Response: append(json.RawMessage(nil), existing.response...)}, nil + } + if time.Now().Before(existing.lease) { + return ReplanLeaseV3{State: ReplanLeaseInFlightV3}, nil + } + if existing.base != baseReplanRequestID { + return ReplanLeaseV3{}, ErrStaleReplanLeaseV3 + } + existing.lease = leaseUntil + s.replans[key] = existing + return ReplanLeaseV3{State: ReplanLeaseOwnedV3}, nil +} + +func (s *MemoryPlanStoreV3) CompleteReplan(_ context.Context, sessionID, requestID, baseReplanRequestID string, response json.RawMessage, record AttemptRecordV3) error { + s.mu.Lock() + defer s.mu.Unlock() + existing, ok := s.attempts[sessionID] + if !ok { + return ErrSessionNotFound + } + if existing.CurrentReplanRequestID != baseReplanRequestID { + return ErrReplanSupersededV3 + } + key := sessionID + ":" + requestID + entry, ok := s.replans[key] + if !ok { + return ErrSessionNotFound + } + entry.completed = true + entry.response = append(json.RawMessage(nil), response...) + s.replans[key] = entry + s.attempts[sessionID] = record + return nil +} + +func (s *MemoryPlanStoreV3) GetAttemptIdentity(ctx context.Context, sessionID string) (*AttemptIdentityV3, error) { + record, err := s.GetAttempt(ctx, sessionID) + if err != nil { + return nil, err + } + return &AttemptIdentityV3{PlaybackAttemptID: record.PlaybackAttemptID, SessionID: record.SessionID, UserID: record.UserID, ProfileID: record.ProfileID}, nil +} + +func (s *MemoryPlanStoreV3) GetAttemptIdentityByPlaybackAttemptID(ctx context.Context, attemptID string) (*AttemptIdentityV3, error) { + record, err := s.GetAttemptByPlaybackAttemptID(ctx, attemptID) + if err != nil { + return nil, err + } + return &AttemptIdentityV3{PlaybackAttemptID: record.PlaybackAttemptID, SessionID: record.SessionID, UserID: record.UserID, ProfileID: record.ProfileID}, nil +} + +func (s *MemoryPlanStoreV3) RecordRouteEvent(_ context.Context, record RouteEventRecordV3) error { + s.mu.Lock() + defer s.mu.Unlock() + s.events = append(s.events, record) + return nil +} + +func (s *MemoryPlanStoreV3) CleanupExpired(_ context.Context, now time.Time) (int64, error) { + s.mu.Lock() + defer s.mu.Unlock() + var count int64 + for sessionID, record := range s.attempts { + if !record.ExpiresAt.After(now) { + delete(s.attempts, sessionID) + for key := range s.replans { + if strings.HasPrefix(key, sessionID+":") { + delete(s.replans, key) + } + } + count++ + } + } + return count, nil +} diff --git a/internal/playback/protocol_v3.go b/internal/playback/protocol_v3.go new file mode 100644 index 00000000..20efbce3 --- /dev/null +++ b/internal/playback/protocol_v3.go @@ -0,0 +1,825 @@ +package playback + +import ( + "errors" + "fmt" + "math" + "regexp" + "slices" + "strings" + "time" +) + +const ( + ProtocolV3 = 3 + FeaturePlaybackPlanV3 = "playback_plan_v3" + FeatureMedia3Only = "media3_only" + FeatureDetailedDecodeV3 = "detailed_decode_capabilities" + FeatureLayoutPassthrough = "layout_aware_passthrough" + FeatureClientVideoTransforms = "client_video_transformations_v1" + FeatureRouteDiagnostics = "playback_route_diagnostics" + FeatureDeviceQuirksV3 = "device_quirks_v1" + FeatureSeekReanchorV3 = "seek_reanchor_v1" + PlanRecipeVersionV3 = "v3.2" + ClientDV7ToDV81V3 = "client_dv7_to_dv81" + ClientDV7ToHDR10V3 = "client_dv7_to_hdr10" + ClientDVTransformVersionV3 = "1" + ClientDV8HDR10PlusSanitizerV3 = "client_dv8_hdr10plus_sanitizer_v1" + ClientPostResumeRecoveryV3 = "client_post_resume_video_recovery_v1" + ClientSurfaceRecoveryV3 = "client_surface_recovery_v1" + DeviceQuirkRegistryRevisionV3 = "2026-07-13.1" +) + +type DecisionOutcomeV3 string + +const ( + OutcomePlayableV3 DecisionOutcomeV3 = "playable" + OutcomeAdaptationUnavailableV3 DecisionOutcomeV3 = "adaptation_unavailable" +) + +type DeliveryV3 string + +const ( + DeliveryOriginalHTTPV3 DeliveryV3 = "original_http" + DeliveryRemuxHLSV3 DeliveryV3 = "server_remux_hls" + DeliveryRemuxProgressiveV3 DeliveryV3 = "server_remux_progressive" + DeliveryTranscodeHLSV3 DeliveryV3 = "server_transcode_hls" +) + +func (d DeliveryV3) KotlinName() string { + switch d { + case DeliveryOriginalHTTPV3: + return "ORIGINAL_HTTP" + case DeliveryRemuxHLSV3: + return "SERVER_REMUX_HLS" + case DeliveryRemuxProgressiveV3: + return "SERVER_REMUX_PROGRESSIVE" + case DeliveryTranscodeHLSV3: + return "SERVER_TRANSCODE_HLS" + default: + return strings.ToUpper(string(d)) + } +} + +type EngineV3 string + +const ( + EngineMedia3DirectV3 EngineV3 = "media3_direct" + EngineMedia3ProgressiveRemuxV3 EngineV3 = "media3_progressive_remux" + EngineMedia3HLSV3 EngineV3 = "media3_hls" +) + +type StreamProtocolV3 string + +const ( + StreamHTTPProgressiveV3 StreamProtocolV3 = "http_progressive" + StreamHLSV3 StreamProtocolV3 = "hls" +) + +func (p StreamProtocolV3) KotlinName() string { + switch p { + case StreamHTTPProgressiveV3: + return "HTTP_PROGRESSIVE" + case StreamHLSV3: + return "HLS" + default: + return strings.ToUpper(string(p)) + } +} + +type HeaderRefreshModeV3 string + +const ( + HeaderRefreshNoneV3 HeaderRefreshModeV3 = "none" + HeaderRefreshSessionV3 HeaderRefreshModeV3 = "session" +) + +type SubtitleModeV3 string + +const ( + SubtitleOffV3 SubtitleModeV3 = "off" + SubtitleRenderV3 SubtitleModeV3 = "render" + SubtitleConvertV3 SubtitleModeV3 = "convert" + SubtitleBurnInV3 SubtitleModeV3 = "burn_in" +) + +func (m SubtitleModeV3) KotlinName() string { + switch m { + case SubtitleOffV3: + return "OFF" + case SubtitleRenderV3: + return "RENDER" + case SubtitleConvertV3: + return "CONVERT" + case SubtitleBurnInV3: + return "BURN_IN" + default: + return strings.ToUpper(string(m)) + } +} + +type SubtitleFidelityV3 string + +const ( + SubtitleFidelityPreserveV3 SubtitleFidelityV3 = "preserve" + SubtitleFidelityCompatibleV3 SubtitleFidelityV3 = "compatible" +) + +type EnhancementLayerV3 string + +const ( + EnhancementNoneV3 EnhancementLayerV3 = "none" + EnhancementMELV3 EnhancementLayerV3 = "mel" + EnhancementFELV3 EnhancementLayerV3 = "fel" + EnhancementUnknownV3 EnhancementLayerV3 = "unknown" +) + +type HDRCapabilitiesV3 struct { + HDR10 bool `json:"hdr10"` + HDR10Plus bool `json:"hdr10_plus"` + HLG bool `json:"hlg"` + DolbyVisionProfiles []int `json:"dolby_vision_profiles"` +} + +type AudioPassthroughV3 struct { + PassthroughCodecs []string `json:"passthrough_codecs"` + SpatializerEnabled bool `json:"spatializer_enabled"` + MaxChannels int `json:"max_channels"` + Entries []AudioPassthroughEntryV3 `json:"entries,omitempty"` +} + +type AudioPassthroughEntryV3 struct { + Codec string `json:"codec"` + ChannelCounts []int `json:"channel_counts,omitempty"` + Layouts []string `json:"layouts,omitempty"` +} + +type VideoDecodeCapabilityV3 struct { + Codec string `json:"codec"` + DecoderName string `json:"decoder_name,omitempty"` + Profiles []string `json:"profiles,omitempty"` + Levels []int `json:"levels,omitempty"` + BitDepths []int `json:"bit_depths,omitempty"` + MaxWidth int `json:"max_width,omitempty"` + MaxHeight int `json:"max_height,omitempty"` + MaxFrameRate float64 `json:"max_frame_rate,omitempty"` + MaxBitrateKbps int `json:"max_bitrate_kbps,omitempty"` + Hardware bool `json:"hardware"` +} + +type ClientCodecCapabilitiesV3 struct { + CodecsVideo []string `json:"codecs_video"` + CodecsVideoHardware []string `json:"codecs_video_hardware"` + CodecsAudio []string `json:"codecs_audio"` + Containers []string `json:"containers"` + MaxResolution string `json:"max_resolution,omitempty"` + HDR bool `json:"hdr"` + HDRDetails *HDRCapabilitiesV3 `json:"hdr_details,omitempty"` + AudioPassthrough *AudioPassthroughV3 `json:"audio_passthrough,omitempty"` + VideoDecode []VideoDecodeCapabilityV3 `json:"video_decode,omitempty"` +} + +type DeviceContextV3 struct { + Manufacturer string `json:"manufacturer,omitempty"` + Model string `json:"model,omitempty"` + Brand string `json:"brand,omitempty"` + Device string `json:"device,omitempty"` + Product string `json:"product,omitempty"` + SoCManufacturer string `json:"soc_manufacturer,omitempty"` + SoCModel string `json:"soc_model,omitempty"` + BuildID string `json:"build_id,omitempty"` + BuildDisplay string `json:"build_display,omitempty"` + SecurityPatch string `json:"security_patch,omitempty"` + SDKInt int `json:"sdk_int,omitempty"` + ABIs []string `json:"abis,omitempty"` +} + +type OutputContextV3 struct { + HDRDetails *HDRCapabilitiesV3 `json:"hdr_details,omitempty"` + AudioPassthrough *AudioPassthroughV3 `json:"audio_passthrough,omitempty"` + CurrentSink string `json:"current_sink,omitempty"` + SinkType string `json:"sink_type,omitempty"` + OutputRouteGeneration int64 `json:"output_route_generation"` +} + +type EngineSubtitleCapabilitiesV3 struct { + EmbeddedText bool `json:"embedded_text"` + SidecarText bool `json:"sidecar_text"` + ASSStyling bool `json:"ass_styling"` + EmbeddedBitmap bool `json:"embedded_bitmap"` + SidecarBitmap bool `json:"sidecar_bitmap"` + FontAttachments bool `json:"font_attachments"` +} + +type EngineCapabilityV3 struct { + Enabled bool `json:"enabled"` + SupportedOnDevice bool `json:"supported_on_device"` + FailureReason string `json:"failure_reason,omitempty"` + Containers []string `json:"containers"` + VideoCodecs []string `json:"video_codecs"` + AudioDecodeCodecs []string `json:"audio_decode_codecs"` + AudioPassthroughCodecs []string `json:"audio_passthrough_codecs"` + MaxChannels *int `json:"max_channels,omitempty"` + HDRDetails *HDRCapabilitiesV3 `json:"hdr_details,omitempty"` + Subtitles EngineSubtitleCapabilitiesV3 `json:"subtitles"` + Features []string `json:"features"` + AuthHeaderRefresh bool `json:"auth_header_refresh"` + ValidatedClaims []string `json:"validated_claims"` + Transformations []TransformationV3 `json:"transformations"` +} + +type ClientPlaybackContextV3 struct { + ProtocolVersion int `json:"protocol_version"` + Features []string `json:"features"` + Platform string `json:"platform"` + FormFactor string `json:"form_factor"` + AppVersion string `json:"app_version"` + Device DeviceContextV3 `json:"device"` + Output OutputContextV3 `json:"output"` + Engines map[string]EngineCapabilityV3 `json:"engines"` +} + +type StartRequestV3 struct { + ProtocolVersion int `json:"protocol_version"` + ClientFeatures []string `json:"client_features"` + FileID int `json:"file_id"` + ProfileID string `json:"profile_id"` + PlaybackAttemptID string `json:"playback_attempt_id"` + QualityPreference string `json:"quality_preference"` + SubtitleFidelityPreference SubtitleFidelityV3 `json:"subtitle_fidelity_preference"` + StartPosition *float64 `json:"start_position,omitempty"` + AudioTrackID string `json:"audio_track_id,omitempty"` + AudioTrackIndex *int `json:"audio_track_index,omitempty"` + SubtitleTrackID string `json:"subtitle_track_id,omitempty"` + SubtitleTrackIndex *int `json:"subtitle_track_index,omitempty"` + OutputRouteGeneration int64 `json:"output_route_generation"` + Metered bool `json:"metered"` + BandwidthEstimateKbps *int `json:"bandwidth_estimate_kbps,omitempty"` + BandwidthCapKbps *int `json:"bandwidth_cap_kbps,omitempty"` + Capabilities ClientCodecCapabilitiesV3 `json:"client_capabilities"` + ClientPlaybackContext ClientPlaybackContextV3 `json:"client_playback_context"` +} + +type TrackIdentityV3 struct { + ID string `json:"id"` + Index *int `json:"index,omitempty"` +} + +type SelectedTracksV3 struct { + Audio *TrackIdentityV3 `json:"audio,omitempty"` + Subtitle *TrackIdentityV3 `json:"subtitle,omitempty"` +} + +type FailureV3 struct { + Classification string `json:"classification"` + Message string `json:"message,omitempty"` + DecoderName string `json:"decoder_name,omitempty"` +} + +type ReplanOperationV3 string + +const ( + ReplanOperationFailureRecoveryV3 ReplanOperationV3 = "failure_recovery" + ReplanOperationSeekReanchorV3 ReplanOperationV3 = "seek_reanchor" + ReplanOperationSeekFailureRecoveryV3 ReplanOperationV3 = "seek_failure_recovery" +) + +type ReplanRequestV3 struct { + ProtocolVersion int `json:"protocol_version"` + Operation ReplanOperationV3 `json:"operation,omitempty"` + PlaybackAttemptID string `json:"playback_attempt_id"` + ReplanRequestID string `json:"replan_request_id"` + FailedPlanID string `json:"failed_plan_id"` + PlanAttemptID string `json:"plan_attempt_id"` + PlanAttemptKey string `json:"plan_attempt_key"` + AttemptedPlanKeys []string `json:"attempted_plan_keys"` + AttemptCount int `json:"attempt_count"` + QualityPreference string `json:"quality_preference"` + PositionSeconds float64 `json:"position_seconds"` + OutputRouteGeneration int64 `json:"output_route_generation"` + Metered bool `json:"metered"` + BandwidthEstimateKbps *int `json:"bandwidth_estimate_kbps,omitempty"` + BandwidthCapKbps *int `json:"bandwidth_cap_kbps,omitempty"` + SelectedTracks SelectedTracksV3 `json:"selected_tracks"` + Failure FailureV3 `json:"failure"` + Capabilities ClientCodecCapabilitiesV3 `json:"client_capabilities"` + ClientPlaybackContext ClientPlaybackContextV3 `json:"client_playback_context"` +} + +const ( + RouteEventPlanSelectedV3 = "plan_selected" + RouteEventPlanInvalidatedV3 = "plan_invalidated" + RouteEventPlanFailedV3 = "plan_failed" + RouteEventFirstFrameV3 = "first_frame" + RouteEventTerminalV3 = "terminal" + RouteEventStoppedV3 = "stopped" + RouteEventRuntimeCorrectionAppliedV3 = "runtime_correction_applied" + RouteEventRuntimeCorrectionSucceededV3 = "runtime_correction_succeeded" + RouteEventRuntimeCorrectionFailedV3 = "runtime_correction_failed" + RouteEventSeekReanchorRequestedV3 = "seek_reanchor_requested" + RouteEventSeekReanchoredV3 = "seek_reanchored" +) + +var routeEventNamesV3 = []string{ + RouteEventPlanSelectedV3, + RouteEventPlanInvalidatedV3, + RouteEventPlanFailedV3, + RouteEventFirstFrameV3, + RouteEventTerminalV3, + RouteEventStoppedV3, + RouteEventRuntimeCorrectionAppliedV3, + RouteEventRuntimeCorrectionSucceededV3, + RouteEventRuntimeCorrectionFailedV3, + RouteEventSeekReanchorRequestedV3, + RouteEventSeekReanchoredV3, +} + +// RouteEventNamesV3 returns the complete protocol-v3 telemetry event contract. +func RouteEventNamesV3() []string { + return append([]string(nil), routeEventNamesV3...) +} + +// ValidRouteEventNameV3 reports whether name is part of the protocol-v3 +// telemetry contract shared by handlers, persistence, and clients. +func ValidRouteEventNameV3(name string) bool { + return slices.Contains(routeEventNamesV3, name) +} + +// EffectiveOperation keeps clients which predate the explicit operation field +// on the ordinary failure-recovery path. Seek operations are deliberately +// opt-in because both pin the current media version and user intent; an exact +// reanchor also preserves the current route instead of walking the ladder. +func (r ReplanRequestV3) EffectiveOperation() ReplanOperationV3 { + if r.Operation == "" { + return ReplanOperationFailureRecoveryV3 + } + return r.Operation +} + +type RouteEventV3 struct { + ProtocolVersion int `json:"protocol_version"` + PlaybackAttemptID string `json:"playback_attempt_id"` + SessionID string `json:"session_id,omitempty"` + PlanID string `json:"plan_id,omitempty"` + PlanAttemptID string `json:"plan_attempt_id,omitempty"` + PlanAttemptKey string `json:"plan_attempt_key,omitempty"` + Event string `json:"event"` + FailureClassification string `json:"failure_classification,omitempty"` + FallbackReason string `json:"fallback_reason,omitempty"` + AppliedQuirkIDs []string `json:"applied_quirk_ids,omitempty"` + QuirkRegistryRevision string `json:"quirk_registry_revision,omitempty"` + OutputRouteGeneration int64 `json:"output_route_generation"` + Diagnostics map[string]string `json:"diagnostics"` +} + +type StreamV3 struct { + URL string `json:"url"` + Protocol StreamProtocolV3 `json:"protocol"` + Container string `json:"container,omitempty"` + MIMEType string `json:"mime_type,omitempty"` + Headers map[string]string `json:"headers"` + HeaderRefresh HeaderRefreshModeV3 `json:"header_refresh"` + HeaderRefreshURL string `json:"header_refresh_url,omitempty"` +} + +type TimelineV3 struct { + SourceStartSeconds float64 `json:"source_start_seconds"` + StreamOriginSeconds float64 `json:"stream_origin_seconds"` + PlayerStartSeconds float64 `json:"player_start_seconds"` + TimelineOffsetSeconds float64 `json:"timeline_offset_seconds"` + SeekWindowStartSeconds *float64 `json:"seek_window_start_seconds,omitempty"` + SeekWindowEndSeconds *float64 `json:"seek_window_end_seconds,omitempty"` + CanSeekAnywhere bool `json:"can_seek_anywhere"` + SeekRestoration string `json:"seek_restoration"` +} + +type EffectiveRecipeV3 struct { + VideoCodec string `json:"video_codec,omitempty"` + AudioCodec string `json:"audio_codec,omitempty"` + Width *int `json:"width,omitempty"` + Height *int `json:"height,omitempty"` + FrameRate *float64 `json:"frame_rate,omitempty"` + BitrateKbps *int `json:"bitrate_kbps,omitempty"` + DynamicRange string `json:"dynamic_range,omitempty"` + AudioChannels *int `json:"audio_channels,omitempty"` + AudioLayout string `json:"audio_layout,omitempty"` +} + +type SourceDescriptorV3 struct { + MediaFileID int `json:"media_file_id"` + Container string `json:"container,omitempty"` + VideoCodec string `json:"video_codec,omitempty"` + VideoProfile string `json:"video_profile,omitempty"` + VideoLevel int `json:"video_level,omitempty"` + BitDepth int `json:"bit_depth,omitempty"` + ColorRange string `json:"color_range,omitempty"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` + FrameRate float64 `json:"frame_rate,omitempty"` + BitrateKbps int `json:"bitrate_kbps,omitempty"` + DynamicRange string `json:"dynamic_range,omitempty"` + HDR10Plus bool `json:"hdr10_plus"` + DVProfile int `json:"dolby_vision_profile,omitempty"` + DVBLCompatID int `json:"dv_bl_compat_id,omitempty"` + DVEnhancementLayer EnhancementLayerV3 `json:"dv_enhancement_layer"` + AudioCodec string `json:"audio_codec,omitempty"` + AudioChannels int `json:"audio_channels,omitempty"` + AudioLayout string `json:"audio_layout,omitempty"` +} + +type VideoClaimsV3 struct { + HDR10 bool `json:"hdr10"` + HDR10Plus bool `json:"hdr10_plus"` + HLG bool `json:"hlg"` + DolbyVision bool `json:"dolby_vision"` + DolbyVisionReason string `json:"dolby_vision_reason,omitempty"` +} + +type AudioClaimsV3 struct { + Codec string `json:"codec,omitempty"` + Passthrough bool `json:"passthrough"` + AtmosPreserved bool `json:"atmos_preserved"` + DTSVariant string `json:"dts_variant,omitempty"` + Reason string `json:"reason,omitempty"` +} + +type SubtitleClaimsV3 struct { + ASSStylingPreserved bool `json:"ass_styling_preserved"` + BitmapOverlay bool `json:"bitmap_overlay"` + BitmapSidecar bool `json:"bitmap_sidecar"` + Reason string `json:"reason,omitempty"` +} + +type ValidationClaimsV3 struct { + Video VideoClaimsV3 `json:"video"` + Audio AudioClaimsV3 `json:"audio"` + Subtitles SubtitleClaimsV3 `json:"subtitles"` +} + +type SubtitleArtifactV3 struct { + URL string `json:"url"` + MIMEType string `json:"mime_type"` + Format string `json:"format"` + TimingOriginSeconds float64 `json:"timing_origin_seconds"` +} + +type SubtitleDecisionV3 struct { + Mode SubtitleModeV3 `json:"mode"` + TrackID string `json:"track_id,omitempty"` + Artifact *SubtitleArtifactV3 `json:"artifact,omitempty"` +} + +type TransformationV3 struct { + Name string `json:"name"` + Executor string `json:"executor"` + RecipeVersion string `json:"recipe_version"` + ValidatedClaims []string `json:"validated_claims"` +} + +type AppliedQuirkV3 struct { + ID string `json:"id"` + RegistryRevision string `json:"registry_revision"` + Action string `json:"action"` + Reason string `json:"reason,omitempty"` +} + +type DegradationWarningV3 struct { + Code string `json:"code"` + Message string `json:"message"` +} + +type PlanV3 struct { + ProtocolVersion int `json:"protocol_version"` + PlanID string `json:"plan_id"` + SessionID string `json:"session_id,omitempty"` + ExpiresAt string `json:"expires_at,omitempty"` + Delivery DeliveryV3 `json:"delivery"` + Engine EngineV3 `json:"engine"` + Stream StreamV3 `json:"stream"` + Timeline TimelineV3 `json:"timeline"` + SelectedTracks SelectedTracksV3 `json:"selected_tracks"` + EffectiveRecipe EffectiveRecipeV3 `json:"effective_recipe"` + Claims ValidationClaimsV3 `json:"claims"` + Subtitle SubtitleDecisionV3 `json:"subtitle"` + Transformations []TransformationV3 `json:"transformations"` + AppliedQuirks []AppliedQuirkV3 `json:"applied_quirks"` + RuntimeCorrections []string `json:"runtime_corrections"` + DegradationWarnings []DegradationWarningV3 `json:"degradation_warnings"` + DecisionReason string `json:"decision_reason"` + RequestedMediaFileID int `json:"requested_media_file_id"` + EffectiveMediaFileID int `json:"effective_media_file_id"` + Source SourceDescriptorV3 `json:"source"` + SubtitleFidelityPolicy string `json:"subtitle_fidelity_policy"` +} + +type TerminalV3 struct { + Reason string `json:"reason"` + Message string `json:"message"` + Retryable bool `json:"retryable"` +} + +type DecisionResponseV3 struct { + ProtocolVersion int `json:"protocol_version,omitempty"` + ServerFeatures []string `json:"server_features"` + Outcome DecisionOutcomeV3 `json:"outcome,omitempty"` + SessionID string `json:"session_id,omitempty"` + PlaybackPlan *PlanV3 `json:"playback_plan,omitempty"` + Terminal *TerminalV3 `json:"terminal,omitempty"` +} + +type CapabilityResponseV3 struct { + Enabled bool `json:"enabled"` + ProtocolVersions []int `json:"protocol_versions"` + Features []string `json:"features"` + Deliveries []DeliveryV3 `json:"deliveries"` + Transformations []TransformationV3 `json:"transformations"` + Reason string `json:"reason,omitempty"` +} + +var boundedIdentifierV3 = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._:-]{7,127}$`) + +func (r *StartRequestV3) NormalizeAndValidate() ([]DegradationWarningV3, error) { + if r.ProtocolVersion != ProtocolV3 { + return nil, fmt.Errorf("protocol_version must be %d", ProtocolV3) + } + if r.FileID <= 0 || strings.TrimSpace(r.ProfileID) == "" { + return nil, errors.New("file_id and profile_id are required") + } + if !boundedIdentifierV3.MatchString(r.PlaybackAttemptID) { + return nil, errors.New("playback_attempt_id is invalid") + } + if r.OutputRouteGeneration < 0 || r.ClientPlaybackContext.Output.OutputRouteGeneration != r.OutputRouteGeneration { + return nil, errors.New("output_route_generation is invalid or inconsistent") + } + if r.ClientPlaybackContext.ProtocolVersion != ProtocolV3 { + return nil, errors.New("client_playback_context.protocol_version must be 3") + } + if r.StartPosition != nil && (!isFiniteV3(*r.StartPosition) || *r.StartPosition < 0 || *r.StartPosition > 31_536_000) { + return nil, errors.New("start_position is outside the supported range") + } + if err := validateOptionalBoundedIntV3(r.BandwidthEstimateKbps, 100, 1_000_000, "bandwidth_estimate_kbps"); err != nil { + return nil, err + } + if err := validateOptionalBoundedIntV3(r.BandwidthCapKbps, 100, 1_000_000, "bandwidth_cap_kbps"); err != nil { + return nil, err + } + if r.SubtitleFidelityPreference != SubtitleFidelityPreserveV3 && r.SubtitleFidelityPreference != SubtitleFidelityCompatibleV3 { + return nil, errors.New("subtitle_fidelity_preference is invalid") + } + if len(r.ClientFeatures) > 64 { + return nil, errors.New("client_features exceeds supported size") + } + for _, feature := range r.ClientFeatures { + if len(feature) > 128 { + return nil, errors.New("client feature exceeds supported size") + } + } + if err := validateCapabilitiesV3(&r.Capabilities, &r.ClientPlaybackContext, r.ClientFeatures); err != nil { + return nil, err + } + if err := validateTrackPairV3(r.FileID, "audio", r.AudioTrackID, r.AudioTrackIndex); err != nil { + return nil, err + } + if err := validateTrackPairV3(r.FileID, "subtitle", r.SubtitleTrackID, r.SubtitleTrackIndex); err != nil { + return nil, err + } + quality, changed := NormalizeQualityV3(r.QualityPreference) + r.QualityPreference = quality + if changed { + return []DegradationWarningV3{{Code: "quality_preference_normalized", Message: "Unknown quality preference was normalized to auto."}}, nil + } + return nil, nil +} + +func NormalizeQualityV3(value string) (string, bool) { + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "auto": + return "auto", false + case "original", "source", "max": + return "original", false + case "2160p", "4k", "uhd": + return "2160p", false + case "1080p", "fhd": + return "1080p", false + case "720p", "hd": + return "720p", false + case "480p", "sd": + return "480p", false + default: + return "auto", true + } +} + +func (r ReplanRequestV3) Validate() error { + if r.ProtocolVersion != ProtocolV3 || !boundedIdentifierV3.MatchString(r.PlaybackAttemptID) || !boundedIdentifierV3.MatchString(r.ReplanRequestID) { + return errors.New("invalid replan identity") + } + if len(r.FailedPlanID) < 8 || len(r.FailedPlanID) > 128 || len(r.PlanAttemptID) < 8 || len(r.PlanAttemptID) > 128 || len(r.PlanAttemptKey) < 8 || len(r.PlanAttemptKey) > 128 || !strings.HasPrefix(r.PlanAttemptKey, "v3:") || r.AttemptCount < 1 || r.AttemptCount > 8 { + return errors.New("invalid replan attempt") + } + if r.OutputRouteGeneration < 0 || r.ClientPlaybackContext.Output.OutputRouteGeneration != r.OutputRouteGeneration { + return errors.New("invalid output route generation") + } + if len(r.AttemptedPlanKeys) > 16 || len(r.Failure.Classification) > 64 || len(r.Failure.Message) > 512 || len(r.Failure.DecoderName) > 128 || !isFiniteV3(r.PositionSeconds) || r.PositionSeconds < 0 || r.PositionSeconds > 31_536_000 { + return errors.New("replan bounds exceeded") + } + if err := validateOptionalBoundedIntV3(r.BandwidthEstimateKbps, 100, 1_000_000, "bandwidth_estimate_kbps"); err != nil { + return err + } + if err := validateOptionalBoundedIntV3(r.BandwidthCapKbps, 100, 1_000_000, "bandwidth_cap_kbps"); err != nil { + return err + } + if err := validateSelectedTrackIdentityV3("audio", r.SelectedTracks.Audio); err != nil { + return err + } + if err := validateSelectedTrackIdentityV3("subtitle", r.SelectedTracks.Subtitle); err != nil { + return err + } + switch r.EffectiveOperation() { + case ReplanOperationFailureRecoveryV3, ReplanOperationSeekFailureRecoveryV3: + if r.Failure.Classification == "" { + return errors.New("failure recovery requires a failure classification") + } + case ReplanOperationSeekReanchorV3: + // An exact seek reanchor is a timeline operation, not a failed recipe. + // Classification remains accepted for older callers but is not required + // and never selects seek semantics. + default: + return errors.New("invalid replan operation") + } + for _, key := range r.AttemptedPlanKeys { + if len(key) > 128 || !strings.HasPrefix(key, "v3:") { + return errors.New("invalid attempted plan key") + } + } + return validateCapabilitiesV3(&r.Capabilities, &r.ClientPlaybackContext, nil) +} + +func validateSelectedTrackIdentityV3(kind string, track *TrackIdentityV3) error { + if track == nil { + return nil + } + if len(track.ID) > 128 { + return fmt.Errorf("%s track id exceeds supported size", kind) + } + if track.Index != nil && (*track.Index < 0 || *track.Index > 10_000) { + return fmt.Errorf("%s track index is invalid", kind) + } + return nil +} + +// validateCapabilitiesV3 validates and normalizes the shared capability +// payload. topLevelFeatures carries the request's client_features when the +// request type has one (replans do not); feature checks accept either +// location, matching the planner's dual-source reads. +func validateCapabilitiesV3(c *ClientCodecCapabilitiesV3, ctx *ClientPlaybackContextV3, topLevelFeatures []string) error { + if len(c.CodecsVideo) > 64 || len(c.CodecsVideoHardware) > 64 || len(c.CodecsAudio) > 64 || len(c.Containers) > 64 || len(c.VideoDecode) > 64 || len(ctx.Features) > 64 || len(ctx.Engines) > 16 || len(ctx.Device.ABIs) > 16 || len(ctx.Platform) > 32 || len(ctx.FormFactor) > 32 || len(ctx.AppVersion) > 64 { + return errors.New("capability list exceeds supported size") + } + deviceValues := []string{ + ctx.Device.Manufacturer, ctx.Device.Model, ctx.Device.Brand, ctx.Device.Device, + ctx.Device.Product, ctx.Device.SoCManufacturer, ctx.Device.SoCModel, ctx.Device.BuildID, + ctx.Device.BuildDisplay, ctx.Device.SecurityPatch, ctx.Output.CurrentSink, ctx.Output.SinkType, + } + for _, value := range deviceValues { + if len(value) > 128 { + return errors.New("device capability value exceeds supported size") + } + } + for _, values := range [][]string{c.CodecsVideo, c.CodecsVideoHardware, c.CodecsAudio, c.Containers, ctx.Features} { + for i := range values { + values[i] = strings.ToLower(strings.TrimSpace(values[i])) + if len(values[i]) > 128 { + return errors.New("capability value exceeds supported size") + } + } + } + for i := range c.VideoDecode { + c.VideoDecode[i].Codec = strings.ToLower(strings.TrimSpace(c.VideoDecode[i].Codec)) + if c.VideoDecode[i].Codec == "" || len(c.VideoDecode[i].DecoderName) > 128 || c.VideoDecode[i].MaxWidth < 0 || c.VideoDecode[i].MaxHeight < 0 || c.VideoDecode[i].MaxFrameRate < 0 || c.VideoDecode[i].MaxBitrateKbps < 0 { + return errors.New("invalid detailed video capability") + } + if len(c.VideoDecode[i].Profiles) > 64 || len(c.VideoDecode[i].Levels) > 64 || len(c.VideoDecode[i].BitDepths) > 64 { + return errors.New("detailed video capability exceeds supported size") + } + for _, profile := range c.VideoDecode[i].Profiles { + if len(profile) > 64 { + return errors.New("detailed video capability value exceeds supported size") + } + } + } + for _, hdr := range []*HDRCapabilitiesV3{c.HDRDetails, ctx.Output.HDRDetails} { + if hdr != nil && len(hdr.DolbyVisionProfiles) > 16 { + return errors.New("dolby vision profile list exceeds supported size") + } + } + for name, engine := range ctx.Engines { + if len(name) > 64 || len(engine.Containers) > 64 || len(engine.VideoCodecs) > 64 || len(engine.AudioDecodeCodecs) > 64 || len(engine.AudioPassthroughCodecs) > 64 || len(engine.Features) > 64 || len(engine.ValidatedClaims) > 64 || len(engine.Transformations) > 16 { + return errors.New("engine capability exceeds supported size") + } + if engine.HDRDetails != nil && len(engine.HDRDetails.DolbyVisionProfiles) > 16 { + return errors.New("dolby vision profile list exceeds supported size") + } + for _, values := range [][]string{engine.Containers, engine.VideoCodecs, engine.AudioDecodeCodecs, engine.AudioPassthroughCodecs, engine.Features, engine.ValidatedClaims} { + for _, value := range values { + if len(value) > 64 { + return errors.New("engine capability value exceeds supported size") + } + } + } + seenTransformations := make(map[string]struct{}, len(engine.Transformations)) + for i := range engine.Transformations { + transformation := &engine.Transformations[i] + transformation.Name = strings.ToLower(strings.TrimSpace(transformation.Name)) + transformation.Executor = strings.ToLower(strings.TrimSpace(transformation.Executor)) + transformation.RecipeVersion = strings.TrimSpace(transformation.RecipeVersion) + if transformation.Name == "" || len(transformation.Name) > 64 || + (transformation.Executor != "client" && transformation.Executor != "server") || + transformation.RecipeVersion == "" || len(transformation.RecipeVersion) > 32 || + len(transformation.ValidatedClaims) > 32 { + return errors.New("invalid engine transformation capability") + } + if transformation.Executor == "client" { + if !engine.Enabled || !engine.SupportedOnDevice || + (!HasFeatureV3(ctx.Features, FeatureClientVideoTransforms) && !HasFeatureV3(topLevelFeatures, FeatureClientVideoTransforms)) { + return errors.New("client transformation capability is not enabled") + } + } + key := transformation.Executor + ":" + transformation.Name + ":" + transformation.RecipeVersion + if _, exists := seenTransformations[key]; exists { + return errors.New("duplicate engine transformation capability") + } + seenTransformations[key] = struct{}{} + for _, claim := range transformation.ValidatedClaims { + if len(claim) > 128 { + return errors.New("transformation claim exceeds supported size") + } + } + } + ctx.Engines[name] = engine + } + for _, abi := range ctx.Device.ABIs { + if len(abi) > 64 { + return errors.New("device ABI exceeds supported size") + } + } + for _, passthrough := range []*AudioPassthroughV3{c.AudioPassthrough, ctx.Output.AudioPassthrough} { + if passthrough == nil { + continue + } + if len(passthrough.PassthroughCodecs) > 64 || len(passthrough.Entries) > 64 || passthrough.MaxChannels < 0 || passthrough.MaxChannels > 64 { + return errors.New("audio passthrough capability exceeds supported size") + } + for _, entry := range passthrough.Entries { + if len(entry.Codec) > 64 || len(entry.ChannelCounts) > 32 || len(entry.Layouts) > 32 { + return errors.New("audio passthrough entry exceeds supported size") + } + } + } + return nil +} + +func validateTrackPairV3(fileID int, kind, id string, index *int) error { + if len(id) > 128 { + return fmt.Errorf("%s_track_id exceeds supported size", kind) + } + if index != nil && (*index < 0 || *index > 10_000) { + return fmt.Errorf("%s_track_index is invalid", kind) + } + if id == "" || index == nil { + return nil + } + want := TrackIDV3(fileID, kind, *index) + if id != want { + return fmt.Errorf("%s track id and index disagree", kind) + } + return nil +} + +func validateOptionalBoundedIntV3(v *int, min, max int, name string) error { + if v != nil && (*v < min || *v > max) { + return fmt.Errorf("%s is outside the supported range", name) + } + return nil +} + +func isFiniteV3(v float64) bool { return !math.IsNaN(v) && !math.IsInf(v, 0) } + +func HasFeatureV3(features []string, wanted string) bool { + return slices.ContainsFunc(features, func(v string) bool { return strings.EqualFold(strings.TrimSpace(v), wanted) }) +} + +func NewTerminalResponseV3(reason, message string, retryable bool) DecisionResponseV3 { + return DecisionResponseV3{ + ProtocolVersion: ProtocolV3, + ServerFeatures: []string{FeaturePlaybackPlanV3, FeatureMedia3Only, FeatureDeviceQuirksV3, FeatureSeekReanchorV3}, + Outcome: OutcomeAdaptationUnavailableV3, + Terminal: &TerminalV3{Reason: reason, Message: message, Retryable: retryable}, + } +} + +func DisabledResponseV3() DecisionResponseV3 { + return DecisionResponseV3{ProtocolVersion: ProtocolV3, ServerFeatures: []string{}} +} + +func NewPlanExpiryV3(now time.Time) string { return now.Add(MaxTokenTTL).UTC().Format(time.RFC3339) } diff --git a/internal/playback/protocol_v3_session_test.go b/internal/playback/protocol_v3_session_test.go new file mode 100644 index 00000000..f5eeabd7 --- /dev/null +++ b/internal/playback/protocol_v3_session_test.go @@ -0,0 +1,143 @@ +package playback + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "testing" + "time" +) + +func TestCheckReplacementAllowedTransfersExistingCapacitySlot(t *testing.T) { + manager := NewSessionManager(2, 1) + direct, err := manager.StartSession(1, "profile-1", 1, PlayDirect, false) + if err != nil { + t.Fatal(err) + } + if err := manager.CheckReplacementAllowed(context.Background(), direct.ID, PlayTranscode, true); err != nil { + t.Fatalf("direct to first transcode denied: %v", err) + } + if _, err := manager.StartSession(1, "profile-1", 2, PlayTranscode, true); !errors.Is(err, ErrTooManyTranscodes) { + t.Fatalf("reservation did not hold the transcode slot: %v", err) + } + manager.CancelReplacementReservation(direct.ID) + if _, err := manager.StartSession(1, "profile-1", 2, PlayTranscode, true); err != nil { + t.Fatal(err) + } + if err := manager.CheckReplacementAllowed(context.Background(), direct.ID, PlayTranscode, true); !errors.Is(err, ErrTooManyTranscodes) { + t.Fatalf("replacement error = %v, want ErrTooManyTranscodes", err) + } +} + +func TestCheckReplacementAllowedRechecksDynamicTranscodePolicy(t *testing.T) { + manager := NewSessionManager(0, 0) + session, err := manager.StartSession(1, "profile-1", 1, PlayDirect, false) + if err != nil { + t.Fatal(err) + } + manager.SetLimitProvider(func(context.Context, int) (SessionLimits, error) { + return SessionLimits{TranscodingDisabled: true}, nil + }) + if err := manager.CheckReplacementAllowed(context.Background(), session.ID, PlayTranscode, true); !errors.Is(err, ErrTranscodingDisabled) { + t.Fatalf("replacement error = %v, want ErrTranscodingDisabled", err) + } +} + +func TestReconstructionOutputDirPreservesPlanScopedIsolation(t *testing.T) { + root := t.TempDir() + sessionID := "session-1" + want := filepath.Join(root, sessionID+"-plan-deadbeef") + if got := reconstructionOutputDir(root, sessionID, sessionID+"-plan-deadbeef"); got != want { + t.Fatalf("output dir = %q, want %q", got, want) + } + if got := reconstructionOutputDir(root, sessionID, filepath.Join("..", "escape")); got != filepath.Join(root, sessionID) { + t.Fatalf("unsafe output dir = %q", got) + } +} + +func TestRecipeCardRoundTripPreservesRemoteTransportIdentity(t *testing.T) { + card := NewRecipeCard(1, "profile-1", 42, "https://node.example", TranscodeOpts{SessionID: "public-session", TranscodeTransportID: "public-session-plan-deadbeef", InputPath: "/media/movie.mkv", TargetCodecVideo: "h264", SegmentDuration: 2}) + claims := card.ToClaims() + if claims.SessionID != "public-session" || claims.TranscodeTransportID != "public-session-plan-deadbeef" { + t.Fatalf("claims = %#v", claims) + } + roundTrip := RecipeCardFromClaims(&claims) + if roundTrip.SessionID != card.SessionID || roundTrip.TranscodeTransportID != card.TranscodeTransportID { + t.Fatalf("round trip = %#v, want %#v", roundTrip, card) + } +} + +func TestMemoryPlanStoreV3StartAndReplanIdempotency(t *testing.T) { + store := NewMemoryPlanStoreV3() + record := AttemptRecordV3{PlaybackAttemptID: "attempt-0001", SessionID: "session-1", UserID: 1, ProfileID: "profile-1", CurrentPlanID: "plan-1", ExpiresAt: time.Now().Add(time.Hour)} + if err := store.SaveAttempt(context.Background(), record); err != nil { + t.Fatal(err) + } + duplicate := record + duplicate.SessionID = "session-2" + if err := store.SaveAttempt(context.Background(), duplicate); !errors.Is(err, ErrPlaybackAttemptExistsV3) { + t.Fatalf("duplicate start error = %v", err) + } + lease, err := store.BeginReplan(context.Background(), record.SessionID, "replan-0001", "digest-a", record.CurrentReplanRequestID, time.Now().Add(time.Minute)) + if err != nil || lease.State != ReplanLeaseOwnedV3 { + t.Fatalf("first lease = %#v, err=%v", lease, err) + } + lease, err = store.BeginReplan(context.Background(), record.SessionID, "replan-0001", "digest-a", record.CurrentReplanRequestID, time.Now().Add(time.Minute)) + if err != nil || lease.State != ReplanLeaseInFlightV3 { + t.Fatalf("in-flight lease = %#v, err=%v", lease, err) + } + if _, err := store.BeginReplan(context.Background(), record.SessionID, "replan-0001", "digest-b", record.CurrentReplanRequestID, time.Now().Add(time.Minute)); !errors.Is(err, ErrIdempotencyKeyReusedV3) { + t.Fatalf("digest conflict = %v", err) + } + response := json.RawMessage(`{"protocol_version":3}`) + if err := store.CompleteReplan(context.Background(), record.SessionID, "replan-0001", record.CurrentReplanRequestID, response, record); err != nil { + t.Fatal(err) + } + lease, err = store.BeginReplan(context.Background(), record.SessionID, "replan-0001", "digest-a", record.CurrentReplanRequestID, time.Now().Add(time.Minute)) + if err != nil || lease.State != ReplanLeaseCompletedV3 || string(lease.Response) != string(response) { + t.Fatalf("completed lease = %#v, err=%v", lease, err) + } +} + +func TestMemoryPlanStoreV3RejectsExpiredLeaseAfterNewerCommit(t *testing.T) { + store := NewMemoryPlanStoreV3() + record := AttemptRecordV3{ + PlaybackAttemptID: "attempt-stale-lease", + SessionID: "session-stale-lease", + UserID: 1, + ProfileID: "profile-1", + CurrentPlanID: "plan-same", + CurrentReplanRequestID: "replan-base", + ExpiresAt: time.Now().Add(time.Hour), + } + if err := store.SaveAttempt(context.Background(), record); err != nil { + t.Fatal(err) + } + if lease, err := store.BeginReplan( + context.Background(), record.SessionID, "replan-abandoned", "digest-a", + record.CurrentReplanRequestID, time.Now().Add(-time.Second), + ); err != nil || lease.State != ReplanLeaseOwnedV3 { + t.Fatalf("abandoned lease = %#v, err=%v", lease, err) + } + if lease, err := store.BeginReplan( + context.Background(), record.SessionID, "replan-newer", "digest-b", + record.CurrentReplanRequestID, time.Now().Add(time.Minute), + ); err != nil || lease.State != ReplanLeaseOwnedV3 { + t.Fatalf("newer lease = %#v, err=%v", lease, err) + } + base := record.CurrentReplanRequestID + record.CurrentReplanRequestID = "replan-newer" + if err := store.CompleteReplan( + context.Background(), record.SessionID, "replan-newer", base, + json.RawMessage(`{"protocol_version":3}`), record, + ); err != nil { + t.Fatal(err) + } + if _, err := store.BeginReplan( + context.Background(), record.SessionID, "replan-abandoned", "digest-a", + record.CurrentReplanRequestID, time.Now().Add(time.Minute), + ); !errors.Is(err, ErrStaleReplanLeaseV3) { + t.Fatalf("expired stale lease error = %v", err) + } +} diff --git a/internal/playback/protocol_v3_test.go b/internal/playback/protocol_v3_test.go new file mode 100644 index 00000000..c1dd2946 --- /dev/null +++ b/internal/playback/protocol_v3_test.go @@ -0,0 +1,967 @@ +package playback + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func hasDegradationWarningV3(warnings []DegradationWarningV3, code string) bool { + for _, warning := range warnings { + if warning.Code == code { + return true + } + } + return false +} + +func TestStartRequestV3Validation(t *testing.T) { + index := 1 + req := validStartRequestV3() + req.AudioTrackIndex = &index + req.AudioTrackID = TrackIDV3(req.FileID, "audio", index) + warnings, err := req.NormalizeAndValidate() + if err != nil { + t.Fatalf("NormalizeAndValidate: %v", err) + } + if len(warnings) != 0 { + t.Fatalf("warnings = %#v, want none", warnings) + } + + req.AudioTrackID = TrackIDV3(req.FileID, "audio", 2) + if _, err := req.NormalizeAndValidate(); err == nil { + t.Fatal("mismatched track id/index accepted") + } +} + +func TestStartRequestV3UnknownQualityFallsBackToAuto(t *testing.T) { + req := validStartRequestV3() + req.QualityPreference = "future-super-quality" + warnings, err := req.NormalizeAndValidate() + if err != nil { + t.Fatalf("NormalizeAndValidate: %v", err) + } + if req.QualityPreference != "auto" || len(warnings) != 1 || warnings[0].Code != "quality_preference_normalized" { + t.Fatalf("quality=%q warnings=%#v", req.QualityPreference, warnings) + } +} + +func TestReplanRequestV3OperationDefaultsAndValidates(t *testing.T) { + start := validStartRequestV3() + request := ReplanRequestV3{ + ProtocolVersion: ProtocolV3, + PlaybackAttemptID: start.PlaybackAttemptID, + ReplanRequestID: "replan-operation-0001", + FailedPlanID: "plan:operation-0001", + PlanAttemptID: "plan-attempt-operation-0001", + PlanAttemptKey: "v3:0000000000000001", + AttemptCount: 1, + QualityPreference: start.QualityPreference, + OutputRouteGeneration: start.OutputRouteGeneration, + Failure: FailureV3{Classification: "parser_failure"}, + Capabilities: start.Capabilities, + ClientPlaybackContext: start.ClientPlaybackContext, + } + if request.EffectiveOperation() != ReplanOperationFailureRecoveryV3 { + t.Fatalf("missing operation = %q", request.EffectiveOperation()) + } + if err := request.Validate(); err != nil { + t.Fatalf("backward-compatible default operation: %v", err) + } + + request.Operation = ReplanOperationSeekReanchorV3 + request.Failure.Classification = "" + if err := request.Validate(); err != nil { + t.Fatalf("seek reanchor operation without fake failure: %v", err) + } + body, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + var wire map[string]any + if err := json.Unmarshal(body, &wire); err != nil { + t.Fatal(err) + } + if wire["operation"] != string(ReplanOperationSeekReanchorV3) { + t.Fatalf("serialized operation = %#v", wire["operation"]) + } + + request.Operation = ReplanOperationSeekFailureRecoveryV3 + if err := request.Validate(); err == nil { + t.Fatal("seek failure recovery without a classification was accepted") + } + request.Failure.Classification = "decoder_failure" + if err := request.Validate(); err != nil { + t.Fatalf("seek failure recovery operation: %v", err) + } + + request.Operation = "future_operation" + if err := request.Validate(); err == nil { + t.Fatal("unknown replan operation was accepted") + } +} + +func TestReplanRequestV3RejectsInvalidNetworkAndTrackEvidence(t *testing.T) { + start := validStartRequestV3() + request := ReplanRequestV3{ + ProtocolVersion: ProtocolV3, + PlaybackAttemptID: start.PlaybackAttemptID, + ReplanRequestID: "replan-validation-0001", + FailedPlanID: "plan:validation-0001", + PlanAttemptID: "plan-attempt-validation-0001", + PlanAttemptKey: "v3:0000000000000001", + AttemptCount: 1, + QualityPreference: start.QualityPreference, + OutputRouteGeneration: start.OutputRouteGeneration, + Failure: FailureV3{Classification: "parser_failure"}, + Capabilities: start.Capabilities, + ClientPlaybackContext: start.ClientPlaybackContext, + } + + negative := -1 + request.SelectedTracks.Subtitle = &TrackIdentityV3{ID: "file:42:subtitle:-1", Index: &negative} + if err := request.Validate(); err == nil { + t.Fatal("negative subtitle index was accepted") + } + + request.SelectedTracks.Subtitle = nil + tooLow := 99 + request.BandwidthEstimateKbps = &tooLow + if err := request.Validate(); err == nil { + t.Fatal("out-of-range bandwidth estimate was accepted") + } +} + +func TestPlanAttemptKeyV3KotlinFixture(t *testing.T) { + type fixture struct { + Name string `json:"name"` + PlanID string `json:"plan_id"` + Delivery DeliveryV3 `json:"delivery"` + StreamProtocol StreamProtocolV3 `json:"stream_protocol"` + Container string `json:"container"` + VideoCodec string `json:"video_codec"` + AudioCodec string `json:"audio_codec"` + Width int `json:"width"` + Height int `json:"height"` + BitrateKbps int `json:"bitrate_kbps"` + DynamicRange string `json:"dynamic_range"` + SubtitleMode SubtitleModeV3 `json:"subtitle_mode"` + Transformations []TransformationV3 `json:"transformations"` + AppliedQuirks []AppliedQuirkV3 `json:"applied_quirks"` + RuntimeCorrections []string `json:"runtime_corrections"` + OutputRouteGeneration int64 `json:"output_route_generation"` + LocalMutations []string `json:"local_mutations"` + Expected string `json:"expected"` + } + body, err := os.ReadFile("testdata/protocol_v3/attempt_keys.json") + if err != nil { + t.Fatal(err) + } + var fixtures []fixture + if err := json.Unmarshal(body, &fixtures); err != nil { + t.Fatal(err) + } + for _, value := range fixtures { + t.Run(value.Name, func(t *testing.T) { + plan := PlanV3{ + PlanID: value.PlanID, + Delivery: value.Delivery, + Stream: StreamV3{Protocol: value.StreamProtocol, Container: value.Container}, + EffectiveRecipe: EffectiveRecipeV3{VideoCodec: value.VideoCodec, AudioCodec: value.AudioCodec, Width: &value.Width, Height: &value.Height, BitrateKbps: &value.BitrateKbps, DynamicRange: value.DynamicRange}, + Subtitle: SubtitleDecisionV3{Mode: value.SubtitleMode}, + } + plan.Transformations = append(plan.Transformations, value.Transformations...) + // The quirk identity is conditionally omitted from the preimage + // when no quirks or runtime corrections apply; fixtures pin both + // arities so the Kotlin client reproduces the omission exactly. + plan.AppliedQuirks = append(plan.AppliedQuirks, value.AppliedQuirks...) + plan.RuntimeCorrections = append(plan.RuntimeCorrections, value.RuntimeCorrections...) + if got := PlanAttemptKeyV3(plan, value.OutputRouteGeneration, value.LocalMutations); got != value.Expected { + t.Fatalf("key = %q, want %q", got, value.Expected) + } + }) + } +} + +func TestProtocolV3GoldenWireFixtures(t *testing.T) { + startBody, err := os.ReadFile("testdata/protocol_v3/start_request.json") + if err != nil { + t.Fatal(err) + } + var start StartRequestV3 + if err := json.Unmarshal(startBody, &start); err != nil { + t.Fatal(err) + } + if _, err := start.NormalizeAndValidate(); err != nil { + t.Fatalf("golden start request: %v", err) + } + replanBody, err := os.ReadFile("testdata/protocol_v3/replan_request.json") + if err != nil { + t.Fatal(err) + } + var replan ReplanRequestV3 + if err := json.Unmarshal(replanBody, &replan); err != nil { + t.Fatal(err) + } + if err := replan.Validate(); err != nil { + t.Fatalf("golden replan request: %v", err) + } + if replan.BandwidthEstimateKbps == nil || *replan.BandwidthEstimateKbps != 3_500 || + replan.BandwidthCapKbps == nil || *replan.BandwidthCapKbps != 4_000 || !replan.Metered { + t.Fatalf("golden replan network evidence = %#v", replan) + } + responseBody, err := os.ReadFile("testdata/protocol_v3/decision_response.json") + if err != nil { + t.Fatal(err) + } + var response DecisionResponseV3 + if err := json.Unmarshal(responseBody, &response); err != nil { + t.Fatal(err) + } + if response.ProtocolVersion != ProtocolV3 || response.Outcome != OutcomePlayableV3 || response.PlaybackPlan == nil || response.PlaybackPlan.Stream.Protocol != StreamHTTPProgressiveV3 { + t.Fatalf("golden response = %#v", response) + } +} + +func TestPlanPlaybackV3DirectRequiresDetailedEvidence(t *testing.T) { + file := detailedFixtureFileV3() + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3, FeatureLayoutPassthrough) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3, FeatureLayoutPassthrough) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + req.ClientPlaybackContext.Output.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } + + req.ClientFeatures = []string{FeaturePlaybackPlanV3, FeatureMedia3Only} + req.ClientPlaybackContext.Features = req.ClientFeatures + result = PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: false, Allow4KTranscode: true}}) + if result.Terminal == nil || result.Terminal.Reason != "transcoding_disabled" { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } +} + +func TestSourceDescriptorV3NormalizesLegacyHEVCMetadata(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].BitDepth = 0 + file.VideoTracks[0].PixelFormat = "yuv420p10le" + file.VideoTracks[0].DVProfile = 7 + file.VideoTracks[0].DVELPresent = false + file.VideoTracks[0].DVEnhancementLayer = "" + file.VideoTracks[0].VideoRangeType = "DOVIWithEL" + + source := SourceDescriptorFromFileV3(file, 0) + if source.BitDepth != 10 { + t.Fatalf("bit depth = %d, want inferred 10", source.BitDepth) + } + if source.DVEnhancementLayer != EnhancementUnknownV3 { + t.Fatalf("enhancement layer = %q, want unknown", source.DVEnhancementLayer) + } +} + +func TestSourceDescriptorV3PreservesCanonicalColorRange(t *testing.T) { + for _, test := range []struct { + name string + input string + want string + }{ + {name: "limited", input: "tv", want: "tv"}, + {name: "full", input: "pc", want: "pc"}, + {name: "unspecified", input: "unknown", want: "unknown"}, + {name: "normalizes case and whitespace", input: " PC ", want: "pc"}, + {name: "rejects non-ffmpeg value", input: "limited", want: ""}, + } { + t.Run(test.name, func(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].ColorRange = test.input + if got := SourceDescriptorFromFileV3(file, 0).ColorRange; got != test.want { + t.Fatalf("color range = %q, want %q", got, test.want) + } + }) + } +} + +func TestPlanPlaybackV3DirectPlaysLegacyHDR10WithInferredBitDepth(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].BitDepth = 0 + file.VideoTracks[0].PixelFormat = "yuv420p10le" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + req.ClientPlaybackContext.Output.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || !result.Plan.Claims.Video.HDR10 { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } +} + +func TestPlanPlaybackV3DirectPlaysLegacyDolbyVisionProfile8(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].BitDepth = 0 + file.VideoTracks[0].PixelFormat = "yuv420p10le" + file.VideoTracks[0].DVProfile = 8 + file.VideoTracks[0].DVBLCompatID = 1 + file.VideoTracks[0].VideoRange = "DolbyVision" + file.VideoTracks[0].VideoRangeType = "DOVIWithHDR10" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true, DolbyVisionProfiles: []int{8}} + req.ClientPlaybackContext.Output.HDRDetails = req.Capabilities.HDRDetails + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || !result.Plan.Claims.Video.DolbyVision { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } + if result.Plan.RequestedMediaFileID != file.ID || result.Plan.EffectiveMediaFileID != file.ID { + t.Fatalf("source ids = requested %d effective %d", result.Plan.RequestedMediaFileID, result.Plan.EffectiveMediaFileID) + } +} + +func TestPlanPlaybackV3RejectsTrulyIncompleteVideoMetadata(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].BitDepth = 0 + file.VideoTracks[0].PixelFormat = "" + file.VideoTracks[0].Profile = "" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Terminal == nil || result.Terminal.Reason != "source_metadata_incomplete" { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } +} + +func TestPlanPlaybackV3BlocksUltrawide4KTranscode(t *testing.T) { + file := detailedFixtureFileV3() + file.Resolution = "2160p" + file.VideoTracks[0].Width = 3840 + file.VideoTracks[0].Height = 1626 + file.VideoTracks[0].VideoRange = "SDR" + file.VideoTracks[0].VideoRangeType = "SDR" + file.VideoTracks[0].ColorTransfer = "bt709" + req := validStartRequestV3() + req.QualityPreference = "1080p" + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: false}, Registry: testTransformationRegistryV3()}) + if result.Terminal == nil || result.Terminal.Reason != "no_alternate_version" { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } +} + +func TestPlanPlaybackV3Profile7FallsBackToHDR10WithoutNativeP7(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].DVProfile = 7 + file.VideoTracks[0].DVBLCompatID = 6 + file.VideoTracks[0].DVELPresent = false + file.VideoTracks[0].DVEnhancementLayer = "" + file.VideoTracks[0].VideoRange = "DolbyVision" + file.VideoTracks[0].VideoRangeType = "DOVIWithEL" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true, DolbyVisionProfiles: []int{5, 8}} + req.ClientPlaybackContext.Output.HDRDetails = req.Capabilities.HDRDetails + registry := NewTransformationRegistryV3([]TransformationSpecV3{{Name: "server_dv7_to_hdr10", Available: true}}) + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true}, Registry: registry}) + if result.Plan == nil || result.Plan.Delivery != DeliveryRemuxProgressiveV3 || !result.Plan.Claims.Video.HDR10 || result.Plan.Claims.Video.DolbyVision { + t.Fatalf("result = %#v", result) + } + if len(result.Plan.Transformations) != 1 || result.Plan.Transformations[0].Name != "server_dv7_to_hdr10" { + t.Fatalf("transformations = %#v", result.Plan.Transformations) + } +} + +func TestPlanPlaybackV3Profile7UsesVersionedClientTransformationsOnSameFile(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].DVProfile = 7 + file.VideoTracks[0].DVBLCompatID = 1 + file.VideoTracks[0].DVELPresent = true + file.VideoTracks[0].DVEnhancementLayer = "unknown" + file.VideoTracks[0].VideoRange = "DolbyVision" + file.VideoTracks[0].VideoRangeType = "DOVIWithEL" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3, FeatureClientVideoTransforms) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3, FeatureClientVideoTransforms) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true, DolbyVisionProfiles: []int{8}} + req.ClientPlaybackContext.Output.HDRDetails = req.Capabilities.HDRDetails + direct := req.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] + direct.Transformations = []TransformationV3{ + {Name: ClientDV7ToDV81V3, Executor: "client", RecipeVersion: ClientDVTransformVersionV3}, + {Name: ClientDV7ToHDR10V3, Executor: "client", RecipeVersion: ClientDVTransformVersionV3}, + } + req.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] = direct + + first := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true}, Registry: testTransformationRegistryV3()}) + if first.Plan == nil || first.Plan.Delivery != DeliveryOriginalHTTPV3 || first.Plan.EffectiveMediaFileID != file.ID || first.Plan.DecisionReason != "client_dv7_to_dv81" { + t.Fatalf("first = %#v", first) + } + if got := first.Plan.Transformations; len(got) != 1 || got[0].Name != ClientDV7ToDV81V3 || got[0].Executor != "client" || got[0].RecipeVersion != "1" { + t.Fatalf("first transformations = %#v", got) + } + + failedKey := PlanAttemptKeyV3(*first.Plan, req.OutputRouteGeneration, nil) + second := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true}, Registry: testTransformationRegistryV3(), AttemptedKeys: []string{failedKey}}) + if second.Plan == nil || second.Plan.Delivery != DeliveryOriginalHTTPV3 || second.Plan.EffectiveMediaFileID != file.ID || second.Plan.DecisionReason != "client_dv7_to_hdr10" || !second.Plan.Claims.Video.HDR10 { + t.Fatalf("second = %#v", second) + } +} + +func TestPlanPlaybackV3Profile7DoesNotInferClientTransformFromHDRProfile(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].DVProfile = 7 + file.VideoTracks[0].VideoRange = "DolbyVision" + file.VideoTracks[0].VideoRangeType = "DOVIWithEL" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true, DolbyVisionProfiles: []int{7, 8}} + req.ClientPlaybackContext.Output.HDRDetails = req.Capabilities.HDRDetails + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: false}}) + if result.Plan != nil && result.Plan.Delivery == DeliveryOriginalHTTPV3 { + t.Fatalf("Profile 7 must not direct-play from codec claims alone: %#v", result.Plan) + } +} + +func TestPlanPlaybackV3AudioAdaptationCopiesVideo(t *testing.T) { + file := detailedFixtureFileV3() + file.AudioTracks[0] = models.AudioTrack{Codec: "truehd", Channels: 8, Layout: "7.1"} + file.CodecAudio = "truehd" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryRemuxProgressiveV3 || !result.TranscodeAudio || result.TargetVideoCodec != "" { + t.Fatalf("result = %#v", result) + } +} + +func TestPlanPlaybackV3FallsBackFromProgressiveToHLSWithoutRepeatingKey(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].VideoRange = "SDR" + file.VideoTracks[0].VideoRangeType = "SDR" + file.VideoTracks[0].ColorTransfer = "bt709" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.Containers = []string{"mp4"} + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + first := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}}) + if first.Plan == nil || first.Plan.Delivery != DeliveryRemuxProgressiveV3 { + t.Fatalf("first = %s", ExplainPlannerResultV3(first)) + } + failedKey := PlanAttemptKeyV3(*first.Plan, req.OutputRouteGeneration, nil) + second := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, AttemptedKeys: []string{failedKey}}) + if second.Plan == nil || second.Plan.Delivery != DeliveryRemuxHLSV3 || second.TargetVideoCodec != "copy" { + t.Fatalf("second = %#v", second) + } + secondKey := PlanAttemptKeyV3(*second.Plan, req.OutputRouteGeneration, nil) + third := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3(), AttemptedKeys: []string{failedKey, secondKey}}) + if third.Plan == nil || third.Plan.Delivery != DeliveryTranscodeHLSV3 || third.Plan.DecisionReason != "copy_routes_exhausted" { + t.Fatalf("third = %#v", third) + } +} + +func TestPlanPlaybackV3NeverClaimsUnimplementedHDRTranscode(t *testing.T) { + file := detailedFixtureFileV3() + req := validStartRequestV3() + req.QualityPreference = "1080p" + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}}) + if result.Terminal == nil || result.Terminal.Reason != "hdr_transcode_unsupported" { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } +} + +func TestPlanPlaybackV3Profile7StripFallsBackToValidatedHLSCopy(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].DVProfile = 7 + file.VideoTracks[0].DVBLCompatID = 1 + file.VideoTracks[0].DVELPresent = true + file.VideoTracks[0].DVEnhancementLayer = "mel" + file.VideoTracks[0].VideoRange = "DolbyVision" + file.VideoTracks[0].VideoRangeType = "DOVIWithEL" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.Containers = []string{"mp4"} + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true, DolbyVisionProfiles: []int{7}} + registry := NewTransformationRegistryV3([]TransformationSpecV3{{Name: "server_dv7_to_hdr10", Available: true}}) + first := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: registry}) + if first.Plan == nil || first.Plan.Delivery != DeliveryRemuxProgressiveV3 || len(first.Plan.Transformations) != 1 { + t.Fatalf("first = %#v", first) + } + failedKey := PlanAttemptKeyV3(*first.Plan, req.OutputRouteGeneration, nil) + second := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: registry, AttemptedKeys: []string{failedKey}}) + if second.Plan == nil || second.Plan.Delivery != DeliveryRemuxHLSV3 || second.TargetVideoCodec != "copy" || len(second.Plan.Transformations) != 1 || second.Plan.Transformations[0].Name != "server_dv7_to_hdr10" { + t.Fatalf("second = %#v", second) + } +} + +func TestPlanPlaybackV3Profile8CompatibleBaseLayerStripsToHDR10(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].DVProfile = 8 + file.VideoTracks[0].DVBLCompatID = 1 + file.VideoTracks[0].VideoRange = "DolbyVision" + file.VideoTracks[0].VideoRangeType = "DOVI" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + registry := NewTransformationRegistryV3([]TransformationSpecV3{{Name: "server_dv7_to_hdr10", Available: true}}) + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: registry}) + if result.Plan == nil || result.Plan.Delivery != DeliveryRemuxProgressiveV3 || result.Plan.EffectiveRecipe.DynamicRange != "hdr10" || len(result.Plan.Transformations) != 1 || result.Plan.Transformations[0].Name != "server_dv7_to_hdr10" { + t.Fatalf("result = %#v", result) + } +} + +func TestPlanPlaybackV3PassthroughRequiresExactLayoutEntry(t *testing.T) { + file := detailedFixtureFileV3() + file.CodecAudio = "truehd" + file.AudioTracks[0] = models.AudioTrack{Codec: "truehd", Channels: 8, Layout: "7.1"} + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3, FeatureLayoutPassthrough) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3, FeatureLayoutPassthrough) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + req.Capabilities.AudioPassthrough = &AudioPassthroughV3{PassthroughCodecs: []string{"truehd"}, MaxChannels: 8, Entries: []AudioPassthroughEntryV3{{Codec: "truehd"}}} + withoutLayout := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if withoutLayout.Plan == nil || !withoutLayout.TranscodeAudio { + t.Fatalf("without exact layout = %#v", withoutLayout) + } + req.Capabilities.AudioPassthrough.Entries[0].ChannelCounts = []int{8} + req.Capabilities.AudioPassthrough.Entries[0].Layouts = []string{"7.1"} + withLayout := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}}) + if withLayout.Plan == nil || withLayout.Plan.Delivery != DeliveryOriginalHTTPV3 || !withLayout.Plan.Claims.Audio.Passthrough { + t.Fatalf("with exact layout = %#v", withLayout) + } +} + +func TestPlanPlaybackV3DownloadedSubtitleUsesFrozenCombinedOrdinal(t *testing.T) { + file := detailedFixtureFileV3() + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + index := 0 + req.SubtitleTrackIndex = &index + req.SubtitleTrackID = TrackIDV3(file.ID, "subtitle", index) + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, AdditionalSubtitles: []SubtitleInventoryEntryV3{{CombinedIndex: 0, Codec: "srt", Source: "downloaded"}}}) + if result.Plan == nil || result.Plan.Subtitle.Mode != SubtitleRenderV3 || result.Plan.SelectedTracks.Subtitle == nil || result.Plan.SelectedTracks.Subtitle.ID != req.SubtitleTrackID { + t.Fatalf("result = %#v", result) + } +} + +func TestSubtitleBurnInUsesEmbeddedOrdinalAndRejectsUnsupportedSources(t *testing.T) { + file := detailedFixtureFileV3() + file.ExternalSubtitles = []models.ExternalSubtitle{{Format: "ass"}} + file.SubtitleTracks = []models.SubtitleTrack{{Codec: "hdmv_pgs_subtitle"}} + req := validStartRequestV3() + embeddedCombinedIndex := 1 + req.SubtitleTrackIndex = &embeddedCombinedIndex + req.SubtitleTrackID = TrackIDV3(file.ID, "subtitle", embeddedCombinedIndex) + result := ResolveSubtitlePolicyV3(file, req, true, EngineMedia3DirectV3, nil) + if !result.RequiresBurn || result.SelectedIndex != 1 || result.TransportIndex != 0 { + t.Fatalf("embedded burn-in result = %#v", result) + } + + externalIndex := 0 + req.SubtitleTrackIndex = &externalIndex + req.SubtitleTrackID = TrackIDV3(file.ID, "subtitle", externalIndex) + req.SubtitleFidelityPreference = SubtitleFidelityPreserveV3 + result = ResolveSubtitlePolicyV3(file, req, true, EngineMedia3DirectV3, nil) + if result.Terminal == nil || result.Terminal.Reason != "subtitle_burn_in_source_unsupported" { + t.Fatalf("external burn-in result = %#v", result) + } +} + +func TestResolveQualityPolicyV3HonorsBandwidthCapInAllModes(t *testing.T) { + source := SourceDescriptorV3{Width: 3840, Height: 2160, BitrateKbps: 20_000} + cap := 5_000 + + req := validStartRequestV3() + req.QualityPreference = "original" + req.BandwidthCapKbps = &cap + result := ResolveQualityPolicyV3(req, source) + if !result.RequiresTranscode || result.Height != 720 || result.Reason != "quality_bandwidth_cap" || result.ExplicitRung { + t.Fatalf("original over cap = %#v", result) + } + if !hasDegradationWarningV3(result.Warnings, "bandwidth_cap_applied") { + t.Fatalf("missing cap warning: %#v", result.Warnings) + } + + lowBitrateSource := SourceDescriptorV3{Width: 1920, Height: 1080, BitrateKbps: 4_000} + result = ResolveQualityPolicyV3(req, lowBitrateSource) + if !result.PreservesSource || result.RequiresTranscode || len(result.Warnings) != 0 { + t.Fatalf("original under cap = %#v", result) + } + + req.QualityPreference = "1080p" + result = ResolveQualityPolicyV3(req, source) + if result.Height != 720 || result.Reason != "quality_bandwidth_cap" || !result.ExplicitRung || !result.RequiresTranscode { + t.Fatalf("fixed rung over cap = %#v", result) + } + if !hasDegradationWarningV3(result.Warnings, "bandwidth_cap_applied") { + t.Fatalf("missing cap warning: %#v", result.Warnings) + } + + req.QualityPreference = "480p" + result = ResolveQualityPolicyV3(req, source) + if result.Height != 480 || result.Reason != "quality_fixed_rung" || hasDegradationWarningV3(result.Warnings, "bandwidth_cap_applied") { + t.Fatalf("fixed rung under cap = %#v", result) + } + + req.QualityPreference = "auto" + result = ResolveQualityPolicyV3(req, source) + if !result.RequiresTranscode || result.Height != 720 || result.PreservesSource { + t.Fatalf("auto with cap = %#v", result) + } +} + +func TestResolveQualityPolicyV3MeteredWithoutEvidencePrefersConservativeRung(t *testing.T) { + source := SourceDescriptorV3{Width: 3840, Height: 2160, BitrateKbps: 20_000} + req := validStartRequestV3() + req.QualityPreference = "auto" + req.Metered = true + result := ResolveQualityPolicyV3(req, source) + if result.Height != 720 || !result.RequiresTranscode || result.Reason != "quality_metered_limit" || result.ExplicitRung { + t.Fatalf("metered auto = %#v", result) + } + + estimate := 30_000 + req.BandwidthEstimateKbps = &estimate + result = ResolveQualityPolicyV3(req, source) + if !result.PreservesSource || result.Reason != "quality_bandwidth_limit" { + t.Fatalf("metered with estimate = %#v", result) + } + + req.BandwidthEstimateKbps = nil + req.Metered = false + result = ResolveQualityPolicyV3(req, source) + if !result.PreservesSource || result.RequiresTranscode { + t.Fatalf("unmetered auto = %#v", result) + } +} + +func TestPlanPlaybackV3HDRDeviceCapFallsBackToOriginalQuality(t *testing.T) { + file := detailedFixtureFileV3() + req := validStartRequestV3() + req.QualityPreference = "auto" + req.Capabilities.MaxResolution = "1080p" + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + req.ClientPlaybackContext.Output.HDRDetails = req.Capabilities.HDRDetails + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || !result.Plan.Claims.Video.HDR10 { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } + if result.Plan.EffectiveRecipe.Height == nil || *result.Plan.EffectiveRecipe.Height != 2160 { + t.Fatalf("recipe = %#v", result.Plan.EffectiveRecipe) + } + if !hasDegradationWarningV3(result.Plan.DegradationWarnings, "quality_reduction_unavailable") { + t.Fatalf("warnings = %#v", result.Plan.DegradationWarnings) + } +} + +func TestPlanPlaybackV3HDRBandwidthCapFallsBackToOriginalQuality(t *testing.T) { + file := detailedFixtureFileV3() + req := validStartRequestV3() + req.QualityPreference = "auto" + cap := 5_000 + req.BandwidthCapKbps = &cap + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + req.ClientPlaybackContext.Output.HDRDetails = req.Capabilities.HDRDetails + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || !result.Plan.Claims.Video.HDR10 { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } + if !hasDegradationWarningV3(result.Plan.DegradationWarnings, "quality_reduction_unavailable") { + t.Fatalf("warnings = %#v", result.Plan.DegradationWarnings) + } +} + +func TestPlanPlaybackV3CapWithoutTranscodeRouteFallsBackToOriginal(t *testing.T) { + file := detailedFixtureFileV3() + file.Resolution = "1080p" + file.VideoTracks[0].Width = 1920 + file.VideoTracks[0].Height = 1080 + file.VideoTracks[0].Bitrate = 8_000 + file.VideoTracks[0].VideoRange = "SDR" + file.VideoTracks[0].VideoRangeType = "SDR" + file.VideoTracks[0].ColorTransfer = "bt709" + req := validStartRequestV3() + req.QualityPreference = "original" + cap := 4_000 + req.BandwidthCapKbps = &cap + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + // The client has no HLS engine, so the cap-induced transcode cannot run. + delete(req.ClientPlaybackContext.Engines, string(EngineMedia3HLSV3)) + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } + if !hasDegradationWarningV3(result.Plan.DegradationWarnings, "quality_reduction_unavailable") { + t.Fatalf("warnings = %#v", result.Plan.DegradationWarnings) + } +} + +func TestPlanPlaybackV3LegacyHDRUnknownAssumesHDR10ForCapableClients(t *testing.T) { + file := detailedFixtureFileV3() + file.HDR = true + file.VideoTracks[0].VideoRange = "" + file.VideoTracks[0].VideoRangeType = "" + file.VideoTracks[0].ColorTransfer = "" + req := validStartRequestV3() + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.Capabilities.HDRDetails = &HDRCapabilitiesV3{HDR10: true} + req.ClientPlaybackContext.Output.HDRDetails = req.Capabilities.HDRDetails + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryOriginalHTTPV3 || !result.Plan.Claims.Video.HDR10 { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } + if !hasDegradationWarningV3(result.Plan.DegradationWarnings, "hdr_range_assumed_hdr10") { + t.Fatalf("warnings = %#v", result.Plan.DegradationWarnings) + } + + // Without HDR10 output support the legacy row keeps the previous + // (ineligible) behavior instead of guessing at the client's range. + req.Capabilities.HDRDetails = nil + req.ClientPlaybackContext.Output.HDRDetails = nil + result = PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Terminal == nil || result.Terminal.Reason != "hdr_transcode_unsupported" { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } +} + +func TestSourceDescriptorV3NormalizesLegacyFileBitrateFallback(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].Bitrate = 0 + file.Bitrate = 60_000_000 // legacy rows stored bps + + source := SourceDescriptorFromFileV3(file, 0) + if source.BitrateKbps != 60_000 { + t.Fatalf("file-level bitrate = %d, want normalized 60000", source.BitrateKbps) + } + + file.VideoTracks[0].Bitrate = 45_000_000 + source = SourceDescriptorFromFileV3(file, 0) + if source.BitrateKbps != 45_000 { + t.Fatalf("track-level bitrate = %d, want normalized 45000", source.BitrateKbps) + } +} + +func TestPlanAttemptedV3RequiresExactKeyMatch(t *testing.T) { + plan := PlanV3{PlanID: "plan:exact", Delivery: DeliveryOriginalHTTPV3, Stream: StreamV3{Protocol: StreamHTTPProgressiveV3, Container: "mkv"}, Subtitle: SubtitleDecisionV3{Mode: SubtitleOffV3}} + key := PlanAttemptKeyV3(plan, 1, nil) + if !planAttemptedV3(plan, 1, []string{" " + key + " "}) { + t.Fatal("whitespace-trimmed exact key must match") + } + if planAttemptedV3(plan, 1, []string{strings.ToUpper(key)}) { + t.Fatal("case-folded attempt key must not match an exact hash") + } +} + +func TestStartRequestV3ValidationBoundsInnerLists(t *testing.T) { + longValue := strings.Repeat("x", 65) + cases := []struct { + name string + mutate func(*StartRequestV3) + }{ + {"video_decode_profile_count", func(r *StartRequestV3) { + r.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "h264", Hardware: true, Profiles: make([]string, 65)}} + }}, + {"video_decode_profile_length", func(r *StartRequestV3) { + r.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "h264", Hardware: true, Profiles: []string{longValue}}} + }}, + {"video_decode_level_count", func(r *StartRequestV3) { + r.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "h264", Hardware: true, Levels: make([]int, 65)}} + }}, + {"video_decode_bit_depth_count", func(r *StartRequestV3) { + r.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "h264", Hardware: true, BitDepths: make([]int, 65)}} + }}, + {"capability_dolby_vision_profiles", func(r *StartRequestV3) { + r.Capabilities.HDRDetails = &HDRCapabilitiesV3{DolbyVisionProfiles: make([]int, 17)} + }}, + {"output_dolby_vision_profiles", func(r *StartRequestV3) { + r.ClientPlaybackContext.Output.HDRDetails = &HDRCapabilitiesV3{DolbyVisionProfiles: make([]int, 17)} + }}, + {"engine_dolby_vision_profiles", func(r *StartRequestV3) { + engine := r.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] + engine.HDRDetails = &HDRCapabilitiesV3{DolbyVisionProfiles: make([]int, 17)} + r.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] = engine + }}, + {"engine_container_length", func(r *StartRequestV3) { + engine := r.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] + engine.Containers = []string{longValue} + r.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] = engine + }}, + {"engine_validated_claim_length", func(r *StartRequestV3) { + engine := r.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] + engine.ValidatedClaims = []string{longValue} + r.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] = engine + }}, + {"engine_feature_length", func(r *StartRequestV3) { + engine := r.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] + engine.Features = []string{longValue} + r.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] = engine + }}, + {"audio_track_id_length", func(r *StartRequestV3) { + r.AudioTrackID = strings.Repeat("a", 129) + }}, + {"subtitle_track_id_length", func(r *StartRequestV3) { + r.SubtitleTrackID = strings.Repeat("s", 129) + }}, + } + for _, value := range cases { + t.Run(value.name, func(t *testing.T) { + req := validStartRequestV3() + value.mutate(&req) + if _, err := req.NormalizeAndValidate(); err == nil { + t.Fatal("oversized capability accepted") + } + }) + } +} + +func TestResolveQualityPolicyV3PreservesNonStandardSourceHeight(t *testing.T) { + req := validStartRequestV3() + req.QualityPreference = "auto" + result := ResolveQualityPolicyV3(req, SourceDescriptorV3{Width: 2560, Height: 1440, BitrateKbps: 12_000}) + if !result.PreservesSource || result.RequiresTranscode || result.Width != 2560 || result.Height != 1440 || result.Label != "1440p" { + t.Fatalf("quality result = %#v", result) + } +} + +func TestPlanPlaybackV3SeekedHLSCopyPreservesVideo(t *testing.T) { + file := detailedFixtureFileV3() + file.Resolution = "1080p" + file.VideoTracks[0].Width = 1920 + file.VideoTracks[0].Height = 1080 + file.VideoTracks[0].BitDepth = 8 + file.VideoTracks[0].VideoRange = "SDR" + file.VideoTracks[0].VideoRangeType = "SDR" + file.VideoTracks[0].ColorTransfer = "bt709" + req := validStartRequestV3() + start := 20.0 + req.StartPosition = &start + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.Containers = []string{"mp4"} + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{8}, MaxWidth: 1920, MaxHeight: 1080, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + req.ClientPlaybackContext.Engines[string(EngineMedia3DirectV3)] = EngineCapabilityV3{} + req.ClientPlaybackContext.Engines[string(EngineMedia3ProgressiveRemuxV3)] = EngineCapabilityV3{} + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryRemuxHLSV3 || result.TargetVideoCodec != "copy" || result.Plan.Timeline.SourceStartSeconds != start { + t.Fatalf("result = %#v", result) + } +} + +func TestPlanPlaybackV3TimelineChangePreservesRouteIdentity(t *testing.T) { + file := detailedFixtureFileV3() + request := validStartRequestV3() + request.ClientFeatures = append(request.ClientFeatures, FeatureDetailedDecodeV3) + request.ClientPlaybackContext.Features = append(request.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + request.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + hdr := &HDRCapabilitiesV3{HDR10: true} + request.Capabilities.HDR = true + request.Capabilities.HDRDetails = hdr + request.ClientPlaybackContext.Output.HDRDetails = hdr + startAtZero := 0.0 + request.StartPosition = &startAtZero + first := PlanPlaybackV3(PlannerInputV3{ + Request: request, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3(), + }) + if first.Plan == nil { + t.Fatalf("initial plan = %#v", first) + } + + startAtSeek := 321.25 + request.StartPosition = &startAtSeek + second := PlanPlaybackV3(PlannerInputV3{ + Request: request, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, + Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3(), + }) + if second.Plan == nil { + t.Fatalf("seeked plan = %#v", second) + } + if first.Plan.PlanID != second.Plan.PlanID || + PlanAttemptKeyV3(*first.Plan, request.OutputRouteGeneration, nil) != PlanAttemptKeyV3(*second.Plan, request.OutputRouteGeneration, nil) { + t.Fatalf("timeline changed route identity: first=%#v second=%#v", first.Plan, second.Plan) + } + if first.Plan.Timeline.SourceStartSeconds != 0 || second.Plan.Timeline.SourceStartSeconds != startAtSeek { + t.Fatalf("timeline positions: first=%#v second=%#v", first.Plan.Timeline, second.Plan.Timeline) + } +} + +func validStartRequestV3() StartRequestV3 { + return StartRequestV3{ + ProtocolVersion: ProtocolV3, + ClientFeatures: []string{FeaturePlaybackPlanV3, FeatureMedia3Only}, + FileID: 42, + ProfileID: "profile-1", + PlaybackAttemptID: "attempt-0001", + QualityPreference: "original", + SubtitleFidelityPreference: SubtitleFidelityCompatibleV3, + OutputRouteGeneration: 1, + Capabilities: ClientCodecCapabilitiesV3{CodecsVideo: []string{"hevc"}, CodecsAudio: []string{"aac"}, Containers: []string{"mkv"}, MaxResolution: "2160p"}, + ClientPlaybackContext: ClientPlaybackContextV3{ProtocolVersion: ProtocolV3, Features: []string{FeaturePlaybackPlanV3, FeatureMedia3Only}, Platform: "android", FormFactor: "tv", AppVersion: "test", Output: OutputContextV3{OutputRouteGeneration: 1}, Engines: map[string]EngineCapabilityV3{ + string(EngineMedia3DirectV3): {Enabled: true, SupportedOnDevice: true, Subtitles: EngineSubtitleCapabilitiesV3{EmbeddedText: true, SidecarText: true}}, + string(EngineMedia3ProgressiveRemuxV3): {Enabled: true, SupportedOnDevice: true}, + string(EngineMedia3HLSV3): {Enabled: true, SupportedOnDevice: true}, + }}, + } +} + +func detailedFixtureFileV3() *models.MediaFile { + return &models.MediaFile{ID: 42, FilePath: "/media/movie.mkv", Container: "mkv", CodecVideo: "hevc", CodecAudio: "aac", Resolution: "2160p", Bitrate: 60_000, AudioChannels: 2, VideoTracks: []models.VideoTrack{{Codec: "hevc", Profile: "Main 10", Level: 153, Width: 3840, Height: 2160, FrameRate: "24000/1001", Bitrate: 60_000, BitDepth: 10, VideoRange: "HDR", VideoRangeType: "HDR10", ColorRange: "tv"}}, AudioTracks: []models.AudioTrack{{Codec: "aac", Channels: 2, Layout: "stereo"}}} +} + +func testTransformationRegistryV3() *TransformationRegistryV3 { + return NewTransformationRegistryV3([]TransformationSpecV3{ + {Name: "audio_to_aac", Available: true}, + {Name: "video_to_h264", Available: true}, + {Name: "server_dv7_to_hdr10", Available: true}, + }) +} diff --git a/internal/playback/recipecard.go b/internal/playback/recipecard.go index 9e97d834..1a830513 100644 --- a/internal/playback/recipecard.go +++ b/internal/playback/recipecard.go @@ -16,39 +16,55 @@ import ( // context, channels, log sink). Those are re-wired on reconstruct from the // live config and request. type RecipeCard struct { - SessionID string `json:"session_id"` - UserID int `json:"user_id"` - ProfileID string `json:"profile_id"` - MediaFileID int `json:"media_file_id"` - TranscodeNodeURL string `json:"transcode_node_url,omitempty"` + SessionID string `json:"session_id"` + UserID int `json:"user_id"` + ProfileID string `json:"profile_id"` + MediaFileID int `json:"media_file_id"` + TranscodeNodeURL string `json:"transcode_node_url,omitempty"` + TranscodeTransportID string `json:"transcode_transport_id,omitempty"` // PlayMethod discriminates which serve path reconstructs this session // (direct / remux / transcode). Empty decodes as PlayTranscode for // back-compat with cards written before direct/remux were reconstructable. PlayMethod PlayMethod `json:"play_method,omitempty"` // TranscodeAudio mirrors Session.TranscodeAudio; used by the remux path to - // re-spawn ffmpeg with the same audio handling on reconstruct. - TranscodeAudio bool `json:"transcode_audio,omitempty"` + // re-spawn ffmpeg with the same audio handling on reconstruct, and by the + // admin activity views to classify the reconstructed session (audio + // re-encode vs repackage). + TranscodeAudio bool `json:"transcode_audio,omitempty"` + RemuxDVMode RemuxDVMode `json:"remux_dv_mode,omitempty"` + + // Client metadata mirrored from the session so admin views (client label, + // Jellyfin pill) survive reconstruction. Carried only by stored cards — + // deliberately NOT projected into stream-token claims, where a user agent + // would bloat every stream URL. + ClientName string `json:"client_name,omitempty"` + ClientVersion string `json:"client_version,omitempty"` + ClientUserAgent string `json:"client_user_agent,omitempty"` // Encode parameters — mirror of the byte-affecting TranscodeOpts fields. // Unused (zero) for direct/remux cards, which carry no segment-based encode. - InputPath string `json:"input_path"` - SourceVideoCodec string `json:"source_video_codec,omitempty"` - SeekSeconds float64 `json:"seek_seconds"` - TargetResolution string `json:"target_resolution,omitempty"` - TargetCodecVideo string `json:"target_codec_video,omitempty"` - TargetCodecAudio string `json:"target_codec_audio,omitempty"` - SegmentDuration int `json:"segment_duration"` - StartSegmentNumber int `json:"start_segment_number"` - HWAccel string `json:"hw_accel,omitempty"` - HWDevice string `json:"hw_device,omitempty"` - SubtitleTrackIndex int `json:"subtitle_track_index"` - SubtitleBurnIn bool `json:"subtitle_burn_in,omitempty"` - SubtitleCodec string `json:"subtitle_codec,omitempty"` - AudioTrackIndex int `json:"audio_track_index"` - TargetBitrateKbps int `json:"target_bitrate_kbps,omitempty"` - TotalDuration float64 `json:"total_duration"` - FastStart bool `json:"fast_start,omitempty"` + InputPath string `json:"input_path"` + OutputSubdir string `json:"output_subdir,omitempty"` + SourceVideoCodec string `json:"source_video_codec,omitempty"` + VideoBitstreamFilter string `json:"video_bitstream_filter,omitempty"` + SeekSeconds float64 `json:"seek_seconds"` + StreamOriginSeconds float64 `json:"stream_origin_seconds,omitempty"` + CopySeekAnchorResolved bool `json:"copy_seek_anchor_resolved,omitempty"` + TargetResolution string `json:"target_resolution,omitempty"` + TargetCodecVideo string `json:"target_codec_video,omitempty"` + TargetCodecAudio string `json:"target_codec_audio,omitempty"` + SegmentDuration int `json:"segment_duration"` + StartSegmentNumber int `json:"start_segment_number"` + HWAccel string `json:"hw_accel,omitempty"` + HWDevice string `json:"hw_device,omitempty"` + SubtitleTrackIndex int `json:"subtitle_track_index"` + SubtitleBurnIn bool `json:"subtitle_burn_in,omitempty"` + SubtitleCodec string `json:"subtitle_codec,omitempty"` + AudioTrackIndex int `json:"audio_track_index"` + TargetBitrateKbps int `json:"target_bitrate_kbps,omitempty"` + TotalDuration float64 `json:"total_duration"` + FastStart bool `json:"fast_start,omitempty"` } // NewRecipeCard builds a RecipeCard from the durable identity fields plus the @@ -58,29 +74,35 @@ type RecipeCard struct { // operator's config change applies to reconstructed sessions too. func NewRecipeCard(userID int, profileID string, mediaFileID int, transcodeNodeURL string, opts TranscodeOpts) RecipeCard { return RecipeCard{ - SessionID: opts.SessionID, - UserID: userID, - ProfileID: profileID, - MediaFileID: mediaFileID, - TranscodeNodeURL: transcodeNodeURL, - PlayMethod: PlayTranscode, - InputPath: opts.InputPath, - SourceVideoCodec: opts.SourceVideoCodec, - SeekSeconds: opts.SeekSeconds, - TargetResolution: opts.TargetResolution, - TargetCodecVideo: opts.TargetCodecVideo, - TargetCodecAudio: opts.TargetCodecAudio, - SegmentDuration: opts.SegmentDuration, - StartSegmentNumber: opts.StartSegmentNumber, - HWAccel: opts.HWAccel, - HWDevice: opts.HWDevice, - SubtitleTrackIndex: opts.SubtitleTrackIndex, - SubtitleBurnIn: opts.SubtitleBurnIn, - SubtitleCodec: opts.SubtitleCodec, - AudioTrackIndex: opts.AudioTrackIndex, - TargetBitrateKbps: opts.TargetBitrateKbps, - TotalDuration: opts.TotalDuration, - FastStart: opts.FastStart, + SessionID: opts.SessionID, + UserID: userID, + ProfileID: profileID, + MediaFileID: mediaFileID, + TranscodeNodeURL: transcodeNodeURL, + TranscodeTransportID: opts.TranscodeTransportID, + PlayMethod: PlayTranscode, + TranscodeAudio: TranscodesAudio(opts.TargetCodecAudio), + InputPath: opts.InputPath, + OutputSubdir: opts.OutputSubdir, + SourceVideoCodec: opts.SourceVideoCodec, + VideoBitstreamFilter: opts.VideoBitstreamFilter, + SeekSeconds: opts.SeekSeconds, + StreamOriginSeconds: opts.StreamOriginSeconds, + CopySeekAnchorResolved: opts.CopySeekAnchorResolved, + TargetResolution: opts.TargetResolution, + TargetCodecVideo: opts.TargetCodecVideo, + TargetCodecAudio: opts.TargetCodecAudio, + SegmentDuration: opts.SegmentDuration, + StartSegmentNumber: opts.StartSegmentNumber, + HWAccel: opts.HWAccel, + HWDevice: opts.HWDevice, + SubtitleTrackIndex: opts.SubtitleTrackIndex, + SubtitleBurnIn: opts.SubtitleBurnIn, + SubtitleCodec: opts.SubtitleCodec, + AudioTrackIndex: opts.AudioTrackIndex, + TargetBitrateKbps: opts.TargetBitrateKbps, + TotalDuration: opts.TotalDuration, + FastStart: opts.FastStart, } } @@ -101,7 +123,11 @@ func NewDirectRecipeCard(sessionID string, userID int, profileID string, mediaFi // NewRemuxRecipeCard builds a card for a remux session: identity plus the audio // selection. The remux ffmpeg is a single pipe re-spawned at the client-supplied // ?seek= on the next request, so no segment/encode parameters are pinned. -func NewRemuxRecipeCard(sessionID string, userID int, profileID string, mediaFileID int, transcodeAudio bool, audioTrackIndex int) RecipeCard { +func NewRemuxRecipeCard(sessionID string, userID int, profileID string, mediaFileID int, transcodeAudio bool, audioTrackIndex int, dvMode ...RemuxDVMode) RecipeCard { + mode := RemuxDVMode("") + if len(dvMode) > 0 { + mode = dvMode[0] + } return RecipeCard{ SessionID: sessionID, UserID: userID, @@ -109,6 +135,7 @@ func NewRemuxRecipeCard(sessionID string, userID int, profileID string, mediaFil MediaFileID: mediaFileID, PlayMethod: PlayRemux, TranscodeAudio: transcodeAudio, + RemuxDVMode: mode, AudioTrackIndex: audioTrackIndex, } } @@ -118,29 +145,34 @@ func NewRemuxRecipeCard(sessionID string, userID int, profileID string, mediaFil // they are environment-specific and not pinned in the card. func (c RecipeCard) TranscodeOpts(outputDir, ffmpegPath string, logSink FFmpegLogSink) TranscodeOpts { return TranscodeOpts{ - InputPath: c.InputPath, - OutputDir: outputDir, - SessionID: c.SessionID, - SourceVideoCodec: c.SourceVideoCodec, - SeekSeconds: c.SeekSeconds, - TargetResolution: c.TargetResolution, - TargetCodecVideo: c.TargetCodecVideo, - TargetCodecAudio: c.TargetCodecAudio, - SegmentDuration: c.SegmentDuration, - StartSegmentNumber: c.StartSegmentNumber, - FFmpegPath: ffmpegPath, - HWAccel: c.HWAccel, - HWDevice: c.HWDevice, - SubtitleTrackIndex: c.SubtitleTrackIndex, - SubtitleBurnIn: c.SubtitleBurnIn, - SubtitleCodec: c.SubtitleCodec, - AudioTrackIndex: c.AudioTrackIndex, - TargetBitrateKbps: c.TargetBitrateKbps, - TotalDuration: c.TotalDuration, - FastStart: c.FastStart, - NodeType: "integrated", - ExecutionMode: "integrated", - FFmpegLogSink: logSink, + InputPath: c.InputPath, + OutputSubdir: c.OutputSubdir, + OutputDir: outputDir, + SessionID: c.SessionID, + TranscodeTransportID: c.TranscodeTransportID, + SourceVideoCodec: c.SourceVideoCodec, + VideoBitstreamFilter: c.VideoBitstreamFilter, + SeekSeconds: c.SeekSeconds, + StreamOriginSeconds: c.StreamOriginSeconds, + CopySeekAnchorResolved: c.CopySeekAnchorResolved, + TargetResolution: c.TargetResolution, + TargetCodecVideo: c.TargetCodecVideo, + TargetCodecAudio: c.TargetCodecAudio, + SegmentDuration: c.SegmentDuration, + StartSegmentNumber: c.StartSegmentNumber, + FFmpegPath: ffmpegPath, + HWAccel: c.HWAccel, + HWDevice: c.HWDevice, + SubtitleTrackIndex: c.SubtitleTrackIndex, + SubtitleBurnIn: c.SubtitleBurnIn, + SubtitleCodec: c.SubtitleCodec, + AudioTrackIndex: c.AudioTrackIndex, + TargetBitrateKbps: c.TargetBitrateKbps, + TotalDuration: c.TotalDuration, + FastStart: c.FastStart, + NodeType: "integrated", + ExecutionMode: "integrated", + FFmpegLogSink: logSink, } } @@ -159,28 +191,34 @@ const MaxTokenTTL = 24 * time.Hour // change applies to reconstructed sessions too. func (c RecipeCard) ToClaims() streamtoken.Claims { return streamtoken.Claims{ - SessionID: c.SessionID, - MediaPath: c.InputPath, - PlayMethod: string(c.PlayMethod), - TranscodeAudio: c.TranscodeAudio, - TranscodeNode: c.TranscodeNodeURL, - TargetCodec: c.TargetCodecVideo, - TargetRes: c.TargetResolution, - AudioTrackIndex: c.AudioTrackIndex, - UserID: c.UserID, - ProfileID: c.ProfileID, - MediaFileID: c.MediaFileID, - SourceVideoCodec: c.SourceVideoCodec, - SeekSeconds: c.SeekSeconds, - SegmentDuration: c.SegmentDuration, - StartSegmentNumber: c.StartSegmentNumber, - SubtitleTrackIndex: c.SubtitleTrackIndex, - SubtitleBurnIn: c.SubtitleBurnIn, - SubtitleCodec: c.SubtitleCodec, - TargetBitrateKbps: c.TargetBitrateKbps, - TotalDuration: c.TotalDuration, - FastStart: c.FastStart, - TargetCodecAudio: c.TargetCodecAudio, + SessionID: c.SessionID, + MediaPath: c.InputPath, + OutputSubdir: c.OutputSubdir, + PlayMethod: string(c.PlayMethod), + TranscodeAudio: c.TranscodeAudio, + RemuxDVMode: string(c.RemuxDVMode), + TranscodeNode: c.TranscodeNodeURL, + TranscodeTransportID: c.TranscodeTransportID, + TargetCodec: c.TargetCodecVideo, + TargetRes: c.TargetResolution, + AudioTrackIndex: c.AudioTrackIndex, + UserID: c.UserID, + ProfileID: c.ProfileID, + MediaFileID: c.MediaFileID, + SourceVideoCodec: c.SourceVideoCodec, + VideoBitstreamFilter: c.VideoBitstreamFilter, + SeekSeconds: c.SeekSeconds, + StreamOriginSeconds: c.StreamOriginSeconds, + CopySeekAnchorResolved: c.CopySeekAnchorResolved, + SegmentDuration: c.SegmentDuration, + StartSegmentNumber: c.StartSegmentNumber, + SubtitleTrackIndex: c.SubtitleTrackIndex, + SubtitleBurnIn: c.SubtitleBurnIn, + SubtitleCodec: c.SubtitleCodec, + TargetBitrateKbps: c.TargetBitrateKbps, + TotalDuration: c.TotalDuration, + FastStart: c.FastStart, + TargetCodecAudio: c.TargetCodecAudio, } } @@ -197,27 +235,33 @@ func RecipeCardFromClaims(c *streamtoken.Claims) RecipeCard { method = PlayTranscode } return RecipeCard{ - SessionID: c.SessionID, - UserID: c.UserID, - ProfileID: c.ProfileID, - MediaFileID: c.MediaFileID, - TranscodeNodeURL: c.TranscodeNode, - PlayMethod: method, - TranscodeAudio: c.TranscodeAudio, - InputPath: c.MediaPath, - SourceVideoCodec: c.SourceVideoCodec, - SeekSeconds: c.SeekSeconds, - TargetResolution: c.TargetRes, - TargetCodecVideo: c.TargetCodec, - TargetCodecAudio: c.TargetCodecAudio, - SegmentDuration: c.SegmentDuration, - StartSegmentNumber: c.StartSegmentNumber, - SubtitleTrackIndex: c.SubtitleTrackIndex, - SubtitleBurnIn: c.SubtitleBurnIn, - SubtitleCodec: c.SubtitleCodec, - AudioTrackIndex: c.AudioTrackIndex, - TargetBitrateKbps: c.TargetBitrateKbps, - TotalDuration: c.TotalDuration, - FastStart: c.FastStart, + SessionID: c.SessionID, + UserID: c.UserID, + ProfileID: c.ProfileID, + MediaFileID: c.MediaFileID, + TranscodeNodeURL: c.TranscodeNode, + TranscodeTransportID: c.TranscodeTransportID, + PlayMethod: method, + TranscodeAudio: c.TranscodeAudio, + RemuxDVMode: RemuxDVMode(c.RemuxDVMode), + InputPath: c.MediaPath, + OutputSubdir: c.OutputSubdir, + SourceVideoCodec: c.SourceVideoCodec, + VideoBitstreamFilter: c.VideoBitstreamFilter, + SeekSeconds: c.SeekSeconds, + StreamOriginSeconds: c.StreamOriginSeconds, + CopySeekAnchorResolved: c.CopySeekAnchorResolved, + TargetResolution: c.TargetRes, + TargetCodecVideo: c.TargetCodec, + TargetCodecAudio: c.TargetCodecAudio, + SegmentDuration: c.SegmentDuration, + StartSegmentNumber: c.StartSegmentNumber, + SubtitleTrackIndex: c.SubtitleTrackIndex, + SubtitleBurnIn: c.SubtitleBurnIn, + SubtitleCodec: c.SubtitleCodec, + AudioTrackIndex: c.AudioTrackIndex, + TargetBitrateKbps: c.TargetBitrateKbps, + TotalDuration: c.TotalDuration, + FastStart: c.FastStart, } } diff --git a/internal/playback/recipecard_test.go b/internal/playback/recipecard_test.go index c83d926d..85660138 100644 --- a/internal/playback/recipecard_test.go +++ b/internal/playback/recipecard_test.go @@ -9,25 +9,28 @@ import ( func TestRecipeCardRoundTripOpts(t *testing.T) { opts := TranscodeOpts{ - InputPath: "/media/movie.mkv", - OutputDir: "/tmp/silo-transcode/abc", - SessionID: "abc", - SourceVideoCodec: "hevc", - SeekSeconds: 900, - TargetResolution: "1080p", - TargetCodecVideo: "h264", - TargetCodecAudio: "aac", - SegmentDuration: 2, - StartSegmentNumber: 450, - HWAccel: "qsv", - HWDevice: "/dev/dri/renderD128", - SubtitleTrackIndex: 3, - SubtitleBurnIn: true, - SubtitleCodec: "hdmv_pgs_subtitle", - AudioTrackIndex: 1, - TargetBitrateKbps: 8000, - TotalDuration: 7200, - FastStart: true, + InputPath: "/media/movie.mkv", + OutputDir: "/tmp/silo-transcode/abc", + SessionID: "abc", + SourceVideoCodec: "hevc", + VideoBitstreamFilter: "dovi_rpu=strip=1", + SeekSeconds: 900, + StreamOriginSeconds: 896, + CopySeekAnchorResolved: true, + TargetResolution: "1080p", + TargetCodecVideo: "h264", + TargetCodecAudio: "aac", + SegmentDuration: 2, + StartSegmentNumber: 450, + HWAccel: "qsv", + HWDevice: "/dev/dri/renderD128", + SubtitleTrackIndex: 3, + SubtitleBurnIn: true, + SubtitleCodec: "hdmv_pgs_subtitle", + AudioTrackIndex: 1, + TargetBitrateKbps: 8000, + TotalDuration: 7200, + FastStart: true, } card := NewRecipeCard(42, "profile-1", 77, "", opts) @@ -43,6 +46,9 @@ func TestRecipeCardRoundTripOpts(t *testing.T) { if got.SeekSeconds != 900 { t.Errorf("SeekSeconds = %v, want 900", got.SeekSeconds) } + if got.StreamOriginSeconds != 896 || !got.CopySeekAnchorResolved { + t.Errorf("copy seek anchor lost: origin=%v resolved=%v", got.StreamOriginSeconds, got.CopySeekAnchorResolved) + } if !got.SubtitleBurnIn { t.Errorf("SubtitleBurnIn lost in round trip") } @@ -55,6 +61,9 @@ func TestRecipeCardRoundTripOpts(t *testing.T) { if got.TargetCodecVideo != "h264" || got.TargetBitrateKbps != 8000 { t.Errorf("encode params wrong: %+v", got) } + if got.VideoBitstreamFilter != "dovi_rpu=strip=1" { + t.Errorf("VideoBitstreamFilter = %q", got.VideoBitstreamFilter) + } if got.FFmpegPath != "/usr/bin/ffmpeg" { t.Errorf("FFmpegPath not re-supplied: %q", got.FFmpegPath) } @@ -74,6 +83,101 @@ func TestRecipeCardPlayMethodConstructors(t *testing.T) { } } +// A transcode card must record whether audio is actually re-encoded so the +// reconstructed session classifies correctly in admin views: only an explicit +// "copy" leaves the audio untouched (an empty codec runs ffmpeg's aac default). +func TestRecipeCardDerivesTranscodeAudioFromOpts(t *testing.T) { + cases := []struct { + codec string + want bool + }{ + {"aac", true}, + {"eac3", true}, + {"", true}, + {"copy", false}, + {"COPY", false}, + } + for _, tc := range cases { + card := NewRecipeCard(1, "p", 2, "", TranscodeOpts{SessionID: "t", TargetCodecAudio: tc.codec}) + if card.TranscodeAudio != tc.want { + t.Errorf("TargetCodecAudio %q: TranscodeAudio = %v, want %v", tc.codec, card.TranscodeAudio, tc.want) + } + } +} + +// Client metadata rides in stored cards (label + JF pill survive restarts) but +// must NOT leak into stream-token claims, where a user agent would bloat every +// stream URL. +func TestRecipeCardClientMetadataStoredNotInClaims(t *testing.T) { + card := NewRecipeCard(1, "p", 2, "", TranscodeOpts{SessionID: "t"}) + card.ClientName = "Findroid" + card.ClientVersion = "0.15" + card.ClientUserAgent = "Findroid/0.15" + + encoded, err := json.Marshal(card) + if err != nil { + t.Fatalf("marshal card: %v", err) + } + var back RecipeCard + if err := json.Unmarshal(encoded, &back); err != nil { + t.Fatalf("unmarshal card: %v", err) + } + if back.ClientName != "Findroid" || back.ClientVersion != "0.15" || back.ClientUserAgent != "Findroid/0.15" { + t.Fatalf("client metadata lost in stored-card round trip: %+v", back) + } + + claims := card.ToClaims() + fromClaims := RecipeCardFromClaims(&claims) + if fromClaims.ClientName != "" || fromClaims.ClientUserAgent != "" { + t.Fatalf("client metadata must not travel via token claims: %+v", fromClaims) + } +} + +// A session rebuilt from a card keeps the client identity for its lifetime, +// so the admin client label and Jellyfin pill survive a server restart. +func TestReconstructSessionRestoresClientMetadata(t *testing.T) { + tm := NewTranscodeManager() + tm.Sessions = NewSessionManager(0, 0) + + card := NewRecipeCard(42, "profile-1", 77, "", TranscodeOpts{SessionID: "sess-jf", InputPath: "/media/movie.mkv"}) + card.ClientName = "Findroid" + card.ClientVersion = "0.15" + card.ClientUserAgent = "Findroid/0.15 (Android)" + + session := tm.ReconstructSession(t.Context(), "sess-jf", 42, card) + if session == nil { + t.Fatal("reconstruct returned nil") + } + if session.ClientName != "Findroid" || session.ClientVersion != "0.15" || session.ClientUserAgent != "Findroid/0.15 (Android)" { + t.Fatalf("client metadata not restored: %+v", session) + } + if !session.TranscodeAudio { + t.Fatal("TranscodeAudio must be restored from the card (aac default re-encodes)") + } +} + +// SetTranscodeStreamDetails records the running transcode's encode decisions +// on the live session so sync rows classify by actual work (video copy = +// repackage) rather than the transport method. +func TestSetTranscodeStreamDetails(t *testing.T) { + m := NewSessionManager(0, 0) + m.RegisterReconstructed(&Session{ID: "sess-1", UserID: 7, PlayMethod: PlayTranscode}) + + if err := m.SetTranscodeStreamDetails("sess-1", "copy", "aac", true); err != nil { + t.Fatalf("SetTranscodeStreamDetails: %v", err) + } + s, err := m.GetSession("sess-1") + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if s.TargetVideoCodec != "copy" || s.TargetAudioCodec != "aac" || !s.TranscodeAudio { + t.Fatalf("details not recorded: %+v", s) + } + if err := m.SetTranscodeStreamDetails("missing", "h264", "aac", true); err == nil { + t.Fatal("expected ErrSessionNotFound for unknown session") + } +} + // A card persisted before the play_method discriminator existed must decode with // an empty PlayMethod so reconstruct can treat it as a transcode (back-compat). func TestRecipeCardLegacyDecodeHasEmptyPlayMethod(t *testing.T) { @@ -97,22 +201,25 @@ func TestRecipeCardLegacyDecodeHasEmptyPlayMethod(t *testing.T) { // not asserted here. func TestRecipeCardClaimsRoundTrip(t *testing.T) { card := NewRecipeCard(42, "profile-1", 77, "http://node:9000", TranscodeOpts{ - InputPath: "/media/movie.mkv", - SessionID: "abc", - SourceVideoCodec: "hevc", - SeekSeconds: 900, - TargetResolution: "1080p", - TargetCodecVideo: "h264", - TargetCodecAudio: "aac", - SegmentDuration: 2, - StartSegmentNumber: 450, - SubtitleTrackIndex: 3, - SubtitleBurnIn: true, - SubtitleCodec: "hdmv_pgs_subtitle", - AudioTrackIndex: 1, - TargetBitrateKbps: 8000, - TotalDuration: 7200, - FastStart: true, + InputPath: "/media/movie.mkv", + SessionID: "abc", + SourceVideoCodec: "hevc", + VideoBitstreamFilter: "dovi_rpu=strip=1", + SeekSeconds: 900, + StreamOriginSeconds: 896, + CopySeekAnchorResolved: true, + TargetResolution: "1080p", + TargetCodecVideo: "h264", + TargetCodecAudio: "aac", + SegmentDuration: 2, + StartSegmentNumber: 450, + SubtitleTrackIndex: 3, + SubtitleBurnIn: true, + SubtitleCodec: "hdmv_pgs_subtitle", + AudioTrackIndex: 1, + TargetBitrateKbps: 8000, + TotalDuration: 7200, + FastStart: true, }) claims := card.ToClaims() @@ -126,7 +233,9 @@ func TestRecipeCardClaimsRoundTrip(t *testing.T) { } // Byte-affecting encode parameters. if got.InputPath != card.InputPath || got.SourceVideoCodec != card.SourceVideoCodec || - got.SeekSeconds != card.SeekSeconds || got.TargetResolution != card.TargetResolution || + got.VideoBitstreamFilter != card.VideoBitstreamFilter || + got.SeekSeconds != card.SeekSeconds || got.StreamOriginSeconds != card.StreamOriginSeconds || + got.CopySeekAnchorResolved != card.CopySeekAnchorResolved || got.TargetResolution != card.TargetResolution || got.TargetCodecVideo != card.TargetCodecVideo || got.TargetCodecAudio != card.TargetCodecAudio || got.SegmentDuration != card.SegmentDuration || got.StartSegmentNumber != card.StartSegmentNumber || got.SubtitleTrackIndex != card.SubtitleTrackIndex || got.SubtitleBurnIn != card.SubtitleBurnIn || diff --git a/internal/playback/remux.go b/internal/playback/remux.go index f0135f45..bdc0d7c6 100644 --- a/internal/playback/remux.go +++ b/internal/playback/remux.go @@ -10,6 +10,7 @@ import ( "os" "os/exec" "strconv" + "strings" "sync" "github.com/Silo-Server/silo-server/internal/httpstream" @@ -19,8 +20,8 @@ var ( resolvedFFmpegPath string ffmpegOnce sync.Once - doviRPUAvailable bool - doviRPUOnce sync.Once + doviRPUMu sync.Mutex + doviRPUCache map[string]bool ) // ffmpegBinary returns the path to the ffmpeg binary. @@ -37,18 +38,38 @@ func ffmpegBinary() string { return resolvedFFmpegPath } -// supportsDoviRPUFilter reports whether the resolved ffmpeg binary ships the -// dovi_rpu bitstream filter (FFmpeg 7.1+). Probed once per process. -func supportsDoviRPUFilter() bool { - doviRPUOnce.Do(func() { - out, err := exec.Command(ffmpegBinary(), "-hide_banner", "-bsfs").Output() - doviRPUAvailable = err == nil && bytes.Contains(out, []byte("dovi_rpu")) - if !doviRPUAvailable { - slog.Warn("ffmpeg lacks the dovi_rpu bitstream filter (needs FFmpeg 7.1+); " + - "Dolby Vision profile 7 remuxes will keep their dangling dual-layer RPUs") - } - }) - return doviRPUAvailable +// ResolveFFmpegPath returns the ffmpeg binary the playback pipeline executes +// for the given configured path: the configured path when set, otherwise the +// process-global discovery (jellyfin-ffmpeg install, then PATH). Capability +// probes must resolve through this same function so a feature advertised at +// planning time is guaranteed present in the binary that later runs. +func ResolveFFmpegPath(configured string) string { + if strings.TrimSpace(configured) != "" { + return configured + } + return ffmpegBinary() +} + +// supportsDoviRPUFilter reports whether the given FFmpeg binary can strip +// Dolby Vision RPU metadata via the dovi_rpu bitstream filter (FFmpeg 7.1+). +// The enhancement layer itself is dropped by stream mapping, so stripping the +// RPUs yields a clean HDR10 base layer. Probed once per binary path. +func supportsDoviRPUFilter(bin string) bool { + doviRPUMu.Lock() + defer doviRPUMu.Unlock() + if available, ok := doviRPUCache[bin]; ok { + return available + } + out, err := exec.Command(bin, "-hide_banner", "-bsfs").Output() + available := err == nil && bytes.Contains(out, []byte("dovi_rpu")) + if !available { + slog.Warn("ffmpeg lacks the dovi_rpu bitstream filter (needs FFmpeg 7.1+); validated Profile 7 HDR10 remux is disabled", "ffmpeg", bin) + } + if doviRPUCache == nil { + doviRPUCache = make(map[string]bool) + } + doviRPUCache[bin] = available + return available } // remuxDVProfile neutralizes a Dolby Vision profile the local ffmpeg cannot @@ -71,9 +92,20 @@ type RemuxSession struct { outputPipe io.ReadCloser } +// RemuxDVMode makes Profile 7 handling an explicit byte-level recipe choice. +// The empty/legacy mode exists only for pre-v3 callers and old stream tokens. +type RemuxDVMode string + +const ( + RemuxDVLegacyAutoV3 RemuxDVMode = "legacy_auto" + RemuxDVPreserveV3 RemuxDVMode = "preserve" + RemuxDVStripToHDR10V3 RemuxDVMode = "strip_to_hdr10" + RemuxDVRejectP7V3 RemuxDVMode = "reject_profile_7" +) + // buildRemuxArgs constructs the ffmpeg argument list for a remux operation. // The args perform codec copy (-c copy) into the target container format, -// using fragmented output for streaming (frag_keyframe+empty_moov+default_base_moof) and +// using fragmented output for streaming (frag_keyframe+delay_moov+default_base_moof) and // pipe:1 for stdout output. // When transcodeAudio is true, video is copied but audio is transcoded to // stereo AAC (handles cases like DTS/TrueHD that browsers cannot decode). @@ -82,7 +114,7 @@ type RemuxSession struct { // the RPUs would dangle — stripping yields a clean HDR10 base layer (the // Apple-parity fallback for devices without a P7 decoder). Profile 8 RPUs // stay: the base layer is self-contained and DV clients can render it. -func buildRemuxArgs(filePath, outputFormat string, seekSeconds float64, transcodeAudio bool, audioTrackIndex int, dvProfile int) []string { +func buildRemuxArgs(filePath, outputFormat string, seekSeconds float64, transcodeAudio bool, audioTrackIndex int, dvProfile int, tagDVSampleEntry bool) []string { args := []string{ "-nostdin", "-hide_banner", @@ -128,6 +160,14 @@ func buildRemuxArgs(filePath, outputFormat string, seekSeconds float64, transcod if dvProfile == 7 { args = append(args, "-bsf:v", "dovi_rpu=strip=1") + } else if (dvProfile == 5 || dvProfile == 8) && tagDVSampleEntry { + // FFmpeg carries the DOVI configuration record into MP4 but otherwise + // labels copied HEVC as hev1. Media3 keys decoder selection from the + // sample entry, so retain an explicit Dolby Vision tag as well. + // dvhe keeps FFmpeg's dvvC box; forcing dvh1 makes FFmpeg 7.1 omit it. + // Only the explicit v3 preserve recipe opts in: legacy web/jellycompat + // consumers keep the pre-v3 hev1 labeling their demuxers accept. + args = append(args, "-tag:v", "dvhe") } if transcodeAudio { @@ -150,7 +190,10 @@ func buildRemuxArgs(filePath, outputFormat string, seekSeconds float64, transcod args = append(args, "-avoid_negative_ts", "make_zero", "-f", outputFormat, - "-movflags", "frag_keyframe+empty_moov+default_base_moof", + // delay_moov lets the MP4 muxer inspect the first audio packet before + // writing codec configuration. empty_moov fails immediately for copied + // E-AC-3/Atmos tracks because their frame size is not known at header time. + "-movflags", "frag_keyframe+delay_moov+default_base_moof", "pipe:1", ) @@ -160,16 +203,58 @@ func buildRemuxArgs(filePath, outputFormat string, seekSeconds float64, transcod // StartRemux starts an ffmpeg process that copies codecs to a new container. // When transcodeAudio is false the command is: // -// ffmpeg -i {input} -c copy -f {format} -movflags frag_keyframe+empty_moov+default_base_moof pipe:1 +// ffmpeg -i {input} -c copy -f {format} -movflags frag_keyframe+delay_moov+default_base_moof pipe:1 // // When transcodeAudio is true video is copied but audio is transcoded to AAC. // The caller must call Close() when done to clean up resources. func StartRemux(ctx context.Context, filePath, outputFormat string, seekSeconds float64, transcodeAudio bool, audioTrackIndex int, dvProfile int) (*RemuxSession, error) { + return StartRemuxWithDVMode(ctx, filePath, outputFormat, seekSeconds, transcodeAudio, audioTrackIndex, dvProfile, RemuxDVLegacyAutoV3, "") +} + +// StartRemuxWithDVMode starts a remux with explicit Dolby Vision behavior. +// ffmpegPath selects the binary to execute (empty = process-global discovery); +// v3 callers must pass the configured playback path so the strip capability +// promised by the planner's probe holds for the binary that actually runs. +func StartRemuxWithDVMode(ctx context.Context, filePath, outputFormat string, seekSeconds float64, transcodeAudio bool, audioTrackIndex int, dvProfile int, mode RemuxDVMode, ffmpegPath string) (*RemuxSession, error) { ctx, cancel := context.WithCancel(ctx) - bin := ffmpegBinary() - args := buildRemuxArgs(filePath, outputFormat, seekSeconds, transcodeAudio, audioTrackIndex, - remuxDVProfile(dvProfile, supportsDoviRPUFilter())) + bin := ResolveFFmpegPath(ffmpegPath) + effectiveProfile := dvProfile + tagDVSampleEntry := false + switch mode { + case "", RemuxDVLegacyAutoV3: + effectiveProfile = remuxDVProfile(dvProfile, supportsDoviRPUFilter(bin)) + case RemuxDVStripToHDR10V3: + if dvProfile != 7 && dvProfile != 8 { + cancel() + return nil, fmt.Errorf("Dolby Vision HDR10 strip requires profile 7 or 8") + } + if !supportsDoviRPUFilter(bin) { + cancel() + return nil, fmt.Errorf("Dolby Vision HDR10 remux requires the dovi_rpu bitstream filter") + } + // buildRemuxArgs uses profile 7 as the explicit strip sentinel; the + // filter is equally required for a compatible profile 8 base layer. + effectiveProfile = 7 + case RemuxDVPreserveV3: + if dvProfile == 7 { + // The remux maps only the base-layer stream, so dual-layer P7 + // cannot be preserved: the EL is dropped and its RPUs would + // dangle. Callers must strip to HDR10 or transcode instead. + cancel() + return nil, fmt.Errorf("Dolby Vision profile 7 cannot be preserved in a progressive remux") + } + tagDVSampleEntry = true + case RemuxDVRejectP7V3: + if dvProfile == 7 { + cancel() + return nil, fmt.Errorf("profile 7 remux is not eligible") + } + default: + cancel() + return nil, fmt.Errorf("unknown remux Dolby Vision mode %q", mode) + } + args := buildRemuxArgs(filePath, outputFormat, seekSeconds, transcodeAudio, audioTrackIndex, effectiveProfile, tagDVSampleEntry) cmd := exec.CommandContext(ctx, bin, args...) stdout, err := cmd.StdoutPipe() @@ -226,6 +311,12 @@ func containerMIME(format string) string { // total size is not known in advance. // When transcodeAudio is true, audio is transcoded to AAC while video is copied. func ServeRemux(w http.ResponseWriter, r *http.Request, filePath, outputFormat string, seekSeconds float64, transcodeAudio bool, audioTrackIndex int, dvProfile int) error { + return ServeRemuxWithDVMode(w, r, filePath, outputFormat, seekSeconds, transcodeAudio, audioTrackIndex, dvProfile, RemuxDVLegacyAutoV3, "") +} + +// ServeRemuxWithDVMode streams an explicitly declared Dolby Vision recipe. +// ffmpegPath selects the binary to execute (empty = process-global discovery). +func ServeRemuxWithDVMode(w http.ResponseWriter, r *http.Request, filePath, outputFormat string, seekSeconds float64, transcodeAudio bool, audioTrackIndex int, dvProfile int, mode RemuxDVMode, ffmpegPath string) error { // Remux output streams for the length of the title; roll the write // deadline with progress instead of the server's absolute WriteTimeout. w = httpstream.NewRollingDeadlineWriter(w) @@ -241,7 +332,7 @@ func ServeRemux(w http.ResponseWriter, r *http.Request, filePath, outputFormat s return err } - session, err := StartRemux(r.Context(), filePath, outputFormat, seekSeconds, transcodeAudio, audioTrackIndex, dvProfile) + session, err := StartRemuxWithDVMode(r.Context(), filePath, outputFormat, seekSeconds, transcodeAudio, audioTrackIndex, dvProfile, mode, ffmpegPath) if err != nil { http.Error(w, "failed to start remux", http.StatusInternalServerError) return err diff --git a/internal/playback/remux_dv_test.go b/internal/playback/remux_dv_test.go index afa61008..a1e19e20 100644 --- a/internal/playback/remux_dv_test.go +++ b/internal/playback/remux_dv_test.go @@ -19,9 +19,9 @@ func argsContainPair(args []string, a, b string) bool { // Stripping the RPUs yields a clean HDR10 stream — both a correctness fix and // the Apple-parity fallback presentation for devices without a P7 decoder. func TestBuildRemuxArgsStripsDolbyVisionRPUForProfile7(t *testing.T) { - args := buildRemuxArgs("/x.mkv", "mp4", 0, false, -1, 7) + args := buildRemuxArgs("/x.mkv", "mp4", 0, false, -1, 7, false) if !argsContainPair(args, "-bsf:v", "dovi_rpu=strip=1") { - t.Fatalf("profile 7 remux must strip DV RPUs, args=%v", strings.Join(args, " ")) + t.Fatalf("profile 7 remux must strip DV RPUs from the base layer, args=%v", strings.Join(args, " ")) } } @@ -48,9 +48,52 @@ func TestRemuxDVProfileFallsBackWithoutFilterSupport(t *testing.T) { // enhancement layer and DV-capable clients can render it. Never strip. func TestBuildRemuxArgsKeepsRPUForProfile8AndPlainFiles(t *testing.T) { for _, profile := range []int{0, 5, 8} { - args := buildRemuxArgs("/x.mkv", "mp4", 0, false, -1, profile) + args := buildRemuxArgs("/x.mkv", "mp4", 0, false, -1, profile, false) if argsContainPair(args, "-bsf:v", "dovi_rpu=strip=1") { t.Fatalf("profile %d remux must not strip DV RPUs, args=%v", profile, strings.Join(args, " ")) } } } + +// The DV sample entry is an explicit opt-in for the v3 preserve recipe: +// Media3 keys decoder selection from it, but legacy web/jellycompat consumers +// rely on the pre-v3 hev1 labeling their demuxers accept. +func TestBuildRemuxArgsTagsPreservedDolbyVisionOnlyWhenRequested(t *testing.T) { + args := buildRemuxArgs("/x.mkv", "mp4", 0, false, -1, 8, true) + if !argsContainPair(args, "-tag:v", "dvhe") { + t.Fatalf("preserved Dolby Vision must retain a DV sample entry, args=%v", strings.Join(args, " ")) + } + legacy := buildRemuxArgs("/x.mkv", "mp4", 0, false, -1, 8, false) + if argsContainPair(legacy, "-tag:v", "dvhe") { + t.Fatalf("legacy remux consumers must keep hev1 labeling, args=%v", strings.Join(legacy, " ")) + } + stripped := buildRemuxArgs("/x.mkv", "mp4", 0, false, -1, 7, false) + if argsContainPair(stripped, "-tag:v", "dvhe") { + t.Fatalf("HDR10 fallback must not retain a DV sample entry, args=%v", strings.Join(stripped, " ")) + } +} + +// Dual-layer Profile 7 cannot be preserved by a base-layer-only remux; the +// explicit preserve recipe must fail fast instead of emitting a stream with +// dangling RPUs and no enhancement layer. +func TestStartRemuxRejectsPreservedProfile7(t *testing.T) { + if _, err := StartRemuxWithDVMode(t.Context(), "/nonexistent.mkv", "mp4", 0, false, -1, 7, RemuxDVPreserveV3, ""); err == nil { + t.Fatal("preserve mode accepted a profile 7 source") + } +} + +// An unknown mode must fail for every profile, not only Profile 7 sources. +func TestStartRemuxRejectsUnknownModeForAllProfiles(t *testing.T) { + for _, profile := range []int{0, 5, 7, 8} { + if _, err := StartRemuxWithDVMode(t.Context(), "/nonexistent.mkv", "mp4", 0, false, -1, profile, RemuxDVMode("bogus"), ""); err == nil { + t.Fatalf("unknown remux DV mode accepted for profile %d", profile) + } + } +} + +func TestBuildRemuxArgsDelaysMoovForCopiedAtmosConfiguration(t *testing.T) { + args := buildRemuxArgs("/x.mkv", "mp4", 0, false, -1, 8, false) + if !argsContainPair(args, "-movflags", "frag_keyframe+delay_moov+default_base_moof") { + t.Fatalf("remux must delay moov until copied audio is parsed, args=%v", strings.Join(args, " ")) + } +} diff --git a/internal/playback/session.go b/internal/playback/session.go index 615b00d0..57d824ba 100644 --- a/internal/playback/session.go +++ b/internal/playback/session.go @@ -21,14 +21,16 @@ type Session struct { RequestedMediaFileID int PlayMethod PlayMethod BasePlayMethod PlayMethod - TranscodeAudio bool // when true, remux should transcode audio to AAC + TranscodeAudio bool // when true, remux should transcode audio to AAC + RemuxDVMode RemuxDVMode ClientIP string // resolved client IP for the playback session ClientName string // reported playback client name, when available ClientVersion string // reported playback client version, when available ClientUserAgent string // trimmed request user agent for the playback session - TranscodeNodeURL string // URL of assigned transcode node (empty = local/integrated) - AudioTrackIndex int + TranscodeNodeURL string // URL of assigned transcode node (empty = local/integrated) + TranscodeTransportID string // remote node process identity; empty means session ID + AudioTrackIndex int StreamBitrateKbps int // currently delivered bitrate, when known TargetResolution string // requested output resolution for transcodes @@ -54,26 +56,32 @@ type Session struct { UpdatedAt time.Time LastActivityAt time.Time activeTransportCount int + replacementPlayMethod PlayMethod + streamRevision uint64 } // SessionStreamState stores the mutable stream-specific details that can // change after a session is created (audio track, client IP, transcode target, // and reported bitrate). type SessionStreamState struct { - PlayMethod PlayMethod - BasePlayMethod PlayMethod - AudioTrackIndex int - TranscodeAudio bool - ClientIP string - ClientName string - ClientVersion string - ClientUserAgent string - StreamBitrateKbps int - TargetResolution string - TargetVideoCodec string - TargetAudioCodec string - TargetBitrateKbps int - TranscodeHWAccel string + PlayMethod PlayMethod + BasePlayMethod PlayMethod + AudioTrackIndex int + TranscodeAudio bool + RemuxDVMode RemuxDVMode + ClientIP string + ClientName string + ClientVersion string + ClientUserAgent string + StreamBitrateKbps int + TargetResolution string + TargetVideoCodec string + TargetAudioCodec string + TargetBitrateKbps int + TranscodeHWAccel string + TranscodeNodeURL string + TranscodeTransportID string + TranscodeRouteSet bool // Byte-affecting transcode recipe fields preserved so an offloaded restart // (e.g. audio switch) can rebuild the exact same stream. SubtitleTrackIndex @@ -85,6 +93,45 @@ type SessionStreamState struct { SegmentDuration int } +// TranscodeRoute identifies the process serving a playback session. An empty +// NodeURL means the integrated server owns the process; an empty TransportID +// means a remote process uses the public playback session ID. +type TranscodeRoute struct { + NodeURL string + TransportID string +} + +// SessionReplacement is the complete mutable session state associated with a +// protocol-v3 replacement plan. Position is optional because ordinary failure +// recovery must preserve the player's latest progress while seek recovery +// intentionally moves the authoritative timeline. +type SessionReplacement struct { + EffectiveMediaFileID int + StreamState SessionStreamState + PositionSeconds *float64 + IsPaused bool + PreservePaused bool +} + +// SessionReplacementRollback is an opaque compare-and-swap token returned by +// ApplyReplacement. It can restore the previous session state only while no +// newer stream or progress mutation has superseded the replacement. +type SessionReplacementRollback struct { + sessionID string + appliedRevision uint64 + previousEffectiveMediaFileID int + previousStreamState SessionStreamState + previousPosition float64 + previousPaused bool + restoreProgress bool + previousReplacementMethod PlayMethod +} + +// ErrSessionReplacementSuperseded means a replacement rollback would overwrite +// a newer session mutation. Callers should terminate the session rather than +// expose state that disagrees with the durable playback plan. +var ErrSessionReplacementSuperseded = errors.New("session replacement was superseded") + type clientInfoContextKey struct{} // ClientInfo carries best-effort client metadata from request handling into @@ -533,6 +580,93 @@ func (m *SessionManager) CheckTranscodingAllowed(ctx context.Context, userID int return transcodingDisabledError(requiresVideoTranscode, !requiresVideoTranscode, limits) } +// CheckReplacementAllowed applies current user limits and admission policy to +// an in-place protocol-v3 recipe replacement. The existing session is excluded +// from the counts because the replacement inherits its stream slot; a direct +// to transcode change still has to acquire an available transcode slot. +func (m *SessionManager) CheckReplacementAllowed(ctx context.Context, sessionID string, method PlayMethod, transcodeAudio bool) error { + if ctx == nil { + ctx = context.Background() + } + // Bounded CAS: persistent count churn means the user is actively starting + // and stopping sessions; failing closed after a few rounds beats spinning + // with a limit-provider DB call per iteration. + const maxAdmissionRetries = 8 + for attempt := 0; attempt < maxAdmissionRetries; attempt++ { + m.mu.Lock() + current, ok := m.sessions[sessionID] + if !ok { + m.mu.Unlock() + return ErrSessionNotFound + } + userID := current.UserID + currentMethod := current.PlayMethod + // Exclude the replaced session from both counts instead of decrementing + // the totals: a failed session idle past the liveness grace is already + // absent from the count, and a blind decrement would free a slot that + // belongs to another live session. + otherStreams := m.activeCountExcludingLocked(userID, sessionID) + otherTranscodes := m.transcodeCountExcludingLocked(userID, sessionID) + decider := m.admissionDecider + m.mu.Unlock() + + limits, err := m.limitsForUser(ctx, userID) + if err != nil { + return err + } + if err := transcodingDisabledError(method == PlayTranscode, transcodeAudio, limits); err != nil { + return err + } + if decider == nil { + m.mu.Lock() + stillCurrent, stillExists := m.sessions[sessionID] + countsStable := stillExists && stillCurrent.PlayMethod == currentMethod && otherStreams == m.activeCountExcludingLocked(userID, sessionID) && otherTranscodes == m.transcodeCountExcludingLocked(userID, sessionID) + if !countsStable { + m.mu.Unlock() + continue + } + if limits.MaxTranscodes > 0 && method == PlayTranscode && otherTranscodes >= limits.MaxTranscodes { + m.mu.Unlock() + return ErrTooManyTranscodes + } + stillCurrent.replacementPlayMethod = method + m.mu.Unlock() + return nil + } + decision, err := decider(ctx, AdmissionRequest{UserID: userID, Limits: limits, CurrentActiveStreams: otherStreams, CurrentActiveTranscodes: otherTranscodes, RequestedMethod: method, RequiresVideoTranscode: method == PlayTranscode, RequiresAudioTranscode: transcodeAudio}) + if err != nil { + // Fail closed, but make an engine outage distinguishable from a + // genuine concurrency-limit denial in the logs. + slog.WarnContext(ctx, "playback replacement admission decider error; denying replacement", "component", "playback", + "user_id", userID, "session", sessionID, "method", method, "error", err) + return ErrPlaybackNotAllowed + } + if !decision.Allowed { + return admissionDenyError(decision.ReasonCode) + } + m.mu.Lock() + stillCurrent, stillExists := m.sessions[sessionID] + countsStable := stillExists && stillCurrent.PlayMethod == currentMethod && otherStreams == m.activeCountExcludingLocked(userID, sessionID) && otherTranscodes == m.transcodeCountExcludingLocked(userID, sessionID) + if countsStable { + stillCurrent.replacementPlayMethod = method + m.mu.Unlock() + return nil + } + m.mu.Unlock() + } + return ErrPlaybackNotAllowed +} + +// CancelReplacementReservation releases a protocol-v3 capacity reservation +// after a replacement fails before UpdateStreamState commits its new method. +func (m *SessionManager) CancelReplacementReservation(sessionID string) { + m.mu.Lock() + defer m.mu.Unlock() + if session := m.sessions[sessionID]; session != nil { + session.replacementPlayMethod = "" + } +} + func transcodingDisabledError(requiresVideoTranscode, requiresAudioTranscode bool, limits SessionLimits) error { if requiresVideoTranscode && limits.TranscodingDisabled { return ErrTranscodingDisabled @@ -555,6 +689,7 @@ func (m *SessionManager) UpdateProgress(sessionID string, position float64, isPa s.Position = position s.IsPaused = isPaused + s.streamRevision++ m.touchSessionLocked(s) return nil } @@ -575,6 +710,7 @@ func (m *SessionManager) UpdateAudioTrack(sessionID string, audioTrackIndex int, if s.PlayMethod != PlayTranscode || method == PlayTranscode { s.PlayMethod = method } + s.streamRevision++ m.touchSessionLocked(s) return nil } @@ -591,6 +727,13 @@ func (m *SessionManager) UpdateStreamState(sessionID string, state SessionStream return ErrSessionNotFound } + applySessionStreamStateLocked(s, state) + s.streamRevision++ + m.touchSessionLocked(s) + return nil +} + +func applySessionStreamStateLocked(s *Session, state SessionStreamState) { if state.PlayMethod != "" { s.PlayMethod = state.PlayMethod } @@ -599,6 +742,15 @@ func (m *SessionManager) UpdateStreamState(sessionID string, state SessionStream } s.AudioTrackIndex = state.AudioTrackIndex s.TranscodeAudio = state.TranscodeAudio + if state.TranscodeRouteSet { + // A full v3 route description owns the DV mode outright: a replan from + // a DV strip remux to an SDR source must clear the stale mode or every + // later remux request fails the profile check. Legacy partial updates + // never carry a mode and must not clobber one. + s.RemuxDVMode = state.RemuxDVMode + } else if state.RemuxDVMode != "" { + s.RemuxDVMode = state.RemuxDVMode + } s.ClientIP = state.ClientIP if value := normalizeClientMetadataValue(state.ClientName, 128); value != "" { s.ClientName = value @@ -615,9 +767,183 @@ func (m *SessionManager) UpdateStreamState(sessionID string, state SessionStream s.TargetAudioCodec = state.TargetAudioCodec s.TargetBitrateKbps = state.TargetBitrateKbps s.TranscodeHWAccel = state.TranscodeHWAccel + if state.TranscodeRouteSet { + s.TranscodeNodeURL = state.TranscodeNodeURL + s.TranscodeTransportID = state.TranscodeTransportID + } s.SubtitleTrackIndex = state.SubtitleTrackIndex s.SubtitleBurnIn = state.SubtitleBurnIn s.SegmentDuration = state.SegmentDuration + if state.TranscodeRouteSet { + // Only the replacement commit consumes the v3 capacity reservation; + // unrelated legacy stream updates arriving mid-replan must not release + // the slot and let a concurrent admission race past the transcode cap. + s.replacementPlayMethod = "" + } +} + +func snapshotSessionStreamStateLocked(s *Session) SessionStreamState { + return SessionStreamState{ + PlayMethod: s.PlayMethod, + BasePlayMethod: s.BasePlayMethod, + AudioTrackIndex: s.AudioTrackIndex, + TranscodeAudio: s.TranscodeAudio, + RemuxDVMode: s.RemuxDVMode, + ClientIP: s.ClientIP, + ClientName: s.ClientName, + ClientVersion: s.ClientVersion, + ClientUserAgent: s.ClientUserAgent, + StreamBitrateKbps: s.StreamBitrateKbps, + TargetResolution: s.TargetResolution, + TargetVideoCodec: s.TargetVideoCodec, + TargetAudioCodec: s.TargetAudioCodec, + TargetBitrateKbps: s.TargetBitrateKbps, + TranscodeHWAccel: s.TranscodeHWAccel, + TranscodeNodeURL: s.TranscodeNodeURL, + TranscodeTransportID: s.TranscodeTransportID, + TranscodeRouteSet: true, + SubtitleTrackIndex: s.SubtitleTrackIndex, + SubtitleBurnIn: s.SubtitleBurnIn, + SegmentDuration: s.SegmentDuration, + } +} + +func restoreSessionStreamStateLocked(s *Session, state SessionStreamState) { + s.PlayMethod = state.PlayMethod + s.BasePlayMethod = state.BasePlayMethod + s.AudioTrackIndex = state.AudioTrackIndex + s.TranscodeAudio = state.TranscodeAudio + s.RemuxDVMode = state.RemuxDVMode + s.ClientIP = state.ClientIP + s.ClientName = state.ClientName + s.ClientVersion = state.ClientVersion + s.ClientUserAgent = state.ClientUserAgent + s.StreamBitrateKbps = state.StreamBitrateKbps + s.TargetResolution = state.TargetResolution + s.TargetVideoCodec = state.TargetVideoCodec + s.TargetAudioCodec = state.TargetAudioCodec + s.TargetBitrateKbps = state.TargetBitrateKbps + s.TranscodeHWAccel = state.TranscodeHWAccel + s.TranscodeNodeURL = state.TranscodeNodeURL + s.TranscodeTransportID = state.TranscodeTransportID + s.SubtitleTrackIndex = state.SubtitleTrackIndex + s.SubtitleBurnIn = state.SubtitleBurnIn + s.SegmentDuration = state.SegmentDuration +} + +// ApplyReplacement atomically updates every live-session field owned by a +// protocol-v3 plan and returns a CAS rollback token for a later persistence +// failure. +func (m *SessionManager) ApplyReplacement(sessionID string, replacement SessionReplacement) (SessionReplacementRollback, error) { + m.mu.Lock() + defer m.mu.Unlock() + + s, ok := m.sessions[sessionID] + if !ok { + return SessionReplacementRollback{}, ErrSessionNotFound + } + return m.applyReplacementLocked(s, sessionID, replacement) +} + +// ApplyReplacementIfRoute applies a complete replacement only while the +// session still routes to expected. It publishes route and stream state in one +// critical section so callers never expose a successor with predecessor state. +func (m *SessionManager) ApplyReplacementIfRoute( + sessionID string, + expected TranscodeRoute, + replacement SessionReplacement, +) (SessionReplacementRollback, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + + s, ok := m.sessions[sessionID] + if !ok { + return SessionReplacementRollback{}, false, ErrSessionNotFound + } + if s.TranscodeNodeURL != expected.NodeURL || s.TranscodeTransportID != expected.TransportID { + return SessionReplacementRollback{}, false, nil + } + rollback, err := m.applyReplacementLocked(s, sessionID, replacement) + return rollback, err == nil, err +} + +func (m *SessionManager) applyReplacementLocked( + s *Session, + sessionID string, + replacement SessionReplacement, +) (SessionReplacementRollback, error) { + if replacement.EffectiveMediaFileID <= 0 { + return SessionReplacementRollback{}, errors.New("replacement effective media file id is invalid") + } + + rollback := SessionReplacementRollback{ + sessionID: sessionID, + previousEffectiveMediaFileID: s.MediaFileID, + previousStreamState: snapshotSessionStreamStateLocked(s), + previousReplacementMethod: s.replacementPlayMethod, + } + if replacement.PositionSeconds != nil { + rollback.previousPosition = s.Position + rollback.previousPaused = s.IsPaused + rollback.restoreProgress = true + } + + s.MediaFileID = replacement.EffectiveMediaFileID + applySessionStreamStateLocked(s, replacement.StreamState) + if replacement.PositionSeconds != nil { + s.Position = *replacement.PositionSeconds + if !replacement.PreservePaused { + s.IsPaused = replacement.IsPaused + } + } + s.streamRevision++ + rollback.appliedRevision = s.streamRevision + m.touchSessionLocked(s) + return rollback, nil +} + +// RollbackReplacement restores the state captured by ApplyReplacement when no +// newer session mutation has superseded it. +func (m *SessionManager) RollbackReplacement(sessionID string, rollback SessionReplacementRollback) error { + m.mu.Lock() + defer m.mu.Unlock() + + s, ok := m.sessions[sessionID] + if !ok { + return ErrSessionNotFound + } + if rollback.sessionID != sessionID || rollback.appliedRevision == 0 || s.streamRevision != rollback.appliedRevision { + return ErrSessionReplacementSuperseded + } + s.MediaFileID = rollback.previousEffectiveMediaFileID + restoreSessionStreamStateLocked(s, rollback.previousStreamState) + if rollback.restoreProgress { + s.Position = rollback.previousPosition + s.IsPaused = rollback.previousPaused + } + s.replacementPlayMethod = rollback.previousReplacementMethod + s.streamRevision++ + m.touchSessionLocked(s) + return nil +} + +// SetTranscodeStreamDetails records the actual encode decisions of a running +// transcode on the session — video copy vs re-encode, and whether audio is +// re-encoded — so session sync and the admin activity views classify the +// stream by what ffmpeg is doing rather than by the transport method alone +// (an HLS session with copied video is a repackage, not a video transcode). +func (m *SessionManager) SetTranscodeStreamDetails(sessionID, targetVideoCodec, targetAudioCodec string, transcodeAudio bool) error { + m.mu.Lock() + defer m.mu.Unlock() + + s, ok := m.sessions[sessionID] + if !ok { + return ErrSessionNotFound + } + + s.TargetVideoCodec = targetVideoCodec + s.TargetAudioCodec = targetAudioCodec + s.TranscodeAudio = transcodeAudio m.touchSessionLocked(s) return nil } @@ -633,6 +959,25 @@ func (m *SessionManager) SetTranscodeNodeURL(sessionID, url string) error { } s.TranscodeNodeURL = url + s.streamRevision++ + m.touchSessionLocked(s) + return nil +} + +// SetTranscodeRoute atomically assigns the node and process identity used to +// serve a transcode. +func (m *SessionManager) SetTranscodeRoute(sessionID string, route TranscodeRoute) error { + m.mu.Lock() + defer m.mu.Unlock() + + s, ok := m.sessions[sessionID] + if !ok { + return ErrSessionNotFound + } + + s.TranscodeNodeURL = route.NodeURL + s.TranscodeTransportID = route.TransportID + s.streamRevision++ m.touchSessionLocked(s) return nil } @@ -651,6 +996,7 @@ func (m *SessionManager) SetEffectiveMediaFileID(sessionID string, fileID int) e if fileID > 0 { s.MediaFileID = fileID } + s.streamRevision++ m.touchSessionLocked(s) return nil } @@ -832,9 +1178,21 @@ func (m *SessionManager) TranscodeCount(userID int) int { // activeCountLocked counts active sessions for a user. Caller must hold the lock. func (m *SessionManager) activeCountLocked(userID int) int { + return m.activeCountExcludingLocked(userID, "") +} + +// activeCountExcludingLocked counts a user's limit-relevant sessions while +// ignoring one session entirely. Replacement admission uses this instead of +// subtracting one from the total: a failed session awaiting replan is often +// idle past the liveness grace and already absent from the count, so a blind +// decrement would free another session's slot. +func (m *SessionManager) activeCountExcludingLocked(userID int, excludeSessionID string) int { now := time.Now() count := 0 for _, s := range m.sessions { + if excludeSessionID != "" && s.ID == excludeSessionID { + continue + } if s.UserID == userID && m.countsTowardLimitsLocked(s, now) { count++ } @@ -844,10 +1202,17 @@ func (m *SessionManager) activeCountLocked(userID int) int { // transcodeCountLocked counts transcode sessions for a user. Caller must hold the lock. func (m *SessionManager) transcodeCountLocked(userID int) int { + return m.transcodeCountExcludingLocked(userID, "") +} + +func (m *SessionManager) transcodeCountExcludingLocked(userID int, excludeSessionID string) int { now := time.Now() count := 0 for _, s := range m.sessions { - if s.UserID == userID && s.PlayMethod == PlayTranscode && m.countsTowardLimitsLocked(s, now) { + if excludeSessionID != "" && s.ID == excludeSessionID { + continue + } + if s.UserID == userID && (s.PlayMethod == PlayTranscode || s.replacementPlayMethod == PlayTranscode) && m.countsTowardLimitsLocked(s, now) { count++ } } diff --git a/internal/playback/session_test.go b/internal/playback/session_test.go index b73a4cbf..0812abf2 100644 --- a/internal/playback/session_test.go +++ b/internal/playback/session_test.go @@ -661,6 +661,79 @@ func TestSetTranscodeNodeURL_NotFound(t *testing.T) { } } +func TestSetTranscodeRoutePublishesNodeAndTransportTogether(t *testing.T) { + mgr := playback.NewSessionManager(0, 0) + session, err := mgr.StartSession(1, "profile-1", 42, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + + if err := mgr.SetTranscodeRoute(session.ID, playback.TranscodeRoute{NodeURL: "http://node:8070", TransportID: session.ID + "-legacy-next"}); err != nil { + t.Fatalf("SetTranscodeRoute: %v", err) + } + got, err := mgr.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if got.TranscodeNodeURL != "http://node:8070" || got.TranscodeTransportID != session.ID+"-legacy-next" { + t.Fatalf("transcode route = %q/%q", got.TranscodeNodeURL, got.TranscodeTransportID) + } +} + +func TestApplyReplacementIfRouteRejectsStalePredecessor(t *testing.T) { + mgr := playback.NewSessionManager(0, 0) + session, err := mgr.StartSession(1, "profile-1", 42, playback.PlayRemux, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if err := mgr.SetTranscodeRoute(session.ID, playback.TranscodeRoute{NodeURL: "http://old-node", TransportID: "old-transport"}); err != nil { + t.Fatalf("SetTranscodeRoute: %v", err) + } + replacement := playback.SessionReplacement{ + EffectiveMediaFileID: 42, + StreamState: playback.SessionStreamState{ + PlayMethod: playback.PlayTranscode, + BasePlayMethod: playback.PlayRemux, + TranscodeAudio: true, + TranscodeRouteSet: true, + SubtitleTrackIndex: -1, + TranscodeNodeURL: "http://new-node", + TranscodeTransportID: "new-transport", + }, + } + + _, published, err := mgr.ApplyReplacementIfRoute( + session.ID, + playback.TranscodeRoute{NodeURL: "http://stale-node", TransportID: "stale-transport"}, + replacement, + ) + if err != nil { + t.Fatalf("ApplyReplacementIfRoute stale: %v", err) + } + if published { + t.Fatal("stale predecessor unexpectedly published a replacement") + } + + _, published, err = mgr.ApplyReplacementIfRoute( + session.ID, + playback.TranscodeRoute{NodeURL: "http://old-node", TransportID: "old-transport"}, + replacement, + ) + if err != nil { + t.Fatalf("ApplyReplacementIfRoute current: %v", err) + } + if !published { + t.Fatal("current predecessor did not publish its replacement") + } + got, err := mgr.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if got.TranscodeNodeURL != "http://new-node" || got.TranscodeTransportID != "new-transport" { + t.Fatalf("transcode route = %q/%q", got.TranscodeNodeURL, got.TranscodeTransportID) + } +} + func TestSetEffectiveMediaFileID(t *testing.T) { mgr := playback.NewSessionManager(0, 0) session, err := mgr.StartSession(1, "profile-1", 100, playback.PlayDirect, false) @@ -684,6 +757,92 @@ func TestSetEffectiveMediaFileID(t *testing.T) { } } +func TestSessionReplacementAppliesAndRollsBackAtomically(t *testing.T) { + manager := playback.NewSessionManager(0, 0) + session, err := manager.StartSessionWithFiles(7, "profile-1", 42, 42, playback.PlayDirect, false) + if err != nil { + t.Fatal(err) + } + if err := manager.UpdateStreamState(session.ID, playback.SessionStreamState{ + PlayMethod: playback.PlayDirect, + BasePlayMethod: playback.PlayDirect, + AudioTrackIndex: 0, + TranscodeRouteSet: true, + SubtitleTrackIndex: -1, + StreamBitrateKbps: 8_000, + TranscodeNodeURL: "http://old-node", + TranscodeTransportID: "old-transport", + }); err != nil { + t.Fatal(err) + } + position := 321.5 + rollback, err := manager.ApplyReplacement(session.ID, playback.SessionReplacement{ + EffectiveMediaFileID: 84, + StreamState: playback.SessionStreamState{ + PlayMethod: playback.PlayTranscode, + BasePlayMethod: playback.PlayTranscode, + AudioTrackIndex: 2, + TranscodeAudio: true, + TranscodeRouteSet: true, + SubtitleTrackIndex: 1, + StreamBitrateKbps: 3_500, + TranscodeNodeURL: "http://new-node", + TranscodeTransportID: "new-transport", + }, + PositionSeconds: &position, + IsPaused: true, + }) + if err != nil { + t.Fatal(err) + } + replaced, err := manager.GetSession(session.ID) + if err != nil { + t.Fatal(err) + } + if replaced.MediaFileID != 84 || replaced.PlayMethod != playback.PlayTranscode || replaced.AudioTrackIndex != 2 || + replaced.TranscodeNodeURL != "http://new-node" || replaced.Position != position || !replaced.IsPaused { + t.Fatalf("replacement session = %#v", replaced) + } + if err := manager.RollbackReplacement(session.ID, rollback); err != nil { + t.Fatal(err) + } + restored, err := manager.GetSession(session.ID) + if err != nil { + t.Fatal(err) + } + if restored.MediaFileID != 42 || restored.PlayMethod != playback.PlayDirect || restored.AudioTrackIndex != 0 || + restored.TranscodeNodeURL != "http://old-node" || restored.TranscodeTransportID != "old-transport" || + restored.Position != 0 || restored.IsPaused { + t.Fatalf("restored session = %#v", restored) + } +} + +func TestSessionReplacementRollbackRejectsNewerMutation(t *testing.T) { + manager := playback.NewSessionManager(0, 0) + session, err := manager.StartSession(7, "profile-1", 42, playback.PlayDirect, false) + if err != nil { + t.Fatal(err) + } + rollback, err := manager.ApplyReplacement(session.ID, playback.SessionReplacement{ + EffectiveMediaFileID: 84, + StreamState: playback.SessionStreamState{ + PlayMethod: playback.PlayRemux, + BasePlayMethod: playback.PlayRemux, + TranscodeRouteSet: true, + SubtitleTrackIndex: -1, + }, + }) + if err != nil { + t.Fatal(err) + } + if err := manager.UpdateProgress(session.ID, 10, false); err != nil { + t.Fatal(err) + } + if err := manager.RollbackReplacement(session.ID, rollback); !errors.Is(err, playback.ErrSessionReplacementSuperseded) { + t.Fatalf("rollback error = %v, want ErrSessionReplacementSuperseded", err) + } +} + func TestStartSessionWithFiles(t *testing.T) { mgr := playback.NewSessionManager(0, 0) session, err := mgr.StartSessionWithFiles(1, "profile-1", 200, 100, playback.PlayRemux, true) @@ -713,19 +872,22 @@ func TestUpdateStreamState(t *testing.T) { } err = mgr.UpdateStreamState(session.ID, playback.SessionStreamState{ - PlayMethod: playback.PlayTranscode, - BasePlayMethod: playback.PlayRemux, - AudioTrackIndex: 2, - TranscodeAudio: true, - ClientIP: "10.0.0.10", - StreamBitrateKbps: 4200, - TargetResolution: "1080p", - TargetVideoCodec: "h264", - TargetAudioCodec: "aac", - TargetBitrateKbps: 4000, - SubtitleTrackIndex: 3, - SubtitleBurnIn: true, - SegmentDuration: 4, + PlayMethod: playback.PlayTranscode, + BasePlayMethod: playback.PlayRemux, + AudioTrackIndex: 2, + TranscodeAudio: true, + ClientIP: "10.0.0.10", + StreamBitrateKbps: 4200, + TargetResolution: "1080p", + TargetVideoCodec: "h264", + TargetAudioCodec: "aac", + TargetBitrateKbps: 4000, + TranscodeNodeURL: "http://node-1", + TranscodeTransportID: "transport-1", + TranscodeRouteSet: true, + SubtitleTrackIndex: 3, + SubtitleBurnIn: true, + SegmentDuration: 4, }) if err != nil { t.Fatalf("UpdateStreamState: %v", err) @@ -744,6 +906,9 @@ func TestUpdateStreamState(t *testing.T) { if got.AudioTrackIndex != 2 { t.Errorf("AudioTrackIndex = %d, want 2", got.AudioTrackIndex) } + if got.TranscodeNodeURL != "http://node-1" || got.TranscodeTransportID != "transport-1" { + t.Fatalf("transcode route = %q/%q", got.TranscodeNodeURL, got.TranscodeTransportID) + } if !got.TranscodeAudio { t.Error("TranscodeAudio = false, want true") } @@ -774,6 +939,16 @@ func TestUpdateStreamState(t *testing.T) { if got.SegmentDuration != 4 { t.Errorf("SegmentDuration = %d, want 4", got.SegmentDuration) } + if err := mgr.UpdateStreamState(session.ID, playback.SessionStreamState{TranscodeRouteSet: true}); err != nil { + t.Fatal(err) + } + got, err = mgr.GetSession(session.ID) + if err != nil { + t.Fatal(err) + } + if got.TranscodeNodeURL != "" || got.TranscodeTransportID != "" { + t.Fatalf("cleared transcode route = %q/%q", got.TranscodeNodeURL, got.TranscodeTransportID) + } } func TestUpdateAudioTrack_PreservesTranscodeTransportForRemuxBase(t *testing.T) { @@ -1154,3 +1329,102 @@ func TestSessionManager_CleanExpired_PausedGracePeriod(t *testing.T) { t.Errorf("paused session should survive with 3x grace period, got: %v", err) } } + +// A failed transcode session idle past the active grace is already absent from +// the live counts; replacement admission must exclude that session explicitly +// rather than decrement the totals, or the decrement frees a slot that belongs +// to another live transcode. +func TestCheckReplacementAllowedExcludesOnlyTheReplacedSession(t *testing.T) { + sm := playback.NewSessionManager(10, 2) + sm.SetLivenessGracePeriods(25*time.Millisecond, time.Hour) + + failed, err := sm.StartSession(1, "profile-1", 100, playback.PlayTranscode, false) + if err != nil { + t.Fatalf("StartSession(failed): %v", err) + } + // Age the failed session past the active grace so it no longer counts. + time.Sleep(60 * time.Millisecond) + + for i := 0; i < 2; i++ { + if _, err := sm.StartSession(1, "profile-1", 200+i, playback.PlayTranscode, false); err != nil { + t.Fatalf("StartSession(live %d): %v", i, err) + } + } + + err = sm.CheckReplacementAllowed(context.Background(), failed.ID, playback.PlayTranscode, false) + if !errors.Is(err, playback.ErrTooManyTranscodes) { + t.Fatalf("CheckReplacementAllowed = %v, want ErrTooManyTranscodes (both slots are held by live sessions)", err) + } +} + +// Legacy stream updates (audio switches, progress-driven state) arriving while +// a v3 replacement is in flight must not release the capacity reservation; +// only the replacement's own route-set commit consumes it. +func TestLegacyStreamUpdateKeepsReplacementReservation(t *testing.T) { + sm := playback.NewSessionManager(10, 1) + + session, err := sm.StartSession(1, "profile-1", 100, playback.PlayDirect, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if err := sm.CheckReplacementAllowed(context.Background(), session.ID, playback.PlayTranscode, false); err != nil { + t.Fatalf("CheckReplacementAllowed: %v", err) + } + + // A legacy partial update does not carry TranscodeRouteSet. + if err := sm.UpdateStreamState(session.ID, playback.SessionStreamState{AudioTrackIndex: 1}); err != nil { + t.Fatalf("UpdateStreamState: %v", err) + } + + if _, err := sm.StartSession(1, "profile-1", 200, playback.PlayTranscode, false); !errors.Is(err, playback.ErrTooManyTranscodes) { + t.Fatalf("StartSession(transcode) = %v, want ErrTooManyTranscodes (reservation must survive the legacy update)", err) + } + + // The route-set commit consumes the reservation and the slot converts to a + // real transcode, so the cap stays enforced. + if err := sm.UpdateStreamState(session.ID, playback.SessionStreamState{PlayMethod: playback.PlayTranscode, TranscodeRouteSet: true}); err != nil { + t.Fatalf("UpdateStreamState(commit): %v", err) + } + if _, err := sm.StartSession(1, "profile-1", 201, playback.PlayTranscode, false); !errors.Is(err, playback.ErrTooManyTranscodes) { + t.Fatalf("StartSession(transcode) after commit = %v, want ErrTooManyTranscodes", err) + } +} + +// A full v3 route description owns RemuxDVMode: a replan that lands on a +// non-DV source must clear a stale strip mode, while legacy partial updates +// must not clobber one. +func TestUpdateStreamStateClearsRemuxDVModeOnRouteSet(t *testing.T) { + sm := playback.NewSessionManager(5, 2) + + session, err := sm.StartSession(1, "profile-1", 100, playback.PlayRemux, false) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if err := sm.UpdateStreamState(session.ID, playback.SessionStreamState{RemuxDVMode: playback.RemuxDVStripToHDR10V3, TranscodeRouteSet: true}); err != nil { + t.Fatalf("UpdateStreamState(set): %v", err) + } + + // Legacy partial update: mode survives. + if err := sm.UpdateStreamState(session.ID, playback.SessionStreamState{AudioTrackIndex: 1}); err != nil { + t.Fatalf("UpdateStreamState(legacy): %v", err) + } + got, err := sm.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if got.RemuxDVMode != playback.RemuxDVStripToHDR10V3 { + t.Fatalf("RemuxDVMode after legacy update = %q, want strip_to_hdr10", got.RemuxDVMode) + } + + // v3 route-set update for a non-DV source: mode clears. + if err := sm.UpdateStreamState(session.ID, playback.SessionStreamState{TranscodeRouteSet: true}); err != nil { + t.Fatalf("UpdateStreamState(clear): %v", err) + } + got, err = sm.GetSession(session.ID) + if err != nil { + t.Fatalf("GetSession: %v", err) + } + if got.RemuxDVMode != "" { + t.Fatalf("RemuxDVMode after route-set update = %q, want cleared", got.RemuxDVMode) + } +} diff --git a/internal/playback/subtitle_policy_v3.go b/internal/playback/subtitle_policy_v3.go new file mode 100644 index 00000000..00a586c6 --- /dev/null +++ b/internal/playback/subtitle_policy_v3.go @@ -0,0 +1,152 @@ +package playback + +import ( + "fmt" + + "github.com/Silo-Server/silo-server/internal/models" +) + +type SubtitlePolicyResultV3 struct { + Decision SubtitleDecisionV3 + Claims SubtitleClaimsV3 + RequiresBurn bool + SelectedIndex int + TransportIndex int + Codec string + Source string + Terminal *TerminalV3 +} + +type SubtitleInventoryEntryV3 struct { + CombinedIndex int + Codec string + Source string +} + +// ResolveSubtitlePolicyV3 decides how the selected subtitle is delivered when +// the plan executes on the given engine. Renderability is engine-specific, so +// callers must resolve the policy against the engine that will actually run +// the plan rather than assuming the direct engine's capabilities. +func ResolveSubtitlePolicyV3(file *models.MediaFile, request StartRequestV3, transcodeAllowed bool, engine EngineV3, additional []SubtitleInventoryEntryV3) SubtitlePolicyResultV3 { + index := -1 + if request.SubtitleTrackIndex != nil { + index = *request.SubtitleTrackIndex + } else if request.SubtitleTrackID != "" { + fileID, kind, ordinal, ok := ParseTrackIDV3(request.SubtitleTrackID) + if !ok || kind != "subtitle" || file == nil || fileID != file.ID { + return subtitleTerminalV3("subtitle_track_invalid", "The selected subtitle identity is invalid.") + } + index = ordinal + } + if index < 0 { + return SubtitlePolicyResultV3{Decision: SubtitleDecisionV3{Mode: SubtitleOffV3}, SelectedIndex: -1, TransportIndex: -1} + } + if file == nil { + return subtitleTerminalV3("subtitle_track_unavailable", "The selected subtitle inventory is unavailable.") + } + codec, source, ok := subtitleCodecAtCombinedIndexV3(file, index, additional) + if !ok { + return subtitleTerminalV3("subtitle_track_unavailable", "The selected subtitle track is unavailable.") + } + trackID := TrackIDV3(file.ID, "subtitle", index) + transportIndex := -1 + if source == "embedded" { + transportIndex = index - len(file.ExternalSubtitles) + } + engineCaps := request.ClientPlaybackContext.Engines[string(engine)] + text := isTextSubtitleV3(codec) + ass := IsASS(codec) + clientBitmap := isClientRenderableBitmapSubtitleV3(codec) + burnInBitmap := clientBitmap || normalizeCodecV3(codec) == "dvb_teletext" + if !text && !burnInBitmap { + // Unknown codecs (arib_caption, ...) have no validated client render + // path and no validated burn-in filter; promising a silent burn-in + // would produce a plan the transcoder cannot honor. + return subtitleTerminalV3("subtitle_codec_unsupported", fmt.Sprintf("Subtitle format %s has no validated rendering or burn-in route.", codec)) + } + if text { + renderable := source != "embedded" && engineCaps.Subtitles.SidecarText || source == "embedded" && engineCaps.Subtitles.EmbeddedText + if ass && request.SubtitleFidelityPreference == SubtitleFidelityPreserveV3 { + renderable = renderable && engineCaps.Subtitles.ASSStyling && engineCaps.Subtitles.FontAttachments + } + if renderable { + return SubtitlePolicyResultV3{ + Decision: SubtitleDecisionV3{Mode: SubtitleRenderV3, TrackID: trackID}, + Claims: SubtitleClaimsV3{ASSStylingPreserved: !ass || engineCaps.Subtitles.ASSStyling, Reason: "client_render_supported"}, + SelectedIndex: index, TransportIndex: transportIndex, Codec: codec, Source: source, + } + } + if request.SubtitleFidelityPreference == SubtitleFidelityCompatibleV3 { + return SubtitlePolicyResultV3{ + Decision: SubtitleDecisionV3{Mode: SubtitleConvertV3, TrackID: trackID}, + Claims: SubtitleClaimsV3{Reason: "server_text_conversion"}, + SelectedIndex: index, TransportIndex: transportIndex, Codec: codec, Source: source, + } + } + } + // The stream handler raw-serves exactly one bitmap sidecar shape: an + // embedded PGS track as .sup. External/downloaded bitmap tracks go through + // WebVTT conversion (impossible for bitmaps) and embedded DVD/DVB have no + // client-renderable representation at all, so promising a sidecar for + // anything broader publishes an artifact URL that always fails at fetch. + // Everything else falls through to burn-in or its terminal. + if clientBitmap && source == "embedded" && IsPGS(codec) && engineCaps.Subtitles.EmbeddedBitmap { + return SubtitlePolicyResultV3{ + Decision: SubtitleDecisionV3{Mode: SubtitleRenderV3, TrackID: trackID}, + Claims: SubtitleClaimsV3{BitmapSidecar: true, Reason: "client_bitmap_render_supported"}, + SelectedIndex: index, TransportIndex: transportIndex, Codec: codec, Source: source, + } + } + if transcodeAllowed { + if source != "embedded" { + return subtitleTerminalV3("subtitle_burn_in_source_unsupported", "The selected subtitle source cannot be burned in by the installed transport.") + } + return SubtitlePolicyResultV3{ + Decision: SubtitleDecisionV3{Mode: SubtitleBurnInV3, TrackID: trackID}, + Claims: SubtitleClaimsV3{BitmapOverlay: burnInBitmap, Reason: "server_burn_in_required"}, + RequiresBurn: true, SelectedIndex: index, TransportIndex: transportIndex, Codec: codec, Source: source, + } + } + return subtitleTerminalV3("subtitle_conversion_unsupported", fmt.Sprintf("Subtitle format %s cannot meet the selected fidelity policy.", codec)) +} + +func subtitleCodecAtCombinedIndexV3(file *models.MediaFile, index int, additional []SubtitleInventoryEntryV3) (codec, source string, ok bool) { + if index < len(file.ExternalSubtitles) { + return normalizeCodecV3(file.ExternalSubtitles[index].Format), "external", true + } + embedded := index - len(file.ExternalSubtitles) + if embedded >= 0 && embedded < len(file.SubtitleTracks) { + return normalizeCodecV3(file.SubtitleTracks[embedded].Codec), "embedded", true + } + for _, entry := range additional { + if entry.CombinedIndex == index { + return normalizeCodecV3(entry.Codec), entry.Source, true + } + } + return "", "", false +} + +func isTextSubtitleV3(codec string) bool { + switch normalizeCodecV3(codec) { + case "srt", "subrip", "vtt", "webvtt", "ass", "ssa", "mov_text", "tx3g", + "eia_608", "eia608", "cea_608", "cea608": + return true + default: + return false + } +} + +func isClientRenderableBitmapSubtitleV3(codec string) bool { + switch normalizeCodecV3(codec) { + // pgssub, dvdsub, and dvbsub are ffmpeg's short aliases for the same + // bitstreams; scanners and older rows record either spelling. + case "pgs", "pgssub", "hdmv_pgs_subtitle", "dvd_subtitle", "dvdsub", "dvb_subtitle", "dvbsub", "vobsub": + return true + default: + return false + } +} + +func subtitleTerminalV3(reason, message string) SubtitlePolicyResultV3 { + return SubtitlePolicyResultV3{Decision: SubtitleDecisionV3{Mode: SubtitleOffV3}, SelectedIndex: -1, TransportIndex: -1, Terminal: &TerminalV3{Reason: reason, Message: message}} +} diff --git a/internal/playback/subtitle_policy_v3_test.go b/internal/playback/subtitle_policy_v3_test.go new file mode 100644 index 00000000..3fce8c3d --- /dev/null +++ b/internal/playback/subtitle_policy_v3_test.go @@ -0,0 +1,118 @@ +package playback + +import ( + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func TestResolveSubtitlePolicyV3RendersCEA608AsTextArtifact(t *testing.T) { + file := detailedFixtureFileV3() + file.ExternalSubtitles = nil + file.SubtitleTracks = []models.SubtitleTrack{{Codec: "eia_608"}} + req := validStartRequestV3() + index := 0 + req.SubtitleTrackIndex = &index + + result := ResolveSubtitlePolicyV3(file, req, true, EngineMedia3DirectV3, nil) + + if result.Terminal != nil || result.RequiresBurn || result.Decision.Mode != SubtitleRenderV3 { + t.Fatalf("CEA-608 should render as a client-styled text artifact: %#v", result) + } + if result.Claims.Reason != "client_render_supported" || result.Claims.BitmapOverlay { + t.Fatalf("CEA-608 claims = %#v", result.Claims) + } +} + +func TestResolveSubtitlePolicyV3DoesNotOfferDVBTeletextAsClientBitmap(t *testing.T) { + file := detailedFixtureFileV3() + file.ExternalSubtitles = nil + file.SubtitleTracks = []models.SubtitleTrack{{Codec: "dvb_teletext"}} + req := validStartRequestV3() + index := 0 + req.SubtitleTrackIndex = &index + + result := ResolveSubtitlePolicyV3(file, req, true, EngineMedia3DirectV3, nil) + + if result.Terminal != nil || !result.RequiresBurn || result.Decision.Mode != SubtitleBurnInV3 { + t.Fatalf("DVB teletext must stay on the server fallback: %#v", result) + } + if !result.Claims.BitmapOverlay { + t.Fatalf("DVB teletext burn-in must retain bitmap overlay semantics: %#v", result.Claims) + } +} + +func TestResolveSubtitlePolicyV3UnknownCodecIsExplicitlyUnsupported(t *testing.T) { + file := detailedFixtureFileV3() + file.ExternalSubtitles = nil + file.SubtitleTracks = []models.SubtitleTrack{{Codec: "arib_caption"}} + req := validStartRequestV3() + index := 0 + req.SubtitleTrackIndex = &index + + for _, transcodeAllowed := range []bool{true, false} { + result := ResolveSubtitlePolicyV3(file, req, transcodeAllowed, EngineMedia3DirectV3, nil) + if result.Terminal == nil || result.Terminal.Reason != "subtitle_codec_unsupported" { + t.Fatalf("transcodeAllowed=%v: unknown codec must be explicitly unsupported, got %#v", transcodeAllowed, result) + } + } +} + +func TestResolveSubtitlePolicyV3BurnsFFmpegBitmapAliases(t *testing.T) { + file := detailedFixtureFileV3() + file.ExternalSubtitles = nil + file.SubtitleTracks = []models.SubtitleTrack{{Codec: "dvdsub"}} + req := validStartRequestV3() + index := 0 + req.SubtitleTrackIndex = &index + + result := ResolveSubtitlePolicyV3(file, req, true, EngineMedia3DirectV3, nil) + if result.Terminal != nil || !result.RequiresBurn || result.Decision.Mode != SubtitleBurnInV3 || !result.Claims.BitmapOverlay { + t.Fatalf("dvdsub alias must burn in like dvd_subtitle: %#v", result) + } +} + +func TestPlanPlaybackV3TranscodeUsesHLSEngineSubtitleCapabilities(t *testing.T) { + file := detailedFixtureFileV3() + file.VideoTracks[0].VideoRange = "SDR" + file.VideoTracks[0].VideoRangeType = "SDR" + file.VideoTracks[0].ColorTransfer = "bt709" + file.SubtitleTracks = []models.SubtitleTrack{{Codec: "srt"}} + req := validStartRequestV3() + // A fixed rung forces the HLS transcode route; the fixture's direct + // engine renders embedded text while the HLS engine cannot. + req.QualityPreference = "1080p" + req.ClientFeatures = append(req.ClientFeatures, FeatureDetailedDecodeV3) + req.ClientPlaybackContext.Features = append(req.ClientPlaybackContext.Features, FeatureDetailedDecodeV3) + req.Capabilities.VideoDecode = []VideoDecodeCapabilityV3{{Codec: "hevc", Profiles: []string{"main 10"}, Levels: []int{153}, BitDepths: []int{10}, MaxWidth: 3840, MaxHeight: 2160, MaxFrameRate: 60, MaxBitrateKbps: 80_000, Hardware: true}} + index := 0 + req.SubtitleTrackIndex = &index + req.SubtitleTrackID = TrackIDV3(file.ID, "subtitle", index) + + result := PlanPlaybackV3(PlannerInputV3{Request: req, RequestedFile: file, EffectiveFile: file, AudioTrackIndex: 0, Settings: PlannerSettingsV3{TranscodeEnabled: true, Allow4KTranscode: true}, Registry: testTransformationRegistryV3()}) + if result.Plan == nil || result.Plan.Delivery != DeliveryTranscodeHLSV3 { + t.Fatalf("result = %s", ExplainPlannerResultV3(result)) + } + if result.Plan.Subtitle.Mode == SubtitleRenderV3 { + t.Fatalf("HLS transcode plan claims client render the HLS engine cannot honor: %#v", result.Plan.Subtitle) + } + if result.Plan.Subtitle.Mode != SubtitleConvertV3 || result.Plan.Claims.Subtitles.Reason != "server_text_conversion" { + t.Fatalf("subtitle decision = %#v claims = %#v", result.Plan.Subtitle, result.Plan.Claims.Subtitles) + } + if result.SubtitleTrackIndex != index { + t.Fatalf("subtitle track index = %d", result.SubtitleTrackIndex) + } +} + +func TestClientRenderableBitmapSubtitleV3UsesExactCodecFamilies(t *testing.T) { + for _, codec := range []string{"pgs", "pgssub", "hdmv_pgs_subtitle", "dvd_subtitle", "dvdsub", "dvb_subtitle", "dvbsub", "vobsub"} { + if !isClientRenderableBitmapSubtitleV3(codec) { + t.Errorf("expected client-renderable bitmap codec: %s", codec) + } + } + for _, codec := range []string{"dvb_teletext", "hdmv_text_subtitle", "arib_caption", "eia_608"} { + if isClientRenderableBitmapSubtitleV3(codec) { + t.Errorf("must not advertise unsupported client bitmap codec: %s", codec) + } + } +} diff --git a/internal/playback/subtitle_stream.go b/internal/playback/subtitle_stream.go index 2f95a13f..d56aa66f 100644 --- a/internal/playback/subtitle_stream.go +++ b/internal/playback/subtitle_stream.go @@ -26,6 +26,9 @@ type StreamExtractOpts struct { // "ass"). Controls whether we copy the stream (for ASS, which carries // styling) or remux to WebVTT (for everything else). SourceCodec string + // TargetFormat optionally forces a compatible converted artifact (currently + // WebVTT). Empty preserves the legacy source-driven behavior. + TargetFormat string // SeekSeconds asks ffmpeg to start demuxing at this position. For // text-event codecs this is the key win — ffmpeg skips the prefix of // the container instead of scanning from byte 0 to produce earlier @@ -133,7 +136,7 @@ func StreamExtractSubtitle(ctx context.Context, opts StreamExtractOpts) error { // streamExtractArgs builds the ffmpeg argument list for a streaming // subtitle extract. func streamExtractArgs(opts StreamExtractOpts) []string { - outCodec, outFormat := streamExtractOutput(opts.SourceCodec) + outCodec, outFormat := streamExtractOutput(opts.SourceCodec, opts.TargetFormat) args := []string{ "-hide_banner", "-nostats", "-loglevel", "error", @@ -249,7 +252,15 @@ func copyAndFlush(dst io.Writer, src io.Reader) error { // copied into a .sup elementary stream for client-side bitmap rendering // (libpgs); everything else is transmuxed to WebVTT for direct `` // consumption. -func streamExtractOutput(codec string) (outCodec, outFormat string) { +func streamExtractOutput(codec string, targetFormat ...string) (outCodec, outFormat string) { + // A forced WebVTT target only applies to text sources: bitmap codecs + // carry no text for ffmpeg's webvtt encoder, so honoring the override + // would build a command that always fails mid-response. Fall through to + // the source-driven mapping instead (handlers reject bitmap-to-vtt + // requests before headers are written). + if len(targetFormat) > 0 && strings.EqualFold(targetFormat[0], "vtt") && !NeedsBurnIn(codec) { + return "webvtt", "webvtt" + } switch { case IsASS(codec): return "copy", "ass" diff --git a/internal/playback/subtitle_stream_test.go b/internal/playback/subtitle_stream_test.go index 26c2bd07..3f7b2d0b 100644 --- a/internal/playback/subtitle_stream_test.go +++ b/internal/playback/subtitle_stream_test.go @@ -229,6 +229,30 @@ func TestPGSWindowRequest(t *testing.T) { } } +// A forced "vtt" target applies only to text sources: bitmap codecs carry no +// text for ffmpeg's webvtt encoder, so the override must fall back to the +// source-driven mapping instead of building a command that always fails. +func TestStreamExtractOutput_TargetFormatVTTGatedToTextSources(t *testing.T) { + cases := []struct { + codec string + wantCodec string + wantFormat string + }{ + {"subrip", "webvtt", "webvtt"}, + {"mov_text", "webvtt", "webvtt"}, + {"ass", "webvtt", "webvtt"}, + {"pgs", "copy", "sup"}, + {"hdmv_pgs_subtitle", "copy", "sup"}, + } + for _, tc := range cases { + outCodec, outFormat := streamExtractOutput(tc.codec, "vtt") + if outCodec != tc.wantCodec || outFormat != tc.wantFormat { + t.Errorf("streamExtractOutput(%q, \"vtt\") = (%q, %q), want (%q, %q)", + tc.codec, outCodec, outFormat, tc.wantCodec, tc.wantFormat) + } + } +} + func TestStreamExtractArgs_PGSProducesSup(t *testing.T) { args := streamExtractArgs(StreamExtractOpts{ InputPath: "/media/movie.mkv", diff --git a/internal/playback/subtitles.go b/internal/playback/subtitles.go index 75a48c8c..c2aff3d6 100644 --- a/internal/playback/subtitles.go +++ b/internal/playback/subtitles.go @@ -3,6 +3,7 @@ package playback import ( "bytes" "context" + "errors" "fmt" "net/http" "os" @@ -235,6 +236,41 @@ func ConvertToVTT(input []byte, fromFormat string) ([]byte, error) { } } +// ConvertToVTTWithFFmpeg converts subtitle formats that require a real parser +// (notably ASS/SSA) from in-memory data. It keeps downloaded subtitle +// conversion on the same executable path as external sidecar conversion. +func ConvertToVTTWithFFmpeg(ctx context.Context, input []byte, fromFormat, ffmpegPath string) ([]byte, error) { + if converted, err := ConvertToVTT(input, fromFormat); err == nil { + return converted, nil + } + inputFormat := strings.ToLower(strings.TrimSpace(fromFormat)) + if inputFormat == "ssa" { + inputFormat = "ass" + } + if inputFormat == "" { + return nil, errors.New("subtitle format is required") + } + if strings.TrimSpace(ffmpegPath) == "" { + ffmpegPath = "ffmpeg" + } + cmd := exec.CommandContext(ctx, ffmpegPath, + "-hide_banner", "-loglevel", "error", + "-f", inputFormat, "-i", "pipe:0", + "-f", "webvtt", "pipe:1", + ) + cmd.Stdin = bytes.NewReader(input) + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return nil, fmt.Errorf("ffmpeg subtitle conversion failed: %w (stderr: %s)", err, truncateStderr(stderr.String())) + } + if stdout.Len() == 0 { + return nil, errors.New("ffmpeg produced empty subtitle output") + } + return stdout.Bytes(), nil +} + // srtToVTT converts SRT subtitle content to WebVTT format. // SRT uses commas for millisecond separators; VTT uses periods. // VTT requires a "WEBVTT" header. diff --git a/internal/playback/testdata/protocol_v3/attempt_keys.json b/internal/playback/testdata/protocol_v3/attempt_keys.json new file mode 100644 index 00000000..d8d6f243 --- /dev/null +++ b/internal/playback/testdata/protocol_v3/attempt_keys.json @@ -0,0 +1,69 @@ +[ + { + "name": "hls_burn_in_sorted_transformations_and_pcm_mutations", + "plan_id": "plan:fixture", + "delivery": "server_remux_hls", + "stream_protocol": "hls", + "container": "hls", + "video_codec": "hevc", + "audio_codec": "aac", + "width": 3840, + "height": 2160, + "bitrate_kbps": 20000, + "dynamic_range": "hdr10", + "subtitle_mode": "burn_in", + "transformations": [ + {"name": "hdr_to_sdr_tonemap", "executor": "server", "recipe_version": "1"}, + {"name": "audio_to_aac", "executor": "server", "recipe_version": "1"} + ], + "output_route_generation": 7, + "local_mutations": ["transport_reopen", "pcm:truehd:8"], + "expected": "v3:6d4724f01b6ff692" + }, + { + "name": "direct_client_dv81_executor_and_version", + "plan_id": "plan:dv81-fixture", + "delivery": "original_http", + "stream_protocol": "http_progressive", + "container": "mkv", + "video_codec": "hevc", + "audio_codec": "truehd", + "width": 3840, + "height": 2160, + "bitrate_kbps": 65000, + "dynamic_range": "dolby_vision", + "subtitle_mode": "off", + "transformations": [ + {"name": "client_dv7_to_dv81", "executor": "client", "recipe_version": "1"} + ], + "output_route_generation": 9, + "local_mutations": [], + "expected": "v3:2a88b5e686373440" + }, + { + "name": "direct_device_quirk_and_runtime_correction_identity", + "plan_id": "plan:quirk", + "delivery": "original_http", + "stream_protocol": "http_progressive", + "container": "mkv", + "video_codec": "hevc", + "audio_codec": "eac3", + "width": 3840, + "height": 2160, + "bitrate_kbps": 60000, + "dynamic_range": "dolby_vision", + "subtitle_mode": "off", + "transformations": [], + "applied_quirks": [ + { + "id": "android.fire_tv.dv8_hdr10plus_sei_v1", + "registry_revision": "2026-07-13.1", + "action": "client_runtime_correction" + } + ], + "runtime_corrections": ["client_dv8_hdr10plus_sanitizer_v1"], + "output_route_generation": 9, + "local_mutations": [], + "expected": "v3:8d843bfffeb3adc3" + } +] diff --git a/internal/playback/testdata/protocol_v3/decision_response.json b/internal/playback/testdata/protocol_v3/decision_response.json new file mode 100644 index 00000000..a76c05dc --- /dev/null +++ b/internal/playback/testdata/protocol_v3/decision_response.json @@ -0,0 +1,27 @@ +{ + "protocol_version": 3, + "server_features": ["playback_plan_v3", "media3_only", "playback_route_diagnostics"], + "outcome": "playable", + "session_id": "11111111-1111-4111-8111-111111111111", + "playback_plan": { + "protocol_version": 3, + "plan_id": "plan:golden0001", + "session_id": "11111111-1111-4111-8111-111111111111", + "expires_at": "2030-01-01T00:00:00Z", + "delivery": "original_http", + "engine": "media3_direct", + "stream": {"url": "/stream/11111111-1111-4111-8111-111111111111", "protocol": "http_progressive", "container": "mp4", "mime_type": "video/mp4", "headers": {}, "header_refresh": "session"}, + "timeline": {"source_start_seconds": 12.5, "stream_origin_seconds": 0, "player_start_seconds": 12.5, "timeline_offset_seconds": 0, "can_seek_anywhere": true, "seek_restoration": "player_position"}, + "selected_tracks": {"audio": {"id": "file:42:audio:0", "index": 0}}, + "effective_recipe": {"video_codec": "h264", "audio_codec": "aac", "width": 1920, "height": 1080, "frame_rate": 23.976, "bitrate_kbps": 8000, "dynamic_range": "sdr", "audio_channels": 2, "audio_layout": "stereo"}, + "claims": {"video": {"hdr10": false, "hdr10_plus": false, "hlg": false, "dolby_vision": false}, "audio": {"codec": "aac", "passthrough": false, "atmos_preserved": false, "reason": "client_decode_supported"}, "subtitles": {"ass_styling_preserved": false, "bitmap_overlay": false, "bitmap_sidecar": false}}, + "subtitle": {"mode": "off"}, + "transformations": [], + "degradation_warnings": [], + "decision_reason": "validated_original_playback", + "requested_media_file_id": 42, + "effective_media_file_id": 42, + "source": {"media_file_id": 42, "container": "mp4", "video_codec": "h264", "video_profile": "high", "video_level": 41, "bit_depth": 8, "width": 1920, "height": 1080, "frame_rate": 23.976, "bitrate_kbps": 8000, "dynamic_range": "sdr", "dv_enhancement_layer": "none", "audio_codec": "aac", "audio_channels": 2, "audio_layout": "stereo"}, + "subtitle_fidelity_policy": "allow_simplified_rendering" + } +} diff --git a/internal/playback/testdata/protocol_v3/replan_request.json b/internal/playback/testdata/protocol_v3/replan_request.json new file mode 100644 index 00000000..0ec6c927 --- /dev/null +++ b/internal/playback/testdata/protocol_v3/replan_request.json @@ -0,0 +1,76 @@ +{ + "protocol_version": 3, + "operation": "failure_recovery", + "playback_attempt_id": "attempt-golden-0001", + "replan_request_id": "replan-golden-0001", + "failed_plan_id": "plan:golden-0001", + "plan_attempt_id": "plan-attempt-golden-0001", + "plan_attempt_key": "v3:0000000000000001", + "attempted_plan_keys": ["v3:0000000000000001"], + "attempt_count": 1, + "quality_preference": "auto", + "position_seconds": 42.5, + "output_route_generation": 7, + "metered": true, + "bandwidth_estimate_kbps": 3500, + "bandwidth_cap_kbps": 4000, + "selected_tracks": { + "audio": { + "id": "file:42:audio:0", + "index": 0 + } + }, + "failure": { + "classification": "network_degraded" + }, + "client_capabilities": { + "codecs_video": ["h264"], + "codecs_video_hardware": ["h264"], + "codecs_audio": ["aac"], + "containers": ["mp4"], + "max_resolution": "1080p", + "video_decode": [ + { + "codec": "h264", + "profiles": ["high"], + "levels": [41], + "bit_depths": [8], + "max_width": 1920, + "max_height": 1080, + "max_frame_rate": 60, + "max_bitrate_kbps": 20000, + "hardware": true + } + ] + }, + "client_playback_context": { + "protocol_version": 3, + "features": ["playback_plan_v3", "media3_only", "detailed_decode_capabilities"], + "platform": "android", + "form_factor": "tv", + "app_version": "3.0-test", + "device": { + "manufacturer": "NVIDIA", + "model": "SHIELD Android TV", + "sdk_int": 35, + "abis": ["arm64-v8a"] + }, + "output": { + "output_route_generation": 7 + }, + "engines": { + "media3_direct": { + "enabled": true, + "supported_on_device": true, + "containers": ["mp4"], + "video_codecs": ["h264"], + "audio_decode_codecs": ["aac"], + "subtitles": { + "embedded_text": true, + "sidecar_text": true + }, + "auth_header_refresh": true + } + } + } +} diff --git a/internal/playback/testdata/protocol_v3/start_request.json b/internal/playback/testdata/protocol_v3/start_request.json new file mode 100644 index 00000000..716b9159 --- /dev/null +++ b/internal/playback/testdata/protocol_v3/start_request.json @@ -0,0 +1,45 @@ +{ + "protocol_version": 3, + "client_features": ["playback_plan_v3", "media3_only", "detailed_decode_capabilities"], + "file_id": 42, + "profile_id": "profile-1", + "playback_attempt_id": "attempt-golden-0001", + "quality_preference": "original", + "subtitle_fidelity_preference": "compatible", + "start_position": 12.5, + "audio_track_id": "file:42:audio:0", + "audio_track_index": 0, + "output_route_generation": 7, + "metered": false, + "client_capabilities": { + "codecs_video": ["h264"], + "codecs_video_hardware": ["h264"], + "codecs_audio": ["aac"], + "containers": ["mp4"], + "max_resolution": "1080p", + "hdr": false, + "video_decode": [{ + "codec": "h264", + "profiles": ["high"], + "levels": [41], + "bit_depths": [8], + "max_width": 1920, + "max_height": 1080, + "max_frame_rate": 60, + "max_bitrate_kbps": 20000, + "hardware": true + }] + }, + "client_playback_context": { + "protocol_version": 3, + "features": ["playback_plan_v3", "media3_only", "detailed_decode_capabilities"], + "platform": "android", + "form_factor": "tv", + "app_version": "3.0-test", + "device": {"manufacturer": "NVIDIA", "model": "SHIELD Android TV", "sdk_int": 35, "abis": ["arm64-v8a"]}, + "output": {"output_route_generation": 7}, + "engines": { + "media3_direct": {"enabled": true, "supported_on_device": true, "containers": ["mp4"], "video_codecs": ["h264"], "audio_decode_codecs": ["aac"], "audio_passthrough_codecs": [], "subtitles": {"embedded_text": true, "sidecar_text": true, "ass_styling": false, "embedded_bitmap": false, "sidecar_bitmap": false, "font_attachments": false}, "features": [], "auth_header_refresh": true, "validated_claims": []} + } + } +} diff --git a/internal/playback/tracks_v3.go b/internal/playback/tracks_v3.go new file mode 100644 index 00000000..568847ec --- /dev/null +++ b/internal/playback/tracks_v3.go @@ -0,0 +1,46 @@ +package playback + +import ( + "fmt" + "strconv" + "strings" +) + +func TrackIDV3(fileID int, kind string, ordinal int) string { + return fmt.Sprintf("file:%d:%s:%d", fileID, kind, ordinal) +} + +func ParseTrackIDV3(value string) (fileID int, kind string, ordinal int, ok bool) { + parts := strings.Split(value, ":") + if len(parts) != 4 || parts[0] != "file" || (parts[2] != "audio" && parts[2] != "subtitle") { + return 0, "", 0, false + } + fileID, ok = parseCanonicalTrackNumberV3(parts[1]) + if !ok || fileID <= 0 { + return 0, "", 0, false + } + ordinal, ok = parseCanonicalTrackNumberV3(parts[3]) + if !ok { + return 0, "", 0, false + } + return fileID, parts[2], ordinal, true +} + +// parseCanonicalTrackNumberV3 accepts only the exact decimal form emitted by +// TrackIDV3: digits without a sign and without leading zeros, so every parsed +// identity round-trips byte-for-byte. +func parseCanonicalTrackNumberV3(value string) (int, bool) { + if value == "" || (len(value) > 1 && value[0] == '0') { + return 0, false + } + for _, c := range value { + if c < '0' || c > '9' { + return 0, false + } + } + parsed, err := strconv.Atoi(value) + if err != nil { + return 0, false + } + return parsed, true +} diff --git a/internal/playback/tracks_v3_test.go b/internal/playback/tracks_v3_test.go new file mode 100644 index 00000000..d26e407a --- /dev/null +++ b/internal/playback/tracks_v3_test.go @@ -0,0 +1,38 @@ +package playback + +import "testing" + +func TestParseTrackIDV3StrictCanonicalRoundTrip(t *testing.T) { + for _, id := range []string{"file:1:audio:0", "file:42:subtitle:7", "file:10:audio:10", "file:987:subtitle:100"} { + fileID, kind, ordinal, ok := ParseTrackIDV3(id) + if !ok { + t.Fatalf("canonical id rejected: %s", id) + } + if rebuilt := TrackIDV3(fileID, kind, ordinal); rebuilt != id { + t.Fatalf("round trip: parsed %s rebuilt as %s", id, rebuilt) + } + } +} + +func TestParseTrackIDV3RejectsNonCanonicalNumerics(t *testing.T) { + invalid := []string{ + "file:007:audio:1", // leading zeros in file id + "file:7:audio:01", // leading zeros in ordinal + "file:7:audio:+1", // sign + "file:+7:audio:1", // sign + "file:7:audio:-1", // sign + "file:0:audio:1", // file ids are positive + "file:7:video:1", // unknown kind + "file:7:audio:", // empty ordinal + "file::audio:1", // empty file id + "file:7:audio:1:2", // extra segment + "file:7:audio: 1", // whitespace + "file:7:audio:1e2", // exponent form + "file:99999999999999999999:audio:1", // overflow + } + for _, id := range invalid { + if _, _, _, ok := ParseTrackIDV3(id); ok { + t.Errorf("non-canonical id accepted: %q", id) + } + } +} diff --git a/internal/playback/transcode.go b/internal/playback/transcode.go index 07515fdd..9dd94804 100644 --- a/internal/playback/transcode.go +++ b/internal/playback/transcode.go @@ -28,35 +28,56 @@ func init() { // TranscodeOpts holds configuration for an HLS transcode session. type TranscodeOpts struct { - InputPath string - OutputDir string // e.g., /tmp/silo-transcode/{session_id}/ - SessionID string - SourceVideoCodec string - SeekSeconds float64 - TargetResolution string // e.g., 1080p, 720p - TargetCodecVideo string // e.g., h264 (or hevc if allowed) - TargetCodecAudio string // e.g., aac - SegmentDuration int // seconds, default 6 - StartSegmentNumber int // -hls_segment_start_number, default 0 - FFmpegPath string // optional explicit ffmpeg binary path - HWAccel string // auto, qsv, vaapi, nvenc, none - HWDevice string // e.g., /dev/dri/renderD128 (default if empty) - SubtitleTrackIndex int // -1 = no subtitles - SubtitleBurnIn bool + InputPath string + OutputDir string // e.g., /tmp/silo-transcode/{session_id}/ + // OutputSubdir is the signed, root-relative reconstruction directory. Empty + // retains the legacy flat {session_id} layout. + OutputSubdir string + TranscodeTransportID string + SessionID string + SourceVideoCodec string + VideoBitstreamFilter string // validated copy-mode BSF, e.g. dovi_rpu=strip=1 + SeekSeconds float64 + // StreamOriginSeconds is the keyframe timestamp at which a copy-video + // stream actually begins. SeekSeconds remains the client-requested -ss so + // FFmpeg performs exactly one demuxer seek; this origin keeps response and + // reconstruction timelines aligned with the resulting media pre-roll. + StreamOriginSeconds float64 + // CopySeekAnchorResolved distinguishes a valid zero-second origin from + // older/shared recipes that never resolved a copy seek anchor. + CopySeekAnchorResolved bool + TargetResolution string // e.g., 1080p, 720p + TargetCodecVideo string // e.g., h264 (or hevc if allowed) + TargetCodecAudio string // e.g., aac + SegmentDuration int // seconds, default 6 + StartSegmentNumber int // -hls_segment_start_number, default 0 + FFmpegPath string // optional explicit ffmpeg binary path + HWAccel string // auto, qsv, vaapi, nvenc, none + HWDevice string // e.g., /dev/dri/renderD128 (default if empty) + SubtitleTrackIndex int // -1 = no subtitles + SubtitleBurnIn bool // SubtitleCodec is the probed codec of the burn-in track (e.g. "subrip", // "hdmv_pgs_subtitle"). Bitmap codecs (PGS/DVD/DVB) select the overlay // filter_complex pipeline; text codecs use the libass subtitles filter. // Empty preserves the legacy text path for callers minted before the field. - SubtitleCodec string - AudioTrackIndex int // -1 = default (first track), >= 0 = specific track - TargetBitrateKbps int // max video bitrate in kbps; 0 = CRF-only (no cap) - TotalDuration float64 // total media duration in seconds (for VOD manifest) - FastStart bool // use superfast preset for faster first-segment production - NodeType string - ExecutionMode string - FFmpegLogSink FFmpegLogSink + SubtitleCodec string + AudioTrackIndex int // -1 = default (first track), >= 0 = specific track + // TargetAudioChannels caps the re-encoded channel count. 0 (or anything + // below 3) keeps the historical stereo downmix; 6 preserves 5.1 from a + // surround source. Ignored for copy/passthrough audio targets. + TargetAudioChannels int + TargetBitrateKbps int // max video bitrate in kbps; 0 = CRF-only (no cap) + TotalDuration float64 // total media duration in seconds (for VOD manifest) + FastStart bool // use superfast preset for faster first-segment production + NodeType string + ExecutionMode string + FFmpegLogSink FFmpegLogSink } +// DV7ToHDR10BitstreamFilter strips Dolby Vision RPU metadata during a +// copy-mode HLS remux; the enhancement layer is dropped by stream mapping. +const DV7ToHDR10BitstreamFilter = "dovi_rpu=strip=1" + // TranscodeSession manages a running ffmpeg HLS transcode process. type TranscodeSession struct { cmd *exec.Cmd @@ -142,6 +163,10 @@ const ( // StartTranscode launches an ffmpeg process that produces HLS segments. func StartTranscode(ctx context.Context, opts TranscodeOpts) (*TranscodeSession, error) { + if opts.VideoBitstreamFilter != "" && + (opts.VideoBitstreamFilter != DV7ToHDR10BitstreamFilter || !strings.EqualFold(opts.TargetCodecVideo, "copy")) { + return nil, fmt.Errorf("unsupported video bitstream filter recipe") + } if opts.SegmentDuration <= 0 { opts.SegmentDuration = defaultSegmentDuration } @@ -290,6 +315,9 @@ func buildFFmpegArgs(opts TranscodeOpts) []string { // Video codec and encoding settings. if isVideoCopy { args = append(args, "-c:v", "copy") + if opts.VideoBitstreamFilter == DV7ToHDR10BitstreamFilter { + args = append(args, "-bsf:v", opts.VideoBitstreamFilter) + } } else { args = appendVideoArgs(args, opts) } @@ -632,13 +660,25 @@ func appendVideoFilterArgs(args []string, opts TranscodeOpts) []string { return args } +// TranscodesAudio reports whether a transcode with the given target audio +// codec re-encodes the audio stream. Only an explicit "copy" passes audio +// through; an empty codec runs ffmpeg's AAC default (see appendAudioArgs), so +// every consumer of the session's audio decision — live stream state, recipe +// cards, and the compat mirror — must share this predicate or the activity +// bucket flips between remux and audio across restarts. +func TranscodesAudio(targetCodecAudio string) bool { + return !strings.EqualFold(targetCodecAudio, "copy") +} + // appendAudioArgs adds audio codec arguments. Supports "copy" for passthrough, // plus opus / aac / eac3 / ac3 as re-encode targets. EAC3 and AC3 are useful // when we must transcode video but want to preserve surround channels for an // HDMI receiver — both are legal in HLS fMP4 (not MPEG-TS; ensure the HLS // packager is fMP4 when emitting these). func appendAudioArgs(args []string, opts TranscodeOpts) []string { - codec := opts.TargetCodecAudio + // Case-insensitive so the switch agrees with TranscodesAudio for any + // client-supplied spelling. + codec := strings.ToLower(opts.TargetCodecAudio) if codec == "" { codec = "aac" } @@ -656,7 +696,14 @@ func appendAudioArgs(args []string, opts TranscodeOpts) []string { // Legacy Dolby Digital; universal AVR support. args = append(args, "-c:a", "ac3", "-b:a", "448k") default: - args = append(args, "-c:a", "aac", "-b:a", "192k", "-ac", "2") + // Preserve surround from multichannel sources when the planner asked + // for it (AAC 5.1 decodes universally in Media3); the historical + // default stays a stereo 192k downmix. + if opts.TargetAudioChannels >= 6 { + args = append(args, "-c:a", "aac", "-b:a", "384k", "-ac", "6") + } else { + args = append(args, "-c:a", "aac", "-b:a", "192k", "-ac", "2") + } } return args @@ -1494,6 +1541,28 @@ func (s *TranscodeSession) cleanStaleSegments(startSegment int) { // copy-mode sessions, stale segments at or after the restart point are // cleaned to prevent serving data from the wrong timeline position. func (s *TranscodeSession) Restart(ctx context.Context, seekSeconds float64, startSegment int) error { + return s.restart(ctx, seekSeconds, startSegment, 0, false) +} + +// RestartWithCopySeekAnchor restarts a copy-video stream with the keyframe +// origin resolved for this specific seek. Keeping this metadata explicit +// prevents a prior seek's origin from being reused after an audio switch. +func (s *TranscodeSession) RestartWithCopySeekAnchor( + ctx context.Context, + seekSeconds float64, + startSegment int, + streamOriginSeconds float64, +) error { + return s.restart(ctx, seekSeconds, startSegment, streamOriginSeconds, true) +} + +func (s *TranscodeSession) restart( + ctx context.Context, + seekSeconds float64, + startSegment int, + streamOriginSeconds float64, + copySeekAnchorResolved bool, +) error { s.mu.Lock() // Single-flight: a second caller arriving while a restart is in // progress must not kill the process the first restart just started. @@ -1535,6 +1604,14 @@ func (s *TranscodeSession) Restart(ctx context.Context, seekSeconds float64, sta opts.SeekSeconds = seekSeconds opts.StartSegmentNumber = startSegment + if strings.EqualFold(opts.TargetCodecVideo, "copy") { + // A seek restart describes a new emitted timeline. Never retain the + // previous copy seek's keyframe origin when the caller has not resolved + // a replacement; falling back to SeekSeconds is conservative and matches + // the behavior of recipes created before copy anchors were introduced. + opts.StreamOriginSeconds = streamOriginSeconds + opts.CopySeekAnchorResolved = copySeekAnchorResolved + } opts.FastStart = false // seek-restarts use veryfast for better quality args := buildFFmpegArgs(opts) @@ -1832,6 +1909,9 @@ func (s *TranscodeSession) SegmentStartTime(segNum int) (float64, bool, error) { s.mu.Lock() manifestPath := filepath.Join(s.outputDir, "stream.m3u8") baseSeekSeconds := s.opts.SeekSeconds + if strings.EqualFold(s.opts.TargetCodecVideo, "copy") && s.opts.CopySeekAnchorResolved { + baseSeekSeconds = s.opts.StreamOriginSeconds + } s.mu.Unlock() manifest, err := os.ReadFile(manifestPath) diff --git a/internal/playback/transcode_args_test.go b/internal/playback/transcode_args_test.go index b72f82aa..b7a65f6e 100644 --- a/internal/playback/transcode_args_test.go +++ b/internal/playback/transcode_args_test.go @@ -1,10 +1,28 @@ package playback import ( + "context" "strings" "testing" ) +func TestStartTranscodeRejectsUnvalidatedBitstreamFilter(t *testing.T) { + _, err := StartTranscode(context.Background(), TranscodeOpts{ + VideoBitstreamFilter: "arbitrary_filter=1", + TargetCodecVideo: "copy", + }) + if err == nil { + t.Fatal("unvalidated bitstream filter was accepted") + } + _, err = StartTranscode(context.Background(), TranscodeOpts{ + VideoBitstreamFilter: DV7ToHDR10BitstreamFilter, + TargetCodecVideo: "h264", + }) + if err == nil { + t.Fatal("DV copy filter was accepted for encoded video") + } +} + func TestBuildFFmpegArgs_QSVDropsSuperfastPreset(t *testing.T) { args := buildFFmpegArgs(TranscodeOpts{ InputPath: "/media/movie.mkv", @@ -66,6 +84,23 @@ func TestBuildFFmpegArgs_CopyVideoFromStartUsesZeroBasedTimestamps(t *testing.T) } } +func TestBuildFFmpegArgs_CopyVideoAppliesValidatedBitstreamFilter(t *testing.T) { + args := buildFFmpegArgs(TranscodeOpts{ + InputPath: "/media/movie.mkv", + OutputDir: "/tmp/out", + SessionID: "session-dv7", + TargetCodecVideo: "copy", + TargetCodecAudio: "copy", + VideoBitstreamFilter: DV7ToHDR10BitstreamFilter, + SegmentDuration: 2, + }) + + joined := strings.Join(args, " ") + if !strings.Contains(joined, "-c:v copy -bsf:v dovi_rpu=strip=1") { + t.Fatalf("copy-video args should apply the validated DV bitstream filter: %s", joined) + } +} + func TestBuildFFmpegArgs_CopyVideoResumePreservesSourceTimestamps(t *testing.T) { args := buildFFmpegArgs(TranscodeOpts{ InputPath: "/media/movie.mkv", @@ -504,3 +539,28 @@ func TestBuildFFmpegArgs_EncodedTranscodePreservesExistingTimestampPolicy(t *tes t.Fatalf("encoded args should keep avoid_negative_ts disabled: %s", joined) } } + +// TranscodesAudio must agree with appendAudioArgs: only an explicit "copy" +// passes audio through; an empty codec runs ffmpeg's AAC default. +func TestTranscodesAudioMatchesFFmpegDefault(t *testing.T) { + cases := []struct { + codec string + want bool + }{ + {"copy", false}, + {"COPY", false}, + {"", true}, + {"aac", true}, + {"opus", true}, + } + for _, tc := range cases { + if got := TranscodesAudio(tc.codec); got != tc.want { + t.Errorf("TranscodesAudio(%q) = %v, want %v", tc.codec, got, tc.want) + } + args := appendAudioArgs(nil, TranscodeOpts{TargetCodecAudio: tc.codec}) + copied := strings.Contains(strings.Join(args, " "), "-c:a copy") + if copied != !tc.want { + t.Errorf("appendAudioArgs(%q) copy=%v disagrees with TranscodesAudio=%v", tc.codec, copied, tc.want) + } + } +} diff --git a/internal/playback/transcode_cleanup.go b/internal/playback/transcode_cleanup.go index 1d9b3487..1cae53a5 100644 --- a/internal/playback/transcode_cleanup.go +++ b/internal/playback/transcode_cleanup.go @@ -1,12 +1,19 @@ package playback import ( + "context" "fmt" + "log/slog" "os" "path/filepath" + "runtime/debug" + "strings" + "sync" "time" ) +var orphanCleanupMu sync.Mutex + // CleanupOrphanedTranscodeDirs removes per-session transcode directories that // are not associated with any currently active session IDs. // @@ -19,6 +26,11 @@ import ( // Pass 0 to disable age-sparing (e.g. a dedicated node's boot-time full wipe, // where node restart is an accepted session loss). func CleanupOrphanedTranscodeDirs(root string, activeSessionIDs map[string]struct{}, minAge time.Duration) (int, error) { + // Serialize concurrent orphan sweeps of the shared transcode root so two + // rare startup sweeps cannot race on os.RemoveAll; one global mutex is fine. + orphanCleanupMu.Lock() + defer orphanCleanupMu.Unlock() + if root == "" { return 0, nil } @@ -38,7 +50,12 @@ func CleanupOrphanedTranscodeDirs(root string, activeSessionIDs map[string]struc continue } - if _, ok := activeSessionIDs[entry.Name()]; ok { + if transcodeDirBelongsToActiveSession(entry.Name(), activeSessionIDs) { + continue + } + + // The subtitle cache is not session state; it manages its own eviction. + if entry.Name() == subtitleCacheDirName { continue } @@ -63,3 +80,73 @@ func CleanupOrphanedTranscodeDirs(root string, activeSessionIDs map[string]struc return removed, nil } + +// OrphanCleanupInterval is how often the periodic orphan sweep re-runs. A dir is +// only reapable once it is older than MaxTokenTTL (24h), so sweeping much more +// often than hourly buys nothing; hourly bounds how long an untracked orphan (a +// dir whose owning session vanished without its RemoveAll succeeding) lingers on +// a process that is never restarted, without adding meaningful load. +const OrphanCleanupInterval = time.Hour + +// runOrphanCleanup executes one sweep, recovering from a panic (a background +// goroutine's unrecovered panic would crash the process) and logging the result. +func runOrphanCleanup(component, dir string, cleanup func() (int, error)) { + defer func() { + if r := recover(); r != nil { + slog.Error("transcode cleanup panicked", "component", component, "dir", dir, "panic", r, "stack", string(debug.Stack())) + } + }() + if cleaned, err := cleanup(); err != nil { + slog.Warn("transcode cleanup failed", "component", component, "dir", dir, "error", err) + } else if cleaned > 0 { + slog.Info("transcode cleanup removed orphaned dirs", "component", component, "dir", dir, "count", cleaned) + } +} + +// StartBackgroundOrphanCleanup runs a single orphaned-transcode sweep in its own +// goroutine so a slow network-filesystem delete never blocks server startup. +// The sweep is already safe to run concurrently with request handling: it +// spares live/in-flight sessions and any dir younger than MaxTokenTTL, and +// CleanupOrphanedTranscodeDirs serializes concurrent sweeps of the same root. +func StartBackgroundOrphanCleanup(component, dir string, cleanup func() (int, error)) { + go runOrphanCleanup(component, dir, cleanup) +} + +// StartPeriodicOrphanCleanup runs an immediate background sweep and then repeats +// it every interval until ctx is cancelled. The startup sweep only reclaims dirs +// orphaned by an ungraceful prior shutdown; the periodic re-run additionally +// bounds "untracked orphan" accumulation (a dir whose owning session was dropped +// without its RemoveAll succeeding) on a process that stays up for weeks. When +// ctx is nil or interval is non-positive it degrades to a single boot-time sweep +// so no ticker goroutine outlives a caller with no lifecycle handle (e.g. tests). +func StartPeriodicOrphanCleanup(ctx context.Context, component, dir string, cleanup func() (int, error), interval time.Duration) { + if ctx == nil || interval <= 0 { + StartBackgroundOrphanCleanup(component, dir, cleanup) + return + } + go func() { + runOrphanCleanup(component, dir, cleanup) + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + runOrphanCleanup(component, dir, cleanup) + } + } + }() +} + +func transcodeDirBelongsToActiveSession(name string, activeSessionIDs map[string]struct{}) bool { + if _, ok := activeSessionIDs[name]; ok { + return true + } + for sessionID := range activeSessionIDs { + if sessionID != "" && strings.HasPrefix(name, sessionID+"-") { + return true + } + } + return false +} diff --git a/internal/playback/transcode_cleanup_test.go b/internal/playback/transcode_cleanup_test.go new file mode 100644 index 00000000..2ae14746 --- /dev/null +++ b/internal/playback/transcode_cleanup_test.go @@ -0,0 +1,63 @@ +package playback + +import ( + "os" + "path/filepath" + "testing" +) + +func TestCleanupOrphanedTranscodeDirsPreservesPlanScopedActiveDirectory(t *testing.T) { + root := t.TempDir() + active := "session-1-plan-abc-generation" + orphan := "session-2-plan-def-generation" + for _, name := range []string{active, orphan} { + if err := os.MkdirAll(filepath.Join(root, name), 0o755); err != nil { + t.Fatal(err) + } + } + removed, err := CleanupOrphanedTranscodeDirs(root, map[string]struct{}{"session-1": {}}, 0) + if err != nil { + t.Fatal(err) + } + if removed != 1 { + t.Fatalf("removed = %d, want 1", removed) + } + if _, err := os.Stat(filepath.Join(root, active)); err != nil { + t.Fatalf("active plan directory was removed: %v", err) + } + if _, err := os.Stat(filepath.Join(root, orphan)); !os.IsNotExist(err) { + t.Fatalf("orphan directory still exists: %v", err) + } +} + +// An active session owns a dir only when the name is the session id exactly or +// the id followed by the generation separator ('-', see transportGenerationV3). +// A foreign session whose id merely shares the active id as a raw string +// prefix must not have its dirs retained. +func TestCleanupOrphanedTranscodeDirsRequiresSeparatorBoundary(t *testing.T) { + root := t.TempDir() + keep := []string{"session-1", "session-1-plan-abc"} + reap := []string{"session-10", "session-10-plan-def", "session-1extra"} + for _, name := range append(append([]string{}, keep...), reap...) { + if err := os.MkdirAll(filepath.Join(root, name), 0o755); err != nil { + t.Fatal(err) + } + } + removed, err := CleanupOrphanedTranscodeDirs(root, map[string]struct{}{"session-1": {}}, 0) + if err != nil { + t.Fatal(err) + } + if removed != len(reap) { + t.Fatalf("removed = %d, want %d", removed, len(reap)) + } + for _, name := range keep { + if _, err := os.Stat(filepath.Join(root, name)); err != nil { + t.Fatalf("active session directory %q was removed: %v", name, err) + } + } + for _, name := range reap { + if _, err := os.Stat(filepath.Join(root, name)); !os.IsNotExist(err) { + t.Fatalf("foreign directory %q still exists: %v", name, err) + } + } +} diff --git a/internal/playback/transcode_manager.go b/internal/playback/transcode_manager.go index 633c6d6c..3264f064 100644 --- a/internal/playback/transcode_manager.go +++ b/internal/playback/transcode_manager.go @@ -185,6 +185,43 @@ func (m *TranscodeManager) RegisterTranscodeSession(sessionID string, ts *Transc m.transcodeMu.Unlock() } +// SwapTranscodeSession atomically publishes a prepared successor and returns +// the predecessor without closing it. Protocol-v3 replans use plan-scoped +// output directories, so the caller can commit state first, publish the new +// process, and only then reap the old process without either process writing to +// the other's directory. +func (m *TranscodeManager) SwapTranscodeSession(sessionID string, successor *TranscodeSession) *TranscodeSession { + m.transcodeMu.Lock() + predecessor := m.transcodes[sessionID] + m.transcodes[sessionID] = successor + m.transcodeMu.Unlock() + return predecessor +} + +// SwapTranscodeSessionIf publishes successor only while the live map still +// contains expected. Callers use it after staging a process in a distinct +// output directory so a stale replacement cannot overwrite a newer process. +func (m *TranscodeManager) SwapTranscodeSessionIf( + sessionID string, + expected *TranscodeSession, + successor *TranscodeSession, +) bool { + m.transcodeMu.Lock() + defer m.transcodeMu.Unlock() + if m.transcodes[sessionID] != expected { + return false + } + m.transcodes[sessionID] = successor + return true +} + +// StopRemoteTranscode removes only the remote node process. It deliberately +// leaves the local live-map entry untouched for an atomic remote-to-local v3 +// replacement. +func (m *TranscodeManager) StopRemoteTranscode(sessionID, transcodeNodeURL string) { + m.deleteRemoteTranscode(sessionID, transcodeNodeURL) +} + // LockSessionLifecycle acquires the per-session lifecycle mutex and returns a // release func. Every path that spawns ffmpeg into a session's output directory // (fresh start, restart, reconstruct) must hold it across "check existing → spawn @@ -226,12 +263,33 @@ func (m *TranscodeManager) LockSessionLifecycle(sessionID string) func() { // replaced it, the stale handle is not re-spawned and ErrSessionSuperseded is // returned. func (m *TranscodeManager) RestartSessionLocked(ctx context.Context, sessionID string, ts *TranscodeSession, seekSeconds float64, startSegment int) error { + return m.restartSessionLocked(sessionID, ts, func() error { + return ts.Restart(ctx, seekSeconds, startSegment) + }) +} + +func (m *TranscodeManager) restartSessionLocked(sessionID string, ts *TranscodeSession, restart func() error) error { unlock := m.LockSessionLifecycle(sessionID) defer unlock() if live := m.GetTranscodeSession(sessionID); live != ts { return ErrSessionSuperseded } - return ts.Restart(ctx, seekSeconds, startSegment) + return restart() +} + +// RestartSessionLockedWithCopySeekAnchor is RestartSessionLocked with the +// resolved keyframe origin for a legacy copy-video seek restart. +func (m *TranscodeManager) RestartSessionLockedWithCopySeekAnchor( + ctx context.Context, + sessionID string, + ts *TranscodeSession, + seekSeconds float64, + startSegment int, + streamOriginSeconds float64, +) error { + return m.restartSessionLocked(sessionID, ts, func() error { + return ts.RestartWithCopySeekAnchor(ctx, seekSeconds, startSegment, streamOriginSeconds) + }) } // markReconstructing records that sessionID's ffmpeg is mid-reconstruct and @@ -348,20 +406,27 @@ func (m *TranscodeManager) ReconstructSession(ctx context.Context, sessionID str } s := &Session{ - ID: card.SessionID, - UserID: card.UserID, - ProfileID: card.ProfileID, - MediaFileID: card.MediaFileID, - PlayMethod: method, - BasePlayMethod: method, - TranscodeNodeURL: card.TranscodeNodeURL, - AudioTrackIndex: card.AudioTrackIndex, - TranscodeAudio: card.TranscodeAudio, - TargetResolution: card.TargetResolution, - TargetVideoCodec: card.TargetCodecVideo, - TargetAudioCodec: card.TargetCodecAudio, - TargetBitrateKbps: card.TargetBitrateKbps, - TranscodeHWAccel: card.HWAccel, + ID: card.SessionID, + UserID: card.UserID, + ProfileID: card.ProfileID, + MediaFileID: card.MediaFileID, + PlayMethod: method, + BasePlayMethod: method, + TranscodeNodeURL: card.TranscodeNodeURL, + TranscodeTransportID: card.TranscodeTransportID, + AudioTrackIndex: card.AudioTrackIndex, + TranscodeAudio: card.TranscodeAudio, + RemuxDVMode: card.RemuxDVMode, + TargetResolution: card.TargetResolution, + TargetVideoCodec: card.TargetCodecVideo, + TargetAudioCodec: card.TargetCodecAudio, + TargetBitrateKbps: card.TargetBitrateKbps, + TranscodeHWAccel: card.HWAccel, + // Client metadata survives the restart so the admin views keep the + // client label and Jellyfin identification for the session's lifetime. + ClientName: normalizeClientMetadataValue(card.ClientName, 128), + ClientVersion: normalizeClientMetadataValue(card.ClientVersion, 64), + ClientUserAgent: normalizeClientMetadataValue(card.ClientUserAgent, 512), // Preserve the byte-affecting recipe so an audio switch after a restart // rebuilds the same stream (subtitles/cadence) instead of dropping them. SubtitleTrackIndex: card.SubtitleTrackIndex, @@ -505,7 +570,7 @@ func (m *TranscodeManager) doReconstructTranscode(ctx context.Context, sessionID defer release() cfg := m.runtimeConfig() - outputDir := filepath.Join(cfg.TranscodeDir, sessionID) + outputDir := reconstructionOutputDir(cfg.TranscodeDir, sessionID, card.OutputSubdir) opts := card.TranscodeOpts(outputDir, cfg.FFmpegPath, m.logSink()) // Re-resolve environment-specific encode knobs from current config so an // operator config change applies to reconstructed sessions too. @@ -586,6 +651,17 @@ func (m *TranscodeManager) doReconstructTranscode(ctx context.Context, sessionID return transcodeSession } +func reconstructionOutputDir(root, sessionID, signedSubdir string) string { + outputSubdir := sessionID + if candidate := filepath.Clean(signedSubdir); candidate != "." && !filepath.IsAbs(candidate) && candidate != ".." && !strings.HasPrefix(candidate, ".."+string(filepath.Separator)) { + first, _, _ := strings.Cut(candidate, string(filepath.Separator)) + if first == sessionID || strings.HasPrefix(first, sessionID+"-") { + outputSubdir = candidate + } + } + return filepath.Join(root, outputSubdir) +} + // MonitorLocalTranscodeExit watches a local ffmpeg process and, on an error exit, // invokes OnFFmpegCrash so the embedding handler tears down the playback session. // A clean exit (no error) leaves the segments servable until the client stops. diff --git a/internal/playback/transcode_manifest_test.go b/internal/playback/transcode_manifest_test.go index 8b7f11b0..21509bd7 100644 --- a/internal/playback/transcode_manifest_test.go +++ b/internal/playback/transcode_manifest_test.go @@ -300,9 +300,12 @@ func TestRestartSeekTarget_CopyModeUsesManifestTimelineWhenAvailable(t *testing. session := &TranscodeSession{ outputDir: tempDir, opts: TranscodeOpts{ - SeekSeconds: 18.261, - TargetCodecVideo: "copy", - SegmentDuration: 2, + SeekSeconds: 18.261, + StreamOriginSeconds: 18, + CopySeekAnchorResolved: true, + TargetCodecVideo: "copy", + SegmentDuration: 2, + StartSegmentNumber: 9, }, } @@ -313,8 +316,8 @@ func TestRestartSeekTarget_CopyModeUsesManifestTimelineWhenAvailable(t *testing. if !ok { t.Fatal("RestartSeekTarget returned ok=false") } - if math.Abs(got-20.93) > 0.0001 { - t.Fatalf("RestartSeekTarget(10) = %.6f, want 20.93", got) + if math.Abs(got-20.669) > 0.0001 { + t.Fatalf("RestartSeekTarget(10) = %.6f, want 20.669", got) } } diff --git a/internal/playback/transcode_restart_guard_test.go b/internal/playback/transcode_restart_guard_test.go index 12422d38..cdcd76ab 100644 --- a/internal/playback/transcode_restart_guard_test.go +++ b/internal/playback/transcode_restart_guard_test.go @@ -74,6 +74,47 @@ func TestRestartInvokesRestartHook(t *testing.T) { } } +func TestRestartCopySeekOriginIsReplacedOrCleared(t *testing.T) { + truePath, err := exec.LookPath("true") + if err != nil { + t.Skipf("`true` not found in PATH: %v", err) + } + + newSession := func() *TranscodeSession { + return &TranscodeSession{ + outputDir: t.TempDir(), + opts: TranscodeOpts{ + TargetCodecVideo: "copy", + SegmentDuration: 2, + SeekSeconds: 18, + StreamOriginSeconds: 10, + CopySeekAnchorResolved: true, + StartSegmentNumber: 5, + FFmpegPath: truePath, + }, + } + } + + resolved := newSession() + if err := resolved.RestartWithCopySeekAnchor(context.Background(), 100, 48, 96); err != nil { + t.Fatalf("RestartWithCopySeekAnchor: %v", err) + } + resolvedOpts := resolved.Opts() + if resolvedOpts.SeekSeconds != 100 || resolvedOpts.StreamOriginSeconds != 96 || + !resolvedOpts.CopySeekAnchorResolved || resolvedOpts.StartSegmentNumber != 48 { + t.Fatalf("resolved restart opts = %+v", resolvedOpts) + } + + unresolved := newSession() + if err := unresolved.Restart(context.Background(), 100, 50); err != nil { + t.Fatalf("Restart: %v", err) + } + unresolvedOpts := unresolved.Opts() + if unresolvedOpts.StreamOriginSeconds != 0 || unresolvedOpts.CopySeekAnchorResolved { + t.Fatalf("generic restart retained stale copy origin: %+v", unresolvedOpts) + } +} + // TestRestartIsSingleFlight covers the other half: Restart must be // single-flight per session. A second caller arriving while a restart is in // progress must return immediately without killing the process the first diff --git a/internal/playback/transformations_v3.go b/internal/playback/transformations_v3.go new file mode 100644 index 00000000..ba964ba1 --- /dev/null +++ b/internal/playback/transformations_v3.go @@ -0,0 +1,121 @@ +package playback + +import ( + "bytes" + "context" + "os/exec" + "sort" + "strings" + "time" +) + +type TransformationSpecV3 struct { + Name string + RecipeVersion string + Available bool + RequiredCapability string + PromisedDynamicRange string + ValidatedClaims []string + TerminalReason string +} + +type TransformationRegistryV3 struct { + entries map[string]TransformationSpecV3 +} + +func ProbeTransformationRegistryV3(ctx context.Context, ffmpegPath string) *TransformationRegistryV3 { + // Resolve exactly like the execution paths (remux and transcode) so every + // capability advertised here holds for the binary that later runs. + ffmpegPath = ResolveFFmpegPath(ffmpegPath) + bsfCtx, cancelBSF := context.WithTimeout(ctx, 3*time.Second) + bsfs, _ := exec.CommandContext(bsfCtx, ffmpegPath, "-hide_banner", "-bsfs").Output() + cancelBSF() + encoderCtx, cancelEncoders := context.WithTimeout(ctx, 3*time.Second) + encoders, _ := exec.CommandContext(encoderCtx, ffmpegPath, "-hide_banner", "-encoders").Output() + cancelEncoders() + _, ffmpegErr := exec.LookPath(ffmpegPath) + return NewTransformationRegistryV3([]TransformationSpecV3{ + {Name: "server_dv7_to_hdr10", RecipeVersion: "1", Available: bytes.Contains(bsfs, []byte("dovi_rpu")), RequiredCapability: "ffmpeg_bsf:dovi_rpu", PromisedDynamicRange: "hdr10", ValidatedClaims: []string{"dolby_vision_metadata_removed", "hdr10_base_layer_preserved", "enhancement_layer_discarded"}, TerminalReason: "dv_conversion_unsupported"}, + {Name: "audio_to_aac", RecipeVersion: "1", Available: ffmpegErr == nil && bytes.Contains(encoders, []byte(" aac ")), RequiredCapability: "ffmpeg_encoder:aac", ValidatedClaims: []string{"media3_audio_decode"}, TerminalReason: "audio_conversion_unsupported"}, + {Name: "video_to_h264", RecipeVersion: "1", Available: ffmpegErr == nil && h264EncoderAvailableV3(encoders), RequiredCapability: "ffmpeg_encoder:h264", PromisedDynamicRange: "sdr", ValidatedClaims: []string{"media3_h264_decode"}, TerminalReason: "video_conversion_unsupported"}, + }) +} + +// h264EncodersV3 lists every H.264 encoder the transcode pipeline can select +// (see buildTranscodeArgs' hardware ladder in transcode.go); any one of them +// satisfies the video_to_h264 transformation. +var h264EncodersV3 = []string{"libx264", "h264_qsv", "h264_vaapi", "h264_nvenc", "h264_videotoolbox"} + +func h264EncoderAvailableV3(encoders []byte) bool { + for _, encoder := range h264EncodersV3 { + if bytes.Contains(encoders, []byte(encoder)) { + return true + } + } + return false +} + +func NewTransformationRegistryV3(specs []TransformationSpecV3) *TransformationRegistryV3 { + r := &TransformationRegistryV3{entries: make(map[string]TransformationSpecV3, len(specs))} + for _, spec := range specs { + if spec.Name != "" { + r.entries[spec.Name] = spec + } + } + return r +} + +func (r *TransformationRegistryV3) Available(name string) bool { + if r == nil { + return false + } + spec, ok := r.entries[name] + return ok && spec.Available +} + +// WithAdvertised returns a registry whose known specs are additionally marked +// available when a pooled transcode node advertises the same server-executed +// transformation at the same recipe version. Advertisements never introduce +// new specs: the planner only selects transformations this server defines, +// and pinning versions to the local spec guarantees a plan built from the +// widened registry passes the per-node advertisement validation at transport +// time. Returns the receiver unchanged when nothing new becomes available. +func (r *TransformationRegistryV3) WithAdvertised(advertised []TransformationV3) *TransformationRegistryV3 { + if r == nil || len(advertised) == 0 { + return r + } + specs := make([]TransformationSpecV3, 0, len(r.entries)) + changed := false + for _, spec := range r.entries { + if !spec.Available { + for _, remote := range advertised { + if strings.EqualFold(strings.TrimSpace(remote.Name), spec.Name) && + strings.TrimSpace(remote.RecipeVersion) == spec.RecipeVersion && + strings.EqualFold(strings.TrimSpace(remote.Executor), "server") { + spec.Available = true + changed = true + break + } + } + } + specs = append(specs, spec) + } + if !changed { + return r + } + return NewTransformationRegistryV3(specs) +} + +func (r *TransformationRegistryV3) Advertised() []TransformationV3 { + if r == nil { + return nil + } + result := make([]TransformationV3, 0, len(r.entries)) + for _, spec := range r.entries { + if spec.Available { + result = append(result, TransformationV3{Name: spec.Name, Executor: "server", RecipeVersion: spec.RecipeVersion, ValidatedClaims: append([]string(nil), spec.ValidatedClaims...)}) + } + } + sort.Slice(result, func(i, j int) bool { return result[i].Name < result[j].Name }) + return result +} diff --git a/internal/playback/transformations_v3_test.go b/internal/playback/transformations_v3_test.go new file mode 100644 index 00000000..68136062 --- /dev/null +++ b/internal/playback/transformations_v3_test.go @@ -0,0 +1,26 @@ +package playback + +import "testing" + +func TestH264EncoderAvailabilityAcceptsAnyPipelineEncoder(t *testing.T) { + cases := []struct { + name string + listing string + want bool + }{ + {"software", " V....D libx264 libx264 H.264 / AVC / MPEG-4 AVC", true}, + {"qsv", " V..... h264_qsv H.264 / AVC / MPEG-4 AVC (Intel Quick Sync Video acceleration)", true}, + {"vaapi", " V..... h264_vaapi H.264/AVC (VAAPI)", true}, + {"nvenc", " V....D h264_nvenc NVIDIA NVENC H.264 encoder", true}, + {"videotoolbox", " V..... h264_videotoolbox VideoToolbox H.264 Encoder", true}, + {"hevc_only", " V..... libx265\n V..... hevc_videotoolbox", false}, + {"empty", "", false}, + } + for _, value := range cases { + t.Run(value.name, func(t *testing.T) { + if got := h264EncoderAvailableV3([]byte(value.listing)); got != value.want { + t.Fatalf("h264EncoderAvailableV3 = %v, want %v", got, value.want) + } + }) + } +} diff --git a/internal/plugins/auto_update.go b/internal/plugins/auto_update.go index 408090e5..a273cd15 100644 --- a/internal/plugins/auto_update.go +++ b/internal/plugins/auto_update.go @@ -188,6 +188,14 @@ func (s *AutoUpdateService) Check(ctx context.Context, opts AutoUpdateOptions) ( if existing == nil || existing.RepositoryID == nil { continue } + // The reserved builtin row must never be matched against catalog + // entries: a catalog plugin named after it could otherwise rewrite its + // version/install_path and convert it into a launchable plugin. + // update_policy='manual' on the row and the reserved-plugin-id install + // rejection are the other layers. + if existing.IsBuiltin() { + continue + } entry, ok := latestByRepositoryPlugin[repositoryPluginKey{ RepositoryID: *existing.RepositoryID, PluginID: existing.PluginID, diff --git a/internal/plugins/builtin_guard_test.go b/internal/plugins/builtin_guard_test.go new file mode 100644 index 00000000..77371834 --- /dev/null +++ b/internal/plugins/builtin_guard_test.go @@ -0,0 +1,111 @@ +package plugins + +import ( + "context" + "errors" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func TestInstallationIsBuiltin(t *testing.T) { + if (&Installation{Kind: KindBuiltin}).IsBuiltin() != true { + t.Error("kind=builtin must report IsBuiltin") + } + if (&Installation{Kind: KindPlugin}).IsBuiltin() { + t.Error("kind=plugin must not report IsBuiltin") + } + if (&Installation{}).IsBuiltin() { + t.Error("zero-value kind must not report IsBuiltin") + } +} + +// The reserved builtin plugin id must be rejected at install time so a +// malicious or accidental catalog entry cannot hijack the reserved row. +func TestReservedPluginIDRejected(t *testing.T) { + if !isReservedPluginID("silo.builtin") { + t.Error("silo.builtin must be reserved") + } + if isReservedPluginID("silo.tmdb") || isReservedPluginID("silo.tvdb") { + t.Error("first-party plugin ids must stay installable") + } + if isReservedPluginID("community.example") { + t.Error("community plugin ids must stay installable") + } +} + +// PreloadEnabled must skip builtin installations explicitly: the sentinel +// install path has no manifest on disk, and without the skip preload would +// fail (its caller is log.Fatalf) or at best emit spurious warnings. +func TestPreloadEnabledSkipsBuiltinInstallation(t *testing.T) { + store := newFakeServiceInstallationStore(&Installation{ + ID: 1, + PluginID: "silo.builtin", + Kind: KindBuiltin, + Enabled: true, + InstallPath: "/nonexistent/silo-builtin", + }) + svc := &Service{installations: store} + + if err := svc.PreloadEnabled(context.Background()); err != nil { + t.Fatalf("PreloadEnabled with builtin-only store = %v, want nil (builtin must be skipped)", err) + } +} + +// InstallationStore.Delete must reject builtin rows at the store layer: the +// delete cascades through every NFO chain row and RemoveAlls the install dir. +func TestInstallationStoreDeleteRejectsBuiltin(t *testing.T) { + pool := builtinGuardTestPool(t) + ctx := context.Background() + + id := seedBuiltinTestInstallation(t, pool, "test.builtin.delete") + + store := NewInstallationStore(pool) + err := store.Delete(ctx, id) + if err == nil { + t.Fatal("Delete(builtin) = nil, want error") + } + if !errors.Is(err, ErrBuiltinInstallationImmutable) { + t.Fatalf("Delete(builtin) error = %v, want ErrBuiltinInstallationImmutable", err) + } + + var count int + if qErr := pool.QueryRow(ctx, `SELECT COUNT(*) FROM plugin_installations WHERE id = $1`, id).Scan(&count); qErr != nil { + t.Fatalf("count row: %v", qErr) + } + if count != 1 { + t.Fatalf("builtin row count after rejected delete = %d, want 1", count) + } +} + +func builtinGuardTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func seedBuiltinTestInstallation(t *testing.T, pool *pgxpool.Pool, pluginID string) int { + t.Helper() + var id int + err := pool.QueryRow(context.Background(), + `INSERT INTO plugin_installations (plugin_id, version, install_path, enabled, update_policy, kind) + VALUES ($1, '0', '/nonexistent/silo-builtin-test', true, 'manual', 'builtin') + RETURNING id`, pluginID+time.Now().UTC().Format("-20060102150405.000000000")).Scan(&id) + if err != nil { + t.Fatalf("seed builtin installation: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(context.Background(), `DELETE FROM plugin_installations WHERE id = $1`, id) + }) + return id +} diff --git a/internal/plugins/event_dispatcher.go b/internal/plugins/event_dispatcher.go index 93147fe2..118c7263 100644 --- a/internal/plugins/event_dispatcher.go +++ b/internal/plugins/event_dispatcher.go @@ -175,6 +175,11 @@ func (d *EventDispatcher) fanOut(ctx context.Context, env events.Envelope, paylo var wg sync.WaitGroup for _, installation := range installations { + // Builtin installations are metadata-only and never launchable; + // defense-in-depth alongside the capability-type filter below. + if installation.IsBuiltin() { + continue + } if env.TargetPluginID != "" && installation.PluginID != env.TargetPluginID { continue } diff --git a/internal/plugins/installation.go b/internal/plugins/installation.go index 0406ec85..1a8481e0 100644 --- a/internal/plugins/installation.go +++ b/internal/plugins/installation.go @@ -19,6 +19,20 @@ var ErrInstallationNotFound = errors.New("plugin installation not found") var ErrArchiveNotFound = errors.New("plugin archive not found") var ErrInstallationDisabled = errors.New("plugin installation is disabled") +// ErrBuiltinInstallationImmutable is returned when a caller tries to mutate +// the reserved builtin-host installation row (delete, update, config, ...). +var ErrBuiltinInstallationImmutable = errors.New("builtin installation cannot be modified") + +// Installation kinds. A 'builtin' installation is the reserved row that +// anchors built-in host provider capabilities (silo.builtin); it has no +// archive, manifest, or binary and must never be launched, updated, or +// deleted. Generic reads must NOT filter builtins — the metadata chain's +// enabled-check depends on reading them. +const ( + KindPlugin = "plugin" + KindBuiltin = "builtin" +) + type Installation struct { ID int RepositoryID *int @@ -26,12 +40,18 @@ type Installation struct { Version string InstallPath string Enabled bool + Kind string `json:"kind"` UpdatePolicy string `json:"update_policy"` AvailableVersion *string `json:"available_version,omitempty"` CreatedAt time.Time UpdatedAt time.Time } +// IsBuiltin reports whether this is the reserved builtin-host installation. +func (i *Installation) IsBuiltin() bool { + return i != nil && i.Kind == KindBuiltin +} + type Capability struct { InstallationID int Type string @@ -77,7 +97,7 @@ func NewInstallationStore(pool *pgxpool.Pool) *InstallationStore { return &InstallationStore{pool: pool} } -const installationColumns = `id, repository_id, plugin_id, version, install_path, enabled, update_policy, available_version, created_at, updated_at` +const installationColumns = `id, repository_id, plugin_id, version, install_path, enabled, kind, update_policy, available_version, created_at, updated_at` const capabilityColumns = `plugin_installation_id, capability_type, capability_id, metadata, created_at, updated_at` const archiveColumns = `plugin_installation_id, manifest_json, checksum, archive_bytes, created_at, updated_at` @@ -91,6 +111,7 @@ func scanInstallation(row pgx.Row) (*Installation, error) { &installation.Version, &installation.InstallPath, &installation.Enabled, + &installation.Kind, &installation.UpdatePolicy, &installation.AvailableVersion, &installation.CreatedAt, @@ -263,6 +284,17 @@ func (s *InstallationStore) ListByPluginID(ctx context.Context, pluginID string) } func (s *InstallationStore) Update(ctx context.Context, id int, input UpdateInstallationInput) error { + // Guard at the store, not only the HTTP handler: the reserved builtin row + // carries no archive/binary and must never have its version, path, enabled + // flag, or capabilities rewritten, or its chain participation breaks. + installation, err := s.GetByID(ctx, id) + if err != nil { + return err + } + if installation.IsBuiltin() { + return ErrBuiltinInstallationImmutable + } + tx, err := s.pool.Begin(ctx) if err != nil { return fmt.Errorf("begin update installation transaction: %w", err) @@ -314,10 +346,6 @@ func (s *InstallationStore) Update(ctx context.Context, id int, input UpdateInst if tag.RowsAffected() == 0 { return ErrInstallationNotFound } - } else { - if _, err := s.GetByID(ctx, id); err != nil { - return err - } } if input.Capabilities != nil { @@ -391,6 +419,12 @@ func (s *InstallationStore) Delete(ctx context.Context, id int) error { if err != nil { return err } + // Guard at the store, not only the HTTP handler: deleting the reserved + // builtin row would cascade through every builtin chain row and RemoveAll + // the (sentinel) install dir. + if installation.IsBuiltin() { + return ErrBuiltinInstallationImmutable + } tag, err := s.pool.Exec(ctx, `DELETE FROM plugin_installations WHERE id = $1`, id) if err != nil { diff --git a/internal/plugins/manifest.go b/internal/plugins/manifest.go index 0763aa04..bd50d364 100644 --- a/internal/plugins/manifest.go +++ b/internal/plugins/manifest.go @@ -61,10 +61,21 @@ func capabilityKey(kind, id string) string { return kind + "\x00" + id } +// isReservedPluginID reports whether a plugin id is reserved for the host and +// must never be installable. Only the exact builtin registration id is +// reserved: first-party plugins legitimately use the "silo." prefix +// (silo.tmdb, silo.tvdb). +func isReservedPluginID(pluginID string) bool { + return pluginID == "silo.builtin" +} + func validateManifestShared(manifest *pluginv1.PluginManifest) error { if err := publicmanifest.Validate(manifest); err != nil { return err } + if isReservedPluginID(manifest.GetPluginId()) { + return fmt.Errorf("plugin id %q is reserved for built-in host providers", manifest.GetPluginId()) + } if manifest.GetSiloApiVersion() == "" { return fmt.Errorf("plugin manifest silo_api_version is required") } diff --git a/internal/plugins/service.go b/internal/plugins/service.go index 2c3fe1ea..93ddfe11 100644 --- a/internal/plugins/service.go +++ b/internal/plugins/service.go @@ -462,6 +462,12 @@ func (s *Service) PreloadEnabled(ctx context.Context) error { if installation == nil { continue } + // Builtin installations have no archive or binary; skip them explicitly + // instead of leaning on the tolerated ErrArchiveNotFound branch below + // (any other load error here is fatal to startup). + if installation.IsBuiltin() { + continue + } if _, err := s.ensureLoadedInstallation(ctx, installation); err != nil { if errors.Is(err, ErrArchiveNotFound) { slog.WarnContext(ctx, @@ -806,6 +812,15 @@ func (s *Service) loadInstallation(ctx context.Context, installationID int, requ if requireEnabled && !installation.Enabled { return nil, ErrInstallationDisabled } + // requireEnabled marks paths that intend to launch or serve the plugin + // (start, manifest routes/assets, gRPC clients, HTTP proxy). The reserved + // builtin row has no binary behind it and must never reach those paths; + // not-found gives the proxy and API a clean 4xx. Reads with + // requireEnabled=false (IsInstallationEnabled for the metadata chain, + // generic listings) still see the row. + if requireEnabled && installation.IsBuiltin() { + return nil, ErrInstallationNotFound + } return installation, nil } @@ -869,6 +884,17 @@ func (s *Service) IsInstallationEnabled(ctx context.Context, installationID int) return installation.Enabled, nil } +// InstallationKind returns the installation's kind ("plugin" or "builtin") from +// the same in-memory cache IsInstallationEnabled reads, so metadata chain +// resolution can identify builtin rows without a per-capability DB query. +func (s *Service) InstallationKind(ctx context.Context, installationID int) (string, error) { + installation, err := s.loadInstallation(ctx, installationID, false) + if err != nil { + return "", err + } + return installation.Kind, nil +} + func (s *Service) ensureLoadedInstallation( ctx context.Context, installation *Installation, diff --git a/internal/plugins/task_registry.go b/internal/plugins/task_registry.go index 866c7d41..123ee456 100644 --- a/internal/plugins/task_registry.go +++ b/internal/plugins/task_registry.go @@ -69,6 +69,11 @@ func (r *TaskRegistry) Tasks(ctx context.Context) ([]taskmanager.Task, error) { var tasks []taskmanager.Task for _, installation := range installations { + // Builtin installations expose no scheduled tasks and cannot be + // launched; defense-in-depth alongside the capability-type filter. + if installation.IsBuiltin() { + continue + } capabilities, err := r.installations.ListCapabilities(ctx, installation.ID) if err != nil { return nil, fmt.Errorf("list plugin capabilities for installation %d: %w", installation.ID, err) diff --git a/internal/proxy/playback_v3_test.go b/internal/proxy/playback_v3_test.go new file mode 100644 index 00000000..370b8f52 --- /dev/null +++ b/internal/proxy/playback_v3_test.go @@ -0,0 +1,18 @@ +package proxy + +import ( + "testing" + + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +func TestTranscodeTransportIDFromClaims(t *testing.T) { + claims := &streamtoken.Claims{SessionID: "public-session", TranscodeTransportID: "public-session-plan-a"} + if got := transcodeTransportIDFromClaims(claims); got != claims.TranscodeTransportID { + t.Fatalf("transport id = %q", got) + } + claims.TranscodeTransportID = "" + if got := transcodeTransportIDFromClaims(claims); got != claims.SessionID { + t.Fatalf("legacy transport id = %q", got) + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 46801c67..9c5562dc 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -169,7 +169,10 @@ func (s *Server) handleRemux(w http.ResponseWriter, r *http.Request) { seekSeconds = v } } - _ = playback.ServeRemux(w, r, claims.MediaPath, "mp4", seekSeconds, claims.TranscodeAudio, claims.AudioTrackIndex, claims.DVProfile) + // Honor the Dolby Vision mode frozen in the token (empty decodes as the + // legacy auto behavior for old tokens), mirroring how the integrated + // server's stream handler serves the same claims. + _ = playback.ServeRemuxWithDVMode(w, r, claims.MediaPath, "mp4", seekSeconds, claims.TranscodeAudio, claims.AudioTrackIndex, claims.DVProfile, playback.RemuxDVMode(claims.RemuxDVMode), s.watcher.Config().Playback.FFmpegPath) } func (s *Server) handleTranscodeManifest(w http.ResponseWriter, r *http.Request) { @@ -178,7 +181,7 @@ func (s *Server) handleTranscodeManifest(w http.ResponseWriter, r *http.Request) return } s.touchTranscodeSession(r, claims) - s.proxyToTranscodeNode(w, r, claims, "/transcode/"+claims.SessionID+"/master.m3u8") + s.proxyToTranscodeNode(w, r, claims, "/transcode/"+transcodeTransportIDFromClaims(claims)+"/master.m3u8") } func (s *Server) handleTranscodeSegment(w http.ResponseWriter, r *http.Request) { @@ -188,7 +191,17 @@ func (s *Server) handleTranscodeSegment(w http.ResponseWriter, r *http.Request) } s.touchTranscodeSession(r, claims) name := chi.URLParam(r, "name") - s.proxyToTranscodeNode(w, r, claims, "/transcode/"+claims.SessionID+"/segment/"+name) + s.proxyToTranscodeNode(w, r, claims, "/transcode/"+transcodeTransportIDFromClaims(claims)+"/segment/"+name) +} + +func transcodeTransportIDFromClaims(claims *streamtoken.Claims) string { + if claims != nil && claims.TranscodeTransportID != "" { + return claims.TranscodeTransportID + } + if claims == nil { + return "" + } + return claims.SessionID } // touchTranscodeSession keeps HLS sessions visible in the active stream count. diff --git a/internal/rootcheck/rootcheck.go b/internal/rootcheck/rootcheck.go new file mode 100644 index 00000000..6fa504c1 --- /dev/null +++ b/internal/rootcheck/rootcheck.go @@ -0,0 +1,223 @@ +// Package rootcheck probes library root paths for reachability: a root is +// reachable when it exists, is a directory, and can be listed. It is shared +// by the scanner's dead-root protection and the admin mount-check endpoint so +// both agree on what "unreachable" means. +package rootcheck + +import ( + "context" + "errors" + "io" + "os" + "sync" + "time" +) + +// Error codes reported by Probe. They are part of the admin mount-check API +// response contract. +const ( + ErrCodeNotFound = "not_found" + ErrCodePermissionDenied = "permission_denied" + ErrCodeNotDirectory = "not_directory" + ErrCodeReadFailed = "read_failed" + ErrCodeStatFailed = "stat_failed" + ErrCodeTimeout = "probe_timeout" +) + +// DefaultProbeTimeout bounds how long a single probe may block. A dead mount +// usually errors within milliseconds, but a hung network filesystem +// (hard-mounted NFS, wedged SMB/FUSE) blocks stat/readdir indefinitely — +// probes run on scan and request hot paths, so a hung mount must degrade +// into "unreachable" rather than stall the caller. +const DefaultProbeTimeout = 5 * time.Second + +// DefaultProbeConcurrency bounds the number of distinct roots started by one +// batch. Repeated probes for the same path are also coalesced process-wide. +const DefaultProbeConcurrency = 8 + +// Result describes the outcome of probing a single root path. +type Result struct { + Reachable bool + // Empty is set for a reachable directory with zero entries. A completely + // empty root is the on-disk signature of a lost mount (the mountpoint + // directory remains, its contents vanished with the mount), which a + // reachability check alone cannot detect. + Empty bool + ErrorCode string // empty when Reachable + ErrorMessage string // empty when Reachable +} + +// Probe checks that path exists, is a directory, and can be listed. +func Probe(path string) Result { + res := Result{Reachable: true} + info, err := os.Stat(path) + switch { + case err != nil: + res.Reachable = false + res.ErrorCode, res.ErrorMessage = classify(err, false) + case !info.IsDir(): + res.Reachable = false + res.ErrorCode, res.ErrorMessage = ErrCodeNotDirectory, "Path is not a directory" + default: + dir, err := os.Open(path) + if err == nil { + _, err = dir.Readdirnames(1) + _ = dir.Close() + } + if errors.Is(err, io.EOF) { + res.Empty = true + } else if err != nil { + res.Reachable = false + res.ErrorCode, res.ErrorMessage = classify(err, true) + } + } + return res +} + +type probeCall struct { + done chan struct{} + result Result +} + +type probeCoordinator struct { + mu sync.Mutex + inFlight map[string]*probeCall +} + +func newProbeCoordinator() *probeCoordinator { + return &probeCoordinator{inFlight: make(map[string]*probeCall)} +} + +var sharedProbes = newProbeCoordinator() + +// ProbeWithTimeout runs Probe but gives up once timeout elapses or ctx is +// done, reporting the root unreachable with ErrCodeTimeout. Concurrent calls +// for the same path share one underlying syscall so a wedged mount cannot +// accumulate one blocked goroutine per scan or mount check. +func ProbeWithTimeout(ctx context.Context, path string, timeout time.Duration) Result { + return sharedProbes.probe(ctx, path, timeout, func(path string) Result { return Probe(path) }) +} + +func (c *probeCoordinator) probe( + ctx context.Context, + path string, + timeout time.Duration, + probe func(string) Result, +) Result { + if timeout <= 0 { + timeout = DefaultProbeTimeout + } + c.mu.Lock() + call := c.inFlight[path] + if call == nil { + call = &probeCall{done: make(chan struct{})} + c.inFlight[path] = call + go func() { + call.result = probe(path) + close(call.done) + c.mu.Lock() + delete(c.inFlight, path) + c.mu.Unlock() + }() + } + c.mu.Unlock() + + return awaitProbe(ctx, timeout, call.done, func() Result { return call.result }) +} + +// ProbeManyWithTimeout probes paths concurrently while preserving input order. +func ProbeManyWithTimeout(ctx context.Context, paths []string, timeout time.Duration) []Result { + return probeMany(ctx, paths, timeout, DefaultProbeConcurrency, ProbeWithTimeout) +} + +func probeMany( + ctx context.Context, + paths []string, + timeout time.Duration, + limit int, + probe func(context.Context, string, time.Duration) Result, +) []Result { + results := make([]Result, len(paths)) + if len(paths) == 0 { + return results + } + if limit <= 0 || limit > len(paths) { + limit = len(paths) + } + jobs := make(chan int) + var workers sync.WaitGroup + workers.Add(limit) + for range limit { + go func() { + defer workers.Done() + for i := range jobs { + results[i] = probe(ctx, paths[i], timeout) + } + }() + } + for i := range paths { + jobs <- i + } + close(jobs) + workers.Wait() + return results +} + +func probeBounded(ctx context.Context, timeout time.Duration, probe func() Result) Result { + if timeout <= 0 { + timeout = DefaultProbeTimeout + } + done := make(chan Result, 1) + go func() { done <- probe() }() + + var ctxDone <-chan struct{} + if ctx != nil { + ctxDone = ctx.Done() + } + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case res := <-done: + return res + case <-ctxDone: + case <-timer.C: + } + return timeoutResult() +} + +func awaitProbe(ctx context.Context, timeout time.Duration, done <-chan struct{}, result func() Result) Result { + var ctxDone <-chan struct{} + if ctx != nil { + ctxDone = ctx.Done() + } + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-done: + return result() + case <-ctxDone: + case <-timer.C: + } + return timeoutResult() +} + +func timeoutResult() Result { + return Result{ + Reachable: false, + ErrorCode: ErrCodeTimeout, + ErrorMessage: "Probe timed out; filesystem is not responding", + } +} + +func classify(err error, isRead bool) (string, string) { + switch { + case errors.Is(err, os.ErrNotExist): + return ErrCodeNotFound, "Path does not exist" + case errors.Is(err, os.ErrPermission): + return ErrCodePermissionDenied, "Permission denied" + case isRead: + return ErrCodeReadFailed, "Failed to read directory" + default: + return ErrCodeStatFailed, "Failed to stat path" + } +} diff --git a/internal/rootcheck/rootcheck_test.go b/internal/rootcheck/rootcheck_test.go new file mode 100644 index 00000000..e7192dbb --- /dev/null +++ b/internal/rootcheck/rootcheck_test.go @@ -0,0 +1,202 @@ +package rootcheck + +import ( + "context" + "os" + "path/filepath" + "sync/atomic" + "testing" + "time" +) + +func TestProbeReachableDirectory(t *testing.T) { + t.Parallel() + + res := Probe(t.TempDir()) + if !res.Reachable { + t.Fatalf("Probe(temp dir) = %+v, want reachable", res) + } + if res.ErrorCode != "" || res.ErrorMessage != "" { + t.Fatalf("Probe(temp dir) error fields = %q/%q, want empty", res.ErrorCode, res.ErrorMessage) + } +} + +func TestProbeMissingPath(t *testing.T) { + t.Parallel() + + res := Probe(filepath.Join(t.TempDir(), "does-not-exist")) + if res.Reachable { + t.Fatal("Probe(missing path) reported reachable") + } + if res.ErrorCode != ErrCodeNotFound { + t.Fatalf("ErrorCode = %q, want %q", res.ErrorCode, ErrCodeNotFound) + } +} + +func TestProbeRegularFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "file.txt") + if err := os.WriteFile(path, []byte("x"), 0o644); err != nil { + t.Fatalf("write file: %v", err) + } + + res := Probe(path) + if res.Reachable { + t.Fatal("Probe(regular file) reported reachable") + } + if res.ErrorCode != ErrCodeNotDirectory { + t.Fatalf("ErrorCode = %q, want %q", res.ErrorCode, ErrCodeNotDirectory) + } +} + +func TestProbeUnreadableDirectory(t *testing.T) { + t.Parallel() + if os.Getuid() == 0 { + t.Skip("running as root; permission bits are not enforced") + } + + path := filepath.Join(t.TempDir(), "locked") + if err := os.Mkdir(path, 0o000); err != nil { + t.Fatalf("mkdir: %v", err) + } + t.Cleanup(func() { _ = os.Chmod(path, 0o755) }) + + res := Probe(path) + if res.Reachable { + t.Fatal("Probe(unreadable dir) reported reachable") + } + if res.ErrorCode != ErrCodePermissionDenied { + t.Fatalf("ErrorCode = %q, want %q", res.ErrorCode, ErrCodePermissionDenied) + } +} + +func TestProbeWithTimeoutReachableDirectory(t *testing.T) { + t.Parallel() + + res := ProbeWithTimeout(context.Background(), t.TempDir(), DefaultProbeTimeout) + if !res.Reachable { + t.Fatalf("ProbeWithTimeout(temp dir) = %+v, want reachable", res) + } +} + +func TestProbeBoundedTimesOutOnHungProbe(t *testing.T) { + t.Parallel() + + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + res := probeBounded(context.Background(), 10*time.Millisecond, func() Result { + <-release // simulate a stat/readdir blocked on a hung mount + return Result{Reachable: true} + }) + if res.Reachable { + t.Fatal("hung probe reported reachable") + } + if res.ErrorCode != ErrCodeTimeout { + t.Fatalf("ErrorCode = %q, want %q", res.ErrorCode, ErrCodeTimeout) + } +} + +func TestProbeBoundedHonorsContextCancellation(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + res := probeBounded(ctx, time.Minute, func() Result { + <-release + return Result{Reachable: true} + }) + if res.Reachable || res.ErrorCode != ErrCodeTimeout { + t.Fatalf("canceled probe = %+v, want unreachable/%s", res, ErrCodeTimeout) + } +} + +func TestProbeBoundedReturnsFastResult(t *testing.T) { + t.Parallel() + + res := probeBounded(context.Background(), time.Minute, func() Result { + return Result{Reachable: false, ErrorCode: ErrCodeNotFound, ErrorMessage: "Path does not exist"} + }) + if res.Reachable || res.ErrorCode != ErrCodeNotFound { + t.Fatalf("probeBounded passthrough = %+v, want the probe's own result", res) + } +} + +func TestProbeCoordinatorCoalescesBlockedPath(t *testing.T) { + t.Parallel() + + var calls atomic.Int32 + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + coordinator := newProbeCoordinator() + probe := func(string) Result { + calls.Add(1) + <-release + return Result{Reachable: true} + } + + results := make(chan Result, 2) + for range 2 { + go func() { + results <- coordinator.probe(context.Background(), "/hung", 20*time.Millisecond, probe) + }() + } + for range 2 { + if result := <-results; result.Reachable || result.ErrorCode != ErrCodeTimeout { + t.Fatalf("coalesced blocked probe = %+v, want timeout", result) + } + } + if got := calls.Load(); got != 1 { + t.Fatalf("underlying probe calls = %d, want 1", got) + } +} + +func TestProbeManyPreservesOrderAndBoundsConcurrency(t *testing.T) { + t.Parallel() + + var active atomic.Int32 + var peak atomic.Int32 + probe := func(_ context.Context, path string, _ time.Duration) Result { + current := active.Add(1) + for { + observed := peak.Load() + if current <= observed || peak.CompareAndSwap(observed, current) { + break + } + } + time.Sleep(10 * time.Millisecond) + active.Add(-1) + return Result{Reachable: true, ErrorMessage: path} + } + + paths := []string{"one", "two", "three", "four"} + results := probeMany(context.Background(), paths, time.Second, 2, probe) + if got := peak.Load(); got > 2 { + t.Fatalf("peak concurrent probes = %d, want at most 2", got) + } + for i, result := range results { + if result.ErrorMessage != paths[i] { + t.Fatalf("result[%d] = %q, want %q", i, result.ErrorMessage, paths[i]) + } + } +} + +func TestProbeReportsEmptyDirectory(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + res := Probe(dir) + if !res.Reachable || !res.Empty { + t.Fatalf("Probe(empty dir) = %+v, want reachable and empty", res) + } + + if err := os.Mkdir(filepath.Join(dir, "sub"), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + res = Probe(dir) + if !res.Reachable || res.Empty { + t.Fatalf("Probe(non-empty dir) = %+v, want reachable and not empty", res) + } +} diff --git a/internal/s3client/client.go b/internal/s3client/client.go index 0008eec6..e8094d38 100644 --- a/internal/s3client/client.go +++ b/internal/s3client/client.go @@ -9,6 +9,7 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/base64" + "encoding/hex" "errors" "fmt" "io" @@ -21,6 +22,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" "github.com/aws/aws-sdk-go-v2/service/s3" s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" ) @@ -35,6 +37,10 @@ const ( URLAuthCloudflareToken = "cloudflare_token" // Cloudflare WAF token authentication ) +// streamUploadPartSize bounds per-upload memory for unsized streaming uploads. +// S3-compatible backends require parts of at least 5 MiB (except the last). +const streamUploadPartSize = 8 * 1024 * 1024 + // BucketConfig holds the configuration for connecting to a single S3 bucket. // Each bucket may have different credentials and endpoints, allowing per-bucket // configuration for metadata, operational, and user-db buckets. @@ -150,6 +156,24 @@ func (c *Client) GetObject(ctx context.Context, bucket, key string) ([]byte, err return data, nil } +// GetObjectStream fetches the object at the given key and returns a streaming +// body. The caller must close the returned ReadCloser. +func (c *Client) GetObjectStream(ctx context.Context, bucket, key string) (io.ReadCloser, error) { + objectKey := c.prefixedKey(key) + out, err := c.s3Client.GetObject(ctx, &s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + }) + if err != nil { + if isNotFoundErr(err) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("s3 GetObject %s/%s: %w", bucket, key, err) + } + + return out.Body, nil +} + // PutObject uploads data to the given key, inferring Content-Type from the // file extension so that CDNs and browsers serve files with the correct MIME type. func (c *Client) PutObject(ctx context.Context, bucket, key string, data []byte) error { @@ -160,6 +184,9 @@ func (c *Client) PutObject(ctx context.Context, bucket, key string, data []byte) Bucket: aws.String(bucket), Key: aws.String(objectKey), Body: body, + Metadata: map[string]string{ + "silo-sha256": objectSHA256(data), + }, } if ct := contentTypeFromKey(key); ct != "" { input.ContentType = aws.String(ct) @@ -173,6 +200,37 @@ func (c *Client) PutObject(ctx context.Context, bucket, key string, data []byte) return nil } +// PutObjectStream uploads a streaming body to the given key. When contentType is +// empty, it falls back to the same extension-based inference used by PutObject. +// The body length is untrusted until streamed, so the upload goes through the +// SDK's multipart manager: it buffers fixed-size parts and sends each with a +// known Content-Length, which backends like Cloudflare R2 require (a plain +// PutObject with an unsized stream is rejected with 411 MissingContentLength). +func (c *Client) PutObjectStream(ctx context.Context, bucket, key string, r io.Reader, contentType string) error { + objectKey := c.prefixedKey(key) + + input := &s3.PutObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + Body: r, + } + if contentType != "" { + input.ContentType = aws.String(contentType) + } else if ct := contentTypeFromKey(key); ct != "" { + input.ContentType = aws.String(ct) + } + + uploader := manager.NewUploader(c.s3Client, func(u *manager.Uploader) { + u.PartSize = streamUploadPartSize + u.Concurrency = 1 + }) + if _, err := uploader.Upload(ctx, input); err != nil { + return fmt.Errorf("s3 PutObject stream %s/%s: %w", bucket, key, err) + } + + return nil +} + // MakeObjectPublic updates the object ACL to allow anonymous reads. func (c *Client) MakeObjectPublic(ctx context.Context, bucket, key string) error { objectKey := c.prefixedKey(key) @@ -373,6 +431,32 @@ func (c *Client) ObjectExists(ctx context.Context, bucket, key string) (bool, er return true, nil } +// ObjectMatches checks that an immutable object exists with the expected +// length and application-recorded SHA-256. Objects written before checksums +// were recorded return false and are safely rewritten by the caller. +func (c *Client) ObjectMatches(ctx context.Context, bucket, key string, data []byte) (bool, error) { + objectKey := c.prefixedKey(key) + out, err := c.s3Client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(objectKey), + }) + if err != nil { + if isNotFoundErr(err) { + return false, nil + } + return false, fmt.Errorf("s3 HeadObject %s/%s: %w", bucket, key, err) + } + if out.ContentLength == nil || *out.ContentLength != int64(len(data)) { + return false, nil + } + return strings.EqualFold(out.Metadata["silo-sha256"], objectSHA256(data)), nil +} + +func objectSHA256(data []byte) string { + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]) +} + // DeleteObject deletes the object at the given key. func (c *Client) DeleteObject(ctx context.Context, bucket, key string) error { objectKey := c.prefixedKey(key) diff --git a/internal/scanner/audiobook_cover.go b/internal/scanner/audiobook_cover.go index 323b4f8a..e5d6fd55 100644 --- a/internal/scanner/audiobook_cover.go +++ b/internal/scanner/audiobook_cover.go @@ -21,11 +21,11 @@ const maxAudiobookSidecarCoverSize = 8 * 1024 * 1024 // request struct) to avoid an import cycle: imagecache imports metadata // which imports scanner. type audiobookCoverCacher interface { - CacheAudiobookCover(ctx context.Context, data []byte, contentID string) (basePath string, ext string, thumbhash string, err error) + CacheAudiobookCover(ctx context.Context, data []byte, contentID string) (storedPath string, thumbhash string, err error) } type ebookCoverCacher interface { - CacheEbookCover(ctx context.Context, data []byte, contentID string) (basePath string, ext string, thumbhash string, err error) + CacheEbookCover(ctx context.Context, data []byte, contentID string) (storedPath string, thumbhash string, err error) } type audiobookCoverMetadataStore interface { @@ -84,13 +84,13 @@ func ExtractAndUploadAudiobookCover( if len(data) == 0 { return "", "" } - basePath, ext, thumbhash, err := cacher.CacheAudiobookCover(ctx, data, contentID) + storedPath, thumbhash, err := cacher.CacheAudiobookCover(ctx, data, contentID) if err != nil { slog.WarnContext(ctx, "audiobook cover: imagecache upload failed", "component", "scanner", "path", audioFilePath, "error", err) return "", "" } - return fmt.Sprintf("%s/original%s", basePath, ext), thumbhash + return storedPath, thumbhash } func applyAudiobookSidecarCover(ctx context.Context, store audiobookCoverMetadataStore, cacher audiobookCoverCacher, contentID string, folderPath string) error { @@ -108,11 +108,10 @@ func applyAudiobookSidecarCover(ctx context.Context, store audiobookCoverMetadat if strings.TrimSpace(existingPosterPath) != "" { return nil } - basePath, ext, thumbhash, err := cacher.CacheAudiobookCover(ctx, cover, contentID) + posterPath, thumbhash, err := cacher.CacheAudiobookCover(ctx, cover, contentID) if err != nil { return err } - posterPath := strings.TrimRight(basePath, "/") + "/original" + ext update := &catalog.MetadataUpdate{PosterPath: &posterPath} if thumbhash != "" { update.PosterThumbhash = &thumbhash diff --git a/internal/scanner/audiobook_cover_test.go b/internal/scanner/audiobook_cover_test.go index 5fea0788..94cee004 100644 --- a/internal/scanner/audiobook_cover_test.go +++ b/internal/scanner/audiobook_cover_test.go @@ -9,6 +9,8 @@ import ( "github.com/Silo-Server/silo-server/internal/catalog" ) +const audiobookCoverTestThumbhash = "thumb" + type fakeAudiobookCoverCacher struct { calls int data []byte @@ -16,14 +18,14 @@ type fakeAudiobookCoverCacher struct { err error } -func (f *fakeAudiobookCoverCacher) CacheAudiobookCover(_ context.Context, data []byte, contentID string) (string, string, string, error) { +func (f *fakeAudiobookCoverCacher) CacheAudiobookCover(_ context.Context, data []byte, contentID string) (string, string, error) { f.calls++ f.data = append([]byte(nil), data...) f.contentID = contentID if f.err != nil { - return "", "", "", f.err + return "", "", f.err } - return "local/audiobooks/" + contentID + "/poster", ".webp", "thumb", nil + return "local/audiobooks/" + contentID + "/poster/original.test-revision.webp", audiobookCoverTestThumbhash, nil } type fakeAudiobookCoverStore struct { @@ -60,7 +62,7 @@ func TestApplyAudiobookSidecarCoverCachesFolderCover(t *testing.T) { if cacher.calls != 1 || string(cacher.data) != "audiobook-sidecar-cover" { t.Fatalf("cache call = calls %d data %q", cacher.calls, string(cacher.data)) } - if store.update == nil || store.update.PosterPath == nil || *store.update.PosterPath != "local/audiobooks/content-1/poster/original.webp" { + if store.update == nil || store.update.PosterPath == nil || *store.update.PosterPath != "local/audiobooks/content-1/poster/original.test-revision.webp" { t.Fatalf("poster update = %#v", store.update) } if store.update.PosterThumbhash == nil || *store.update.PosterThumbhash != "thumb" { diff --git a/internal/scanner/audiobook_scan.go b/internal/scanner/audiobook_scan.go index ce2b490c..165b0c17 100644 --- a/internal/scanner/audiobook_scan.go +++ b/internal/scanner/audiobook_scan.go @@ -319,14 +319,14 @@ func (s *Scanner) ScanAudiobookFolder(ctx context.Context, folder *models.MediaF for i := range scans { candidates = append(candidates, scans[i].candidates...) } - reconcileRoots, seenPaths, sawFiles := splitAudiobookReconcileRoots(scans) + reconcileRoots, seenPaths, _ := splitAudiobookReconcileRoots(scans) if len(candidates) == 0 { // No books on disk. Still reconcile (guarded) so a legitimately-emptied // library converges — but only when a root was readable, and the // empty-walk guard requires operator confirmation before deleting. if len(reconcileRoots) > 0 { - if err := s.reconcileAudiobookMissingFiles(ctx, folder, reconcileRoots, seenPaths, sawFiles, fullScan); err != nil { + if err := s.reconcileAudiobookMissingFiles(ctx, folder, reconcileRoots, seenPaths, fullScan); err != nil { slog.WarnContext(ctx, "audiobook scan: missing-file reconcile failed", "component", "scanner", "folder_id", folder.ID, "error", err) } } else if len(scans) > 0 { @@ -431,7 +431,7 @@ func (s *Scanner) ScanAudiobookFolder(ctx context.Context, folder *models.MediaF // Reconcile files that vanished from disk now that the full walk's // seenPaths is known and the scan completed without cancellation. - if err := s.reconcileAudiobookMissingFiles(ctx, folder, reconcileRoots, seenPaths, sawFiles, fullScan); err != nil { + if err := s.reconcileAudiobookMissingFiles(ctx, folder, reconcileRoots, seenPaths, fullScan); err != nil { slog.WarnContext(ctx, "audiobook scan: missing-file reconcile failed", "component", "scanner", "folder_id", folder.ID, "error", err) } return nil @@ -444,38 +444,19 @@ func (s *Scanner) ScanAudiobookFolder(ctx context.Context, folder *models.MediaF // the newly indexed path instead of leaving a stale duplicate). A scan that saw // zero files while the DB still has rows only reconciles when the operator has // confirmed cleanup, so an unmounted source can't wipe the catalog. -func (s *Scanner) reconcileAudiobookMissingFiles(ctx context.Context, folder *models.MediaFolder, roots []string, seenPaths map[string]bool, sawFiles, fullScan bool) error { +func (s *Scanner) reconcileAudiobookMissingFiles(ctx context.Context, folder *models.MediaFolder, roots []string, seenPaths map[string]bool, fullScan bool) error { if s.fileRepo == nil || s.libraryRepo == nil || len(roots) == 0 { return nil } - if !sawFiles { - existingCount := 0 - for _, root := range roots { - existing, err := s.fileRepo.GetByFolderAndPathPrefix(ctx, folder.ID, root) - if err != nil { - return fmt.Errorf("listing existing audiobook files for %q: %w", root, err) - } - existingCount += len(existing) - } - if existingCount > 0 { - var guard ebookCleanupGuardRepo - if s.folderRepo != nil { - guard = s.folderRepo - } - // Only a full scan may consume the operator's one-shot empty-cleanup - // allowance; an incremental/subtree scan that happens to see zero - // files must not (mirrors the ebook path). - allowed, err := ebookEmptyCleanupAllowed(ctx, guard, folder.ID, fullScan) - if err != nil { - return err - } - if !allowed { - slog.WarnContext(ctx, "audiobook scan: walk saw zero files but the database has files under the scanned roots; skipping reconciliation until cleanup is confirmed", "component", "scanner", - "folder_id", folder.ID, "existing_files", existingCount) - return nil - } - } + confirmedCleanup, blockAll, err := s.emptyCleanupDecision( + ctx, folder, roots, seenPaths, fullScan, "audiobook", + ) + if err != nil { + return err + } + if blockAll { + return nil } now := time.Now().UTC() @@ -500,25 +481,12 @@ func (s *Scanner) reconcileAudiobookMissingFiles(ctx context.Context, folder *mo } } - if s.emptyTrashAfterScan { - trashed, err := s.fileRepo.DeleteMissingByFolder(ctx, folder.ID, s.fileRemovalGrace) - if err != nil { - return fmt.Errorf("emptying trash for folder %d: %w", folder.ID, err) - } - if trashed > 0 { - slog.InfoContext(ctx, "audiobook scan: emptied trash", "component", "scanner", "folder_id", folder.ID, "deleted", trashed) - } + trashed, removedMemberships, deletedItems, err := s.sweepMissingAndReconcile(ctx, folder, confirmedCleanup) + if trashed > 0 { + slog.InfoContext(ctx, "audiobook scan: emptied trash", "component", "scanner", "folder_id", folder.ID, "deleted", trashed) } - - removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID) if err != nil { - return fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err) - } - if s.s3Client != nil && len(orphanedImageDirs) > 0 { - bucket := s.s3Client.Bucket() - for _, dir := range orphanedImageDirs { - _, _ = s.s3Client.DeletePrefix(ctx, bucket, dir) - } + return err } if missing > 0 || removedMemberships > 0 || deletedItems > 0 { slog.InfoContext(ctx, "audiobook scan: reconciled missing files", "component", "scanner", diff --git a/internal/scanner/audiobook_test.go b/internal/scanner/audiobook_test.go index 083c307f..5cebd60d 100644 --- a/internal/scanner/audiobook_test.go +++ b/internal/scanner/audiobook_test.go @@ -67,11 +67,11 @@ type fakeScannerCoverCacher struct { contentID string } -func (f *fakeScannerCoverCacher) CacheAudiobookCover(_ context.Context, data []byte, contentID string) (string, string, string, error) { +func (f *fakeScannerCoverCacher) CacheAudiobookCover(_ context.Context, data []byte, contentID string) (string, string, error) { f.calls++ f.data = append([]byte(nil), data...) f.contentID = contentID - return "local/audiobooks/" + contentID + "/poster", ".webp", "thumbhash", nil + return "local/audiobooks/" + contentID + "/poster/original.test-revision.webp", "thumbhash", nil } func TestApplyAudiobookEmbeddedCoverStoresPosterDuringScan(t *testing.T) { @@ -108,7 +108,7 @@ func TestApplyAudiobookEmbeddedCoverStoresPosterDuringScan(t *testing.T) { if len(exec.args) != 3 { t.Fatalf("Exec args = %#v, want 3 args", exec.args) } - if exec.args[0] != "local/audiobooks/content-1/poster/original.webp" { + if exec.args[0] != "local/audiobooks/content-1/poster/original.test-revision.webp" { t.Fatalf("poster arg = %#v", exec.args[0]) } if exec.args[1] != "thumbhash" || exec.args[2] != "content-1" { diff --git a/internal/scanner/dead_root_test.go b/internal/scanner/dead_root_test.go new file mode 100644 index 00000000..af7b5d3b --- /dev/null +++ b/internal/scanner/dead_root_test.go @@ -0,0 +1,965 @@ +package scanner + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func TestRootCoverageClauses(t *testing.T) { + t.Parallel() + + const moviesRoot = "/mnt/movies" + clauses, args := rootCoverageClauses([]string{moviesRoot, "/mnt/tv_shows"}, 3) + if len(clauses) != 2 { + t.Fatalf("clauses len = %d, want 2 (%v)", len(clauses), clauses) + } + if clauses[0] != `(file_path = $3 OR file_path LIKE $4 ESCAPE '\')` { + t.Fatalf("clauses[0] = %q", clauses[0]) + } + if clauses[1] != `(file_path = $5 OR file_path LIKE $6 ESCAPE '\')` { + t.Fatalf("clauses[1] = %q", clauses[1]) + } + + want := []any{moviesRoot, moviesRoot + "/%", "/mnt/tv_shows", `/mnt/tv\_shows/%`} + if len(args) != len(want) { + t.Fatalf("args = %v, want %v", args, want) + } + for i := range want { + if args[i] != want[i] { + t.Fatalf("args[%d] = %v, want %v", i, args[i], want[i]) + } + } + + // The prefix pattern must end with a separator so a sibling root sharing a + // string prefix (/mnt/movies2) can never match /mnt/movies. + pattern, ok := args[1].(string) + if !ok || !strings.HasSuffix(pattern, string(filepath.Separator)+"%") { + t.Fatalf("prefix pattern %v does not enforce a path separator boundary", args[1]) + } + + if clauses, args := rootCoverageClauses(nil, 1); len(clauses) != 0 || len(args) != 0 { + t.Fatalf("rootCoverageClauses(nil) = %v, %v; want empty", clauses, args) + } +} + +func TestDeadRootWarningMessage(t *testing.T) { + t.Parallel() + + got := deadRootWarningMessage(2, []string{"/mnt/movies"}, nil) + want := "1 of 2 roots unreachable: /mnt/movies" + if got != want { + t.Fatalf("deadRootWarningMessage = %q, want %q", got, want) + } + + got = deadRootWarningMessage(3, []string{"/a", "/b"}, nil) + want = "2 of 3 roots unreachable: /a, /b" + if got != want { + t.Fatalf("deadRootWarningMessage = %q, want %q", got, want) + } + + got = deadRootWarningMessage(2, nil, []string{"/mnt/movies"}) + want = "1 of 2 roots returned no files while the library still has cataloged files (lost mount?): /mnt/movies" + if got != want { + t.Fatalf("deadRootWarningMessage = %q, want %q", got, want) + } + + got = deadRootWarningMessage(3, []string{"/a"}, []string{"/b"}) + want = "1 of 3 roots unreachable: /a; 1 of 3 roots returned no files while the library still has cataloged files (lost mount?): /b" + if got != want { + t.Fatalf("deadRootWarningMessage = %q, want %q", got, want) + } +} + +func TestProbeUnreachableRoots(t *testing.T) { + t.Parallel() + + alive := t.TempDir() + dead := filepath.Join(t.TempDir(), "gone") + + got := probeUnreachableRoots(context.Background(), 1, []string{alive, dead}) + if len(got) != 1 || got[0] != dead { + t.Fatalf("probeUnreachableRoots = %v, want [%s]", got, dead) + } + if got := probeUnreachableRoots(context.Background(), 1, []string{alive}); len(got) != 0 { + t.Fatalf("probeUnreachableRoots(all alive) = %v, want empty", got) + } +} + +func newDeadRootTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + pool, err := pgxpool.New(context.Background(), dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + return pool +} + +func seedDeadRootTestFolder(t *testing.T, pool *pgxpool.Pool, folderType, name string) int { + t.Helper() + ctx := context.Background() + var folderID int + if err := pool.QueryRow(ctx, + `INSERT INTO media_folders (type, name, enabled) VALUES ($1, $2, true) RETURNING id`, + folderType, name, + ).Scan(&folderID); err != nil { + t.Fatalf("seed folder: %v", err) + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_files WHERE media_folder_id = $1`, folderID) + _, _ = pool.Exec(ctx, `DELETE FROM media_folders WHERE id = $1`, folderID) + }) + return folderID +} + +// TestDeleteMissingByFolderProtectedRoots covers the trash-sweep guard at the +// repository level: rows under a protected (unreachable) root survive the +// sweep no matter how stale their missing_since is, sibling roots that merely +// share a string prefix are NOT protected, and an empty protected set +// preserves the historical folder-wide sweep. +func TestDeleteMissingByFolderProtectedRoots(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Dead Root Sweep Test") + + base := fmt.Sprintf("/drp-sweep-%d", time.Now().UnixNano()) + protectedRoot := base + "/movies" + staleSince := time.Now().UTC().Add(-48 * time.Hour) + + seed := func(path string) int { + var id int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_files (media_folder_id, file_path, file_size, missing_since) + VALUES ($1, $2, 1024, $3) RETURNING id + `, folderID, path, staleSince).Scan(&id); err != nil { + t.Fatalf("seed media file %s: %v", path, err) + } + return id + } + protectedID := seed(protectedRoot + "/Alpha (2020)/Alpha (2020).mkv") + seed(base + "/movies2/Beta (2021)/Beta (2021).mkv") // sibling string prefix + seed(base + "/other/Gamma (2022)/Gamma (2022).mkv") + + repo := NewFileRepository(pool) + deleted, err := repo.DeleteMissingByFolder(ctx, folderID, 24*time.Hour, []string{protectedRoot}) + if err != nil { + t.Fatalf("DeleteMissingByFolder with protection: %v", err) + } + if deleted != 2 { + t.Fatalf("deleted = %d, want 2 (sibling-prefix and unrelated rows)", deleted) + } + var remaining int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM media_files WHERE media_folder_id = $1`, folderID, + ).Scan(&remaining); err != nil { + t.Fatalf("count remaining: %v", err) + } + if remaining != 1 { + t.Fatalf("remaining rows = %d, want 1 (the protected row)", remaining) + } + var stillThere bool + if err := pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM media_files WHERE id = $1)`, protectedID, + ).Scan(&stillThere); err != nil { + t.Fatalf("check protected row: %v", err) + } + if !stillThere { + t.Fatal("protected row was deleted") + } + + // Without protection the sweep behaves exactly as before and removes it. + deleted, err = repo.DeleteMissingByFolder(ctx, folderID, 24*time.Hour, nil) + if err != nil { + t.Fatalf("DeleteMissingByFolder without protection: %v", err) + } + if deleted != 1 { + t.Fatalf("deleted = %d, want 1", deleted) + } +} + +// TestScanFolderDeadRootProtection walks the real scan pipeline end to end +// with two on-disk roots and verifies the full dead-root story: +// +// 1. a root that disappears has its files marked missing but never +// hard-deleted, even with trash emptying enabled and a zero grace +// (which would delete them in the very same scan without protection), +// and the folder surfaces a dead_root scan warning naming the root; +// 2. when the root comes back the rows resurrect in place (same ids, +// missing_since cleared) and the warning clears; +// 3. deleting a file under a reachable root still purges its row after the +// grace elapses (regression: the historical sweep is untouched). +func TestScanFolderDeadRootProtection(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Dead Root Scan Test") + + base := t.TempDir() + rootA := filepath.Join(base, "libraryA") + rootB := filepath.Join(base, "libraryB") + fileA := filepath.Join(rootA, "Alpha (2020)", "Alpha (2020).mkv") + fileB := filepath.Join(rootB, "Beta (2021)", "Beta (2021).mkv") + writeMovie := func(path string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + writeMovie(fileA) + writeMovie(fileB) + + folder := &models.MediaFolder{ + ID: folderID, + Paths: []string{rootA, rootB}, + Type: "movies", + Name: "Dead Root Scan Test", + Enabled: true, + } + + // emptyTrashAfterScan=true with zero grace: a missing row is eligible for + // deletion in the very scan that marks it missing. + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + fileRow := func(path string) (id int, missingSince *time.Time, found bool) { + t.Helper() + err := pool.QueryRow(ctx, + `SELECT id, missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, path, + ).Scan(&id, &missingSince) + if err != nil { + if strings.Contains(err.Error(), "no rows") { + return 0, nil, false + } + t.Fatalf("query file row %s: %v", path, err) + } + return id, missingSince, true + } + warning := func() (code, message *string) { + t.Helper() + if err := pool.QueryRow(ctx, + `SELECT scan_warning_code, scan_warning_message FROM media_folders WHERE id = $1`, + folderID, + ).Scan(&code, &message); err != nil { + t.Fatalf("query scan warning: %v", err) + } + return code, message + } + + // Scan 1: both roots healthy. + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 1: %v", err) + } + idA, missingA, foundA := fileRow(fileA) + idB, missingB, foundB := fileRow(fileB) + if !foundA || !foundB { + t.Fatalf("after scan 1: foundA=%v foundB=%v, want both rows", foundA, foundB) + } + if missingA != nil || missingB != nil { + t.Fatalf("after scan 1: missingA=%v missingB=%v, want both nil", missingA, missingB) + } + + // Root B dies (unmounted / dead drive). + if err := os.RemoveAll(rootB); err != nil { + t.Fatalf("remove rootB: %v", err) + } + + // Scan 2: files under the dead root are marked missing (hidden) but the + // row must survive the sweep, and the folder must carry a dead_root + // warning naming the root. + result, err := scanner.ScanFolder(ctx, folder) + if err != nil { + t.Fatalf("scan 2: %v", err) + } + if len(result.UnreachableRoots) != 1 || result.UnreachableRoots[0] != rootB { + t.Fatalf("scan 2 UnreachableRoots = %v, want [%s]", result.UnreachableRoots, rootB) + } + if _, missingA, foundA = fileRow(fileA); !foundA || missingA != nil { + t.Fatalf("after scan 2: fileA found=%v missing=%v, want present and not missing", foundA, missingA) + } + gotIDB, missingB, foundB := fileRow(fileB) + if !foundB { + t.Fatal("after scan 2: fileB row was hard-deleted; dead-root protection failed") + } + if missingB == nil { + t.Fatal("after scan 2: fileB not marked missing; it should be hidden") + } + if gotIDB != idB { + t.Fatalf("after scan 2: fileB id changed %d -> %d", idB, gotIDB) + } + code, message := warning() + if code == nil || *code != "dead_root" { + t.Fatalf("after scan 2: scan_warning_code = %v, want dead_root", code) + } + if message == nil || !strings.Contains(*message, rootB) { + t.Fatalf("after scan 2: scan_warning_message = %v, want to contain %q", message, rootB) + } + + // Rescan while still dead: row keeps surviving (grace long since elapsed). + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 2b: %v", err) + } + if _, _, foundB = fileRow(fileB); !foundB { + t.Fatal("after scan 2b: fileB row was hard-deleted on rescan") + } + + // Root B returns: the same row resurrects (same id, missing cleared) and + // the warning clears. + writeMovie(fileB) + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 3: %v", err) + } + gotIDB, missingB, foundB = fileRow(fileB) + if !foundB || missingB != nil { + t.Fatalf("after scan 3: fileB found=%v missing=%v, want resurrected", foundB, missingB) + } + if gotIDB != idB { + t.Fatalf("after scan 3: fileB resurrected under a new id %d, want original %d", gotIDB, idB) + } + if code, _ := warning(); code != nil { + t.Fatalf("after scan 3: scan_warning_code = %q, want cleared", *code) + } + if _, missingA, _ = fileRow(fileA); missingA != nil { + t.Fatalf("after scan 3: fileA missing = %v, want nil", missingA) + } + _ = idA + + // Regression: deleting one FILE under a reachable root still purges its + // row once the grace (zero here) elapses — reachable-root semantics are + // unchanged. + if err := os.Remove(fileB); err != nil { + t.Fatalf("remove fileB: %v", err) + } + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 4: %v", err) + } + if _, _, foundB = fileRow(fileB); foundB { + t.Fatal("after scan 4: fileB row still present; reachable-root purge regressed") + } + if _, _, foundA = fileRow(fileA); !foundA { + t.Fatal("after scan 4: fileA row vanished unexpectedly") + } + if code, _ := warning(); code != nil { + t.Fatalf("after scan 4: scan_warning_code = %q, want none", *code) + } +} + +// TestScanFolderNestedDeadChildRootProtection covers a child mount configured +// INSIDE a reachable parent root (/parent plus /parent/child). Traversal +// compaction drops the child, but it can die independently: its files must be +// protected from the sweep and the folder must warn, even though the parent +// scan is otherwise healthy. +func TestScanFolderNestedDeadChildRootProtection(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Nested Dead Root Scan Test") + + base := t.TempDir() + parent := filepath.Join(base, "media") + child := filepath.Join(parent, "drive") + fileParent := filepath.Join(parent, "Alpha (2020)", "Alpha (2020).mkv") + fileChild := filepath.Join(child, "Beta (2021)", "Beta (2021).mkv") + writeMovie := func(path string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + writeMovie(fileParent) + writeMovie(fileChild) + + folder := &models.MediaFolder{ + ID: folderID, + Paths: []string{parent, child}, + Type: "movies", + Name: "Nested Dead Root Scan Test", + Enabled: true, + } + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 1: %v", err) + } + var childID int + var childMissing *time.Time + if err := pool.QueryRow(ctx, + `SELECT id, missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, fileChild, + ).Scan(&childID, &childMissing); err != nil { + t.Fatalf("child row after scan 1: %v", err) + } + if childMissing != nil { + t.Fatalf("child missing after scan 1: %v, want nil", childMissing) + } + + // The child mount dies while the parent stays reachable. Compaction hides + // the child from traversal, so only the uncompacted probe can protect it. + if err := os.RemoveAll(child); err != nil { + t.Fatalf("remove child root: %v", err) + } + result, err := scanner.ScanFolder(ctx, folder) + if err != nil { + t.Fatalf("scan 2: %v", err) + } + if len(result.UnreachableRoots) != 1 || result.UnreachableRoots[0] != child { + t.Fatalf("scan 2 UnreachableRoots = %v, want [%s]", result.UnreachableRoots, child) + } + var gotID int + if err := pool.QueryRow(ctx, + `SELECT id, missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, fileChild, + ).Scan(&gotID, &childMissing); err != nil { + t.Fatalf("child row after scan 2 (was it hard-deleted?): %v", err) + } + if childMissing == nil { + t.Fatal("child file not marked missing after its root died") + } + if gotID != childID { + t.Fatalf("child row id changed %d -> %d", childID, gotID) + } + var code, message *string + if err := pool.QueryRow(ctx, + `SELECT scan_warning_code, scan_warning_message FROM media_folders WHERE id = $1`, + folderID, + ).Scan(&code, &message); err != nil { + t.Fatalf("query warning: %v", err) + } + if code == nil || *code != "dead_root" { + t.Fatalf("scan_warning_code = %v, want dead_root", code) + } + if message == nil || !strings.Contains(*message, child) { + t.Fatalf("scan_warning_message = %v, want to contain %q", message, child) + } +} + +func TestScanFolderNestedSuspectEmptyChildRootProtection(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Nested Suspect Root Scan Test") + + base := t.TempDir() + parent := filepath.Join(base, "media") + child := filepath.Join(parent, "drive") + parentFile := filepath.Join(parent, "Alpha (2020)", "Alpha (2020).mkv") + childFile := filepath.Join(child, "Beta (2021)", "Beta (2021).mkv") + for _, path := range []string{parentFile, childFile} { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + + folder := &models.MediaFolder{ + ID: folderID, Paths: []string{parent, child}, Type: "movies", + Name: "Nested Suspect Root Scan Test", Enabled: true, + } + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 1: %v", err) + } + + var childID int + if err := pool.QueryRow(ctx, + `SELECT id FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, childFile, + ).Scan(&childID); err != nil { + t.Fatalf("child row after scan 1: %v", err) + } + if err := os.RemoveAll(filepath.Dir(childFile)); err != nil { + t.Fatalf("empty child mountpoint: %v", err) + } + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatalf("restore empty child mountpoint: %v", err) + } + + result, err := scanner.ScanFolder(ctx, folder) + if err != nil { + t.Fatalf("scan 2: %v", err) + } + if len(result.SuspectEmptyRoots) != 1 || result.SuspectEmptyRoots[0] != child { + t.Fatalf("SuspectEmptyRoots = %v, want [%s]", result.SuspectEmptyRoots, child) + } + var gotID int + var missing *time.Time + if err := pool.QueryRow(ctx, + `SELECT id, missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, childFile, + ).Scan(&gotID, &missing); err != nil { + t.Fatalf("child row after scan 2 (was it deleted?): %v", err) + } + if gotID != childID || missing == nil { + t.Fatalf("child row id/missing = %d/%v, want %d/non-nil", gotID, missing, childID) + } +} + +// TestScanFolderAllRootsDeadOutage covers the single-drive-library outage: +// when EVERY configured root is unreachable, the scan must bypass the +// empty-root confirm flow (without consuming the operator's one-time cleanup +// allowance), mark all files missing so they hide, keep every row, and raise +// dead_root — not empty_root. +func TestScanFolderAllRootsDeadOutage(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "All Roots Dead Scan Test") + + base := t.TempDir() + root := filepath.Join(base, "movies") + file := filepath.Join(root, "Alpha (2020)", "Alpha (2020).mkv") + if err := os.MkdirAll(filepath.Dir(file), 0o755); err != nil { + t.Fatalf("mkdir: %v", err) + } + if err := os.WriteFile(file, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write: %v", err) + } + + folder := &models.MediaFolder{ + ID: folderID, + Paths: []string{root}, + Type: "movies", + Name: "All Roots Dead Scan Test", + Enabled: true, + } + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 1: %v", err) + } + + // Arm the one-time cleanup allowance so we can prove the outage path does + // NOT consume it (it must stay reserved for a deliberate empty-root scan). + if _, err := pool.Exec(ctx, + `UPDATE media_folders SET allow_empty_cleanup_once = true WHERE id = $1`, folderID, + ); err != nil { + t.Fatalf("arm cleanup allowance: %v", err) + } + + if err := os.RemoveAll(root); err != nil { + t.Fatalf("remove root: %v", err) + } + result, err := scanner.ScanFolder(ctx, folder) + if err != nil { + t.Fatalf("scan 2: %v", err) + } + if result.EmptyRootGuarded { + t.Fatal("scan 2 reported EmptyRootGuarded; all-dead outage should take the dead_root path") + } + + var missing *time.Time + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, file, + ).Scan(&missing); err != nil { + t.Fatalf("file row after scan 2 (was it hard-deleted?): %v", err) + } + if missing == nil { + t.Fatal("file not marked missing during all-roots-dead outage") + } + + var code *string + var allowance bool + if err := pool.QueryRow(ctx, + `SELECT scan_warning_code, allow_empty_cleanup_once FROM media_folders WHERE id = $1`, + folderID, + ).Scan(&code, &allowance); err != nil { + t.Fatalf("query folder state: %v", err) + } + if code == nil || *code != "dead_root" { + t.Fatalf("scan_warning_code = %v, want dead_root (not empty_root)", code) + } + if !allowance { + t.Fatal("outage scan consumed the empty-cleanup allowance; it must be preserved") + } +} + +func TestListRootsWithOnlyMissingFiles(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Suspect Root Query Test") + + base := fmt.Sprintf("/drp-suspect-%d", time.Now().UnixNano()) + allMissing := base + "/gone" + mixed := base + "/mixed" + empty := base + "/empty" + stale := time.Now().UTC().Add(-48 * time.Hour) + + seed := func(path string, missing *time.Time) { + t.Helper() + if _, err := pool.Exec(ctx, ` + INSERT INTO media_files (media_folder_id, file_path, file_size, missing_since) + VALUES ($1, $2, 1024, $3) + `, folderID, path, missing); err != nil { + t.Fatalf("seed media file %s: %v", path, err) + } + } + seed(allMissing+"/Alpha (2020)/Alpha (2020).mkv", &stale) + seed(allMissing+"/Beta (2021)/Beta (2021).mkv", &stale) + seed(mixed+"/Gamma (2022)/Gamma (2022).mkv", &stale) + seed(mixed+"/Delta (2023)/Delta (2023).mkv", nil) + + repo := NewFileRepository(pool) + got, err := repo.ListRootsWithOnlyMissingFiles(ctx, folderID, []string{allMissing, mixed, empty}) + if err != nil { + t.Fatalf("ListRootsWithOnlyMissingFiles: %v", err) + } + if len(got) != 1 || got[0] != allMissing { + t.Fatalf("suspect roots = %v, want [%s]", got, allMissing) + } + + if got, err := repo.ListRootsWithOnlyMissingFiles(ctx, folderID, nil); err != nil || len(got) != 0 { + t.Fatalf("ListRootsWithOnlyMissingFiles(nil) = %v, %v; want empty", got, err) + } +} + +// TestScanFolderSuspectEmptyRootProtection covers the most common lost-mount +// presentation: the mount drops out but leaves an empty, stat-able mountpoint +// directory behind, so the reachability probe reports the root healthy. The +// walk finds zero files while rows remain cataloged: those rows must only be +// marked missing (surviving a zero-grace sweep), the folder must raise +// dead_root, and a later scan with the operator's one-time cleanup allowance +// armed must complete the deletion and clear the warning. +func TestScanFolderSuspectEmptyRootProtection(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Suspect Empty Root Scan Test") + + base := t.TempDir() + rootA := filepath.Join(base, "libraryA") + rootB := filepath.Join(base, "libraryB") + fileA := filepath.Join(rootA, "Alpha (2020)", "Alpha (2020).mkv") + fileB := filepath.Join(rootB, "Beta (2021)", "Beta (2021).mkv") + writeMovie := func(path string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + writeMovie(fileA) + writeMovie(fileB) + + folder := &models.MediaFolder{ + ID: folderID, + Paths: []string{rootA, rootB}, + Type: "movies", + Name: "Suspect Empty Root Scan Test", + Enabled: true, + } + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + fileRow := func(path string) (id int, missingSince *time.Time, found bool) { + t.Helper() + err := pool.QueryRow(ctx, + `SELECT id, missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, path, + ).Scan(&id, &missingSince) + if err != nil { + if strings.Contains(err.Error(), "no rows") { + return 0, nil, false + } + t.Fatalf("query file row %s: %v", path, err) + } + return id, missingSince, true + } + warning := func() (code, message *string) { + t.Helper() + if err := pool.QueryRow(ctx, + `SELECT scan_warning_code, scan_warning_message FROM media_folders WHERE id = $1`, + folderID, + ).Scan(&code, &message); err != nil { + t.Fatalf("query scan warning: %v", err) + } + return code, message + } + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 1: %v", err) + } + idB, _, foundB := fileRow(fileB) + if !foundB { + t.Fatal("after scan 1: fileB row not found") + } + + // Root B's mount drops out, leaving the empty mountpoint directory. + if err := os.RemoveAll(filepath.Join(rootB, "Beta (2021)")); err != nil { + t.Fatalf("empty rootB: %v", err) + } + + result, err := scanner.ScanFolder(ctx, folder) + if err != nil { + t.Fatalf("scan 2: %v", err) + } + if len(result.UnreachableRoots) != 0 { + t.Fatalf("scan 2 UnreachableRoots = %v, want empty (root still probes reachable)", result.UnreachableRoots) + } + if len(result.SuspectEmptyRoots) != 1 || result.SuspectEmptyRoots[0] != rootB { + t.Fatalf("scan 2 SuspectEmptyRoots = %v, want [%s]", result.SuspectEmptyRoots, rootB) + } + gotIDB, missingB, foundB := fileRow(fileB) + if !foundB { + t.Fatal("after scan 2: fileB row was hard-deleted; suspect-empty protection failed") + } + if missingB == nil { + t.Fatal("after scan 2: fileB not marked missing; it should be hidden") + } + if gotIDB != idB { + t.Fatalf("after scan 2: fileB id changed %d -> %d", idB, gotIDB) + } + code, message := warning() + if code == nil || *code != "dead_root" { + t.Fatalf("after scan 2: scan_warning_code = %v, want dead_root", code) + } + if message == nil || !strings.Contains(*message, rootB) { + t.Fatalf("after scan 2: scan_warning_message = %v, want to contain %q", message, rootB) + } + + // Rescan without confirmation: the rows keep surviving. + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 2b: %v", err) + } + if _, _, foundB = fileRow(fileB); !foundB { + t.Fatal("after scan 2b: fileB row was hard-deleted on rescan") + } + + // The operator confirms the root really is meant to be empty. + if _, err := pool.Exec(ctx, + `UPDATE media_folders SET allow_empty_cleanup_once = true WHERE id = $1`, folderID, + ); err != nil { + t.Fatalf("arm cleanup allowance: %v", err) + } + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 3: %v", err) + } + if _, _, foundB = fileRow(fileB); foundB { + t.Fatal("after scan 3: fileB row still present; confirmed cleanup did not complete") + } + if _, _, foundA := fileRow(fileA); !foundA { + t.Fatal("after scan 3: fileA row vanished unexpectedly") + } + if code, _ := warning(); code != nil { + t.Fatalf("after scan 3: scan_warning_code = %q, want cleared", *code) + } + var allowance bool + if err := pool.QueryRow(ctx, + `SELECT allow_empty_cleanup_once FROM media_folders WHERE id = $1`, folderID, + ).Scan(&allowance); err != nil { + t.Fatalf("query allowance: %v", err) + } + if allowance { + t.Fatal("after scan 3: one-time cleanup allowance was not consumed") + } +} + +// TestScanFolderConfirmedCleanupPreservesDeadRoot pins the confirmed +// empty-cleanup path against dead roots: arming the one-time allowance to +// clean a reachable, intentionally emptied root must not erase a probe-dead +// sibling root's catalog — an outage is never a confirmation. +func TestScanFolderConfirmedCleanupPreservesDeadRoot(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "movies", "Confirmed Cleanup Dead Root Test") + + base := t.TempDir() + rootA := filepath.Join(base, "libraryA") + rootB := filepath.Join(base, "libraryB") + fileA := filepath.Join(rootA, "Alpha (2020)", "Alpha (2020).mkv") + fileB := filepath.Join(rootB, "Beta (2021)", "Beta (2021).mkv") + writeMovie := func(path string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("mkdir %s: %v", filepath.Dir(path), err) + } + if err := os.WriteFile(path, []byte("fake movie payload"), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + writeMovie(fileA) + writeMovie(fileB) + + folder := &models.MediaFolder{ + ID: folderID, + Paths: []string{rootA, rootB}, + Type: "movies", + Name: "Confirmed Cleanup Dead Root Test", + Enabled: true, + } + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + + if _, err := scanner.ScanFolder(ctx, folder); err != nil { + t.Fatalf("scan 1: %v", err) + } + + // Root A dies outright; root B is intentionally emptied (dir remains). + if err := os.RemoveAll(rootA); err != nil { + t.Fatalf("remove rootA: %v", err) + } + if err := os.RemoveAll(filepath.Join(rootB, "Beta (2021)")); err != nil { + t.Fatalf("empty rootB: %v", err) + } + if _, err := pool.Exec(ctx, + `UPDATE media_folders SET allow_empty_cleanup_once = true WHERE id = $1`, folderID, + ); err != nil { + t.Fatalf("arm cleanup allowance: %v", err) + } + + result, err := scanner.ScanFolder(ctx, folder) + if err != nil { + t.Fatalf("scan 2: %v", err) + } + if result.EmptyRootGuarded { + t.Fatal("scan 2 reported EmptyRootGuarded despite the armed allowance") + } + if len(result.UnreachableRoots) != 1 || result.UnreachableRoots[0] != rootA { + t.Fatalf("scan 2 UnreachableRoots = %v, want [%s]", result.UnreachableRoots, rootA) + } + + var missingA *time.Time + if err := pool.QueryRow(ctx, + `SELECT missing_since FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, fileA, + ).Scan(&missingA); err != nil { + t.Fatalf("fileA row after scan 2 (was the dead root's catalog erased?): %v", err) + } + if missingA == nil { + t.Fatal("fileA not marked missing during the outage") + } + var countB int + if err := pool.QueryRow(ctx, + `SELECT count(*) FROM media_files WHERE media_folder_id = $1 AND file_path = $2`, + folderID, fileB, + ).Scan(&countB); err != nil { + t.Fatalf("count fileB rows: %v", err) + } + if countB != 0 { + t.Fatalf("fileB rows = %d, want 0 (confirmed cleanup of the reachable empty root)", countB) + } + + var code *string + var allowance bool + if err := pool.QueryRow(ctx, + `SELECT scan_warning_code, allow_empty_cleanup_once FROM media_folders WHERE id = $1`, + folderID, + ).Scan(&code, &allowance); err != nil { + t.Fatalf("query folder state: %v", err) + } + if code == nil || *code != "dead_root" { + t.Fatalf("scan_warning_code = %v, want dead_root", code) + } + if allowance { + t.Fatal("allowance was not consumed by the confirmed cleanup") + } +} + +// TestSweepMissingAndReconcileProtectsDeadRootsFromScopedScans pins the +// audiobook/ebook/podcast folder-wide sweep against two regressions at once: +// a scoped scan clone (ScanSubtree/ScanFile) whose Paths holds only the +// scanned subtree must still probe every CONFIGURED root (reloaded from the +// DB), and the probe must use the uncompacted list so a nested child mount +// inside a reachable parent is protected independently. +func TestSweepMissingAndReconcileProtectsDeadRootsFromScopedScans(t *testing.T) { + pool := newDeadRootTestPool(t) + ctx := context.Background() + folderID := seedDeadRootTestFolder(t, pool, "audiobooks", "Scoped Sweep Dead Root Test") + + base := t.TempDir() + parent := filepath.Join(base, "audio") + child := filepath.Join(parent, "drive") + if err := os.MkdirAll(child, 0o755); err != nil { + t.Fatalf("mkdir child: %v", err) + } + for _, p := range []string{parent, child} { + if _, err := pool.Exec(ctx, + `INSERT INTO media_folder_paths (media_folder_id, path) VALUES ($1, $2)`, + folderID, p, + ); err != nil { + t.Fatalf("seed folder path %s: %v", p, err) + } + } + t.Cleanup(func() { + _, _ = pool.Exec(ctx, `DELETE FROM media_folder_paths WHERE media_folder_id = $1`, folderID) + }) + + stale := time.Now().UTC().Add(-48 * time.Hour) + seed := func(path string, missing *time.Time) int { + t.Helper() + var id int + if err := pool.QueryRow(ctx, ` + INSERT INTO media_files (media_folder_id, file_path, file_size, missing_since) + VALUES ($1, $2, 1024, $3) RETURNING id + `, folderID, path, missing).Scan(&id); err != nil { + t.Fatalf("seed media file %s: %v", path, err) + } + return id + } + // A present book keeps the parent root non-suspect, a stale row under the + // parent is a genuine deletion the sweep must still purge, and a stale row + // under the (about to die) child mount must survive. + presentPath := filepath.Join(parent, "Book A", "a.m4b") + if err := os.MkdirAll(filepath.Dir(presentPath), 0o755); err != nil { + t.Fatalf("mkdir book: %v", err) + } + if err := os.WriteFile(presentPath, []byte("fake audio payload"), 0o644); err != nil { + t.Fatalf("write book: %v", err) + } + seed(presentPath, nil) + goneID := seed(filepath.Join(parent, "Book B", "b.m4b"), &stale) + childID := seed(filepath.Join(child, "Book C", "c.m4b"), &stale) + + // The nested child mount dies while the parent stays reachable. + if err := os.RemoveAll(child); err != nil { + t.Fatalf("remove child mount: %v", err) + } + + scanner := NewScanner(NewFileRepository(pool), "", nil, 2, true, 0) + // A scoped clone the way ScanSubtree/ScanFile build one: Paths is just the + // scanned subtree, not the configured roots. + scoped := scopedFolderPaths(&models.MediaFolder{ + ID: folderID, + Paths: []string{parent, child}, + Type: "audiobooks", + Name: "Scoped Sweep Dead Root Test", + Enabled: true, + }, []string{filepath.Join(parent, "Book A")}) + + if _, _, _, err := scanner.sweepMissingAndReconcile(ctx, scoped, false); err != nil { + t.Fatalf("sweepMissingAndReconcile: %v", err) + } + + var exists bool + if err := pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM media_files WHERE id = $1)`, childID, + ).Scan(&exists); err != nil { + t.Fatalf("check child row: %v", err) + } + if !exists { + t.Fatal("row under the dead nested child mount was hard-deleted by a scoped sweep") + } + if err := pool.QueryRow(ctx, + `SELECT EXISTS(SELECT 1 FROM media_files WHERE id = $1)`, goneID, + ).Scan(&exists); err != nil { + t.Fatalf("check gone row: %v", err) + } + if exists { + t.Fatal("genuinely deleted row under the reachable parent survived the sweep") + } +} diff --git a/internal/scanner/ebook_scan.go b/internal/scanner/ebook_scan.go index 2f10c42d..2d12c5c4 100644 --- a/internal/scanner/ebook_scan.go +++ b/internal/scanner/ebook_scan.go @@ -19,6 +19,7 @@ import ( "github.com/Silo-Server/silo-server/internal/idgen" "github.com/Silo-Server/silo-server/internal/imageutil" "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/rootcheck" "github.com/Silo-Server/silo-server/internal/titleutil" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" @@ -282,10 +283,89 @@ func ebookEmptyCleanupAllowed(ctx context.Context, repo ebookCleanupGuardRepo, f return allowed, nil } +// emptyCleanupDecision determines whether an audio/ebook scan may clean roots +// that contain catalog rows but produced no files. A completely empty scan is +// blocked until confirmation; in a mixed multi-root scan only literally empty +// roots need confirmation, while healthy roots continue reconciling normally. +func (s *Scanner) emptyCleanupDecision( + ctx context.Context, + folder *models.MediaFolder, + roots []string, + seenPaths map[string]bool, + fullScan bool, + mediaKind string, +) (confirmed, blockAll bool, err error) { + if s.fileRepo == nil || len(roots) == 0 { + return false, false, nil + } + wholeScanEmpty := len(seenPaths) == 0 + probes := rootcheck.ProbeManyWithTimeout(ctx, roots, rootcheck.DefaultProbeTimeout) + needsConfirmation := false + for i, root := range roots { + if !wholeScanEmpty { + seenUnderRoot := false + for path := range seenPaths { + if pathWithinAnyRoot(path, []string{root}) { + seenUnderRoot = true + break + } + } + if seenUnderRoot || !probes[i].Reachable || !probes[i].Empty { + continue + } + } else if !probes[i].Reachable { + continue + } + existing, listErr := s.fileRepo.GetByFolderAndPathPrefix(ctx, folder.ID, root) + if listErr != nil { + return false, false, fmt.Errorf("listing existing %s files for %q: %w", mediaKind, root, listErr) + } + if len(existing) > 0 { + needsConfirmation = true + break + } + } + if !needsConfirmation { + return false, false, nil + } + var guard ebookCleanupGuardRepo + if s.folderRepo != nil { + guard = s.folderRepo + } + allowed := false + if wholeScanEmpty { + var allowErr error + allowed, allowErr = ebookEmptyCleanupAllowed(ctx, guard, folder.ID, fullScan) + if allowErr != nil { + return false, false, allowErr + } + } else if fullScan && guard != nil { + var allowErr error + allowed, allowErr = guard.ConsumeEmptyCleanupAllowance(ctx, folder.ID) + if allowErr != nil { + return false, false, fmt.Errorf("checking partial-root cleanup confirmation for folder %d: %w", folder.ID, allowErr) + } + if !allowed { + if warnErr := guard.SetScanWarning(ctx, folder.ID, + "dead_root", + "One or more roots are reachable but empty while the library still has cataloged files; cleanup was skipped until deletion is confirmed.", + time.Now().UTC(), + ); warnErr != nil { + return false, false, fmt.Errorf("recording partial-root warning for folder %d: %w", folder.ID, warnErr) + } + } + } + if !allowed { + slog.WarnContext(ctx, mediaKind+" scan: empty roots still have cataloged files; cleanup requires confirmation", + "component", "scanner", "folder_id", folder.ID, "full_scan", fullScan) + } + return allowed, wholeScanEmpty && !allowed, nil +} + // reconcileEbookScan applies the post-walk safety policy and then performs // missing-file reconciliation for the roots that walked cleanly. func (s *Scanner) reconcileEbookScan(ctx context.Context, folder *models.MediaFolder, scans []ebookRootScan, seenPaths map[string]bool, fullScan bool) error { - reconcileRoots, sawFiles := splitEbookReconcileRoots(scans) + reconcileRoots, _ := splitEbookReconcileRoots(scans) if len(reconcileRoots) == 0 { if len(scans) > 0 { slog.WarnContext(ctx, "ebook scan: every root walk failed; skipping missing-file reconciliation", "component", "scanner", @@ -298,36 +378,17 @@ func (s *Scanner) reconcileEbookScan(ctx context.Context, folder *models.MediaFo return nil } - if !sawFiles { - existingCount := 0 - for _, root := range reconcileRoots { - existing, err := s.fileRepo.GetByFolderAndPathPrefix(ctx, folder.ID, root) - if err != nil { - return fmt.Errorf("listing existing ebook files for %q: %w", root, err) - } - existingCount += len(existing) - } - if existingCount > 0 { - var guard ebookCleanupGuardRepo - if s.folderRepo != nil { - guard = s.folderRepo - } - allowed, err := ebookEmptyCleanupAllowed(ctx, guard, folder.ID, fullScan) - if err != nil { - return err - } - if !allowed { - slog.WarnContext(ctx, "ebook scan: walk saw zero ebooks but the database has files under the scanned roots; skipping reconciliation until cleanup is confirmed", "component", "scanner", - "folder_id", folder.ID, - "existing_files", existingCount, - "full_scan", fullScan, - ) - return nil - } - } + confirmedCleanup, blockAll, err := s.emptyCleanupDecision( + ctx, folder, reconcileRoots, seenPaths, fullScan, "ebook", + ) + if err != nil { + return err + } + if blockAll { + return nil } - if err := s.reconcileMissingEbookFiles(ctx, folder, reconcileRoots, seenPaths); err != nil { + if err := s.reconcileMissingEbookFiles(ctx, folder, reconcileRoots, seenPaths, confirmedCleanup); err != nil { return err } if fullScan && s.folderRepo != nil { @@ -345,7 +406,7 @@ func (s *Scanner) reconcileEbookScan(ctx context.Context, folder *models.MediaFo // folder trash is optionally emptied, and library memberships are reconciled // so items with no remaining files are removed (renames therefore converge on // the newly indexed path instead of leaving a stale duplicate item). -func (s *Scanner) reconcileMissingEbookFiles(ctx context.Context, folder *models.MediaFolder, roots []string, seenPaths map[string]bool) error { +func (s *Scanner) reconcileMissingEbookFiles(ctx context.Context, folder *models.MediaFolder, roots []string, seenPaths map[string]bool, confirmedCleanup bool) error { if s.fileRepo == nil || s.libraryRepo == nil || len(roots) == 0 { return nil } @@ -375,36 +436,17 @@ func (s *Scanner) reconcileMissingEbookFiles(ctx context.Context, folder *models } } - if s.emptyTrashAfterScan { - trashed, err := s.fileRepo.DeleteMissingByFolder(ctx, folder.ID, s.fileRemovalGrace) - if err != nil { - return fmt.Errorf("emptying trash for folder %d: %w", folder.ID, err) - } - if trashed > 0 { - slog.InfoContext(ctx, "ebook scan: emptied trash", "component", "scanner", "folder_id", folder.ID, "deleted", trashed) - } + trashed, removedMemberships, deletedItems, err := s.sweepMissingAndReconcile(ctx, folder, confirmedCleanup) + if trashed > 0 { + slog.InfoContext(ctx, "ebook scan: emptied trash", "component", "scanner", "folder_id", folder.ID, "deleted", trashed) } - - removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID) if err != nil { - return fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err) + return err } - - // Best-effort S3 image cleanup for orphaned items. - if s.s3Client != nil && len(orphanedImageDirs) > 0 { - bucket := s.s3Client.Bucket() - for _, dir := range orphanedImageDirs { - _, _ = s.s3Client.DeletePrefix(ctx, bucket, dir) - } - } - if missing > 0 || removedMemberships > 0 || deletedItems > 0 { slog.InfoContext(ctx, "ebook scan: reconciled missing files", "component", "scanner", - "folder_id", folder.ID, - "missing", missing, - "memberships_removed", removedMemberships, - "items_deleted", deletedItems, - ) + "folder_id", folder.ID, "missing", missing, + "memberships_removed", removedMemberships, "items_deleted", deletedItems) } return nil } @@ -1077,11 +1119,10 @@ func cacheEbookCoverBytes(ctx context.Context, store ebookCoverMetadataStore, ca return nil } } - basePath, ext, thumbhash, err := cacher.CacheEbookCover(ctx, data, contentID) + posterPath, thumbhash, err := cacher.CacheEbookCover(ctx, data, contentID) if err != nil { return err } - posterPath := strings.TrimRight(basePath, "/") + "/original" + ext if _, err := store.SetLocalPoster(ctx, contentID, posterPath, thumbhash, localEbookPosterPrefix); err != nil { return fmt.Errorf("set ebook local poster: %w", err) } diff --git a/internal/scanner/ebook_test.go b/internal/scanner/ebook_test.go index d340441a..3c21aaa0 100644 --- a/internal/scanner/ebook_test.go +++ b/internal/scanner/ebook_test.go @@ -16,6 +16,8 @@ import ( "github.com/jackc/pgx/v5/pgconn" ) +const ebookCoverTestThumbhash = "thumb" + type recordingEbookExecutor struct { queries []string args [][]any @@ -43,14 +45,14 @@ type fakeEbookCoverCacher struct { err error } -func (f *fakeEbookCoverCacher) CacheEbookCover(_ context.Context, data []byte, contentID string) (string, string, string, error) { +func (f *fakeEbookCoverCacher) CacheEbookCover(_ context.Context, data []byte, contentID string) (string, string, error) { f.calls++ f.data = append([]byte(nil), data...) f.contentID = contentID if f.err != nil { - return "", "", "", f.err + return "", "", f.err } - return "local/ebooks/" + contentID + "/poster", ".webp", "thumb", nil + return "local/ebooks/" + contentID + "/poster/original.test-revision.webp", ebookCoverTestThumbhash, nil } type fakeEbookMetadataUpdater struct { @@ -1220,7 +1222,7 @@ func TestApplyEbookLocalCoverCachesEmbeddedAndSetsPoster(t *testing.T) { if updater.setCalls != 1 || updater.contentID != "content-1" { t.Fatalf("set call = calls %d content %q", updater.setCalls, updater.contentID) } - if updater.setPosterPath != "local/ebooks/content-1/poster/original.webp" { + if updater.setPosterPath != "local/ebooks/content-1/poster/original.test-revision.webp" { t.Fatalf("poster path = %q", updater.setPosterPath) } if updater.setThumbhash != "thumb" { @@ -1293,7 +1295,7 @@ func TestApplyEbookLocalCoverPrefersSidecarOverEmbedded(t *testing.T) { if cacher.calls != 1 || string(cacher.data) != "ebook-sidecar-cover" { t.Fatalf("cache call = calls %d data %q, want sidecar bytes", cacher.calls, string(cacher.data)) } - if updater.setCalls != 1 || updater.setPosterPath != "local/ebooks/content-1/poster/original.webp" { + if updater.setCalls != 1 || updater.setPosterPath != "local/ebooks/content-1/poster/original.test-revision.webp" { t.Fatalf("poster update = calls %d path %q", updater.setCalls, updater.setPosterPath) } } diff --git a/internal/scanner/file_repo.go b/internal/scanner/file_repo.go index 32ff40b1..82d9c92d 100644 --- a/internal/scanner/file_repo.go +++ b/internal/scanner/file_repo.go @@ -2616,19 +2616,77 @@ func (r *FileRepository) MarkMissing(ctx context.Context, id int, since time.Tim // hidden from clients; the grace only delays deleting the row so a file that // reappears within the window restores without re-probing or re-matching. // A zero grace deletes all missing-marked rows immediately. +// +// Rows whose file_path lies at or under one of protectedRoots are never +// deleted, no matter how long they have been missing: an unreachable library +// root (dead drive, lost mount) is temporarily offline, not removed, so its +// catalog state must survive until the root is reachable again. Passing no +// protected roots preserves the historical folder-wide sweep exactly. // Returns the number of rows deleted. -func (r *FileRepository) DeleteMissingByFolder(ctx context.Context, folderID int, gracePeriod time.Duration) (int, error) { +func (r *FileRepository) DeleteMissingByFolder(ctx context.Context, folderID int, gracePeriod time.Duration, protectedRoots []string) (int, error) { cutoff := time.Now().UTC().Add(-gracePeriod) - tag, err := r.pool.Exec(ctx, - "DELETE FROM media_files WHERE media_folder_id = $1 AND missing_since IS NOT NULL AND missing_since < $2", - folderID, cutoff, - ) + query := "DELETE FROM media_files WHERE media_folder_id = $1 AND missing_since IS NOT NULL AND missing_since < $2" + args := []any{folderID, cutoff} + if clauses, clauseArgs := rootCoverageClauses(protectedRoots, len(args)+1); len(clauses) > 0 { + query += " AND NOT (" + strings.Join(clauses, " OR ") + ")" + args = append(args, clauseArgs...) + } + tag, err := r.pool.Exec(ctx, query, args...) if err != nil { return 0, fmt.Errorf("deleting missing files for folder %d: %w", folderID, err) } return int(tag.RowsAffected()), nil } +// ListRootsWithOnlyMissingFiles returns the subset of roots (in input order) +// that still have media_files rows at or under them in the folder but none +// that are present (missing_since IS NULL). A reachable root in this state is +// "suspect empty": it is the on-disk signature of a mount that dropped out +// while leaving an empty, stat-able mountpoint directory, which a +// reachability probe cannot distinguish from an intentionally emptied root. +func (r *FileRepository) ListRootsWithOnlyMissingFiles(ctx context.Context, folderID int, roots []string) ([]string, error) { + if len(roots) == 0 { + return nil, nil + } + patterns := make([]string, len(roots)) + for i, root := range roots { + patterns[i] = pathscope.PrefixLike(root) + } + rows, err := r.pool.Query(ctx, ` + SELECT r.root + FROM unnest($2::text[], $3::text[]) WITH ORDINALITY AS r(root, pattern, ord) + WHERE EXISTS ( + SELECT 1 FROM media_files mf + WHERE mf.media_folder_id = $1 + AND (mf.file_path = r.root OR mf.file_path LIKE r.pattern ESCAPE '\') + ) + AND NOT EXISTS ( + SELECT 1 FROM media_files mf + WHERE mf.media_folder_id = $1 + AND (mf.file_path = r.root OR mf.file_path LIKE r.pattern ESCAPE '\') + AND mf.missing_since IS NULL + ) + ORDER BY r.ord + `, folderID, roots, patterns) + if err != nil { + return nil, fmt.Errorf("querying roots with only missing files: %w", err) + } + defer rows.Close() + + suspect := make([]string, 0) + for rows.Next() { + var root string + if err := rows.Scan(&root); err != nil { + return nil, fmt.Errorf("scanning suspect-empty root: %w", err) + } + suspect = append(suspect, root) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating suspect-empty roots: %w", err) + } + return suspect, nil +} + // DeleteByIDs removes specific media file rows by primary key. func (r *FileRepository) DeleteByIDs(ctx context.Context, ids []int) (int, error) { if len(ids) == 0 { @@ -2671,13 +2729,8 @@ func (r *FileRepository) ListIDsOutsideRoots(ctx context.Context, folderID int, args := make([]any, 0, 1+len(roots)*2) args = append(args, folderID) - coveredClauses := make([]string, 0, len(roots)) - for i, root := range roots { - pathArg := 2 + (i * 2) - likeArg := pathArg + 1 - coveredClauses = append(coveredClauses, fmt.Sprintf("(file_path = $%d OR file_path LIKE $%d ESCAPE '\\')", pathArg, likeArg)) - args = append(args, root, pathPrefixLike(root)) - } + coveredClauses, coveredArgs := rootCoverageClauses(roots, 2) + args = append(args, coveredArgs...) query := `SELECT id FROM media_files WHERE media_folder_id = $1 AND NOT (` + strings.Join(coveredClauses, " OR ") + `)` rows, err := r.pool.Query(ctx, query, args...) @@ -3449,3 +3502,9 @@ func nilIfZero(n int) *int { func pathPrefixLike(pathPrefix string) string { return pathscope.PrefixLike(pathPrefix) } + +// rootCoverageClauses builds one SQL predicate per root matching file_path +// rows that live at or under that root; see pathscope.CoverageClauses. +func rootCoverageClauses(roots []string, firstArg int) ([]string, []any) { + return pathscope.CoverageClauses("file_path", roots, firstArg) +} diff --git a/internal/scanner/podcast_scan.go b/internal/scanner/podcast_scan.go index fc332bbf..fa88bdee 100644 --- a/internal/scanner/podcast_scan.go +++ b/internal/scanner/podcast_scan.go @@ -84,7 +84,7 @@ func (s *Scanner) ScanPodcastFolder(ctx context.Context, folder *models.MediaFol if attempted > 0 && succeeded == 0 && len(failures) > 0 { return fmt.Errorf("podcast scan failed for every attempted folder_id=%d: %w", folder.ID, errors.Join(failures...)) } - if err := s.reconcilePodcastMissingFiles(ctx, folder, reconcileRoots, seenPaths, len(seenPaths) > 0); err != nil { + if err := s.reconcilePodcastMissingFiles(ctx, folder, reconcileRoots, seenPaths); err != nil { slog.WarnContext(ctx, "podcast scan: missing-file reconcile failed", "component", "scanner", "folder_id", folder.ID, "error", err) } return nil @@ -217,35 +217,19 @@ func applyPodcastShowMetadata(item *models.MediaItem, show *parsedPodcastShow) b return changed } -func (s *Scanner) reconcilePodcastMissingFiles(ctx context.Context, folder *models.MediaFolder, roots []string, seenPaths map[string]bool, sawFiles bool) error { +func (s *Scanner) reconcilePodcastMissingFiles(ctx context.Context, folder *models.MediaFolder, roots []string, seenPaths map[string]bool) error { if s.fileRepo == nil || s.libraryRepo == nil || len(roots) == 0 { return nil } - if !sawFiles { - existingCount := 0 - for _, root := range roots { - existing, err := s.fileRepo.GetByFolderAndPathPrefix(ctx, folder.ID, root) - if err != nil { - return fmt.Errorf("listing existing podcast files for %q: %w", root, err) - } - existingCount += len(existing) - } - if existingCount > 0 { - var guard ebookCleanupGuardRepo - if s.folderRepo != nil { - guard = s.folderRepo - } - allowed, err := ebookEmptyCleanupAllowed(ctx, guard, folder.ID, true) - if err != nil { - return err - } - if !allowed { - slog.WarnContext(ctx, "podcast scan: walk saw zero files but the database has files under the scanned roots; skipping reconciliation until cleanup is confirmed", "component", "scanner", - "folder_id", folder.ID, "existing_files", existingCount) - return nil - } - } + confirmedCleanup, blockAll, err := s.emptyCleanupDecision( + ctx, folder, roots, seenPaths, true, "podcast", + ) + if err != nil { + return err + } + if blockAll { + return nil } now := time.Now().UTC() @@ -270,25 +254,12 @@ func (s *Scanner) reconcilePodcastMissingFiles(ctx context.Context, folder *mode } } - if s.emptyTrashAfterScan { - trashed, err := s.fileRepo.DeleteMissingByFolder(ctx, folder.ID, s.fileRemovalGrace) - if err != nil { - return fmt.Errorf("emptying trash for folder %d: %w", folder.ID, err) - } - if trashed > 0 { - slog.InfoContext(ctx, "podcast scan: emptied trash", "component", "scanner", "folder_id", folder.ID, "deleted", trashed) - } + trashed, removedMemberships, deletedItems, err := s.sweepMissingAndReconcile(ctx, folder, confirmedCleanup) + if trashed > 0 { + slog.InfoContext(ctx, "podcast scan: emptied trash", "component", "scanner", "folder_id", folder.ID, "deleted", trashed) } - - removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID) if err != nil { - return fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err) - } - if s.s3Client != nil && len(orphanedImageDirs) > 0 { - bucket := s.s3Client.Bucket() - for _, dir := range orphanedImageDirs { - _, _ = s.s3Client.DeletePrefix(ctx, bucket, dir) - } + return err } if missing > 0 || removedMemberships > 0 || deletedItems > 0 { slog.InfoContext(ctx, "podcast scan: reconciled missing files", "component", "scanner", diff --git a/internal/scanner/probe.go b/internal/scanner/probe.go index 20d9e088..36376a26 100644 --- a/internal/scanner/probe.go +++ b/internal/scanner/probe.go @@ -1,15 +1,19 @@ package scanner import ( + "bufio" "context" "encoding/json" "fmt" + "io" + "math" "os/exec" "slices" "strconv" "strings" "github.com/Silo-Server/silo-server/internal/lang" + "github.com/Silo-Server/silo-server/internal/models" ) // ffprobeOutput represents the top-level JSON output from ffprobe. @@ -49,6 +53,7 @@ type ffprobeFormat struct { Filename string `json:"filename"` FormatName string `json:"format_name"` FormatLongName string `json:"format_long_name"` + StartTime string `json:"start_time"` Duration string `json:"duration"` Size string `json:"size"` BitRate string `json:"bit_rate"` @@ -68,7 +73,10 @@ type ffprobeStream struct { DisplayAspectRatio string `json:"display_aspect_ratio"` FieldOrder string `json:"field_order"` AvgFrameRate string `json:"avg_frame_rate"` + StartTime string `json:"start_time"` + Duration string `json:"duration"` BitRate string `json:"bit_rate"` + ColorRange string `json:"color_range"` ColorTransfer string `json:"color_transfer"` ColorPrimaries string `json:"color_primaries"` ColorSpace string `json:"color_space"` @@ -104,8 +112,9 @@ type ffprobeSideData struct { // ffprobeDisp represents the disposition flags on a stream. type ffprobeDisp struct { - Default int `json:"default"` - Forced int `json:"forced"` + Default int `json:"default"` + Forced int `json:"forced"` + AttachedPic int `json:"attached_pic"` } // ProbeFile runs ffprobe on the given file and returns parsed ProbeData. @@ -130,7 +139,20 @@ func ProbeFile(ctx context.Context, ffprobePath string, filePath string) (*Probe return nil, fmt.Errorf("ffprobe JSON parse failed for %s: %w", filePath, err) } - return convertProbeData(&raw), nil + probe := convertProbeData(&raw) + if probe.Duration == 0 { + if frameRate, hasVideo := primaryVideoFrameRate(raw.Streams); hasVideo { + // A failed or empty packet scan must not discard the codec and + // track metadata that already parsed successfully: callers persist + // the partial probe, and the repair layer retries rows whose + // duration is still unknown. + if duration, packetErr := probeVideoPacketDuration(ctx, ffprobePath, filePath, frameRate); packetErr == nil && duration > 0 { + probe.Duration = duration + } + } + } + + return probe, nil } // FFprobePathFromFFmpeg derives the sibling ffprobe binary path from a configured ffmpeg path. @@ -150,17 +172,8 @@ func convertProbeData(raw *ffprobeOutput) *ProbeData { Container: detectContainer(raw.Format.FormatName), } - // Parse duration from format (ffprobe reports seconds). - // Some containers (notably MKV) may produce duration in microseconds; - // detect and normalise to seconds when the value is unreasonably large. - if raw.Format.Duration != "" { - if dur, err := strconv.ParseFloat(raw.Format.Duration, 64); err == nil { - const maxReasonableSec = 100_000 // ~27.8 hours - if dur > maxReasonableSec { - dur = dur / 1_000_000 // microseconds → seconds - } - pd.Duration = int(dur) - } + if duration, ok := durationFromProbeMetadata(raw); ok { + pd.Duration = duration } // Parse bitrate from format (bps to kbps). @@ -174,30 +187,35 @@ func convertProbeData(raw *ffprobeOutput) *ProbeData { switch s.CodecType { case "video": dvProfile := dolbyVisionProfileNumber(s.SideDataList) + // ffprobe omits unspecified optional fields by default; "unknown" is + // FFmpeg's canonical name for AVCOL_RANGE_UNSPECIFIED. + colorRange := firstNonEmpty(s.ColorRange, "unknown") track := VideoTrackInfo{ - Title: firstNonEmpty(s.Tags["title"], s.CodecLongName, strings.ToUpper(s.CodecName)), - Codec: s.CodecName, - DolbyVision: dolbyVisionProfile(s.SideDataList), - DVProfile: dvProfile, - DVBLCompatID: dolbyVisionBLCompatID(s.SideDataList), - DVELPresent: dolbyVisionELPresent(s.SideDataList), - HDR10Plus: hasHDR10Plus(s.SideDataList), - Profile: s.Profile, - Level: s.Level, - Width: s.Width, - Height: s.Height, - AspectRatio: s.DisplayAspectRatio, - Interlaced: isInterlaced(s.FieldOrder), - FrameRate: normalizeFrameRate(s.AvgFrameRate), - Bitrate: parseNumeric(s.BitRate) / 1000, - VideoRange: videoRangeLabel(s), - VideoRangeType: videoRangeType(s), - ColorPrimaries: s.ColorPrimaries, - ColorSpace: s.ColorSpace, - ColorTransfer: s.ColorTransfer, - BitDepth: parseBitDepth(s), - PixelFormat: s.PixFmt, - ReferenceFrames: s.Refs, + Title: firstNonEmpty(s.Tags["title"], s.CodecLongName, strings.ToUpper(s.CodecName)), + Codec: s.CodecName, + DolbyVision: dolbyVisionProfile(s.SideDataList), + DVProfile: dvProfile, + DVBLCompatID: dolbyVisionBLCompatID(s.SideDataList), + DVELPresent: dolbyVisionELPresent(s.SideDataList), + DVEnhancementLayer: dolbyVisionEnhancementLayer(dolbyVisionELPresent(s.SideDataList)), + HDR10Plus: hasHDR10Plus(s.SideDataList), + Profile: s.Profile, + Level: s.Level, + Width: s.Width, + Height: s.Height, + AspectRatio: s.DisplayAspectRatio, + Interlaced: isInterlaced(s.FieldOrder), + FrameRate: normalizeFrameRate(s.AvgFrameRate), + Bitrate: parseNumeric(s.BitRate) / 1000, + VideoRange: videoRangeLabel(s), + VideoRangeType: videoRangeType(s), + ColorRange: colorRange, + ColorPrimaries: s.ColorPrimaries, + ColorSpace: s.ColorSpace, + ColorTransfer: s.ColorTransfer, + BitDepth: models.NormalizeVideoBitDepth(parseBitDepth(s), s.PixFmt, s.Profile), + PixelFormat: s.PixFmt, + ReferenceFrames: s.Refs, } pd.VideoTracks = append(pd.VideoTracks, track) if pd.CodecVideo == "" { @@ -246,6 +264,214 @@ func convertProbeData(raw *ffprobeOutput) *ProbeData { return pd } +const ( + maxReasonableMediaDurationSeconds = 100_000 + // Audio-only files (audiobooks, podcasts) legitimately exceed the video + // ceiling, but still need a cap so malformed containers cannot persist + // multi-year durations. + maxReasonableAudioDurationSeconds = 1_000_000 +) + +// A large video file whose derived duration is only a few seconds is the +// signature of malformed container timestamps (and of the legacy probe that +// divided large durations by one million). The shape is shared with the +// repair triggers in probe_repair.go and scanner.go so the probe parser and +// the repair layers cannot drift apart. +const ( + implausiblyShortVideoMaxSeconds = 10 + implausiblyShortVideoMinBytes = 100 * 1024 * 1024 +) + +func videoDurationImplausiblyShort(durationSeconds float64, sizeBytes int64, hasVideo bool) bool { + return hasVideo && + durationSeconds > 0 && durationSeconds <= implausiblyShortVideoMaxSeconds && + sizeBytes >= implausiblyShortVideoMinBytes +} + +func durationFromProbeMetadata(raw *ffprobeOutput) (int, bool) { + if raw == nil { + return 0, false + } + + formatDuration := parseFloat(raw.Format.Duration) + if !hasVideoStream(raw.Streams) && + durationIsPositiveFinite(formatDuration) && formatDuration <= maxReasonableAudioDurationSeconds { + return truncatedDuration(formatDuration), true + } + if durationIsReasonable(formatDuration) && !durationLooksImplausiblyShort(raw, formatDuration) { + return truncatedDuration(formatDuration), true + } + + for _, stream := range raw.Streams { + if !isMainVideoStream(stream) { + continue + } + streamDuration := parseFloat(stream.Duration) + if durationIsReasonable(streamDuration) && !durationLooksImplausiblyShort(raw, streamDuration) { + return truncatedDuration(streamDuration), true + } + duration := durationAfterStart(streamDuration, parseFloat(stream.StartTime)) + if duration > 0 && !durationLooksImplausiblyShort(raw, duration) { + return truncatedDuration(duration), true + } + } + + duration := durationAfterStart(formatDuration, parseFloat(raw.Format.StartTime)) + if duration > 0 && !durationLooksImplausiblyShort(raw, duration) { + return truncatedDuration(duration), true + } + return 0, false +} + +func durationLooksImplausiblyShort(raw *ffprobeOutput, duration float64) bool { + if raw == nil { + return false + } + size := int64(parseFloat(raw.Format.Size)) + return videoDurationImplausiblyShort(duration, size, hasVideoStream(raw.Streams)) +} + +func durationAfterStart(end, start float64) float64 { + if start <= 0 || end <= start { + return 0 + } + duration := end - start + if !durationIsReasonable(duration) { + return 0 + } + return duration +} + +func durationIsReasonable(duration float64) bool { + return durationIsPositiveFinite(duration) && duration <= maxReasonableMediaDurationSeconds +} + +func durationIsPositiveFinite(duration float64) bool { + return duration > 0 && !math.IsNaN(duration) && !math.IsInf(duration, 0) +} + +func roundedDuration(duration float64) int { + return max(1, int(math.Round(duration))) +} + +func truncatedDuration(duration float64) int { + return max(1, int(duration)) +} + +// isMainVideoStream reports whether the stream is a real video stream. +// Embedded cover art (attached_pic) is reported by ffprobe as a video stream +// but must not drive duration decisions: it would route audiobooks and music +// through the video duration gauntlet and packet-scan a single still image. +func isMainVideoStream(stream ffprobeStream) bool { + return stream.CodecType == "video" && stream.Disposition.AttachedPic == 0 +} + +func hasVideoStream(streams []ffprobeStream) bool { + return slices.ContainsFunc(streams, isMainVideoStream) +} + +func primaryVideoFrameRate(streams []ffprobeStream) (string, bool) { + for _, stream := range streams { + if isMainVideoStream(stream) { + return stream.AvgFrameRate, true + } + } + return "", false +} + +// probeVideoPacketDuration derives a duration for files whose duration +// metadata is unusable by scanning video packet timestamps. It intentionally +// demuxes the whole file: the timestamps being repaired are the same ones +// ffprobe would need for reliable interval seeking, so sampling cannot be +// trusted here. The repair layer keeps this one-shot per file. +func probeVideoPacketDuration( + ctx context.Context, + ffprobePath string, + filePath string, + frameRate string, +) (int, error) { + cmd := exec.CommandContext(ctx, ffprobePath, + "-v", "error", + "-select_streams", "v:0", + "-show_entries", "packet=pts_time", + "-of", "csv=p=0", + filePath, + ) + stdout, err := cmd.StdoutPipe() + if err != nil { + return 0, fmt.Errorf("opening ffprobe packet output: %w", err) + } + if err := cmd.Start(); err != nil { + return 0, fmt.Errorf("starting ffprobe packet scan: %w", err) + } + + duration := estimateVideoPacketDuration(stdout, frameRate) + if err := cmd.Wait(); err != nil { + return 0, fmt.Errorf("ffprobe packet scan failed for %s: %w", filePath, err) + } + return duration, nil +} + +func estimateVideoPacketDuration(reader io.Reader, frameRate string) int { + scanner := bufio.NewScanner(reader) + packetCount := 0 + minTimestamp := math.Inf(1) + maxTimestamp := math.Inf(-1) + + for scanner.Scan() { + value := strings.TrimSpace(scanner.Text()) + if value == "" { + continue + } + packetCount++ + timestamp, err := strconv.ParseFloat(value, 64) + if err != nil { + continue + } + minTimestamp = min(minTimestamp, timestamp) + maxTimestamp = max(maxTimestamp, timestamp) + } + + best := 0.0 + if !math.IsInf(minTimestamp, 1) && !math.IsInf(maxTimestamp, -1) { + span := maxTimestamp - minTimestamp + if durationIsReasonable(span) { + best = span + } + } + if fps := parseFrameRate(frameRate); fps > 0 && packetCount > 0 { + frameDuration := float64(packetCount) / fps + if durationIsReasonable(frameDuration) && frameDuration > best { + best = frameDuration + } + } + if best <= 0 { + return 0 + } + return roundedDuration(best) +} + +// parseFrameRate parses ffprobe's rational frame-rate shape ("30000/1001") +// or a plain float, returning 0 when unparsable. normalizeFrameRate formats +// the same parse for persistence; keep the parsing logic here only. +func parseFrameRate(raw string) float64 { + raw = strings.TrimSpace(raw) + parts := strings.SplitN(raw, "/", 2) + if len(parts) != 2 { + fps, _ := strconv.ParseFloat(raw, 64) + return fps + } + numerator, err := strconv.ParseFloat(parts[0], 64) + if err != nil { + return 0 + } + denominator, err := strconv.ParseFloat(parts[1], 64) + if err != nil || denominator == 0 { + return 0 + } + return numerator / denominator +} + func parseNumeric(raw string) int { if raw == "" { return 0 @@ -369,16 +595,14 @@ func normalizeFrameRate(raw string) string { if raw == "" || raw == "0/0" { return "" } - parts := strings.SplitN(raw, "/", 2) - if len(parts) != 2 { + if !strings.Contains(raw, "/") { return raw } - num, err1 := strconv.ParseFloat(parts[0], 64) - den, err2 := strconv.ParseFloat(parts[1], 64) - if err1 != nil || err2 != nil || den == 0 { + fps := parseFrameRate(raw) + if fps == 0 { return raw } - return strconv.FormatFloat(num/den, 'f', 3, 64) + return strconv.FormatFloat(fps, 'f', 3, 64) } func isInterlaced(fieldOrder string) bool { @@ -434,6 +658,16 @@ func dolbyVisionELPresent(sideData []ffprobeSideData) bool { return false } +// dolbyVisionEnhancementLayer remains conservative until a libdovi-backed +// analyzer has inspected the RPU mapping. ffprobe can prove that an enhancement +// layer exists, but it cannot distinguish MEL from FEL. +func dolbyVisionEnhancementLayer(present bool) string { + if !present { + return "none" + } + return "unknown" +} + func hasHDR10Plus(sideData []ffprobeSideData) bool { for _, data := range sideData { typ := strings.ToLower(data.SideDataType) diff --git a/internal/scanner/probe_duration_test.go b/internal/scanner/probe_duration_test.go new file mode 100644 index 00000000..371d62bd --- /dev/null +++ b/internal/scanner/probe_duration_test.go @@ -0,0 +1,239 @@ +package scanner + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" +) + +func TestDurationFromProbeMetadataUsesReasonableVideoDuration(t *testing.T) { + t.Parallel() + + raw := &ffprobeOutput{ + Format: ffprobeFormat{Duration: "1745764.949333"}, + Streams: []ffprobeStream{ + {CodecType: "video", Duration: "4077.708452"}, + {CodecType: "audio", Duration: "1745764.949333"}, + }, + } + + got, ok := durationFromProbeMetadata(raw) + if !ok || got != 4077 { + t.Fatalf("durationFromProbeMetadata() = %d, %v; want 4077, true", got, ok) + } +} + +func TestDurationFromProbeMetadataRemovesAbsoluteTimestampOffset(t *testing.T) { + t.Parallel() + + raw := &ffprobeOutput{ + Format: ffprobeFormat{ + StartTime: "1514891.405000", + Duration: "1520667.605000", + }, + Streams: []ffprobeStream{{ + CodecType: "video", + StartTime: "1514891.405000", + }}, + } + + got, ok := durationFromProbeMetadata(raw) + if !ok || got != 5776 { + t.Fatalf("durationFromProbeMetadata() = %d, %v; want 5776, true", got, ok) + } +} + +func TestDurationFromProbeMetadataRejectsAbsurdSubtitleTimeline(t *testing.T) { + t.Parallel() + + raw := &ffprobeOutput{ + Format: ffprobeFormat{Duration: "4298357.248000"}, + Streams: []ffprobeStream{ + {CodecType: "video", AvgFrameRate: "30000/1001"}, + {CodecType: "subtitle", Duration: "4298357.248000"}, + }, + } + + got, ok := durationFromProbeMetadata(raw) + if ok || got != 0 { + t.Fatalf("durationFromProbeMetadata() = %d, %v; want 0, false", got, ok) + } +} + +func TestDurationFromProbeMetadataRejectsImplausiblyShortLargeVideo(t *testing.T) { + t.Parallel() + + raw := &ffprobeOutput{ + Format: ffprobeFormat{ + Duration: "3.022000", + Size: "2022705152", + }, + Streams: []ffprobeStream{{ + CodecType: "video", + AvgFrameRate: "30/1", + }}, + } + + got, ok := durationFromProbeMetadata(raw) + if ok || got != 0 { + t.Fatalf("durationFromProbeMetadata() = %d, %v; want 0, false", got, ok) + } +} + +func TestDurationFromProbeMetadataKeepsLongAudioDurationInSeconds(t *testing.T) { + t.Parallel() + + raw := &ffprobeOutput{ + Format: ffprobeFormat{Duration: "108000.000000"}, + Streams: []ffprobeStream{{CodecType: "audio"}}, + } + + got, ok := durationFromProbeMetadata(raw) + if !ok || got != 108000 { + t.Fatalf("durationFromProbeMetadata() = %d, %v; want 108000, true", got, ok) + } +} + +func TestEstimateVideoPacketDurationUsesPacketSpan(t *testing.T) { + t.Parallel() + + packets := strings.NewReader("1514891.405000\n1515000.000000\n1520667.605000\n") + got := estimateVideoPacketDuration(packets, "30000/1001") + if got != 5776 { + t.Fatalf("estimateVideoPacketDuration() = %d, want 5776", got) + } +} + +func TestEstimateVideoPacketDurationUsesFrameCountForCollapsedTimestamps(t *testing.T) { + t.Parallel() + + var packets strings.Builder + for i := 0; i < 300; i++ { + packets.WriteString("3.022000\n") + } + + got := estimateVideoPacketDuration(strings.NewReader(packets.String()), "30/1") + if got != 10 { + t.Fatalf("estimateVideoPacketDuration() = %d, want 10", got) + } +} + +func TestProbeFileFallsBackToVideoPacketsForInvalidMetadata(t *testing.T) { + t.Parallel() + + ffprobePath := filepath.Join(t.TempDir(), "ffprobe") + script := `#!/bin/sh +case " $* " in + *" -show_format "*) + printf '%s\n' '{"format":{"duration":"4298357.248000","size":"1258000000"},"streams":[{"codec_type":"video","avg_frame_rate":"30/1"},{"codec_type":"subtitle","duration":"4298357.248000"}]}' + ;; + *) + i=0 + while [ "$i" -lt 300 ]; do + printf '%s\n' '3.022000' + i=$((i + 1)) + done + ;; +esac +` + if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil { + t.Fatalf("writing fake ffprobe: %v", err) + } + + probe, err := ProbeFile(context.Background(), ffprobePath, "broken.mkv") + if err != nil { + t.Fatalf("ProbeFile() returned error: %v", err) + } + if probe.Duration != 10 { + t.Fatalf("ProbeFile() duration = %d, want 10", probe.Duration) + } +} + +// A failed packet fallback must not discard the codec/track metadata that +// already parsed successfully; the file imports with an unknown duration and +// the repair layer retries later. +func TestProbeFileKeepsMetadataWhenPacketFallbackFails(t *testing.T) { + t.Parallel() + + ffprobePath := filepath.Join(t.TempDir(), "ffprobe") + script := `#!/bin/sh +case " $* " in + *" -show_format "*) + printf '%s\n' '{"format":{"duration":"4298357.248000","size":"1258000000"},"streams":[{"codec_type":"video","codec_name":"h264","avg_frame_rate":"30/1"}]}' + ;; + *) + exit 1 + ;; +esac +` + if err := os.WriteFile(ffprobePath, []byte(script), 0o755); err != nil { + t.Fatalf("writing fake ffprobe: %v", err) + } + + probe, err := ProbeFile(context.Background(), ffprobePath, "broken.mkv") + if err != nil { + t.Fatalf("ProbeFile() returned error: %v", err) + } + if probe.Duration != 0 { + t.Fatalf("ProbeFile() duration = %d, want 0 (unknown)", probe.Duration) + } + if probe.CodecVideo != "h264" { + t.Fatalf("ProbeFile() codec = %q, want parsed metadata to survive the failed fallback", probe.CodecVideo) + } +} + +// Embedded cover art is reported by ffprobe as a video stream; it must not +// route audiobooks and music through the video duration gauntlet. +func TestDurationFromProbeMetadataIgnoresCoverArtVideoStream(t *testing.T) { + t.Parallel() + + raw := &ffprobeOutput{ + Format: ffprobeFormat{Duration: "108000.000000"}, + Streams: []ffprobeStream{ + {CodecType: "audio"}, + {CodecType: "video", CodecName: "mjpeg", Disposition: ffprobeDisp{AttachedPic: 1}}, + }, + } + + got, ok := durationFromProbeMetadata(raw) + if !ok || got != 108000 { + t.Fatalf("durationFromProbeMetadata() = %d, %v; want 108000, true", got, ok) + } +} + +func TestDurationFromProbeMetadataRejectsAbsurdAudioDuration(t *testing.T) { + t.Parallel() + + raw := &ffprobeOutput{ + Format: ffprobeFormat{Duration: "4298357.248000"}, + Streams: []ffprobeStream{{CodecType: "audio"}}, + } + + got, ok := durationFromProbeMetadata(raw) + if ok || got != 0 { + t.Fatalf("durationFromProbeMetadata() = %d, %v; want 0, false", got, ok) + } +} + +// The end-minus-start fallbacks must apply the same plausibility guard as the +// direct duration paths: a multi-GB video whose absolute timestamps collapse +// to a few seconds needs the packet fallback, not a persisted 3s duration. +func TestDurationFromProbeMetadataRejectsCollapsedTimestampSpanForLargeVideo(t *testing.T) { + t.Parallel() + + raw := &ffprobeOutput{ + Format: ffprobeFormat{Size: "2022705152"}, + Streams: []ffprobeStream{{ + CodecType: "video", + StartTime: "1514891.405000", + Duration: "1514894.405000", + }}, + } + + got, ok := durationFromProbeMetadata(raw) + if ok || got != 0 { + t.Fatalf("durationFromProbeMetadata() = %d, %v; want 0, false", got, ok) + } +} diff --git a/internal/scanner/probe_repair.go b/internal/scanner/probe_repair.go index 45cc4091..255a76b9 100644 --- a/internal/scanner/probe_repair.go +++ b/internal/scanner/probe_repair.go @@ -28,6 +28,12 @@ func NeedsCriticalProbeRepair(file *models.MediaFile) bool { if file.Duration <= 0 { return true } + // Legacy probes could turn malformed multi-day container timestamps into a + // few seconds by treating ffprobe's seconds as microseconds. Reprobe the + // narrow, physically implausible shape produced by that conversion. + if needsLegacyDurationRepair(file) { + return true + } if strings.TrimSpace(file.Container) == "" { return true } @@ -48,6 +54,9 @@ func NeedsCriticalProbeRepair(file *models.MediaFile) bool { if strings.TrimSpace(file.CodecVideo) == "" || strings.TrimSpace(file.Resolution) == "" { return true } + if videoTracksMissingColorRange(file.VideoTracks) { + return true + } } if file.Chapters == nil { return true @@ -55,6 +64,15 @@ func NeedsCriticalProbeRepair(file *models.MediaFile) bool { return false } +func videoTracksMissingColorRange(tracks []models.VideoTrack) bool { + for _, track := range tracks { + if strings.TrimSpace(track.ColorRange) == "" { + return true + } + } + return false +} + // PlaybackProbeEnsurer repairs missing playback-critical probe metadata on // demand by running a local ffprobe and persisting the result. type PlaybackProbeEnsurer struct { @@ -83,6 +101,9 @@ func (e *PlaybackProbeEnsurer) Ensure(ctx context.Context, file *models.MediaFil if timeout <= 0 { timeout = 5 * time.Second } + if reprobeMayScanPackets(file) && timeout < time.Minute { + timeout = time.Minute + } probeCtx, cancel := context.WithTimeout(ctx, timeout) defer cancel() @@ -95,3 +116,36 @@ func (e *PlaybackProbeEnsurer) Ensure(ctx context.Context, file *models.MediaFil applyProbeData(&updated, probe, "local") return e.fileRepo.Upsert(ctx, updated) } + +// reprobeMayScanPackets reports whether reprobing this file is likely to hit +// ProbeFile's packet-scan fallback, which demuxes the entire file and cannot +// finish inside the default metadata-probe timeout. +func reprobeMayScanPackets(file *models.MediaFile) bool { + if file == nil || len(file.VideoTracks) == 0 { + return false + } + return file.Duration <= 0 || + videoDurationImplausiblyShort(float64(file.Duration), file.FileSize, true) +} + +// legacyProbeDurationFixTime is when the probe duration parser stopped +// treating large ffprobe durations as microseconds. Rows probed before this +// may carry the collapsed durations that conversion produced. Rows probed +// after it are authoritative: a still-short duration was re-derived from +// packet timestamps, and re-flagging it would reprobe genuinely short clips +// on every playback decision forever. Adjust if this fix ships later. +var legacyProbeDurationFixTime = time.Date(2026, time.July, 18, 0, 0, 0, 0, time.UTC) + +func needsLegacyDurationRepair(file *models.MediaFile) bool { + if file == nil { + return false + } + return legacyDurationRepairNeeded(file.Duration, file.FileSize, len(file.VideoTracks) > 0, file.ProbeUpdatedAt) +} + +func legacyDurationRepairNeeded(duration int, sizeBytes int64, hasVideo bool, probeUpdatedAt *time.Time) bool { + if !videoDurationImplausiblyShort(float64(duration), sizeBytes, hasVideo) { + return false + } + return probeUpdatedAt == nil || probeUpdatedAt.Before(legacyProbeDurationFixTime) +} diff --git a/internal/scanner/probe_repair_audio_test.go b/internal/scanner/probe_repair_audio_test.go index 22855c76..03007531 100644 --- a/internal/scanner/probe_repair_audio_test.go +++ b/internal/scanner/probe_repair_audio_test.go @@ -47,8 +47,95 @@ func TestNeedsCriticalProbeRepair_ProbedVideoMissingResolutionStillRepairs(t *te } } +func TestNeedsCriticalProbeRepair_ProbedVideoMissingColorRangeRepairsOnce(t *testing.T) { + now := time.Now() + f := &models.MediaFile{ + ProbeSource: "local", + ProbeUpdatedAt: &now, + Duration: 7200, + Container: "mkv", + CodecAudio: "aac", + AudioTracks: []models.AudioTrack{{Language: "eng"}}, + CodecVideo: "h264", + Resolution: "1080p", + VideoTracks: []models.VideoTrack{{Codec: "h264"}}, + Chapters: []models.MediaChapter{}, + } + + if !NeedsCriticalProbeRepair(f) { + t.Fatal("a legacy video without color range should need probe repair") + } + + f.VideoTracks[0].ColorRange = "unknown" + if NeedsCriticalProbeRepair(f) { + t.Fatal("a reprobed video with unknown color range should not repair again") + } +} + func TestNeedsCriticalProbeRepair_UnprobedFileRepairs(t *testing.T) { if !NeedsCriticalProbeRepair(&models.MediaFile{}) { t.Fatal("an unprobed file must need probe repair") } } + +func implausiblyShortLargeVideoFile(probedAt time.Time) *models.MediaFile { + return &models.MediaFile{ + ProbeSource: "local", + ProbeUpdatedAt: &probedAt, + FileSize: 1_200_000_000, + Duration: 4, + Container: "mkv", + CodecAudio: "aac", + AudioTracks: []models.AudioTrack{{Language: "eng"}}, + CodecVideo: "h264", + Resolution: "720p", + VideoTracks: []models.VideoTrack{{Codec: "h264", ColorRange: "unknown"}}, + Chapters: []models.MediaChapter{}, + } +} + +func TestNeedsCriticalProbeRepair_ImplausiblyShortLargeVideoRepairs(t *testing.T) { + f := implausiblyShortLargeVideoFile(legacyProbeDurationFixTime.Add(-time.Hour)) + + if !NeedsCriticalProbeRepair(f) { + t.Fatal("a large video claiming to be four seconds should need probe repair") + } +} + +// A short duration re-derived by the fixed parser (packet scan) is +// authoritative: re-flagging it would reprobe genuinely short clips on every +// playback decision forever. +func TestNeedsCriticalProbeRepair_ShortLargeVideoReprobedAfterFixConverges(t *testing.T) { + f := implausiblyShortLargeVideoFile(legacyProbeDurationFixTime.Add(time.Hour)) + + if NeedsCriticalProbeRepair(f) { + t.Fatal("a short large video already reprobed by the fixed parser must not repair again") + } +} + +func TestNeedsCriticalProbeRepairScanState_LegacyShortDurationRepairs(t *testing.T) { + probedAt := legacyProbeDurationFixTime.Add(-time.Hour) + f := &scanStateFile{ + ProbeSource: "local", + ProbeUpdatedAt: &probedAt, + FileSize: 1_200_000_000, + Duration: 4, + Container: "mkv", + CodecVideo: "h264", + CodecAudio: "aac", + Resolution: "720p", + HasVideoTracks: true, + HasAudioTracks: true, + HasChapters: true, + } + + if !needsCriticalProbeRepairScanState(f) { + t.Fatal("library scans must flag legacy-collapsed durations for reprobe") + } + + reprobedAt := legacyProbeDurationFixTime.Add(time.Hour) + f.ProbeUpdatedAt = &reprobedAt + if needsCriticalProbeRepairScanState(f) { + t.Fatal("a short large video already reprobed by the fixed parser must not repair again") + } +} diff --git a/internal/scanner/probe_video_range_test.go b/internal/scanner/probe_video_range_test.go index 7529f2d3..7cdcfaad 100644 --- a/internal/scanner/probe_video_range_test.go +++ b/internal/scanner/probe_video_range_test.go @@ -1,6 +1,44 @@ package scanner -import "testing" +import ( + "encoding/json" + "testing" + + "github.com/Silo-Server/silo-server/internal/models" +) + +func TestProbePipelinePreservesVideoColorRange(t *testing.T) { + tests := []struct { + name string + colorRange string + wantColorRange string + }{ + {name: "limited", colorRange: "tv", wantColorRange: "tv"}, + {name: "full", colorRange: "pc", wantColorRange: "pc"}, + {name: "unspecified", wantColorRange: "unknown"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rawJSON := `{"streams":[{"codec_type":"video","codec_name":"h264","color_range":"` + tt.colorRange + `"}]}` + var raw ffprobeOutput + if err := json.Unmarshal([]byte(rawJSON), &raw); err != nil { + t.Fatalf("unmarshal ffprobe output: %v", err) + } + + probe := convertProbeData(&raw) + file := &models.MediaFile{} + applyProbeData(file, probe, "local") + + if len(file.VideoTracks) != 1 { + t.Fatalf("VideoTracks length = %d, want 1", len(file.VideoTracks)) + } + if got := file.VideoTracks[0].ColorRange; got != tt.wantColorRange { + t.Fatalf("ColorRange = %q, want %q", got, tt.wantColorRange) + } + }) + } +} func TestConvertProbeDataVideoRangeTypes(t *testing.T) { tests := []struct { diff --git a/internal/scanner/scanner.go b/internal/scanner/scanner.go index 6f89e033..f9fcb834 100644 --- a/internal/scanner/scanner.go +++ b/internal/scanner/scanner.go @@ -19,6 +19,7 @@ import ( "github.com/Silo-Server/silo-server/internal/librarykind" "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/naming" + "github.com/Silo-Server/silo-server/internal/rootcheck" "github.com/Silo-Server/silo-server/internal/s3client" ) @@ -889,9 +890,22 @@ func (s *Scanner) scanPaths( // Empty trash: delete files marked as missing for longer than the removal // grace for this folder. Safe because the empty-root guard (above) returns // early when 0 files are found on disk, so we only reach here when the - // root is populated. + // root is populated. The sweep is folder-wide, so a scoped scan of a + // healthy subtree must not hard-delete rows an unreachable or + // suspect-empty sibling root left marked missing: probe the configured + // library roots and exempt any that are currently protected. + protectedRoots, err := s.protectedConfiguredRoots(ctx, folder) + if err != nil { + return nil, err + } + removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID, protectedRoots) + if err != nil { + return nil, fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err) + } + result.MembershipsRemoved = removedMemberships + result.ItemsDeleted = deletedItems if s.emptyTrashAfterScan { - trashed, err := s.fileRepo.DeleteMissingByFolder(ctx, folder.ID, s.fileRemovalGrace) + trashed, err := s.fileRepo.DeleteMissingByFolder(ctx, folder.ID, s.fileRemovalGrace, protectedRoots) if err != nil { return nil, fmt.Errorf("emptying trash for folder %d: %w", folder.ID, err) } @@ -914,13 +928,6 @@ func (s *Scanner) scanPaths( } result.FilesDeleted += deletedFiles - removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID) - if err != nil { - return nil, fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err) - } - result.MembershipsRemoved = removedMemberships - result.ItemsDeleted = deletedItems - if s.seriesQueueSyncer != nil { if allowEmptyRootGuard { if err := s.seriesQueueSyncer.SyncForFolder(ctx, folder.ID); err != nil { @@ -999,9 +1006,44 @@ func (s *Scanner) scanFolderByRoots( reconcileRoots []string, ) (*ScanResult, error) { result := &ScanResult{} - roots := compactScanRoots(reconcileRoots) - if len(roots) == 0 { - roots = compactScanRoots(walkRoots) + configuredRoots := cleanScanRoots(reconcileRoots) + if len(configuredRoots) == 0 { + configuredRoots = cleanScanRoots(walkRoots) + } + roots := compactScanRoots(configuredRoots) + + // An unreachable root (dead drive, lost mount) is temporarily offline, not + // removed from the library: its files are still marked missing below so + // clients stop seeing them, but nothing under it may be hard-deleted while + // it stays unreachable — trash emptying and item purging are guarded + // against these roots further down. Probe every CONFIGURED path, not the + // compacted traversal roots: a nested child mount is dropped by compaction + // but can die independently of its reachable parent. + configuredProbes := rootcheck.ProbeManyWithTimeout(ctx, configuredRoots, rootcheck.DefaultProbeTimeout) + unreachableRoots := make([]string, 0) + suspectRoots := make([]string, 0) + for i, root := range configuredRoots { + probe := configuredProbes[i] + if !probe.Reachable { + logUnreachableRoot(ctx, folder.ID, root, probe) + unreachableRoots = append(unreachableRoots, root) + continue + } + if !probe.Empty { + continue + } + existing, err := s.fileRepo.GetByFolderAndPathPrefix(ctx, folder.ID, root) + if err != nil { + return nil, fmt.Errorf("listing files under empty root %q: %w", root, err) + } + if len(existing) > 0 { + suspectRoots = append(suspectRoots, root) + } + } + result.UnreachableRoots = unreachableRoots + unreachableSet := make(map[string]bool, len(unreachableRoots)) + for _, root := range unreachableRoots { + unreachableSet[root] = true } pendingEmptyScopes := make([]*scopedScan, 0) @@ -1015,7 +1057,13 @@ func (s *Scanner) scanFolderByRoots( Message: "Scanning library root", CurrentScope: root, }) - scope, err := s.scanScope(ctx, folder, []string{root}, []string{root}) + scopeWalkRoots := []string{root} + if unreachableSet[root] { + // Skip walking a dead root — it would yield nothing anyway. Its + // scope still reconciles below so existing rows are marked missing. + scopeWalkRoots = nil + } + scope, err := s.scanScope(ctx, folder, scopeWalkRoots, []string{root}) if err != nil { return nil, err } @@ -1028,36 +1076,48 @@ func (s *Scanner) scanFolderByRoots( if len(scope.filePaths) > 0 { seenAnyFiles = true - for _, pending := range pendingEmptyScopes { - beforeErrors := pending.result.Errors - if err := s.applyScopedScan(ctx, folder, pending, false); err != nil { - return nil, err - } - mergeCleanupResult(result, pending.result, beforeErrors) - } - pendingEmptyScopes = pendingEmptyScopes[:0] - beforeErrors := scope.result.Errors - if err := s.applyScopedScan(ctx, folder, scope, false); err != nil { - return nil, err - } - mergeCleanupResult(result, scope.result, beforeErrors) - continue - } - - if seenAnyFiles { - beforeErrors := scope.result.Errors - if err := s.applyScopedScan(ctx, folder, scope, false); err != nil { + if err := s.applyScopedScan(ctx, folder, scope, false, nil); err != nil { return nil, err } mergeCleanupResult(result, scope.result, beforeErrors) continue } + // Empty scopes stay pending until after the loop: whether they may + // force-delete (confirmed cleanup) and whether their roots must be + // treated as suspect depends on what the other roots produced. pendingEmptyScopes = append(pendingEmptyScopes, scope) } - if !seenAnyFiles && totalExisting > 0 { + // Re-probe empty scopes after walking. A root can disconnect after the + // initial check; promote that transition into the protected set before any + // empty-root guard or reconciliation decision is made. + for _, pending := range pendingEmptyScopes { + root := firstScope(pending.reconcileRoots) + if root == "" || unreachableSet[root] || len(pending.existingFiles) == 0 { + continue + } + probe := rootcheck.ProbeWithTimeout(ctx, root, rootcheck.DefaultProbeTimeout) + if !probe.Reachable { + logUnreachableRoot(ctx, folder.ID, root, probe) + unreachableSet[root] = true + unreachableRoots = appendUniquePath(unreachableRoots, root) + result.UnreachableRoots = unreachableRoots + suspectRoots = removePath(suspectRoots, root) + } else if probe.Empty { + suspectRoots = appendUniquePath(suspectRoots, root) + } + } + + // When EVERY configured root is unreachable this is an outage, not an + // intentionally emptied library: skip the empty-root confirm flow (and do + // not consume the operator's one-time cleanup allowance), fall through so + // pending scopes mark files missing (hiding them), and let the dead_root + // warning below explain the state. All deletion paths are protected by + // the unreachable-roots exclusions, so nothing is purged. + allRootsUnreachable := len(unreachableRoots) > 0 && len(unreachableRoots) == len(configuredRoots) + if !seenAnyFiles && totalExisting > 0 && !allRootsUnreachable { var err error allowEmptyCleanup, err = s.folderRepo.ConsumeEmptyCleanupAllowance(ctx, folder.ID) if err != nil { @@ -1076,9 +1136,37 @@ func (s *Scanner) scanFolderByRoots( } } + // A reachable root that is a LITERALLY empty directory while rows remain + // cataloged under it carries the lost-mount signature: the mountpoint + // survived, its contents vanished with the mount (NFS/SMB drop, dead + // bind-mount source), and the reachability probe cannot tell it from an + // intentionally emptied root. Treat such roots as suspect: their rows are + // only marked missing, and the sweep/purge below exempts them until the + // operator confirms cleanup or the files return. A root that still has + // entries but no media files walked its files genuinely deleted and keeps + // the historical purge path. When the whole folder walked empty the + // allowance was already consumed above; the mixed case (other roots + // healthy) consumes it here so a confirmed single-root cleanout still + // completes. + if seenAnyFiles && len(suspectRoots) > 0 { + var err error + allowEmptyCleanup, err = s.folderRepo.ConsumeEmptyCleanupAllowance(ctx, folder.ID) + if err != nil { + return nil, fmt.Errorf("checking empty cleanup confirmation for folder %d: %w", folder.ID, err) + } + } + if allowEmptyCleanup { + suspectRoots = nil + } + result.SuspectEmptyRoots = suspectRoots + + protectedScanRoots := append(append([]string(nil), unreachableRoots...), suspectRoots...) for _, pending := range pendingEmptyScopes { beforeErrors := pending.result.Errors - if err := s.applyScopedScan(ctx, folder, pending, allowEmptyCleanup); err != nil { + // forceDeleteAll only ever fires for confirmed cleanup, and even then + // rows under a probe-dead root must survive: an outage is not a + // confirmation to erase that root's catalog. + if err := s.applyScopedScan(ctx, folder, pending, allowEmptyCleanup, protectedScanRoots); err != nil { return nil, err } mergeCleanupResult(result, pending.result, beforeErrors) @@ -1097,8 +1185,19 @@ func (s *Scanner) scanFolderByRoots( return nil, fmt.Errorf("syncing present library state for folder %d: %w", folder.ID, err) } + protectedRoots := unreachableRoots + if len(suspectRoots) > 0 { + protectedRoots = make([]string, 0, len(unreachableRoots)+len(suspectRoots)) + protectedRoots = append(append(protectedRoots, unreachableRoots...), suspectRoots...) + } + removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID, protectedRoots) + if err != nil { + return nil, fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err) + } + result.MembershipsRemoved = removedMemberships + result.ItemsDeleted = deletedItems if s.emptyTrashAfterScan { - trashed, err := s.fileRepo.DeleteMissingByFolder(ctx, folder.ID, s.fileRemovalGrace) + trashed, err := s.fileRepo.DeleteMissingByFolder(ctx, folder.ID, s.fileRemovalGrace, protectedRoots) if err != nil { return nil, fmt.Errorf("emptying trash for folder %d: %w", folder.ID, err) } @@ -1118,13 +1217,6 @@ func (s *Scanner) scanFolderByRoots( } result.FilesDeleted += deletedOutsideRoots - removedMemberships, deletedItems, orphanedImageDirs, err := s.reconcileLibraryMemberships(ctx, folder.ID) - if err != nil { - return nil, fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err) - } - result.MembershipsRemoved = removedMemberships - result.ItemsDeleted = deletedItems - if s.seriesQueueSyncer != nil { reportProgress(ctx, ProgressUpdate{ Phase: "queue_sync", @@ -1161,7 +1253,19 @@ func (s *Scanner) scanFolderByRoots( } } - if seenAnyFiles || allowEmptyCleanup { + switch { + case len(unreachableRoots) > 0 || len(suspectRoots) > 0: + // Surface the outage so the admin UI explains why part of the library + // vanished; the mount-check endpoint or the next fully-healthy scan + // clears it (both clear "dead_root" the same way as "empty_root"). + if err := s.folderRepo.SetScanWarning(ctx, folder.ID, + "dead_root", + deadRootWarningMessage(len(configuredRoots), unreachableRoots, suspectRoots), + time.Now().UTC(), + ); err != nil { + return nil, fmt.Errorf("recording dead-root warning for folder %d: %w", folder.ID, err) + } + case seenAnyFiles || allowEmptyCleanup: if err := s.folderRepo.ClearScanWarning(ctx, folder.ID); err != nil { return nil, fmt.Errorf("clearing scan warning for folder %d: %w", folder.ID, err) } @@ -1170,6 +1274,138 @@ func (s *Scanner) scanFolderByRoots( return result, nil } +// probeUnreachableRoots returns the subset of roots that are currently +// unreachable (missing, not a directory, unlistable, or hung past the probe +// timeout), in input order. The timeout matters because probes run on scan +// hot paths (including per-file autoscan events): a wedged mount must degrade +// into the protected "unreachable" path instead of stalling the scanner. +func probeUnreachableRoots(ctx context.Context, folderID int, roots []string) []string { + var unreachable []string + probes := rootcheck.ProbeManyWithTimeout(ctx, roots, rootcheck.DefaultProbeTimeout) + for i, root := range roots { + if probe := probes[i]; !probe.Reachable { + logUnreachableRoot(ctx, folderID, root, probe) + unreachable = append(unreachable, root) + } + } + return unreachable +} + +func logUnreachableRoot(ctx context.Context, folderID int, root string, probe rootcheck.Result) { + slog.WarnContext(ctx, "scanner: library root unreachable", "component", "scanner", + "folder_id", folderID, + "root", root, + "error_code", probe.ErrorCode, + "error", probe.ErrorMessage, + ) +} + +func appendUniquePath(paths []string, path string) []string { + for _, existing := range paths { + if existing == path { + return paths + } + } + return append(paths, path) +} + +func removePath(paths []string, path string) []string { + for i, existing := range paths { + if existing == path { + return append(paths[:i], paths[i+1:]...) + } + } + return paths +} + +// suspectEmptyRoots returns configured roots that probe as reachable but +// LITERALLY empty directories while the database still holds rows (all +// missing-marked) under them. That is the signature of a mount that dropped +// out leaving its bare mountpoint directory behind (NFS/SMB drop, dead +// bind-mount source) — a reachability probe cannot tell it apart from an +// intentionally emptied root, so destructive cleanup under these roots is +// deferred until the operator confirms or the files return. A root that +// still has directory entries keeps the historical purge path. +func (s *Scanner) suspectEmptyRoots(ctx context.Context, folderID int, configuredRoots, unreachableRoots []string) ([]string, error) { + unreachableSet := make(map[string]bool, len(unreachableRoots)) + for _, root := range unreachableRoots { + unreachableSet[root] = true + } + emptyRoots := make([]string, 0, len(configuredRoots)) + probes := rootcheck.ProbeManyWithTimeout(ctx, configuredRoots, rootcheck.DefaultProbeTimeout) + for i, root := range configuredRoots { + if unreachableSet[root] { + continue + } + if probe := probes[i]; probe.Reachable && probe.Empty { + emptyRoots = append(emptyRoots, root) + } + } + if len(emptyRoots) == 0 { + return nil, nil + } + suspect, err := s.fileRepo.ListRootsWithOnlyMissingFiles(ctx, folderID, emptyRoots) + if err != nil { + return nil, fmt.Errorf("listing suspect-empty roots for folder %d: %w", folderID, err) + } + if len(suspect) > 0 { + slog.WarnContext(ctx, "scanner: empty roots still hold cataloged files; protecting them from cleanup", "component", "scanner", + "folder_id", folderID, + "roots", suspect, + ) + } + return suspect, nil +} + +// protectedConfiguredRoots probes the folder's configured root paths +// (uncompacted, so a nested child mount is probed independently of its +// reachable parent) and returns every root that must be exempted from +// destructive cleanup right now: probe-unreachable roots plus suspect-empty +// ones. Callers must pass a folder whose Paths is the full configured list. +func (s *Scanner) protectedConfiguredRoots(ctx context.Context, folder *models.MediaFolder) ([]string, error) { + configuredRoots := cleanScanRoots(folder.Paths) + unreachableRoots := probeUnreachableRoots(ctx, folder.ID, configuredRoots) + suspectRoots, err := s.suspectEmptyRoots(ctx, folder.ID, configuredRoots, unreachableRoots) + if err != nil { + return nil, err + } + return append(unreachableRoots, suspectRoots...), nil +} + +// configuredFolderPaths returns the folder's full configured root list for +// dead-root protection. Scoped scans (ScanSubtree, ScanFile) pass a folder +// clone whose Paths holds only the scanned subtree, but the cleanup they +// trigger is folder-wide, so protection must consider every configured root: +// reload them instead of trusting the possibly-scoped clone. +func (s *Scanner) configuredFolderPaths(ctx context.Context, folder *models.MediaFolder) ([]string, error) { + if s.folderRepo == nil { + return folder.Paths, nil + } + full, err := s.folderRepo.GetByID(ctx, folder.ID) + if err != nil { + return nil, fmt.Errorf("loading configured paths for folder %d: %w", folder.ID, err) + } + if full == nil || len(full.Paths) == 0 { + // No persisted paths (fresh row, tests): the caller's view is all + // there is to probe. + return folder.Paths, nil + } + return full.Paths, nil +} + +func deadRootWarningMessage(totalRoots int, unreachableRoots, suspectRoots []string) string { + parts := make([]string, 0, 2) + if len(unreachableRoots) > 0 { + parts = append(parts, fmt.Sprintf("%d of %d roots unreachable: %s", + len(unreachableRoots), totalRoots, strings.Join(unreachableRoots, ", "))) + } + if len(suspectRoots) > 0 { + parts = append(parts, fmt.Sprintf("%d of %d roots returned no files while the library still has cataloged files (lost mount?): %s", + len(suspectRoots), totalRoots, strings.Join(suspectRoots, ", "))) + } + return strings.Join(parts, "; ") +} + func (s *Scanner) scanScope( ctx context.Context, folder *models.MediaFolder, @@ -1354,11 +1590,17 @@ func (s *Scanner) scanScope( }, nil } +// applyScopedScan reconciles one root's scope. forceDeleteAll (confirmed +// empty cleanup) hard-deletes the scope's rows instead of marking them +// missing — except rows under protectedRoots (probe-dead roots, including a +// nested dead child inside this scope's root): an outage is never a +// confirmation to erase that root's catalog. func (s *Scanner) applyScopedScan( ctx context.Context, folder *models.MediaFolder, scope *scopedScan, forceDeleteAll bool, + protectedRoots []string, ) error { if scope == nil || scope.result == nil { return nil @@ -1398,6 +1640,9 @@ func (s *Scanner) applyScopedScan( if forceDeleteAll && len(scope.filePaths) == 0 { staleFileIDs = make([]int, 0, len(scope.existingFiles)) for _, existing := range scope.existingFiles { + if pathWithinAnyRoot(existing.FilePath, protectedRoots) { + continue + } staleFileIDs = append(staleFileIDs, existing.ID) } } @@ -1440,6 +1685,28 @@ func mergeCleanupResult(dst *ScanResult, src *ScanResult, priorErrors int) { } } +// cleanScanRoots normalizes and dedupes configured root paths WITHOUT +// removing nested roots. Reachability must be probed against every +// configured path: a child mount (/media/drive under /media) is dropped by +// compactScanRoots for traversal, but can die independently and its files +// must still be protected from deletion. +func cleanScanRoots(paths []string) []string { + out := make([]string, 0, len(paths)) + seen := make(map[string]bool, len(paths)) + for _, rawPath := range paths { + if strings.TrimSpace(rawPath) == "" { + continue + } + path := filepath.Clean(rawPath) + if path == "" || path == "." || seen[path] { + continue + } + seen[path] = true + out = append(out, path) + } + return out +} + func compactScanRoots(paths []string) []string { out := make([]string, 0, len(paths)) for _, rawPath := range paths { @@ -1569,13 +1836,63 @@ func (s *Scanner) syncFolderScopedAudioLibraryState(ctx context.Context, folderI return nil } -func (s *Scanner) reconcileLibraryMemberships(ctx context.Context, folderID int) (int, int, []string, error) { +// reconcileLibraryMemberships removes memberships for content with no +// remaining non-missing files in the folder and purges orphaned items. +// unreachableRoots exempts items whose files sit under a currently +// unreachable library root from the orphan purge (membership removal still +// happens so the items stay hidden); pass nil when every root is reachable. +func (s *Scanner) reconcileLibraryMemberships(ctx context.Context, folderID int, unreachableRoots []string) (int, int, []string, error) { if s.episodeLibraryRepo != nil { if _, err := s.episodeLibraryRepo.ReconcileFolderMembership(ctx, folderID); err != nil { return 0, 0, nil, err } } - return s.libraryRepo.ReconcileFolderMembership(ctx, folderID) + return s.libraryRepo.ReconcileFolderMembership(ctx, folderID, unreachableRoots) +} + +// sweepMissingAndReconcile finishes an audio/ebook/podcast missing-file pass: +// it reconciles library memberships, empties trash (when enabled), and +// best-effort deletes orphaned S3 image dirs. The folder-wide sweep and +// orphan purge must not destroy rows/items whose files sit under a currently +// unreachable or suspect-empty library root (see the video scanner's +// dead-root protection): unreachable is not removed. The folder argument may +// be a scoped clone (ScanSubtree/ScanFile), so the configured roots are +// reloaded and probed uncompacted — a nested child mount can die +// independently of its reachable parent. confirmedCleanup skips the +// suspect-empty exemption for a scan the operator explicitly confirmed. +// Counts are returned for the caller's flavor-specific logging. +func (s *Scanner) sweepMissingAndReconcile(ctx context.Context, folder *models.MediaFolder, confirmedCleanup bool) (trashed, removedMemberships, deletedItems int, err error) { + configuredPaths, err := s.configuredFolderPaths(ctx, folder) + if err != nil { + return 0, 0, 0, err + } + configuredRoots := cleanScanRoots(configuredPaths) + protectedRoots := probeUnreachableRoots(ctx, folder.ID, configuredRoots) + if !confirmedCleanup { + suspectRoots, serr := s.suspectEmptyRoots(ctx, folder.ID, configuredRoots, protectedRoots) + if serr != nil { + return 0, 0, 0, serr + } + protectedRoots = append(protectedRoots, suspectRoots...) + } + var orphanedImageDirs []string + removedMemberships, deletedItems, orphanedImageDirs, err = s.reconcileLibraryMemberships(ctx, folder.ID, protectedRoots) + if err != nil { + return 0, 0, 0, fmt.Errorf("reconciling library membership for folder %d: %w", folder.ID, err) + } + if s.emptyTrashAfterScan { + trashed, err = s.fileRepo.DeleteMissingByFolder(ctx, folder.ID, s.fileRemovalGrace, protectedRoots) + if err != nil { + return 0, 0, 0, fmt.Errorf("emptying trash for folder %d: %w", folder.ID, err) + } + } + if s.s3Client != nil && len(orphanedImageDirs) > 0 { + bucket := s.s3Client.Bucket() + for _, dir := range orphanedImageDirs { + _, _ = s.s3Client.DeletePrefix(ctx, bucket, dir) + } + } + return trashed, removedMemberships, deletedItems, nil } func collectStaleRemovedPathFileIDs(existingFiles []*scanStateFile, seenPaths map[string]bool, roots []string) []int { @@ -1676,7 +1993,11 @@ func (s *Scanner) ScanFile(ctx context.Context, filePath string, folder *models. if err := s.syncPresentLibraryState(ctx, folder.ID); err != nil { return fmt.Errorf("syncing present library state for extra file: %w", err) } - if _, _, _, err := s.reconcileLibraryMemberships(ctx, folder.ID); err != nil { + protectedRoots, err := s.protectedConfiguredRoots(ctx, folder) + if err != nil { + return err + } + if _, _, _, err := s.reconcileLibraryMemberships(ctx, folder.ID, protectedRoots); err != nil { return fmt.Errorf("reconciling library membership after extra file scan: %w", err) } return nil @@ -1692,7 +2013,11 @@ func (s *Scanner) ScanFile(ctx context.Context, filePath string, folder *models. return clearErr } if cleared > 0 { - if _, _, _, reconcileErr := s.reconcileLibraryMemberships(ctx, folder.ID); reconcileErr != nil { + protectedRoots, protErr := s.protectedConfiguredRoots(ctx, folder) + if protErr != nil { + return protErr + } + if _, _, _, reconcileErr := s.reconcileLibraryMemberships(ctx, folder.ID, protectedRoots); reconcileErr != nil { return fmt.Errorf("reconciling folder membership after clearing legacy links: %w", reconcileErr) } } @@ -2402,6 +2727,9 @@ func needsCriticalProbeRepairScanState(file *scanStateFile) bool { if file.Duration <= 0 { return true } + if legacyDurationRepairNeeded(file.Duration, file.FileSize, file.HasVideoTracks, file.ProbeUpdatedAt) { + return true + } if strings.TrimSpace(file.Container) == "" { return true } @@ -2840,29 +3168,31 @@ func applyProbeData(mf *models.MediaFile, probe *ProbeData, probeSource string) videoTracks := make([]models.VideoTrack, len(probe.VideoTracks)) for i, vt := range probe.VideoTracks { videoTracks[i] = models.VideoTrack{ - Title: vt.Title, - Codec: vt.Codec, - DolbyVision: vt.DolbyVision, - DVProfile: vt.DVProfile, - DVBLCompatID: vt.DVBLCompatID, - DVELPresent: vt.DVELPresent, - HDR10Plus: vt.HDR10Plus, - Profile: vt.Profile, - Level: vt.Level, - Width: vt.Width, - Height: vt.Height, - AspectRatio: vt.AspectRatio, - Interlaced: vt.Interlaced, - FrameRate: vt.FrameRate, - Bitrate: vt.Bitrate, - VideoRange: vt.VideoRange, - VideoRangeType: vt.VideoRangeType, - ColorPrimaries: vt.ColorPrimaries, - ColorSpace: vt.ColorSpace, - ColorTransfer: vt.ColorTransfer, - BitDepth: vt.BitDepth, - PixelFormat: vt.PixelFormat, - ReferenceFrames: vt.ReferenceFrames, + Title: vt.Title, + Codec: vt.Codec, + DolbyVision: vt.DolbyVision, + DVProfile: vt.DVProfile, + DVBLCompatID: vt.DVBLCompatID, + DVELPresent: vt.DVELPresent, + DVEnhancementLayer: vt.DVEnhancementLayer, + HDR10Plus: vt.HDR10Plus, + Profile: vt.Profile, + Level: vt.Level, + Width: vt.Width, + Height: vt.Height, + AspectRatio: vt.AspectRatio, + Interlaced: vt.Interlaced, + FrameRate: vt.FrameRate, + Bitrate: vt.Bitrate, + VideoRange: vt.VideoRange, + VideoRangeType: vt.VideoRangeType, + ColorRange: vt.ColorRange, + ColorPrimaries: vt.ColorPrimaries, + ColorSpace: vt.ColorSpace, + ColorTransfer: vt.ColorTransfer, + BitDepth: vt.BitDepth, + PixelFormat: vt.PixelFormat, + ReferenceFrames: vt.ReferenceFrames, } } mf.VideoTracks = videoTracks diff --git a/internal/scanner/types.go b/internal/scanner/types.go index 4bd01b2f..f969c0aa 100644 --- a/internal/scanner/types.go +++ b/internal/scanner/types.go @@ -11,7 +11,18 @@ type ScanResult struct { ItemsDeleted int Errors int EmptyRootGuarded bool - RootObservations []RootObservation + // UnreachableRoots lists configured library roots that failed the + // reachability probe at scan start. Files under them were marked missing + // (hidden) but were exempted from trash emptying and item purging. + UnreachableRoots []string + // SuspectEmptyRoots lists roots that probed reachable but were literally + // empty directories while the library still holds cataloged files under + // them — the signature of a lost mount exposing its bare mountpoint + // directory. They receive the same cleanup exemptions as + // UnreachableRoots until the operator confirms cleanup or the files + // return. + SuspectEmptyRoots []string + RootObservations []RootObservation } // FileHints contains the OSHash gathered during scanning. @@ -38,29 +49,31 @@ type ProbeData struct { // VideoTrackInfo describes a probed video track. type VideoTrackInfo struct { - Title string - Codec string - DolbyVision string - DVProfile int - DVBLCompatID int - DVELPresent bool - HDR10Plus bool - Profile string - Level int - Width int - Height int - AspectRatio string - Interlaced bool - FrameRate string - Bitrate int - VideoRange string - VideoRangeType string - ColorPrimaries string - ColorSpace string - ColorTransfer string - BitDepth int - PixelFormat string - ReferenceFrames int + Title string + Codec string + DolbyVision string + DVProfile int + DVBLCompatID int + DVELPresent bool + DVEnhancementLayer string + HDR10Plus bool + Profile string + Level int + Width int + Height int + AspectRatio string + Interlaced bool + FrameRate string + Bitrate int + VideoRange string + VideoRangeType string + ColorRange string + ColorPrimaries string + ColorSpace string + ColorTransfer string + BitDepth int + PixelFormat string + ReferenceFrames int } // AudioTrackInfo describes a probed audio track. diff --git a/internal/server/branding_frontend_test.go b/internal/server/branding_frontend_test.go index 7b74cedc..4e2d2b37 100644 --- a/internal/server/branding_frontend_test.go +++ b/internal/server/branding_frontend_test.go @@ -61,6 +61,39 @@ func TestFrontendInjectsServerNameIntoTitle(t *testing.T) { } } +// TestFrontendShellCacheFollowsBrandingChanges guards the rendered-shell +// cache: one handler instance must re-render (and re-tag) the shell when the +// branding snapshot changes, not keep serving the first rendering forever. +func TestFrontendShellCacheFollowsBrandingChanges(t *testing.T) { + settings := fakeSettings{branding.KeyServerName: "Acme Media"} + withBranding(t, settings) + handler := FrontendHandler() + + first := httptest.NewRecorder() + handler.ServeHTTP(first, httptest.NewRequest(http.MethodGet, "/", nil)) + if !strings.Contains(first.Body.String(), "Acme Media") { + t.Fatalf("initial title not branded: %q", first.Body.String()) + } + + // Repeat request with unchanged branding: same ETag (served from cache). + repeat := httptest.NewRecorder() + handler.ServeHTTP(repeat, httptest.NewRequest(http.MethodGet, "/", nil)) + if first.Header().Get("ETag") != repeat.Header().Get("ETag") { + t.Fatalf("etag changed without a branding change: %q vs %q", + first.Header().Get("ETag"), repeat.Header().Get("ETag")) + } + + settings[branding.KeyServerName] = "Renamed Media" + renamed := httptest.NewRecorder() + handler.ServeHTTP(renamed, httptest.NewRequest(http.MethodGet, "/", nil)) + if !strings.Contains(renamed.Body.String(), "Renamed Media") { + t.Fatalf("renamed title not rendered: %q", renamed.Body.String()) + } + if renamed.Header().Get("ETag") == first.Header().Get("ETag") { + t.Fatal("etag must change when the rendered shell changes") + } +} + func TestFrontendServesDynamicManifest(t *testing.T) { withBranding(t, fakeSettings{branding.KeyServerName: "Acme Media"}) rr := httptest.NewRecorder() diff --git a/internal/server/frontend.go b/internal/server/frontend.go index c59f1b4f..cdb28eda 100644 --- a/internal/server/frontend.go +++ b/internal/server/frontend.go @@ -1,9 +1,15 @@ package server import ( + "bytes" + "crypto/sha256" + "encoding/hex" "io/fs" "net/http" "strings" + "sync" + "sync/atomic" + "time" "github.com/Silo-Server/silo-server/internal/branding" ) @@ -68,51 +74,156 @@ func FrontendHandler() http.Handler { }) } - fileServer := http.FileServer(http.FS(WebDistFS)) + // The embedded build output is immutable for the process lifetime, so the + // unbranded shell can be read once here. A read failure falls back to a + // per-request error response. + rawIndex, _ := fs.ReadFile(WebDistFS, "index.html") - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("X-Content-Type-Options", "nosniff") - path := r.URL.Path + return &frontendHandler{ + fileServer: http.FileServer(http.FS(WebDistFS)), + rawIndex: rawIndex, + } +} - // Dynamic branding endpoints must be handled before the static file - // server, which would otherwise serve the bundled defaults shadowing - // them. Both fall through to the static asset when no override applies. - if Branding != nil { - switch path { - case "/site.webmanifest": - serveDynamicManifest(w, r) +// frontendHandler serves the embedded SPA. It is a struct rather than a +// closure so its caches are scoped to one handler instance and reset when a +// new handler is constructed over a different WebDistFS (as tests do). +type frontendHandler struct { + fileServer http.Handler + rawIndex []byte // unbranded index.html, read once at construction + + // staticETags caches content ETags for stable-URL bundled files (sw.js, + // icons, vendor bundles). The embedded FS never changes, so a path's ETag + // is computed at most once. + staticETags sync.Map // path string -> etag string + + // shell caches the branded index.html and its ETag for the last-seen + // branding snapshot, so steady-state shell requests — especially the 304 + // revalidations that no-cache makes the common case — skip re-reading, + // re-rendering, and re-hashing the document. + shell atomic.Pointer[renderedShell] +} + +type renderedShell struct { + brandingKey string + body []byte + etag string +} + +func (h *frontendHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Content-Type-Options", "nosniff") + path := r.URL.Path + + // Dynamic branding endpoints must be handled before the static file + // server, which would otherwise serve the bundled defaults shadowing + // them. Both fall through to the static asset when no override applies. + if Branding != nil { + switch path { + case "/site.webmanifest": + serveDynamicManifest(w, r) + return + case "/favicon.ico": + if serveCustomFavicon(w, r) { return - case "/favicon.ico": - if serveCustomFavicon(w, r) { - return + } + } + } + + // Try to serve the file directly. index.html is excluded so the SPA + // HTML always goes through the fallback below and carries the CSP. + if path != "/" && path != "/index.html" && !strings.HasSuffix(path, "/") { + if f, err := WebDistFS.Open(strings.TrimPrefix(path, "/")); err == nil { + _ = f.Close() + if strings.HasPrefix(path, "/assets/") { + // Vite content-hashes /assets/ filenames, so those URLs are + // immutable: a new build produces new URLs, which is what + // lets browsers cache them for a year yet pick up deploys. + w.Header().Set("Cache-Control", "public, max-age=31536000, immutable") + } else { + // Every other bundled file (service worker, icons, vendor + // bundles) keeps its URL across builds, so it must be + // revalidated. The embedded FS carries no modtimes, meaning + // http.FileServer emits no validator of its own — without + // this ETag, no-cache would force a full re-download on + // every use instead of a 304. + w.Header().Set("Cache-Control", "no-cache") + if etag := h.staticETag(path); etag != "" { + w.Header().Set("ETag", etag) } } - } - - // Try to serve the file directly. index.html is excluded so the SPA - // HTML always goes through the fallback below and carries the CSP. - if path != "/" && path != "/index.html" && !strings.HasSuffix(path, "/") { - if f, err := WebDistFS.Open(strings.TrimPrefix(path, "/")); err == nil { - f.Close() - fileServer.ServeHTTP(w, r) - return - } - } - - // SPA fallback: serve index.html - indexBytes, err := fs.ReadFile(WebDistFS, "index.html") - if err != nil { - http.Error(w, "index.html not found", http.StatusInternalServerError) + h.fileServer.ServeHTTP(w, r) return } - if Branding != nil { - indexBytes = branding.RenderIndexHTML(indexBytes, Branding.Load(r.Context())) + } + + // SPA fallback: serve the (branded) index.html shell. + shell, ok := h.brandedShell(r) + if !ok { + http.Error(w, "index.html not found", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Content-Security-Policy", frontendContentSecurityPolicy) + // The HTML shell keeps a stable URL across builds, so it must never be + // served stale: every deploy changes which content-hashed /assets/* + // bundles it references. no-cache lets browsers and CDNs store it but + // forces revalidation on each load; the ETag turns an unchanged shell + // into a cheap 304. ServeContent implements RFC 9110 conditional + // semantics (weak comparison, ETag lists), so revalidation keeps working + // behind proxies that compress the body and weaken the ETag to W/"...". + w.Header().Set("ETag", shell.etag) + w.Header().Set("Cache-Control", "no-cache") + http.ServeContent(w, r, "index.html", time.Time{}, bytes.NewReader(shell.body)) +} + +// brandedShell returns the branding-rendered index.html and its ETag, reusing +// the cached rendering while the branding snapshot is unchanged. +func (h *frontendHandler) brandedShell(r *http.Request) (*renderedShell, bool) { + if h.rawIndex == nil { + return nil, false + } + var brandingKey string + var snap branding.Snapshot + if Branding != nil { + snap = Branding.Load(r.Context()) + brandingKey = snap.RenderKey() + } + if cached := h.shell.Load(); cached != nil && cached.brandingKey == brandingKey { + return cached, true + } + body := h.rawIndex + if Branding != nil { + body = branding.RenderIndexHTML(h.rawIndex, snap) + } + rendered := &renderedShell{brandingKey: brandingKey, body: body, etag: contentETag(body)} + h.shell.Store(rendered) + return rendered, true +} + +// staticETag returns the content ETag for a bundled static file, computing and +// caching it on first use. Returns "" for paths that can't be read as files +// (directories), which are served without a validator. +func (h *frontendHandler) staticETag(path string) string { + if v, ok := h.staticETags.Load(path); ok { + if etag, ok := v.(string); ok { + return etag } - w.Header().Set("Content-Type", "text/html; charset=utf-8") - w.Header().Set("Content-Security-Policy", frontendContentSecurityPolicy) - w.WriteHeader(http.StatusOK) - _, _ = w.Write(indexBytes) - }) + } + data, err := fs.ReadFile(WebDistFS, strings.TrimPrefix(path, "/")) + if err != nil { + return "" + } + etag := contentETag(data) + h.staticETags.Store(path, etag) + return etag +} + +// contentETag derives a strong validator from response bytes so a no-cache +// resource can answer conditional requests with a 304 instead of re-sending the +// body. Truncated SHA-256 is ample for cache validation (not a security token). +func contentETag(b []byte) string { + sum := sha256.Sum256(b) + return `"` + hex.EncodeToString(sum[:16]) + `"` } // serveDynamicManifest writes the branding-aware web app manifest. @@ -136,18 +247,14 @@ func serveCustomFavicon(w http.ResponseWriter, r *http.Request) bool { // X-Content-Type-Options is already set on the response by the caller. The // favicon may be an admin-uploaded SVG; harden it against script execution // on direct navigation (stored-XSS defense), matching the API asset route. - etag := `"` + ref + `"` w.Header().Set("Content-Type", contentType) w.Header().Set("Content-Security-Policy", branding.AssetContentSecurityPolicy) - w.Header().Set("ETag", etag) + w.Header().Set("ETag", `"`+ref+`"`) // Stable path (no content hash in the URL), so revalidate rather than cache // long-lived; the ETag lets browsers skip the body when unchanged. + // ServeContent handles If-None-Match with RFC 9110 semantics (weak + // comparison, ETag lists) rather than a naive string compare. w.Header().Set("Cache-Control", "public, max-age=300") - if r.Header.Get("If-None-Match") == etag { - w.WriteHeader(http.StatusNotModified) - return true - } - w.WriteHeader(http.StatusOK) - _, _ = w.Write(data) + http.ServeContent(w, r, "", time.Time{}, bytes.NewReader(data)) return true } diff --git a/internal/server/frontend_test.go b/internal/server/frontend_test.go index fbb3570f..05520ddc 100644 --- a/internal/server/frontend_test.go +++ b/internal/server/frontend_test.go @@ -78,3 +78,93 @@ func TestFrontendHandlerServesStaticAssetsWithoutCSP(t *testing.T) { t.Fatalf("x-content-type-options = %q", got) } } + +func newFrontendTestHandler(t *testing.T) http.Handler { + t.Helper() + prev := WebDistFS + WebDistFS = fstest.MapFS{ + "index.html": &fstest.MapFile{Data: []byte("
")}, + "assets/app.js": &fstest.MapFile{Data: []byte("console.log(1)")}, + "sw.js": &fstest.MapFile{Data: []byte("self.addEventListener('fetch', () => {})")}, + } + t.Cleanup(func() { WebDistFS = prev }) + return FrontendHandler() +} + +func TestFrontendShellCacheHeadersAndConditionalGet(t *testing.T) { + handler := newFrontendTestHandler(t) + + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/", nil)) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d", rr.Code) + } + if got := rr.Header().Get("Cache-Control"); got != "no-cache" { + t.Fatalf("shell cache-control = %q, want no-cache", got) + } + etag := rr.Header().Get("ETag") + if etag == "" { + t.Fatal("shell response missing ETag") + } + + // Revalidation with the exact ETag answers 304 with no body. + conditional := func(inm string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, "/", nil) + req.Header.Set("If-None-Match", inm) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + return rec + } + if rec := conditional(etag); rec.Code != http.StatusNotModified { + t.Fatalf("If-None-Match exact: status = %d, want 304", rec.Code) + } + // A fronting proxy that compresses the shell weakens the ETag to W/"..."; + // RFC 9110 weak comparison must still produce the 304 (a naive string + // compare here silently kills revalidation behind nginx gzip). + if rec := conditional("W/" + etag); rec.Code != http.StatusNotModified { + t.Fatalf("If-None-Match weakened: status = %d, want 304", rec.Code) + } + // ETag lists must match too. + if rec := conditional(`"stale-etag", ` + etag); rec.Code != http.StatusNotModified { + t.Fatalf("If-None-Match list: status = %d, want 304", rec.Code) + } + // A stale validator gets the full document. + if rec := conditional(`"stale-etag"`); rec.Code != http.StatusOK || rec.Body.Len() == 0 { + t.Fatalf("If-None-Match stale: status = %d body = %d bytes, want 200 with body", rec.Code, rec.Body.Len()) + } +} + +func TestFrontendStaticFilesCarryValidators(t *testing.T) { + handler := newFrontendTestHandler(t) + + // Content-hashed bundles are immutable; the URL is the validator. + rr := httptest.NewRecorder() + handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/assets/app.js", nil)) + if got := rr.Header().Get("Cache-Control"); got != "public, max-age=31536000, immutable" { + t.Fatalf("asset cache-control = %q", got) + } + + // Stable-URL files must revalidate — and the embedded FS has no modtimes, + // so without an explicit ETag no-cache would force a full re-download on + // every use (there would be nothing to revalidate against). + rr = httptest.NewRecorder() + handler.ServeHTTP(rr, httptest.NewRequest(http.MethodGet, "/sw.js", nil)) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d", rr.Code) + } + if got := rr.Header().Get("Cache-Control"); got != "no-cache" { + t.Fatalf("sw.js cache-control = %q, want no-cache", got) + } + etag := rr.Header().Get("ETag") + if etag == "" { + t.Fatal("stable-path static file missing ETag validator") + } + + req := httptest.NewRequest(http.MethodGet, "/sw.js", nil) + req.Header.Set("If-None-Match", etag) + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, req) + if rec.Code != http.StatusNotModified { + t.Fatalf("sw.js If-None-Match: status = %d, want 304", rec.Code) + } +} diff --git a/internal/streamtoken/token.go b/internal/streamtoken/token.go index f6b78964..4df90711 100644 --- a/internal/streamtoken/token.go +++ b/internal/streamtoken/token.go @@ -18,20 +18,24 @@ import ( // are lookup keys re-resolved against the authority on reconstruct; they are // never trusted on their own. type Claims struct { - SessionID string `json:"sid"` - MediaPath string `json:"path"` - PlayMethod string `json:"method"` - TranscodeAudio bool `json:"ta,omitempty"` - TranscodeNode string `json:"tnode,omitempty"` - TargetCodec string `json:"tc,omitempty"` - TargetRes string `json:"tres,omitempty"` - AudioCodec string `json:"ac,omitempty"` - AudioChannels int `json:"ach,omitempty"` - AudioTrackIndex int `json:"ati,omitempty"` + SessionID string `json:"sid"` + MediaPath string `json:"path"` + PlayMethod string `json:"method"` + TranscodeAudio bool `json:"ta,omitempty"` + TranscodeNode string `json:"tnode,omitempty"` + TranscodeTransportID string `json:"tid,omitempty"` + TargetCodec string `json:"tc,omitempty"` + TargetRes string `json:"tres,omitempty"` + AudioCodec string `json:"ac,omitempty"` + AudioChannels int `json:"ach,omitempty"` + AudioTrackIndex int `json:"ati,omitempty"` // DVProfile is the file's Dolby Vision profile (0 = none); remux nodes // use it to strip dangling profile 7 RPUs. Absent in older tokens, which // decodes as 0 (no strip — the pre-existing behavior). DVProfile int `json:"dvp,omitempty"` + // RemuxDVMode freezes whether a Profile 7 remux preserves or strips DV + // metadata. Empty is the legacy auto behavior for old tokens. + RemuxDVMode string `json:"dvm,omitempty"` // Ownership / authorization lookup keys (re-resolved at reconstruct). // Not trust assertions. @@ -42,17 +46,21 @@ type Claims struct { // Reconstruction recipe — the byte-affecting encode parameters, mirroring the // former playback.RecipeCard. Zero for direct/remux tokens, which reconstruct // from identity alone plus the client-supplied position. - SourceVideoCodec string `json:"svc,omitempty"` - SeekSeconds float64 `json:"seek,omitempty"` - SegmentDuration int `json:"segd,omitempty"` - StartSegmentNumber int `json:"ssn,omitempty"` - SubtitleTrackIndex int `json:"sti,omitempty"` - SubtitleBurnIn bool `json:"sbi,omitempty"` - SubtitleCodec string `json:"sbc,omitempty"` - TargetBitrateKbps int `json:"tbr,omitempty"` - TotalDuration float64 `json:"dur,omitempty"` - FastStart bool `json:"fs,omitempty"` - TargetCodecAudio string `json:"tca,omitempty"` + SourceVideoCodec string `json:"svc,omitempty"` + VideoBitstreamFilter string `json:"vbsf,omitempty"` + OutputSubdir string `json:"osd,omitempty"` + SeekSeconds float64 `json:"seek,omitempty"` + StreamOriginSeconds float64 `json:"origin,omitempty"` + CopySeekAnchorResolved bool `json:"origin_ok,omitempty"` + SegmentDuration int `json:"segd,omitempty"` + StartSegmentNumber int `json:"ssn,omitempty"` + SubtitleTrackIndex int `json:"sti,omitempty"` + SubtitleBurnIn bool `json:"sbi,omitempty"` + SubtitleCodec string `json:"sbc,omitempty"` + TargetBitrateKbps int `json:"tbr,omitempty"` + TotalDuration float64 `json:"dur,omitempty"` + FastStart bool `json:"fs,omitempty"` + TargetCodecAudio string `json:"tca,omitempty"` // Recipe staleness hint, bumped on each re-mint after a recipe mutation // (audio/quality/seek switch). An optional client-side hint only. diff --git a/internal/taskmanager/tasks/cleanup_artwork_revisions.go b/internal/taskmanager/tasks/cleanup_artwork_revisions.go new file mode 100644 index 00000000..ad7c5660 --- /dev/null +++ b/internal/taskmanager/tasks/cleanup_artwork_revisions.go @@ -0,0 +1,56 @@ +package tasks + +import ( + "context" + "fmt" + "time" + + "github.com/Silo-Server/silo-server/internal/metadata" + "github.com/Silo-Server/silo-server/internal/taskmanager" +) + +type ArtworkRevisionGCRunner interface { + Run(ctx context.Context) (metadata.ArtworkRevisionGCStats, error) +} + +type CleanupArtworkRevisionsTask struct { + runner ArtworkRevisionGCRunner +} + +func NewCleanupArtworkRevisionsTask(runner ArtworkRevisionGCRunner) *CleanupArtworkRevisionsTask { + return &CleanupArtworkRevisionsTask{runner: runner} +} + +func (t *CleanupArtworkRevisionsTask) Key() string { return "cleanup_artwork_revisions" } +func (t *CleanupArtworkRevisionsTask) Name() string { return "Clean Artwork Revisions" } +func (t *CleanupArtworkRevisionsTask) Description() string { + return "Deletes unpublished or displaced immutable artwork revisions after a grace period when no catalog record references them." +} +func (t *CleanupArtworkRevisionsTask) Category() taskmanager.TaskCategory { + return taskmanager.TaskCategoryMetadata +} +func (t *CleanupArtworkRevisionsTask) IsHidden() bool { return false } +func (t *CleanupArtworkRevisionsTask) DefaultTriggers() []taskmanager.TriggerConfig { + return []taskmanager.TriggerConfig{ + {Type: taskmanager.TriggerTypeStartup}, + {Type: taskmanager.TriggerTypeInterval, IntervalMs: int64(time.Hour / time.Millisecond)}, + } +} + +func (t *CleanupArtworkRevisionsTask) Execute(ctx context.Context, progress taskmanager.ProgressReporter) error { + if t == nil || t.runner == nil { + progress.Report(100, "Artwork revision cleanup is not configured") + return nil + } + progress.Report(0, "Checking displaced artwork revisions") + stats, err := t.runner.Run(ctx) + progress.SetResultData(stats.JSON()) + if err != nil { + return fmt.Errorf("cleaning artwork revisions: %w", err) + } + progress.Report(100, fmt.Sprintf( + "Processed %d revisions: deleted %d, retained %d referenced, scheduled %d retries", + stats.Claimed, stats.Deleted, stats.Referenced, stats.Retried, + )) + return nil +} diff --git a/internal/taskmanager/tasks/cleanup_client_diagnostics.go b/internal/taskmanager/tasks/cleanup_client_diagnostics.go new file mode 100644 index 00000000..cb63cdc3 --- /dev/null +++ b/internal/taskmanager/tasks/cleanup_client_diagnostics.go @@ -0,0 +1,73 @@ +package tasks + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/Silo-Server/silo-server/internal/diagnostics" + "github.com/Silo-Server/silo-server/internal/taskmanager" +) + +// ClientDiagnosticsCleanupTask prunes expired client diagnostics and reconciles +// object-store leftovers from interrupted uploads. +type ClientDiagnosticsCleanupTask struct { + repo diagnostics.CleanupRepository + settings diagnostics.SettingsStore + store diagnostics.ObjectStore + logger *slog.Logger +} + +func NewClientDiagnosticsCleanupTask( + repo diagnostics.CleanupRepository, + settings diagnostics.SettingsStore, + store diagnostics.ObjectStore, +) *ClientDiagnosticsCleanupTask { + return &ClientDiagnosticsCleanupTask{ + repo: repo, + settings: settings, + store: store, + logger: slog.Default(), + } +} + +func (t *ClientDiagnosticsCleanupTask) Key() string { return "cleanup_client_diagnostics" } +func (t *ClientDiagnosticsCleanupTask) Name() string { + return "Cleanup Client Diagnostics" +} +func (t *ClientDiagnosticsCleanupTask) Description() string { + return "Prunes expired client diagnostic reports and orphaned bundles" +} +func (t *ClientDiagnosticsCleanupTask) Category() taskmanager.TaskCategory { + return taskmanager.TaskCategorySystem +} +func (t *ClientDiagnosticsCleanupTask) IsHidden() bool { return false } + +func (t *ClientDiagnosticsCleanupTask) DefaultTriggers() []taskmanager.TriggerConfig { + // DefaultTriggers runs on the startup path, so bound the settings read rather + // than letting a slow store stall trigger setup before the default applies. + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return []taskmanager.TriggerConfig{ + {Type: taskmanager.TriggerTypeStartup}, + { + Type: taskmanager.TriggerTypeInterval, + IntervalMs: int64(diagnostics.LoadCleanupInterval(ctx, t.settings) / time.Millisecond), + }, + } +} + +func (t *ClientDiagnosticsCleanupTask) Execute(ctx context.Context, progress taskmanager.ProgressReporter) error { + progress.Report(0, "Pruning client diagnostic reports") + result, err := diagnostics.CleanupOnce(ctx, t.repo, t.settings, t.store, t.logger) + if err != nil { + return err + } + progress.Report(100, fmt.Sprintf( + "Pruned %d client diagnostic reports and %d orphaned bundles", + result.ReportsDeleted(), + result.OrphanObjectsDeleted, + )) + return nil +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index 9719775f..0ac57de6 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -27,22 +27,27 @@ import ( // TranscodeStartRequest is the JSON body for POST /transcode/start. type TranscodeStartRequest struct { - SessionID string `json:"session_id"` - InputPath string `json:"input_path"` - SourceVideoCodec string `json:"source_video_codec"` - SeekSeconds float64 `json:"seek_seconds"` - StartSegmentNumber int `json:"start_segment_number"` - TargetResolution string `json:"target_resolution"` - TargetCodecVideo string `json:"target_codec_video"` - TargetCodecAudio string `json:"target_codec_audio"` - TargetBitrateKbps int `json:"target_bitrate_kbps"` - SegmentDuration int `json:"segment_duration"` - HWAccel string `json:"hw_accel"` - AudioTrackIndex int `json:"audio_track_index"` - SubtitleTrackIndex int `json:"subtitle_track_index"` - SubtitleBurnIn bool `json:"subtitle_burn_in"` - SubtitleCodec string `json:"subtitle_codec,omitempty"` - TotalDuration float64 `json:"total_duration"` + SessionID string `json:"session_id"` + InputPath string `json:"input_path"` + SourceVideoCodec string `json:"source_video_codec"` + VideoBitstreamFilter string `json:"video_bitstream_filter,omitempty"` + SeekSeconds float64 `json:"seek_seconds"` + StreamOriginSeconds float64 `json:"stream_origin_seconds,omitempty"` + CopySeekAnchorResolved bool `json:"copy_seek_anchor_resolved,omitempty"` + StartSegmentNumber int `json:"start_segment_number"` + TargetResolution string `json:"target_resolution"` + TargetCodecVideo string `json:"target_codec_video"` + TargetCodecAudio string `json:"target_codec_audio"` + TargetAudioChannels int `json:"target_audio_channels,omitempty"` + TargetBitrateKbps int `json:"target_bitrate_kbps"` + SegmentDuration int `json:"segment_duration"` + HWAccel string `json:"hw_accel"` + AudioTrackIndex int `json:"audio_track_index"` + SubtitleTrackIndex int `json:"subtitle_track_index"` + SubtitleBurnIn bool `json:"subtitle_burn_in"` + SubtitleCodec string `json:"subtitle_codec,omitempty"` + TotalDuration float64 `json:"total_duration"` + RequireReady bool `json:"require_ready,omitempty"` } // TranscodeStartResponse is the JSON response for POST /transcode/start. @@ -58,12 +63,30 @@ type HealthResponse struct { ActiveJobs int32 `json:"active_jobs"` } +// sessionIdleTTL is how long a job may go without a manifest or segment +// request before the idle reaper closes it. Reaping is safe because a client +// that comes back later re-presents its still-valid stream token and the job +// reconstructs seeked to the requested segment; without the reaper, a job +// whose audience vanished (e.g. a v3 replan retired its transport id and a +// stale in-flight token resurrected the old one) encodes to end-of-file for +// nobody. +const sessionIdleTTL = 10 * time.Minute + +// sessionReapInterval is how often the idle reaper sweeps for stale jobs. +const sessionReapInterval = time.Minute + // Server is the HTTP handler for transcode mode. type Server struct { watcher *nodeconfig.Watcher tracker *nodesessions.Tracker ffmpegSink playback.FFmpegLogSink sessions map[string]*playback.TranscodeSession + // lastAccess records, per registered session id, when a manifest or segment + // request last touched the job (registration counts as the first access). + // Guarded by mu alongside sessions; the idle reaper closes jobs whose entry + // is older than sessionIdleTTL. + lastAccess map[string]time.Time + reaperOnce sync.Once mu sync.RWMutex activeJobs atomic.Int32 @@ -149,24 +172,182 @@ func (s *Server) restartSessionLocked(ctx context.Context, sessionID string, ses // NewServer creates a new transcode server. func NewServer(watcher *nodeconfig.Watcher, tracker *nodesessions.Tracker) *Server { s := &Server{ - watcher: watcher, - tracker: tracker, - sessions: make(map[string]*playback.TranscodeSession), - } - if cfg := watcher.Config(); cfg != nil { - if cleaned, err := playback.CleanupOrphanedTranscodeDirs(cfg.Playback.TranscodeDir, nil, 0); err != nil { - slog.Warn("transcode node cleanup failed", "dir", cfg.Playback.TranscodeDir, "error", err) - } else if cleaned > 0 { - slog.Info("transcode node cleanup removed orphaned dirs", "dir", cfg.Playback.TranscodeDir, "count", cleaned) - } + watcher: watcher, + tracker: tracker, + sessions: make(map[string]*playback.TranscodeSession), + lastAccess: make(map[string]time.Time), } return s } +// StartOrphanSweeper runs the age-guarded orphan-transcode sweep immediately and +// then hourly until ctx is cancelled. It never blocks (a slow network-filesystem +// delete runs in its own goroutine), so it is safe to call before the node binds +// its listener. This is the node's only filesystem-level reclaimer of dirs left +// behind by a session that was dropped without its output dir being removed — the +// idle reaper only deletes dirs it still tracks in s.sessions, so without this +// periodic pass such orphans would linger until the next process restart. The +// MaxTokenTTL age guard keeps a delete from racing a token-carried reconstruct +// writing into TranscodeDir/: a dir younger than the max token +// lifetime may still be reused, while older dirs are never reconstructable. +func (s *Server) StartOrphanSweeper(ctx context.Context) { + dir := "" + if cfg := s.watcher.Config(); cfg != nil { + dir = cfg.Playback.TranscodeDir + } + playback.StartPeriodicOrphanCleanup(ctx, "transcodenode", dir, func() (int, error) { + // Re-read config each run so a hot-reloaded TranscodeDir is honored. + cfg := s.watcher.Config() + if cfg == nil { + return 0, nil + } + // Spare the live registered jobs by id, not by age alone: now that the + // sweep runs during live traffic, a long-lived session that re-serves + // already-written segments stops advancing its dir mtime, so the age + // guard could misclassify it as orphaned. The live set is authoritative + // (in-flight reconstructs are covered by their fresh writes + age guard). + return playback.CleanupOrphanedTranscodeDirs(cfg.Playback.TranscodeDir, s.activeSessionIDs(), playback.MaxTokenTTL) + }, playback.OrphanCleanupInterval) +} + +// activeSessionIDs snapshots the ids of currently registered jobs so the orphan +// sweep spares their output dirs regardless of directory mtime, mirroring the +// central TranscodeManager's live-set snapshot. +func (s *Server) activeSessionIDs() map[string]struct{} { + s.mu.RLock() + defer s.mu.RUnlock() + active := make(map[string]struct{}, len(s.sessions)) + for id := range s.sessions { + active[id] = struct{}{} + } + return active +} + func (s *Server) SetFFmpegLogSink(sink playback.FFmpegLogSink) { s.ffmpegSink = sink } +// noteSessionAccessLocked records an access for a registered job. Callers must +// hold s.mu for writing. Lazily allocates so directly-constructed test servers +// work. +func (s *Server) noteSessionAccessLocked(sessionID string) { + if s.lastAccess == nil { + s.lastAccess = make(map[string]time.Time) + } + s.lastAccess[sessionID] = time.Now() +} + +// touchSession refreshes a registered job's idle clock so the reaper spares +// it. Unknown ids are ignored — a reconstruct records its own first access +// when it registers the rebuilt job. +func (s *Server) touchSession(sessionID string) { + s.mu.Lock() + if _, ok := s.sessions[sessionID]; ok { + s.noteSessionAccessLocked(sessionID) + } + s.mu.Unlock() +} + +// acquireSessionTouched returns the registered job for sessionID and, when +// found, refreshes its idle clock in the same critical section. Doing both +// under one lock closes the gap where the idle reaper could unregister the +// job between a read-lock lookup and a separate touch, leaving the request +// serving a session whose teardown is already removing its output dir. +func (s *Server) acquireSessionTouched(sessionID string) (*playback.TranscodeSession, bool) { + s.mu.Lock() + defer s.mu.Unlock() + session, ok := s.sessions[sessionID] + if ok { + s.noteSessionAccessLocked(sessionID) + } + return session, ok +} + +// startIdleReaper launches the background sweep that closes jobs no client has +// touched for sessionIdleTTL. Called once when the node starts serving; +// subsequent calls are no-ops. The goroutine runs for the process lifetime, +// matching the node's own. +func (s *Server) startIdleReaper() { + s.reaperOnce.Do(func() { + go func() { + ticker := time.NewTicker(sessionReapInterval) + defer ticker.Stop() + for range ticker.C { + s.reapIdleSessions(sessionIdleTTL) + } + }() + }) +} + +// reapIdleSessions closes and unregisters every job whose last manifest or +// segment access is older than ttl. Registration counts as the first access, +// so a job still waiting on its manifest (the RequireReady flow) is never +// reaped mid-wait. Candidates are collected under the map lock, then each is +// re-validated and torn down under its per-session lifecycle lock: Close +// removes the output dir, and without that lock it could race a token +// reconstruct and wipe the segments a fresh ffmpeg is writing. The session's +// recipe is deliberately kept — an idle reap is not a client stop, and a +// still-valid token must be able to reconstruct on the next hit. +func (s *Server) reapIdleSessions(ttl time.Duration) { + cutoff := time.Now().Add(-ttl) + type idleJob struct { + id string + session *playback.TranscodeSession + } + var candidates []idleJob + s.mu.Lock() + for id, session := range s.sessions { + last, ok := s.lastAccess[id] + if !ok { + // Untracked registration (shouldn't happen): start its idle clock + // now rather than closing a job that may be actively serving. + s.noteSessionAccessLocked(id) + continue + } + if last.Before(cutoff) { + candidates = append(candidates, idleJob{id: id, session: session}) + } + } + s.mu.Unlock() + + for _, c := range candidates { + s.reapSession(c.id, c.session, cutoff) + } +} + +// reapSession tears down one idle job under the per-session lifecycle lock so +// its Close can never overlap a start or reconstruct spawning a new ffmpeg +// into the same output dir. Ownership and idleness are re-checked under the +// lock; a job touched or replaced since the sweep scan is spared. +func (s *Server) reapSession(sessionID string, session *playback.TranscodeSession, cutoff time.Time) { + unlock := s.lockSessionLifecycle(sessionID) + defer unlock() + s.mu.Lock() + live, ok := s.sessions[sessionID] + if !ok || live != session { + s.mu.Unlock() + return + } + last, tracked := s.lastAccess[sessionID] + if tracked && !last.Before(cutoff) { + s.mu.Unlock() + return + } + delete(s.sessions, sessionID) + delete(s.lastAccess, sessionID) + s.mu.Unlock() + + s.activeJobs.Add(-1) + if err := session.Close(); err != nil { + slog.Error("close idle transcode session", "component", "transcodenode", "error", err, "session", sessionID, "playback_session_id", sessionID) + } + if s.tracker != nil { + s.tracker.Remove(context.Background(), sessionID) + } + slog.Info("transcode node reaped idle session", "component", "transcodenode", + "session", sessionID, "playback_session_id", sessionID, "idle_ms", time.Since(last).Milliseconds()) +} + // recipeStore reads a remote transcode's reconstruction recipe written by central // at transcode start. The jellycompat node-hop token is identity-only by design — // not because a Jellyfin client can't round-trip it, but because the recipe is @@ -190,6 +371,7 @@ func (s *Server) SetRecipeStore(store recipeStore) { // Handler returns the chi.Router with all transcode routes. func (s *Server) Handler() http.Handler { + s.startIdleReaper() r := chi.NewRouter() r.Get("/api/v1/health", s.handleHealth) @@ -215,12 +397,13 @@ func (s *Server) handleHealth(w http.ResponseWriter, _ *http.Request) { }) } -func (s *Server) handleHWCapabilities(w http.ResponseWriter, _ *http.Request) { +func (s *Server) handleHWCapabilities(w http.ResponseWriter, r *http.Request) { ffmpegPath := "" if cfg := s.watcher.Config(); cfg != nil { ffmpegPath = cfg.Playback.FFmpegPath } info := playback.DetectHWAccelWithFFmpeg(ffmpegPath) + info.Transformations = playback.ProbeTransformationRegistryV3(r.Context(), ffmpegPath).Advertised() w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(info) } @@ -293,29 +476,33 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { outputDir := filepath.Join(cfg.Playback.TranscodeDir, req.SessionID) opts := playback.TranscodeOpts{ - InputPath: req.InputPath, - OutputDir: outputDir, - SessionID: req.SessionID, - SourceVideoCodec: req.SourceVideoCodec, - SeekSeconds: req.SeekSeconds, - StartSegmentNumber: req.StartSegmentNumber, - TargetResolution: req.TargetResolution, - TargetCodecVideo: req.TargetCodecVideo, - TargetCodecAudio: req.TargetCodecAudio, - TargetBitrateKbps: req.TargetBitrateKbps, - SegmentDuration: req.SegmentDuration, - FFmpegPath: cfg.Playback.FFmpegPath, - HWAccel: req.HWAccel, - HWDevice: "", - AudioTrackIndex: req.AudioTrackIndex, - SubtitleTrackIndex: req.SubtitleTrackIndex, - SubtitleBurnIn: req.SubtitleBurnIn, - SubtitleCodec: req.SubtitleCodec, - TotalDuration: req.TotalDuration, - FastStart: true, - NodeType: "transcode", - ExecutionMode: "transcode_node", - FFmpegLogSink: s.ffmpegSink, + InputPath: req.InputPath, + OutputDir: outputDir, + SessionID: req.SessionID, + SourceVideoCodec: req.SourceVideoCodec, + VideoBitstreamFilter: req.VideoBitstreamFilter, + SeekSeconds: req.SeekSeconds, + StreamOriginSeconds: req.StreamOriginSeconds, + CopySeekAnchorResolved: req.CopySeekAnchorResolved, + StartSegmentNumber: req.StartSegmentNumber, + TargetResolution: req.TargetResolution, + TargetCodecVideo: req.TargetCodecVideo, + TargetCodecAudio: req.TargetCodecAudio, + TargetAudioChannels: req.TargetAudioChannels, + TargetBitrateKbps: req.TargetBitrateKbps, + SegmentDuration: req.SegmentDuration, + FFmpegPath: cfg.Playback.FFmpegPath, + HWAccel: req.HWAccel, + HWDevice: "", + AudioTrackIndex: req.AudioTrackIndex, + SubtitleTrackIndex: req.SubtitleTrackIndex, + SubtitleBurnIn: req.SubtitleBurnIn, + SubtitleCodec: req.SubtitleCodec, + TotalDuration: req.TotalDuration, + FastStart: true, + NodeType: "transcode", + ExecutionMode: "transcode_node", + FFmpegLogSink: s.ffmpegSink, } if opts.HWAccel == "" && cfg.Playback.HWAccel != "" { @@ -332,6 +519,7 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { s.mu.Lock() if old, ok := s.sessions[req.SessionID]; ok { delete(s.sessions, req.SessionID) + delete(s.lastAccess, req.SessionID) s.mu.Unlock() s.activeJobs.Add(-1) _ = old.Close() @@ -356,9 +544,19 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { http.Error(w, "failed to start transcode", http.StatusInternalServerError) return } + if req.RequireReady { + if _, err := session.WaitForManifest(8 * time.Second); err != nil { + _ = session.Close() + unlock() + slog.ErrorContext(r.Context(), "transcode failed readiness check", "component", "transcodenode", "error", err, "session", req.SessionID, "playback_session_id", req.SessionID) + http.Error(w, "transcode did not become ready", http.StatusInternalServerError) + return + } + } s.mu.Lock() s.sessions[req.SessionID] = session + s.noteSessionAccessLocked(req.SessionID) s.mu.Unlock() unlock() s.activeJobs.Add(1) @@ -415,7 +613,11 @@ func (s *Server) reconstructFromToken(r *http.Request, sessionID string, request // The token's recipe must be a transcode card for the session id in the URL: a // mismatch is a forged or stale request, and direct/remux cards carry no encode // parameters to rebuild. An empty PlayMethod is a transcode card (back-compat). - if card.SessionID != sessionID || (card.PlayMethod != "" && card.PlayMethod != playback.PlayTranscode) { + expectedTransportID := card.SessionID + if card.TranscodeTransportID != "" { + expectedTransportID = card.TranscodeTransportID + } + if expectedTransportID != sessionID || (card.PlayMethod != "" && card.PlayMethod != playback.PlayTranscode) { return nil } // A native token carries the full byte-affecting recipe. The jellycompat node @@ -486,6 +688,7 @@ func (s *Server) spawnReconstruct(r *http.Request, sessionID string, requestedSe cfg := s.watcher.Config() outputDir := filepath.Join(cfg.Playback.TranscodeDir, sessionID) opts := card.TranscodeOpts(outputDir, cfg.Playback.FFmpegPath, s.ffmpegSink) + opts.SessionID = sessionID // Re-resolve environment-specific encode knobs from this node's live config; the // token deliberately omits HWAccel/HWDevice so an operator change applies on // rebuild. Run as a transcode node, not integrated (card.TranscodeOpts defaults). @@ -528,6 +731,7 @@ func (s *Server) spawnReconstruct(r *http.Request, sessionID string, requestedSe return existing } s.sessions[sessionID] = session + s.noteSessionAccessLocked(sessionID) s.mu.Unlock() s.activeJobs.Add(1) @@ -575,6 +779,15 @@ func (s *Server) acquireReconstructSlot(ctx context.Context) (func(), bool) { func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") + // Serialize against in-flight starts and reconstructs. A RequireReady + // start registers its job only after the readiness wait; a stop racing + // that wait (the API rolling back a timed-out start) would otherwise miss + // the map and 404, orphaning the just-spawned ffmpeg until the idle + // reaper finds it. Blocking here until the start registers turns that + // miss into a normal teardown. + unlock := s.lockSessionLifecycle(sessionID) + defer unlock() + s.mu.Lock() session, ok := s.sessions[sessionID] if !ok { @@ -583,6 +796,7 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { return } delete(s.sessions, sessionID) + delete(s.lastAccess, sessionID) s.mu.Unlock() s.activeJobs.Add(-1) @@ -611,10 +825,10 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { func (s *Server) handleManifest(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") - s.mu.RLock() - session, ok := s.sessions[sessionID] - s.mu.RUnlock() - + // Lookup and liveness refresh happen atomically so the idle reaper can + // never unregister the job between them and tear down a session this + // request is about to serve from. + session, ok := s.acquireSessionTouched(sessionID) if !ok { // Lost the in-memory session (this node restarted): rebuild it from the // stream token the proxy forwarded. The manifest path carries no segment @@ -624,6 +838,9 @@ func (s *Server) handleManifest(w http.ResponseWriter, r *http.Request) { http.Error(w, "session not found", http.StatusNotFound) return } + // A reconstruct that yielded to a concurrently registered winner has + // not recorded this hit; count it so the reaper sees the liveness. + s.touchSession(sessionID) } manifest, err := session.BuildPlaybackManifest("segment/", r.URL.RawQuery) @@ -643,10 +860,10 @@ func (s *Server) handleSegment(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") name := chi.URLParam(r, "name") - s.mu.RLock() - session, ok := s.sessions[sessionID] - s.mu.RUnlock() - + // Lookup and liveness refresh happen atomically so the idle reaper can + // never unregister the job between them and tear down a session this + // request is about to serve from. + session, ok := s.acquireSessionTouched(sessionID) if !ok { // Lost the in-memory session (this node restarted): rebuild it from the // forwarded stream token, seeked to the segment the client is requesting so @@ -660,6 +877,9 @@ func (s *Server) handleSegment(w http.ResponseWriter, r *http.Request) { http.Error(w, "session not found", http.StatusNotFound) return } + // A reconstruct that yielded to a concurrently registered winner has + // not recorded this hit; count it so the reaper sees the liveness. + s.touchSession(sessionID) } segPath, err := session.GetSegment(name) @@ -777,6 +997,7 @@ func (s *Server) handleForceReload(w http.ResponseWriter, r *http.Request) { session.Close() os.RemoveAll(filepath.Join(cfg.Playback.TranscodeDir, id)) delete(s.sessions, id) + delete(s.lastAccess, id) stopped = append(stopped, id) } s.activeJobs.Store(0) diff --git a/internal/transcodenode/server_test.go b/internal/transcodenode/server_test.go index 5ee91b48..2f4f6170 100644 --- a/internal/transcodenode/server_test.go +++ b/internal/transcodenode/server_test.go @@ -1,9 +1,13 @@ package transcodenode import ( + "bytes" "context" + "encoding/json" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" "time" @@ -35,6 +39,81 @@ func newTestServer(t *testing.T) *Server { } } +func TestHandleStartRequireReadyRejectsExitedFFmpeg(t *testing.T) { + server := newTestServer(t) + ffmpegPath := filepath.Join(t.TempDir(), "failing-ffmpeg.sh") + if err := os.WriteFile(ffmpegPath, []byte("#!/bin/sh\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + server.watcher.Config().Playback.FFmpegPath = ffmpegPath + requestBody, err := json.Marshal(TranscodeStartRequest{ + SessionID: "ready-failure-1", + InputPath: "/media/movie.mkv", + TargetCodecVideo: "h264", + TargetCodecAudio: "aac", + SegmentDuration: 2, + RequireReady: true, + }) + if err != nil { + t.Fatal(err) + } + req := httptest.NewRequest(http.MethodPost, "/transcode/start", bytes.NewReader(requestBody)) + rr := httptest.NewRecorder() + server.handleStart(rr, req) + if rr.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + server.mu.RLock() + _, registered := server.sessions["ready-failure-1"] + server.mu.RUnlock() + if registered { + t.Fatal("failed readiness session was registered") + } +} + +func TestHandleStartDistinctReplacementFailurePreservesPredecessor(t *testing.T) { + server := newTestServer(t) + server.sessions["public-session"] = &playback.TranscodeSession{} + server.activeJobs.Store(1) + + ffmpegPath := filepath.Join(t.TempDir(), "failing-ffmpeg.sh") + if err := os.WriteFile(ffmpegPath, []byte("#!/bin/sh\nexit 1\n"), 0o755); err != nil { + t.Fatal(err) + } + server.watcher.Config().Playback.FFmpegPath = ffmpegPath + requestBody, err := json.Marshal(TranscodeStartRequest{ + SessionID: "public-session-legacy-replacement", + InputPath: "/media/movie.mkv", + TargetCodecVideo: "copy", + TargetCodecAudio: "aac", + SegmentDuration: 2, + RequireReady: true, + }) + if err != nil { + t.Fatal(err) + } + + req := httptest.NewRequest(http.MethodPost, "/transcode/start", bytes.NewReader(requestBody)) + rr := httptest.NewRecorder() + server.handleStart(rr, req) + if rr.Code != http.StatusInternalServerError { + t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String()) + } + server.mu.RLock() + predecessor := server.sessions["public-session"] + _, replacementRegistered := server.sessions["public-session-legacy-replacement"] + server.mu.RUnlock() + if predecessor == nil { + t.Fatal("failed distinct replacement removed the active predecessor") + } + if replacementRegistered { + t.Fatal("failed distinct replacement was registered") + } + if got := server.activeJobs.Load(); got != 1 { + t.Fatalf("active jobs = %d, want predecessor only", got) + } +} + func signCard(t *testing.T, card playback.RecipeCard) string { t.Helper() tok, err := streamtoken.Sign(card.ToClaims(), testSecret, time.Hour) @@ -212,6 +291,89 @@ func TestHandleStop_DeletesRecipe(t *testing.T) { } } +// The idle reaper must close only jobs whose last access predates the TTL; +// registration counts as an access, so a just-started job (including one still +// waiting on its manifest in the RequireReady flow) is spared. Zero-value +// TranscodeSessions Close without ffmpeg, so this runs without a real spawn. +func TestReapIdleSessions_ClosesOnlyIdleJobs(t *testing.T) { + s := newTestServer(t) + s.tracker = nodesessions.NewTracker(nil, "node-url", "node-name", "transcode") + + s.sessions["fresh-1"] = &playback.TranscodeSession{} + s.sessions["stale-1"] = &playback.TranscodeSession{} + s.lastAccess = map[string]time.Time{ + "fresh-1": time.Now(), + "stale-1": time.Now().Add(-sessionIdleTTL - time.Minute), + } + s.activeJobs.Store(2) + + s.reapIdleSessions(sessionIdleTTL) + + s.mu.RLock() + _, freshAlive := s.sessions["fresh-1"] + _, staleAlive := s.sessions["stale-1"] + _, staleTracked := s.lastAccess["stale-1"] + s.mu.RUnlock() + if !freshAlive { + t.Fatal("recently accessed session was reaped") + } + if staleAlive { + t.Fatal("idle session survived the reaper") + } + if staleTracked { + t.Fatal("reaped session's idle clock was not dropped") + } + if got := s.activeJobs.Load(); got != 1 { + t.Fatalf("activeJobs = %d, want 1", got) + } +} + +// A registered job with no recorded access (untracked registration) must not +// be closed; the sweep starts its idle clock instead of reaping a job that may +// be actively serving. +func TestReapIdleSessions_StartsClockForUntrackedJob(t *testing.T) { + s := newTestServer(t) + s.sessions["untracked-1"] = &playback.TranscodeSession{} + s.activeJobs.Store(1) + + s.reapIdleSessions(sessionIdleTTL) + + s.mu.RLock() + _, alive := s.sessions["untracked-1"] + last, tracked := s.lastAccess["untracked-1"] + s.mu.RUnlock() + if !alive { + t.Fatal("untracked session was reaped") + } + if !tracked || last.IsZero() { + t.Fatal("sweep did not start the untracked session's idle clock") + } + if got := s.activeJobs.Load(); got != 1 { + t.Fatalf("activeJobs = %d, want 1", got) + } +} + +// touchSession must refresh a registered job's idle clock and ignore ids with +// no live session (a reconstruct records its own first access on register). +func TestTouchSession_RefreshesIdleClock(t *testing.T) { + s := newTestServer(t) + s.sessions["live-1"] = &playback.TranscodeSession{} + stale := time.Now().Add(-sessionIdleTTL - time.Minute) + s.lastAccess = map[string]time.Time{"live-1": stale} + + s.touchSession("live-1") + s.touchSession("ghost-1") + + s.mu.RLock() + defer s.mu.RUnlock() + if !s.lastAccess["live-1"].After(stale) { + t.Fatal("touch did not refresh the live session's idle clock") + } + if _, ok := s.lastAccess["ghost-1"]; ok { + t.Fatal("touch recorded access for an unregistered session") + } +} + // spawnReconstruct must NOT apply the fast seg×dur resume seek for copy-mode // cards: copy-mode segments have variable durations, so seg×dur points at the // wrong source time. The card's original start must stand. Asserting opts off a diff --git a/internal/userdb/collections.go b/internal/userdb/collections.go index 91fb8cf3..cb31a8ff 100644 --- a/internal/userdb/collections.go +++ b/internal/userdb/collections.go @@ -114,13 +114,53 @@ func ListCollections(db *sql.DB, profileID string) ([]Collection, error) { if err := rows.Scan(&c.ID, &c.ProfileID, &c.CreatorProfileID, &c.Name, &c.CollectionType, &c.IsShared, &c.QueryDefinition, &c.SortConfig, &c.CreatedAt, &c.UpdatedAt); err != nil { return nil, err } - c.AllowedProfileIDs, err = listCollectionProfiles(db, c.ID) - if err != nil { - return nil, err - } collections = append(collections, c) } - return collections, rows.Err() + if err := rows.Err(); err != nil { + return nil, err + } + if err := attachCollectionProfiles(db, profileID, collections); err != nil { + return nil, err + } + return collections, nil +} + +// attachCollectionProfiles fills AllowedProfileIDs for every collection with a +// single batched query, avoiding the N+1 round trip a per-collection lookup +// would create. +func attachCollectionProfiles(db *sql.DB, profileID string, collections []Collection) error { + if len(collections) == 0 { + return nil + } + rows, err := db.Query( + `SELECT allowed.collection_id, allowed.profile_id + FROM personal_collection_profiles AS visible + JOIN personal_collection_profiles AS allowed + ON allowed.collection_id = visible.collection_id + WHERE visible.profile_id = ? + ORDER BY allowed.profile_id ASC`, + profileID, + ) + if err != nil { + return err + } + defer rows.Close() + + byCollection := make(map[string][]string, len(collections)) + for rows.Next() { + var collectionID, profileID string + if err := rows.Scan(&collectionID, &profileID); err != nil { + return err + } + byCollection[collectionID] = append(byCollection[collectionID], profileID) + } + if err := rows.Err(); err != nil { + return err + } + for i := range collections { + collections[i].AllowedProfileIDs = byCollection[collections[i].ID] + } + return nil } // UpdateCollection renames a collection and updates its updated_at timestamp. diff --git a/internal/userdb/conformance_test.go b/internal/userdb/conformance_test.go index 6d581080..0f1118cb 100644 --- a/internal/userdb/conformance_test.go +++ b/internal/userdb/conformance_test.go @@ -1,8 +1,10 @@ package userdb import ( + "context" "database/sql" "testing" + "time" "github.com/Silo-Server/silo-server/internal/userstore" "github.com/Silo-Server/silo-server/internal/userstore/storetest" @@ -27,3 +29,27 @@ func newConformanceStore(t *testing.T) userstore.UserStore { func TestSQLiteProgressSince(t *testing.T) { storetest.RunProgressSince(t, newConformanceStore) } + +func TestSQLiteAddFavoriteAtReportsInsertion(t *testing.T) { + ctx := context.Background() + store := newConformanceStore(t) + if err := store.CreateProfile(ctx, userstore.Profile{ID: "p1", Name: "Test"}); err != nil { + t.Fatalf("CreateProfile: %v", err) + } + + addedAt := time.Date(2026, time.July, 16, 12, 0, 0, 0, time.UTC) + inserted, err := store.AddFavoriteAt(ctx, "p1", "movie-1", addedAt) + if err != nil { + t.Fatalf("first AddFavoriteAt: %v", err) + } + if !inserted { + t.Fatal("first AddFavoriteAt reported no insertion") + } + inserted, err = store.AddFavoriteAt(ctx, "p1", "movie-1", addedAt) + if err != nil { + t.Fatalf("duplicate AddFavoriteAt: %v", err) + } + if inserted { + t.Fatal("duplicate AddFavoriteAt reported an insertion") + } +} diff --git a/internal/userdb/favorites.go b/internal/userdb/favorites.go index 5f18fe25..fd33ab1a 100644 --- a/internal/userdb/favorites.go +++ b/internal/userdb/favorites.go @@ -20,19 +20,23 @@ type WatchlistEntry = userstore.WatchlistEntry // AddFavorite adds a media item to a profile's favorites. // If the item is already a favorite, the operation is a no-op. func AddFavorite(db *sql.DB, profileID, mediaItemID string) error { - _, err := db.Exec( - `INSERT OR IGNORE INTO favorites (profile_id, media_item_id, added_at) VALUES (?, ?, ?)`, - profileID, mediaItemID, nowUTC(), - ) + _, err := AddFavoriteAt(db, profileID, mediaItemID, time.Now().UTC()) return err } -func AddFavoriteAt(db *sql.DB, profileID, mediaItemID string, addedAt time.Time) error { - _, err := db.Exec( +func AddFavoriteAt(db *sql.DB, profileID, mediaItemID string, addedAt time.Time) (bool, error) { + result, err := db.Exec( `INSERT OR IGNORE INTO favorites (profile_id, media_item_id, added_at) VALUES (?, ?, ?)`, profileID, mediaItemID, addedAt.UTC().Format(time.RFC3339), ) - return err + if err != nil { + return false, err + } + rows, err := result.RowsAffected() + if err != nil { + return false, err + } + return rows > 0, nil } // RemoveFavorite removes a media item from a profile's favorites. diff --git a/internal/userdb/listing_batch_test.go b/internal/userdb/listing_batch_test.go new file mode 100644 index 00000000..2a11ba80 --- /dev/null +++ b/internal/userdb/listing_batch_test.go @@ -0,0 +1,92 @@ +package userdb + +import ( + "database/sql" + "reflect" + "testing" + + "github.com/Silo-Server/silo-server/internal/userstore" +) + +func newListingTestDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + db.SetMaxOpenConns(1) + t.Cleanup(func() { _ = db.Close() }) + if err := InitSchema(db); err != nil { + t.Fatalf("InitSchema: %v", err) + } + return db +} + +func TestListProfilesAttachesAllowedLibrariesInOrder(t *testing.T) { + db := newListingTestDB(t) + for _, profile := range []Profile{ + {ID: "p1", Name: "One", AllowedLibraryIDs: []int{5, 2}, CreatedAt: "2026-01-01T00:00:00Z"}, + {ID: "p2", Name: "Two", AllowedLibraryIDs: []int{3}, CreatedAt: "2026-01-02T00:00:00Z"}, + {ID: "p3", Name: "Three", CreatedAt: "2026-01-03T00:00:00Z"}, + } { + if err := CreateProfile(db, profile); err != nil { + t.Fatalf("CreateProfile(%s): %v", profile.ID, err) + } + } + + profiles, err := ListProfiles(db) + if err != nil { + t.Fatalf("ListProfiles: %v", err) + } + if len(profiles) != 3 { + t.Fatalf("ListProfiles returned %d profiles, want 3", len(profiles)) + } + if got, want := profiles[0].AllowedLibraryIDs, []int{2, 5}; !reflect.DeepEqual(got, want) { + t.Fatalf("p1 AllowedLibraryIDs = %v, want %v", got, want) + } + if got, want := profiles[1].AllowedLibraryIDs, []int{3}; !reflect.DeepEqual(got, want) { + t.Fatalf("p2 AllowedLibraryIDs = %v, want %v", got, want) + } + if profiles[2].AllowedLibraryIDs != nil { + t.Fatalf("p3 AllowedLibraryIDs = %v, want nil", profiles[2].AllowedLibraryIDs) + } +} + +func TestListCollectionsAttachesAllowedProfilesInOrder(t *testing.T) { + db := newListingTestDB(t) + collection, err := CreateCollection(db, userstore.CreateCollectionInput{ + CreatorProfileID: "p1", + Name: "Shared", + IsShared: true, + AllowedProfileIDs: []string{"p3", "p2"}, + }) + if err != nil { + t.Fatalf("CreateCollection: %v", err) + } + + collections, err := ListCollections(db, "p2") + if err != nil { + t.Fatalf("ListCollections: %v", err) + } + if len(collections) != 1 || collections[0].ID != collection.ID { + t.Fatalf("ListCollections = %+v, want collection %s", collections, collection.ID) + } + if got, want := collections[0].AllowedProfileIDs, []string{"p1", "p2", "p3"}; !reflect.DeepEqual(got, want) { + t.Fatalf("AllowedProfileIDs = %v, want %v", got, want) + } +} + +func TestBatchAttachDoesNotScaleBindParametersWithRows(t *testing.T) { + db := newListingTestDB(t) + const aboveSQLiteVariableLimit = 32767 + + profiles := make([]Profile, aboveSQLiteVariableLimit) + if err := attachAllowedLibraries(db, profiles); err != nil { + t.Fatalf("attachAllowedLibraries: %v", err) + } + + collections := make([]Collection, aboveSQLiteVariableLimit) + if err := attachCollectionProfiles(db, "viewer", collections); err != nil { + t.Fatalf("attachCollectionProfiles: %v", err) + } +} diff --git a/internal/userdb/profile_libraries.go b/internal/userdb/profile_libraries.go index 6bcd316f..0e897c98 100644 --- a/internal/userdb/profile_libraries.go +++ b/internal/userdb/profile_libraries.go @@ -3,6 +3,8 @@ package userdb import ( "database/sql" "fmt" + + "github.com/Silo-Server/silo-server/internal/userstore" ) func listProfileAllowedLibraries(db *sql.DB, profileID string) ([]int, error) { @@ -29,6 +31,38 @@ func listProfileAllowedLibraries(db *sql.DB, profileID string) ([]int, error) { return libraryIDs, nil } +// attachAllowedLibraries fills AllowedLibraryIDs for every profile with a +// single batched query, avoiding the N+1 round trip a per-profile lookup +// would create. +func attachAllowedLibraries(db *sql.DB, profiles []Profile) error { + if len(profiles) == 0 { + return nil + } + rows, err := db.Query( + `SELECT profile_id, library_id + FROM profile_allowed_libraries + ORDER BY library_id ASC`, + ) + if err != nil { + return fmt.Errorf("listing allowed libraries: %w", err) + } + defer rows.Close() + + var allowedLibraries []userstore.ProfileAllowedLibrary + for rows.Next() { + var allowedLibrary userstore.ProfileAllowedLibrary + if err := rows.Scan(&allowedLibrary.ProfileID, &allowedLibrary.LibraryID); err != nil { + return fmt.Errorf("scanning allowed library: %w", err) + } + allowedLibraries = append(allowedLibraries, allowedLibrary) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating allowed libraries: %w", err) + } + userstore.AttachAllowedLibraries(profiles, allowedLibraries) + return nil +} + func replaceProfileAllowedLibrariesTx(tx *sql.Tx, profileID string, libraryIDs []int) error { if _, err := tx.Exec("DELETE FROM profile_allowed_libraries WHERE profile_id = ?", profileID); err != nil { return fmt.Errorf("clearing allowed libraries for profile %s: %w", profileID, err) diff --git a/internal/userdb/profiles.go b/internal/userdb/profiles.go index 09cf606e..54f80b0f 100644 --- a/internal/userdb/profiles.go +++ b/internal/userdb/profiles.go @@ -128,15 +128,14 @@ func ListProfiles(db *sql.DB) ([]Profile, error) { ); err != nil { return nil, fmt.Errorf("scanning profile row: %w", err) } - p.AllowedLibraryIDs, err = listProfileAllowedLibraries(db, p.ID) - if err != nil { - return nil, err - } profiles = append(profiles, p) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("iterating profile rows: %w", err) } + if err := attachAllowedLibraries(db, profiles); err != nil { + return nil, err + } return profiles, nil } diff --git a/internal/userdb/sqlitestore.go b/internal/userdb/sqlitestore.go index 866f63e5..9c78c389 100644 --- a/internal/userdb/sqlitestore.go +++ b/internal/userdb/sqlitestore.go @@ -172,7 +172,7 @@ func (s *SQLiteUserStore) AddFavorite(_ context.Context, profileID, mediaItemID return AddFavorite(s.db, profileID, mediaItemID) } -func (s *SQLiteUserStore) AddFavoriteAt(_ context.Context, profileID, mediaItemID string, addedAt time.Time) error { +func (s *SQLiteUserStore) AddFavoriteAt(_ context.Context, profileID, mediaItemID string, addedAt time.Time) (bool, error) { return AddFavoriteAt(s.db, profileID, mediaItemID, addedAt) } diff --git a/internal/userstore/pgstore/favorites.go b/internal/userstore/pgstore/favorites.go index 35a03ac7..8806b42a 100644 --- a/internal/userstore/pgstore/favorites.go +++ b/internal/userstore/pgstore/favorites.go @@ -15,17 +15,21 @@ import ( // --- Favorites --- func (s *PostgresUserStore) AddFavorite(ctx context.Context, profileID, mediaItemID string) error { - return s.AddFavoriteAt(ctx, profileID, mediaItemID, time.Now().UTC()) + _, err := s.AddFavoriteAt(ctx, profileID, mediaItemID, time.Now().UTC()) + return err } -func (s *PostgresUserStore) AddFavoriteAt(ctx context.Context, profileID, mediaItemID string, addedAt time.Time) error { - _, err := s.pool.Exec(ctx, +func (s *PostgresUserStore) AddFavoriteAt(ctx context.Context, profileID, mediaItemID string, addedAt time.Time) (bool, error) { + tag, err := s.pool.Exec(ctx, `INSERT INTO user_favorites (user_id, profile_id, media_item_id, added_at) VALUES ($1, $2, $3, $4) ON CONFLICT DO NOTHING`, s.userID, profileID, mediaItemID, addedAt.UTC(), ) - return err + if err != nil { + return false, err + } + return tag.RowsAffected() > 0, nil } func (s *PostgresUserStore) RemoveFavorite(ctx context.Context, profileID, mediaItemID string) error { diff --git a/internal/userstore/pgstore/profile_libraries.go b/internal/userstore/pgstore/profile_libraries.go index 534a5b6a..f15d668c 100644 --- a/internal/userstore/pgstore/profile_libraries.go +++ b/internal/userstore/pgstore/profile_libraries.go @@ -6,6 +6,8 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" + + "github.com/Silo-Server/silo-server/internal/userstore" ) type profileLibraryQuerier interface { @@ -41,6 +43,45 @@ func listProfileAllowedLibraries(ctx context.Context, q profileLibraryQuerier, u return libraryIDs, nil } +// attachAllowedLibraries fills AllowedLibraryIDs for every profile with a +// single batched query, avoiding the N+1 round trip a per-profile lookup +// would create. +func (s *PostgresUserStore) attachAllowedLibraries(ctx context.Context, profiles []userstore.Profile) error { + if len(profiles) == 0 { + return nil + } + ids := make([]string, len(profiles)) + for i := range profiles { + ids[i] = profiles[i].ID + } + rows, err := s.pool.Query(ctx, + `SELECT profile_id, library_id + FROM user_profile_allowed_libraries + WHERE user_id = $1 AND profile_id = ANY($2) + ORDER BY library_id ASC`, + s.userID, + ids, + ) + if err != nil { + return fmt.Errorf("listing allowed libraries for user %d: %w", s.userID, err) + } + defer rows.Close() + + var allowedLibraries []userstore.ProfileAllowedLibrary + for rows.Next() { + var allowedLibrary userstore.ProfileAllowedLibrary + if err := rows.Scan(&allowedLibrary.ProfileID, &allowedLibrary.LibraryID); err != nil { + return fmt.Errorf("scanning allowed library for user %d: %w", s.userID, err) + } + allowedLibraries = append(allowedLibraries, allowedLibrary) + } + if err := rows.Err(); err != nil { + return fmt.Errorf("iterating allowed libraries for user %d: %w", s.userID, err) + } + userstore.AttachAllowedLibraries(profiles, allowedLibraries) + return nil +} + func replaceProfileAllowedLibraries(ctx context.Context, q profileLibraryQuerier, userID int, profileID string, libraryIDs []int) error { if _, err := q.Exec(ctx, "DELETE FROM user_profile_allowed_libraries WHERE user_id = $1 AND profile_id = $2", diff --git a/internal/userstore/pgstore/profiles.go b/internal/userstore/pgstore/profiles.go index 4131a23a..44e89fe2 100644 --- a/internal/userstore/pgstore/profiles.go +++ b/internal/userstore/pgstore/profiles.go @@ -126,15 +126,14 @@ func (s *PostgresUserStore) ListProfiles(ctx context.Context) ([]userstore.Profi if err != nil { return nil, fmt.Errorf("scanning profile row: %w", err) } - p.AllowedLibraryIDs, err = listProfileAllowedLibraries(ctx, s.pool, s.userID, p.ID) - if err != nil { - return nil, err - } profiles = append(profiles, *p) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("iterating profile rows: %w", err) } + if err := s.attachAllowedLibraries(ctx, profiles); err != nil { + return nil, err + } return profiles, nil } diff --git a/internal/userstore/profile_libraries.go b/internal/userstore/profile_libraries.go new file mode 100644 index 00000000..6e2e4235 --- /dev/null +++ b/internal/userstore/profile_libraries.go @@ -0,0 +1,22 @@ +package userstore + +// ProfileAllowedLibrary associates one profile with one allowed library. +type ProfileAllowedLibrary struct { + ProfileID string + LibraryID int +} + +// AttachAllowedLibraries replaces each profile's AllowedLibraryIDs with the +// matching associations, preserving their input order. +func AttachAllowedLibraries(profiles []Profile, allowedLibraries []ProfileAllowedLibrary) { + byProfile := make(map[string][]int, len(profiles)) + for _, allowedLibrary := range allowedLibraries { + byProfile[allowedLibrary.ProfileID] = append( + byProfile[allowedLibrary.ProfileID], + allowedLibrary.LibraryID, + ) + } + for i := range profiles { + profiles[i].AllowedLibraryIDs = byProfile[profiles[i].ID] + } +} diff --git a/internal/userstore/profile_libraries_test.go b/internal/userstore/profile_libraries_test.go new file mode 100644 index 00000000..3dcce786 --- /dev/null +++ b/internal/userstore/profile_libraries_test.go @@ -0,0 +1,31 @@ +package userstore + +import ( + "reflect" + "testing" +) + +func TestAttachAllowedLibraries(t *testing.T) { + profiles := []Profile{ + {ID: "p1", AllowedLibraryIDs: []int{99}}, + {ID: "p2", AllowedLibraryIDs: []int{99}}, + {ID: "p3", AllowedLibraryIDs: []int{99}}, + } + allowedLibraries := []ProfileAllowedLibrary{ + {ProfileID: "p2", LibraryID: 1}, + {ProfileID: "p1", LibraryID: 2}, + {ProfileID: "p1", LibraryID: 5}, + } + + AttachAllowedLibraries(profiles, allowedLibraries) + + if got, want := profiles[0].AllowedLibraryIDs, []int{2, 5}; !reflect.DeepEqual(got, want) { + t.Fatalf("p1 AllowedLibraryIDs = %v, want %v", got, want) + } + if got, want := profiles[1].AllowedLibraryIDs, []int{1}; !reflect.DeepEqual(got, want) { + t.Fatalf("p2 AllowedLibraryIDs = %v, want %v", got, want) + } + if profiles[2].AllowedLibraryIDs != nil { + t.Fatalf("p3 AllowedLibraryIDs = %v, want nil", profiles[2].AllowedLibraryIDs) + } +} diff --git a/internal/userstore/store.go b/internal/userstore/store.go index cc511551..61e40b7e 100644 --- a/internal/userstore/store.go +++ b/internal/userstore/store.go @@ -58,7 +58,7 @@ type UserStore interface { // Favorites & Watchlist AddFavorite(ctx context.Context, profileID, mediaItemID string) error - AddFavoriteAt(ctx context.Context, profileID, mediaItemID string, addedAt time.Time) error + AddFavoriteAt(ctx context.Context, profileID, mediaItemID string, addedAt time.Time) (bool, error) RemoveFavorite(ctx context.Context, profileID, mediaItemID string) error ListFavorites(ctx context.Context, profileID string, limit, offset int) ([]Favorite, error) ListFavoritesByMediaItems(ctx context.Context, profileID string, mediaItemIDs []string) (map[string]bool, error) diff --git a/internal/watchsync/lists.go b/internal/watchsync/lists.go index 98c84cbb..e451249c 100644 --- a/internal/watchsync/lists.go +++ b/internal/watchsync/lists.go @@ -101,7 +101,8 @@ func (s *Service) favoritesBinding() listBinding { return res, true, err }, localAdd: func(ctx context.Context, store userstore.UserStore, profileID, mediaItemID string, at time.Time) error { - return store.AddFavoriteAt(ctx, profileID, mediaItemID, at) + _, err := store.AddFavoriteAt(ctx, profileID, mediaItemID, at) + return err }, localRemove: func(ctx context.Context, store userstore.UserStore, profileID, mediaItemID string) error { return store.RemoveFavorite(ctx, profileID, mediaItemID) diff --git a/internal/watchsync/providers/mdblist/provider.go b/internal/watchsync/providers/mdblist/provider.go index d6499f7d..bdc4f24c 100644 --- a/internal/watchsync/providers/mdblist/provider.go +++ b/internal/watchsync/providers/mdblist/provider.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "io" + "math" "net/http" "net/url" "strconv" @@ -38,6 +39,7 @@ const ( // survives burst-limit blips; anything longer defers the sync run. maxInPlaceRetryWait = 15 * time.Second maxRetryAttempts = 2 + maxErrorBodyBytes = 4 << 10 // Without a Retry-After header we cannot tell a burst limit from an // exhausted daily quota, so default to backing off until the next @@ -125,19 +127,23 @@ func (p *Provider) RefreshToken(_ context.Context, _ watchsync.ServerConfig, con func (p *Provider) FetchWatched(ctx context.Context, _ watchsync.ServerConfig, conn watchsync.Connection) ([]watchsync.RemoteWatch, error) { var rows []watchsync.RemoteWatch - for offset := 0; ; { + page := mdblistPageState{} + for { var payload mdblistWatchedResponse - path := fmt.Sprintf("/sync/watched?limit=%d&offset=%d", syncPageLimit, offset) + path := page.path("/sync/watched") if err := p.do(ctx, http.MethodGet, path, conn.AccessToken, nil, &payload); err != nil { return nil, err } - page := watchedRowsFromPayload(p.Key(), payload) - rows = append(rows, page...) + pageRows := watchedRowsFromPayload(p.Key(), payload) + rows = append(rows, pageRows...) fetched := len(payload.Movies) + len(payload.Episodes) - if fetched < syncPageLimit { + done, err := page.advance(payload.Pagination, fetched) + if err != nil { + return nil, fmt.Errorf("mdblist watched pagination: %w", err) + } + if done { break } - offset += fetched } return rows, nil } @@ -302,50 +308,76 @@ func (p *Provider) FetchHistory(ctx context.Context, cfg watchsync.ServerConfig, func (p *Provider) ExportHistory(ctx context.Context, _ watchsync.ServerConfig, conn watchsync.Connection, plays []watchsync.LocalPlay) (watchsync.ExportResult, error) { payload := buildWatchedPayload(plays, true) - if payload.empty() { - return watchsync.ExportResult{}, nil - } return p.sendWatched(ctx, conn, plays, "/sync/watched", payload) } func (p *Provider) RemoveHistory(ctx context.Context, _ watchsync.ServerConfig, conn watchsync.Connection, plays []watchsync.LocalPlay) (watchsync.ExportResult, error) { payload := buildWatchedPayload(plays, false) - if payload.empty() { - return watchsync.ExportResult{}, nil - } return p.sendWatched(ctx, conn, plays, "/sync/watched/remove", payload) } func (p *Provider) sendWatched(ctx context.Context, conn watchsync.Connection, plays []watchsync.LocalPlay, path string, payload mdblistWatchedPayload) (watchsync.ExportResult, error) { + if payload.empty() { + return watchedExportResult(plays, false), nil + } var body bytes.Buffer if err := json.NewEncoder(&body).Encode(payload); err != nil { return watchsync.ExportResult{}, fmt.Errorf("encode mdblist watched payload: %w", err) } - if err := p.do(ctx, http.MethodPost, path, conn.AccessToken, &body, nil); err != nil { + var response mdblistWatchedWriteResponse + if err := p.do(ctx, http.MethodPost, path, conn.AccessToken, &body, &response); err != nil { return watchsync.ExportResult{}, err } - result := watchsync.ExportResult{Sent: make([]string, 0, len(plays))} + return watchedExportResult(plays, !emptyJSONValue(response.NotFound)), nil +} + +func watchedExportResult(plays []watchsync.LocalPlay, responseHasNotFound bool) watchsync.ExportResult { + result := watchsync.ExportResult{ + Sent: make([]string, 0, len(plays)), + Failed: make(map[string]string), + } for _, play := range plays { + if play.HistoryID == "" { + continue + } + if !watchedPlaySupported(play) { + result.Failed[play.HistoryID] = "MDBList watched export requires a supported media kind and external ID" + continue + } + if responseHasNotFound { + // MDBList documents not_found only as an unstructured object. It does + // not guarantee enough identity information to safely attribute a + // partial rejection, so never claim that any item in this batch sent. + result.Failed[play.HistoryID] = "MDBList did not accept one or more watched items in the batch" + continue + } result.Sent = append(result.Sent, play.HistoryID) } - return result, nil + if len(result.Failed) == 0 { + result.Failed = nil + } + return result } func (p *Provider) FetchWatchlist(ctx context.Context, _ watchsync.ServerConfig, conn watchsync.Connection) ([]watchsync.RemoteFavorite, error) { now := time.Now().UTC() var rows []watchsync.RemoteFavorite - for offset := 0; ; { + page := mdblistPageState{} + for { var payload mdblistWatchlistResponse - path := fmt.Sprintf("/watchlist/items?limit=%d&offset=%d", syncPageLimit, offset) + path := page.path("/watchlist/items") if err := p.do(ctx, http.MethodGet, path, conn.AccessToken, nil, &payload); err != nil { return nil, err } rows = append(rows, watchlistRowsFromPayload(p.Key(), payload, now)...) fetched := len(payload.Movies) + len(payload.Shows) - if fetched < syncPageLimit { + done, err := page.advance(payload.Pagination, fetched) + if err != nil { + return nil, fmt.Errorf("mdblist watchlist pagination: %w", err) + } + if done { break } - offset += fetched } return rows, nil } @@ -392,30 +424,65 @@ func watchlistRowsFromPayload(providerKey string, payload mdblistWatchlistRespon } func (p *Provider) ExportWatchlist(ctx context.Context, _ watchsync.ServerConfig, conn watchsync.Connection, items []watchsync.LocalFavorite) (watchsync.ExportResult, error) { - return p.sendWatchlist(ctx, conn, items, "/watchlist/items/add") + return p.sendWatchlist(ctx, conn, items, "/watchlist/items/add", false) } func (p *Provider) RemoveWatchlist(ctx context.Context, _ watchsync.ServerConfig, conn watchsync.Connection, items []watchsync.LocalFavorite) (watchsync.ExportResult, error) { - return p.sendWatchlist(ctx, conn, items, "/watchlist/items/remove") + return p.sendWatchlist(ctx, conn, items, "/watchlist/items/remove", true) } -func (p *Provider) sendWatchlist(ctx context.Context, conn watchsync.Connection, favorites []watchsync.LocalFavorite, path string) (watchsync.ExportResult, error) { +func (p *Provider) sendWatchlist(ctx context.Context, conn watchsync.Connection, favorites []watchsync.LocalFavorite, path string, removing bool) (watchsync.ExportResult, error) { payload := buildWatchlistPayload(favorites) if len(payload.Movies) == 0 && len(payload.Shows) == 0 { - return watchsync.ExportResult{}, nil + return watchlistExportResult(favorites, false, removing), nil } var body bytes.Buffer if err := json.NewEncoder(&body).Encode(payload); err != nil { return watchsync.ExportResult{}, fmt.Errorf("encode mdblist watchlist payload: %w", err) } - if err := p.do(ctx, http.MethodPost, path, conn.AccessToken, &body, nil); err != nil { + var response mdblistWatchlistWriteResponse + if err := p.do(ctx, http.MethodPost, path, conn.AccessToken, &body, &response); err != nil { return watchsync.ExportResult{}, err } - result := watchsync.ExportResult{Sent: make([]string, 0, len(favorites)*2)} + return watchlistExportResult(favorites, !emptyJSONValue(response.NotFound), removing), nil +} + +func watchlistExportResult(favorites []watchsync.LocalFavorite, responseHasNotFound, removing bool) watchsync.ExportResult { + result := watchsync.ExportResult{ + Sent: make([]string, 0, len(favorites)), + NotFound: make([]string, 0, len(favorites)), + Failed: make(map[string]string), + } for _, fav := range favorites { + if fav.MediaItemID == "" { + continue + } + if !watchlistFavoriteSupported(fav) { + result.Failed[fav.MediaItemID] = "MDBList watchlist export requires a movie or series with an external ID" + continue + } + if responseHasNotFound { + if removing { + // A successful removal leaves both removed and already-absent items + // absent remotely. MDBList returns only aggregate counts, so mark + // the batch reconciled without trying to attribute individual rows. + result.NotFound = append(result.NotFound, fav.MediaItemID) + continue + } + // Adds still need item-level identity that the aggregate response + // does not provide, so never claim that any item in the batch sent. + result.Failed[fav.MediaItemID] = "MDBList did not accept one or more watchlist items in the batch" + continue + } result.Sent = append(result.Sent, fav.MediaItemID) } - return result, nil + if len(result.Failed) == 0 { + result.Failed = nil + } + if len(result.NotFound) == 0 { + result.NotFound = nil + } + return result } func (p *Provider) Start(ctx context.Context, _ watchsync.ServerConfig, conn watchsync.Connection, event watchsync.ScrobbleEvent) error { @@ -542,6 +609,10 @@ func (p *Provider) doOnce(ctx context.Context, method, path, target string, payl return -1, fmt.Errorf("mdblist request %s %s rejected: status %d (check api key)", method, path, resp.StatusCode) } if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + detail := responseErrorDetail(resp.Body) + if detail != "" { + return -1, fmt.Errorf("mdblist request %s %s failed: status %d: %s", method, path, resp.StatusCode, detail) + } return -1, fmt.Errorf("mdblist request %s %s failed: status %d", method, path, resp.StatusCode) } if out == nil || resp.StatusCode == http.StatusNoContent { @@ -553,6 +624,28 @@ func (p *Provider) doOnce(ctx context.Context, method, path, target string, payl return -1, nil } +func responseErrorDetail(body io.Reader) string { + raw, err := io.ReadAll(io.LimitReader(body, maxErrorBodyBytes)) + if err != nil { + return "" + } + raw = bytes.TrimSpace(raw) + if len(raw) == 0 { + return "" + } + var envelope struct { + Error json.RawMessage `json:"error"` + } + if json.Unmarshal(raw, &envelope) == nil && len(envelope.Error) > 0 { + raw = bytes.TrimSpace(envelope.Error) + } + var compact bytes.Buffer + if json.Compact(&compact, raw) == nil { + return compact.String() + } + return strings.Join(strings.Fields(string(raw)), " ") +} + // parseRetryAfter reads an RFC 7231 Retry-After value (delay-seconds or // HTTP-date). It returns 0 when the header is absent or unparseable. func parseRetryAfter(value string, now time.Time) time.Duration { @@ -669,8 +762,110 @@ func (e *mdblistWatchedEpisode) UnmarshalJSON(data []byte) error { } type mdblistWatchedResponse struct { - Movies []mdblistWatchedMovie `json:"movies"` - Episodes []mdblistWatchedEpisode `json:"episodes"` + Movies []mdblistWatchedMovie `json:"movies"` + Episodes []mdblistWatchedEpisode `json:"episodes"` + Pagination *mdblistPagination `json:"pagination"` +} + +type mdblistPagination struct { + NextCursor string `json:"next_cursor"` + HasMore bool `json:"has_more"` +} + +type mdblistPageState struct { + cursor string + offset int + legacyOffset bool +} + +func (s mdblistPageState) path(base string) string { + query := url.Values{"limit": {strconv.Itoa(syncPageLimit)}} + if s.cursor != "" { + query.Set("cursor", s.cursor) + } else if s.legacyOffset { + query.Set("offset", strconv.Itoa(s.offset)) + } + return base + "?" + query.Encode() +} + +func (s *mdblistPageState) advance(pagination *mdblistPagination, fetched int) (bool, error) { + if pagination != nil { + next := strings.TrimSpace(pagination.NextCursor) + if next != "" { + if next == s.cursor { + return false, errors.New("provider returned the same cursor twice") + } + s.cursor = next + return false, nil + } + if !pagination.HasMore { + return true, nil + } + if fetched == 0 { + return false, errors.New("provider reported more pages without returning items") + } + s.cursor = "" + s.legacyOffset = true + s.offset += fetched + return false, nil + } + if fetched < syncPageLimit { + return true, nil + } + // Older MDBList deployments omitted pagination metadata. Preserve the + // deprecated offset fallback for those responses while preferring cursors. + s.cursor = "" + s.legacyOffset = true + s.offset += fetched + return false, nil +} + +type mdblistWatchedWriteResponse struct { + NotFound json.RawMessage `json:"not_found"` +} + +type mdblistWatchlistWriteResponse struct { + NotFound json.RawMessage `json:"not_found"` +} + +func emptyJSONValue(raw json.RawMessage) bool { + if len(bytes.TrimSpace(raw)) == 0 { + return true + } + var value any + if err := json.Unmarshal(raw, &value); err != nil { + return false + } + return emptyJSONTree(value) +} + +func emptyJSONTree(value any) bool { + switch typed := value.(type) { + case nil: + return true + case bool: + return !typed + case float64: + return typed == 0 + case string: + return strings.TrimSpace(typed) == "" + case []any: + for _, child := range typed { + if !emptyJSONTree(child) { + return false + } + } + return true + case map[string]any: + for _, child := range typed { + if !emptyJSONTree(child) { + return false + } + } + return true + default: + return false + } } type mdblistPlaybackItem struct { @@ -775,8 +970,9 @@ func (i mdblistWatchlistItem) preferredIDs() mdblistIDs { } type mdblistWatchlistResponse struct { - Movies []mdblistWatchlistItem `json:"movies"` - Shows []mdblistWatchlistItem `json:"shows"` + Movies []mdblistWatchlistItem `json:"movies"` + Shows []mdblistWatchlistItem `json:"shows"` + Pagination *mdblistPagination `json:"pagination"` } type mdblistUser struct { @@ -866,6 +1062,18 @@ func buildWatchedPayload(plays []watchsync.LocalPlay, includeWatchedAt bool) mdb return payload } +func watchedPlaySupported(play watchsync.LocalPlay) bool { + switch play.Kind { + case historyimport.KindMovie: + return idsFromLocal(play.IMDbID, play.TMDBID, play.TVDBID) != (mdblistIDs{}) + case historyimport.KindEpisode: + return idsFromLocal(play.IMDbID, play.TMDBID, play.TVDBID) != (mdblistIDs{}) || + idsFromLocal(play.SeriesIMDbID, play.SeriesTMDBID, play.SeriesTVDBID) != (mdblistIDs{}) + default: + return false + } +} + func buildWatchlistPayload(favorites []watchsync.LocalFavorite) mdblistWatchlistPayload { var payload mdblistWatchlistPayload for _, fav := range favorites { @@ -887,6 +1095,17 @@ func buildWatchlistPayload(favorites []watchsync.LocalFavorite) mdblistWatchlist return payload } +func watchlistFavoriteSupported(fav watchsync.LocalFavorite) bool { + if fav.Kind != historyimport.KindMovie && fav.Kind != historyimport.KindSeries { + return false + } + ids := idsFromLocal(fav.IMDbID, fav.TMDBID, fav.TVDBID) + if ids == (mdblistIDs{}) { + ids = idsFromProviderItemKey(fav.ProviderItemKey) + } + return ids != (mdblistIDs{}) +} + func buildScrobblePayload(event watchsync.ScrobbleEvent) map[string]any { progress := 0.0 if event.DurationSeconds > 0 { @@ -898,6 +1117,10 @@ func buildScrobblePayload(event watchsync.ScrobbleEvent) map[string]any { if progress < 0 { progress = 0 } + // MDBList validates progress as a decimal with at most five total digits. + // Two decimal places keeps every value in the 0-100 range within that + // contract and avoids raw floating-point expansion on the wire. + progress = math.Round(progress*100) / 100 payload := map[string]any{"progress": progress} switch event.Kind { case historyimport.KindEpisode: @@ -905,9 +1128,15 @@ func buildScrobblePayload(event watchsync.ScrobbleEvent) map[string]any { if showIDs == (mdblistIDs{}) { showIDs = idsFromLocal(event.IMDbID, event.TMDBID, event.TVDBID) } - payload["show"] = map[string]any{"ids": showIDs} - payload["season"] = event.SeasonNumber - payload["episode"] = event.EpisodeNumber + payload["show"] = map[string]any{ + "ids": showIDs, + "season": map[string]any{ + "number": event.SeasonNumber, + "episode": map[string]any{ + "number": event.EpisodeNumber, + }, + }, + } default: payload["movie"] = map[string]any{"ids": idsFromLocal(event.IMDbID, event.TMDBID, event.TVDBID)} } diff --git a/internal/watchsync/providers/mdblist/provider_test.go b/internal/watchsync/providers/mdblist/provider_test.go index 3b8240ab..9316e515 100644 --- a/internal/watchsync/providers/mdblist/provider_test.go +++ b/internal/watchsync/providers/mdblist/provider_test.go @@ -123,6 +123,57 @@ func TestFetchWatchedParsesMoviesAndEpisodes(t *testing.T) { } } +func TestFetchWatchedUsesCursorPagination(t *testing.T) { + var queries []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + queries = append(queries, r.URL.RawQuery) + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("cursor") == "next-page" { + _, _ = w.Write([]byte(`{"movies":[],"episodes":[],"pagination":{"next_cursor":null}}`)) + return + } + _, _ = w.Write([]byte(`{"movies":[],"episodes":[],"pagination":{"next_cursor":"next-page"}}`)) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + if _, err := p.FetchWatched(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}); err != nil { + t.Fatalf("fetch watched: %v", err) + } + if len(queries) != 2 { + t.Fatalf("got %d requests, want 2", len(queries)) + } + if strings.Contains(queries[0], "offset=") || !strings.Contains(queries[1], "cursor=next-page") { + t.Fatalf("unexpected pagination queries: %#v", queries) + } +} + +func TestFetchWatchedContinuesOffsetPaginationWhenHasMore(t *testing.T) { + var offsets []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + offsets = append(offsets, r.URL.Query().Get("offset")) + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("offset") == "1" { + _, _ = w.Write([]byte(`{"movies":[],"episodes":[],"pagination":{"has_more":false}}`)) + return + } + _, _ = w.Write([]byte(`{"movies":[{"watched_at":"2025-11-01T08:30:00Z","movie":{"ids":{"imdb":"tt0111161"}}}],"episodes":[],"pagination":{"has_more":true}}`)) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + rows, err := p.FetchWatched(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}) + if err != nil { + t.Fatalf("fetch watched: %v", err) + } + if len(rows) != 1 { + t.Fatalf("got %d rows, want 1", len(rows)) + } + if len(offsets) != 2 || offsets[0] != "" || offsets[1] != "1" { + t.Fatalf("unexpected offsets: %#v", offsets) + } +} + func TestFetchProgressParsesFlatArray(t *testing.T) { pausedAt := time.Date(2025, time.November, 1, 8, 30, 0, 0, time.UTC) server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -241,6 +292,59 @@ func TestExportHistorySendsBulkPayload(t *testing.T) { } } +func TestExportHistoryDoesNotClaimNotFoundBatchWasSent(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"added":{"movies":[]},"updated":{},"not_found":{"movies":[{"ids":{"imdb":"tt-missing"}}]}}`)) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + plays := []watchsync.LocalPlay{{HistoryID: "h1", Kind: historyimport.KindMovie, IMDbID: "tt-missing"}} + result, err := p.ExportHistory(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}, plays) + if err != nil { + t.Fatalf("export: %v", err) + } + if len(result.Sent) != 0 || result.Failed["h1"] == "" { + t.Fatalf("unexpected result: %#v", result) + } +} + +func TestExportHistoryTreatsEmptyNotFoundBucketsAsSuccess(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"added":{"movies":1},"not_found":{"movies":[],"episodes":[]}}`)) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + plays := []watchsync.LocalPlay{{HistoryID: "h1", Kind: historyimport.KindMovie, IMDbID: "tt0111161"}} + result, err := p.ExportHistory(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}, plays) + if err != nil { + t.Fatalf("export: %v", err) + } + if len(result.Sent) != 1 || len(result.Failed) != 0 { + t.Fatalf("unexpected result: %#v", result) + } +} + +func TestExportHistoryReportsUnsupportedItemWithoutCallingMDBList(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + t.Fatal("MDBList should not be called for an unsupported item") + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + plays := []watchsync.LocalPlay{{HistoryID: "h1", Kind: historyimport.KindMovie}} + result, err := p.ExportHistory(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}, plays) + if err != nil { + t.Fatalf("export: %v", err) + } + if len(result.Sent) != 0 || result.Failed["h1"] == "" { + t.Fatalf("unexpected result: %#v", result) + } +} + func TestScrobbleStartUsesEpisodeShape(t *testing.T) { var gotPath string var gotBody map[string]any @@ -270,16 +374,52 @@ func TestScrobbleStartUsesEpisodeShape(t *testing.T) { if progress, _ := gotBody["progress"].(float64); progress != 25 { t.Fatalf("expected progress 25, got %v", gotBody["progress"]) } - if season, _ := gotBody["season"].(float64); int(season) != 3 { - t.Fatalf("expected season 3, got %v", gotBody["season"]) - } show, _ := gotBody["show"].(map[string]any) + season, _ := show["season"].(map[string]any) + if number, _ := season["number"].(float64); int(number) != 3 { + t.Fatalf("expected show season number 3, got %v", season["number"]) + } + episode, _ := season["episode"].(map[string]any) + if number, _ := episode["number"].(float64); int(number) != 7 { + t.Fatalf("expected show episode number 7, got %v", episode["number"]) + } + if _, exists := gotBody["season"]; exists { + t.Fatalf("season must be nested under show: %#v", gotBody) + } + if _, exists := show["episode"]; exists { + t.Fatalf("episode must be nested under show season: %#v", show) + } ids, _ := show["ids"].(map[string]any) if ids["imdb"] != "tt0903747" { t.Fatalf("expected show imdb tt0903747, got %v", ids["imdb"]) } } +func TestScrobbleRoundsProgressToMDBListPrecision(t *testing.T) { + var gotBody map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Fatalf("decode request: %v", err) + } + w.WriteHeader(http.StatusCreated) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + err := p.Start(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}, watchsync.ScrobbleEvent{ + Kind: historyimport.KindMovie, + TMDBID: "950387", + PositionSeconds: 611.8, + DurationSeconds: 2163.4, + }) + if err != nil { + t.Fatalf("scrobble start: %v", err) + } + if got := gotBody["progress"]; got != 28.28 { + t.Fatalf("progress = %v, want 28.28", got) + } +} + func TestExportWatchlistSendsToWatchlistAdd(t *testing.T) { var gotPath string var gotBody map[string]any @@ -288,7 +428,7 @@ func TestExportWatchlistSendsToWatchlistAdd(t *testing.T) { body, _ := io.ReadAll(r.Body) _ = json.Unmarshal(body, &gotBody) w.WriteHeader(http.StatusOK) - _, _ = w.Write([]byte(`{"added":1,"existing":0,"not_found":0}`)) + _, _ = w.Write([]byte(`{"added":{"movies":1,"shows":1},"existing":{"movies":0,"shows":0},"not_found":{"movies":0,"shows":0}}`)) })) defer server.Close() @@ -315,6 +455,110 @@ func TestExportWatchlistSendsToWatchlistAdd(t *testing.T) { } } +func TestExportWatchlistDoesNotClaimNotFoundBatchWasSent(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"added":{"movies":0,"shows":0},"existing":{"movies":0,"shows":0},"not_found":{"movies":1,"shows":0}}`)) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + favorites := []watchsync.LocalFavorite{{MediaItemID: "m1", Kind: historyimport.KindMovie, IMDbID: "tt0111161"}} + result, err := p.ExportWatchlist(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}, favorites) + if err != nil { + t.Fatalf("export watchlist: %v", err) + } + if len(result.Sent) != 0 || result.Failed["m1"] == "" { + t.Fatalf("unexpected result: %#v", result) + } +} + +func TestRemoveWatchlistTreatsNotFoundBatchAsReconciled(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/watchlist/items/remove" { + t.Fatalf("path = %q, want /watchlist/items/remove", r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"added":{"movies":0,"shows":0},"existing":{"movies":0,"shows":0},"not_found":{"movies":1,"shows":0}}`)) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + favorites := []watchsync.LocalFavorite{{MediaItemID: "m1", Kind: historyimport.KindMovie, IMDbID: "tt0111161"}} + result, err := p.RemoveWatchlist(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}, favorites) + if err != nil { + t.Fatalf("remove watchlist: %v", err) + } + if len(result.NotFound) != 1 || result.NotFound[0] != "m1" || len(result.Sent) != 0 || len(result.Failed) != 0 { + t.Fatalf("unexpected result: %#v", result) + } +} + +func TestExportWatchlistReportsUnsupportedItemWithoutCallingMDBList(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + t.Fatal("MDBList should not be called for an unsupported item") + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + favorites := []watchsync.LocalFavorite{{MediaItemID: "m1", Kind: historyimport.KindMovie}} + result, err := p.ExportWatchlist(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}, favorites) + if err != nil { + t.Fatalf("export watchlist: %v", err) + } + if len(result.Sent) != 0 || result.Failed["m1"] == "" { + t.Fatalf("unexpected result: %#v", result) + } +} + +func TestFetchWatchlistUsesCursorPagination(t *testing.T) { + var cursors []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cursors = append(cursors, r.URL.Query().Get("cursor")) + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("cursor") == "watch-next" { + _, _ = w.Write([]byte(`{"movies":[],"shows":[],"pagination":{"next_cursor":null}}`)) + return + } + _, _ = w.Write([]byte(`{"movies":[],"shows":[],"pagination":{"next_cursor":"watch-next"}}`)) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + if _, err := p.FetchWatchlist(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}); err != nil { + t.Fatalf("fetch watchlist: %v", err) + } + if len(cursors) != 2 || cursors[0] != "" || cursors[1] != "watch-next" { + t.Fatalf("unexpected cursors: %#v", cursors) + } +} + +func TestFetchWatchlistContinuesOffsetPaginationWhenHasMore(t *testing.T) { + var offsets []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + offsets = append(offsets, r.URL.Query().Get("offset")) + w.Header().Set("Content-Type", "application/json") + if r.URL.Query().Get("offset") == "1" { + _, _ = w.Write([]byte(`{"movies":[],"shows":[],"pagination":{"has_more":false}}`)) + return + } + _, _ = w.Write([]byte(`{"movies":[{"title":"The Shawshank Redemption","release_year":1994,"ids":{"imdb":"tt0111161"}}],"shows":[],"pagination":{"has_more":true}}`)) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + rows, err := p.FetchWatchlist(context.Background(), watchsync.ServerConfig{}, watchsync.Connection{AccessToken: "k"}) + if err != nil { + t.Fatalf("fetch watchlist: %v", err) + } + if len(rows) != 1 { + t.Fatalf("got %d rows, want 1", len(rows)) + } + if len(offsets) != 2 || offsets[0] != "" || offsets[1] != "1" { + t.Fatalf("unexpected offsets: %#v", offsets) + } +} + func TestDoRetriesShortRateLimitInPlace(t *testing.T) { attempts := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -411,6 +655,21 @@ func TestDoReplaysBodyOnRateLimitRetry(t *testing.T) { } } +func TestDoIncludesMDBListValidationError(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusBadRequest) + _, _ = w.Write([]byte(`{"error":{"progress":["Ensure that there are no more than 5 digits in total."]}}`)) + })) + defer server.Close() + + p := NewProvider(server.Client(), server.URL) + err := p.do(context.Background(), http.MethodPost, "/scrobble/start", "key", strings.NewReader(`{}`), nil) + if err == nil || !strings.Contains(err.Error(), "no more than 5 digits") { + t.Fatalf("expected validation detail, got %v", err) + } +} + func TestParseRetryAfter(t *testing.T) { now := time.Date(2026, time.July, 5, 12, 0, 0, 0, time.UTC) cases := []struct { diff --git a/internal/watchsync/repository.go b/internal/watchsync/repository.go index ec295eba..0b6c0781 100644 --- a/internal/watchsync/repository.go +++ b/internal/watchsync/repository.go @@ -45,10 +45,23 @@ type Repository interface { MarkListItemError(ctx context.Context, connectionID string, kind ListKind, mediaItemID, lastError string) error ListScrobbleConnections(ctx context.Context, userID int, profileID string) ([]Connection, error) UpsertScrobbleSession(ctx context.Context, event ScrobbleEvent, connectionID string, action string) error + PrepareConfirmedScrobbleStop(ctx context.Context, event ScrobbleEvent, connectionID string, staleBefore time.Time) (confirmedStopPreparation, time.Time, error) + CompleteConfirmedScrobbleStop(ctx context.Context, playbackSessionID string, connectionID string, progress float64, historyID string, claimVersion time.Time, stopSentAt time.Time) error + FailConfirmedScrobbleStop(ctx context.Context, playbackSessionID string, connectionID string, progress float64, historyID string, claimVersion time.Time, lastError string) error UpdateScrobbleSession(ctx context.Context, playbackSessionID string, connectionID string, action string, progress float64, historyID string, lastError string, stopSentAt *time.Time) error ListOpenScrobbleSessions(ctx context.Context) ([]ScrobbleSession, error) } +type confirmedStopPreparation uint8 + +const ( + confirmedStopPrepared confirmedStopPreparation = iota + confirmedStopAlreadySent + confirmedStopInProgress +) + +var errConfirmedStopClaimLost = errors.New("confirmed scrobble stop claim lost") + // connectionColumns is the canonical select/returning column list for // watch_provider_connections, in the exact order scanConnection reads. Sharing // it across every read query keeps the column set and scan order in lockstep. @@ -981,6 +994,113 @@ func (r *PostgresRepository) UpdateScrobbleSession(ctx context.Context, playback return nil } +func (r *PostgresRepository) PrepareConfirmedScrobbleStop(ctx context.Context, event ScrobbleEvent, connectionID string, staleBefore time.Time) (confirmedStopPreparation, time.Time, error) { + var claimVersion time.Time + err := r.pool.QueryRow(ctx, ` + INSERT INTO watch_provider_scrobble_sessions ( + playback_session_id, connection_id, media_item_id, provider_item_key, kind, + imdb_id, tmdb_id, tvdb_id, series_imdb_id, series_tmdb_id, series_tvdb_id, + season_number, episode_number, history_id, started_at, last_progress, + duration_seconds, completed, last_action, last_error, stop_sent_at + ) + VALUES ( + $1, $2::uuid, $3, $4, $5, $6, $7, $8, $9, $10, + $11, $12, $13, $14, $15, $16, $17, $18, 'stop_confirming', '', NULL + ) + ON CONFLICT (playback_session_id, connection_id) DO NOTHING + RETURNING updated_at + `, event.PlaybackSessionID, connectionID, event.MediaItemID, event.ProviderItemKey, event.Kind, + event.IMDbID, event.TMDBID, event.TVDBID, event.SeriesIMDbID, event.SeriesTMDBID, + event.SeriesTVDBID, event.SeasonNumber, event.EpisodeNumber, event.HistoryID, + event.OccurredAt, event.PositionSeconds, event.DurationSeconds, event.Completed).Scan(&claimVersion) + if err == nil { + return confirmedStopPrepared, claimVersion, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return confirmedStopInProgress, time.Time{}, fmt.Errorf("insert confirmed scrobble stop: %w", err) + } + + err = r.pool.QueryRow(ctx, ` + UPDATE watch_provider_scrobble_sessions + SET last_action = 'stop_confirming', + last_progress = $3, + history_id = COALESCE(NULLIF($4, ''), history_id), + duration_seconds = $5, + completed = $6, + last_error = '', + stop_sent_at = NULL, + updated_at = now() + WHERE playback_session_id = $1 AND connection_id = $2::uuid + -- A non-null stop_sent_at can be the provisional ActiveEncodings + -- fallback. Confirmed delivery must replace it with the later + -- authoritative Stopped position, so last_action owns deduplication. + AND last_action IS DISTINCT FROM 'stop_confirmed' + AND (last_action IS DISTINCT FROM 'stop_confirming' OR updated_at <= $7) + RETURNING updated_at + `, event.PlaybackSessionID, connectionID, event.PositionSeconds, event.HistoryID, + event.DurationSeconds, event.Completed, staleBefore).Scan(&claimVersion) + if err == nil { + return confirmedStopPrepared, claimVersion, nil + } + if !errors.Is(err, pgx.ErrNoRows) { + return confirmedStopInProgress, time.Time{}, fmt.Errorf("prepare confirmed scrobble stop: %w", err) + } + + var lastAction string + if err := r.pool.QueryRow(ctx, ` + SELECT last_action + FROM watch_provider_scrobble_sessions + WHERE playback_session_id = $1 AND connection_id = $2::uuid + `, event.PlaybackSessionID, connectionID).Scan(&lastAction); err != nil { + return confirmedStopInProgress, time.Time{}, fmt.Errorf("read confirmed scrobble stop state: %w", err) + } + if lastAction == "stop_confirmed" { + return confirmedStopAlreadySent, time.Time{}, nil + } + return confirmedStopInProgress, time.Time{}, nil +} + +func (r *PostgresRepository) CompleteConfirmedScrobbleStop(ctx context.Context, playbackSessionID string, connectionID string, progress float64, historyID string, claimVersion time.Time, stopSentAt time.Time) error { + tag, err := r.pool.Exec(ctx, ` + UPDATE watch_provider_scrobble_sessions + SET last_action = 'stop_confirmed', + last_progress = $3, + history_id = COALESCE(NULLIF($4, ''), history_id), + last_error = '', + stop_sent_at = $5, + updated_at = now() + WHERE playback_session_id = $1 AND connection_id = $2::uuid + AND last_action = 'stop_confirming' AND updated_at = $6 + `, playbackSessionID, connectionID, progress, historyID, stopSentAt, claimVersion) + if err != nil { + return fmt.Errorf("complete confirmed scrobble stop: %w", err) + } + if tag.RowsAffected() == 0 { + return errConfirmedStopClaimLost + } + return nil +} + +func (r *PostgresRepository) FailConfirmedScrobbleStop(ctx context.Context, playbackSessionID string, connectionID string, progress float64, historyID string, claimVersion time.Time, lastError string) error { + tag, err := r.pool.Exec(ctx, ` + UPDATE watch_provider_scrobble_sessions + SET last_action = 'stop_retry', + last_progress = $3, + history_id = COALESCE(NULLIF($4, ''), history_id), + last_error = $6, + updated_at = now() + WHERE playback_session_id = $1 AND connection_id = $2::uuid + AND last_action = 'stop_confirming' AND updated_at = $5 + `, playbackSessionID, connectionID, progress, historyID, claimVersion, lastError) + if err != nil { + return fmt.Errorf("fail confirmed scrobble stop: %w", err) + } + if tag.RowsAffected() == 0 { + return errConfirmedStopClaimLost + } + return nil +} + func (r *PostgresRepository) ListOpenScrobbleSessions(ctx context.Context) ([]ScrobbleSession, error) { rows, err := r.pool.Query(ctx, ` SELECT playback_session_id, connection_id::text, media_item_id, provider_item_key, kind, @@ -989,6 +1109,7 @@ func (r *PostgresRepository) ListOpenScrobbleSessions(ctx context.Context) ([]Sc duration_seconds, completed, last_action, stop_sent_at, last_error FROM watch_provider_scrobble_sessions WHERE stop_sent_at IS NULL + AND last_action NOT IN ('stop_confirming', 'stop_retry') ORDER BY started_at ASC `) if err != nil { diff --git a/internal/watchsync/scrobble_identity.go b/internal/watchsync/scrobble_identity.go new file mode 100644 index 00000000..95ef0623 --- /dev/null +++ b/internal/watchsync/scrobble_identity.go @@ -0,0 +1,55 @@ +package watchsync + +import ( + "context" + + "github.com/Silo-Server/silo-server/internal/historyimport" + "github.com/Silo-Server/silo-server/internal/userstore" +) + +// ScrobbleIdentityResolver resolves a local media item to the stable provider +// identity required by external scrobble APIs. +type ScrobbleIdentityResolver interface { + ResolveHistoryIdentity(ctx context.Context, mediaItemID string) userstore.WatchIdentity +} + +// ResolveScrobbleIdentity enriches an event with stable movie or episode IDs. +// Unknown identities retain the native playback fallback of treating the item +// as a movie; providers then report their normal missing-identity error instead +// of silently dropping the lifecycle event. +func ResolveScrobbleIdentity(ctx context.Context, resolver ScrobbleIdentityResolver, event ScrobbleEvent) ScrobbleEvent { + if resolver == nil { + if event.Kind == "" { + event.Kind = historyimport.KindMovie + } + return event + } + + identity := resolver.ResolveHistoryIdentity(ctx, event.MediaItemID) + if identity.StableType != "" { + event.Kind = identity.StableType + } + if event.Kind == "" { + event.Kind = historyimport.KindMovie + } + event.SeasonNumber = optionalIntValue(identity.Season) + event.EpisodeNumber = optionalIntValue(identity.Episode) + if identity.ProviderIDs != nil { + event.IMDbID = identity.ProviderIDs["imdb"] + event.TMDBID = identity.ProviderIDs["tmdb"] + event.TVDBID = identity.ProviderIDs["tvdb"] + } + if identity.SeriesProviderIDs != nil { + event.SeriesIMDbID = identity.SeriesProviderIDs["imdb"] + event.SeriesTMDBID = identity.SeriesProviderIDs["tmdb"] + event.SeriesTVDBID = identity.SeriesProviderIDs["tvdb"] + } + return event +} + +func optionalIntValue(value *int) int { + if value == nil { + return 0 + } + return *value +} diff --git a/internal/watchsync/scrobble_identity_test.go b/internal/watchsync/scrobble_identity_test.go new file mode 100644 index 00000000..5aff2889 --- /dev/null +++ b/internal/watchsync/scrobble_identity_test.go @@ -0,0 +1,44 @@ +package watchsync + +import ( + "context" + "testing" + + "github.com/Silo-Server/silo-server/internal/userstore" +) + +type fixedScrobbleIdentityResolver struct { + identity userstore.WatchIdentity +} + +func (r fixedScrobbleIdentityResolver) ResolveHistoryIdentity(context.Context, string) userstore.WatchIdentity { + return r.identity +} + +func TestResolveScrobbleIdentityEpisode(t *testing.T) { + season := 2 + episode := 7 + event := ResolveScrobbleIdentity(context.Background(), fixedScrobbleIdentityResolver{ + identity: userstore.WatchIdentity{ + StableType: "episode", + ProviderIDs: map[string]string{"tmdb": "episode-tmdb"}, + SeriesProviderIDs: map[string]string{"tvdb": "series-tvdb"}, + Season: &season, + Episode: &episode, + }, + }, ScrobbleEvent{MediaItemID: "episode-1"}) + + if event.Kind != "episode" || event.TMDBID != "episode-tmdb" || event.SeriesTVDBID != "series-tvdb" { + t.Fatalf("resolved identity = %+v", event) + } + if event.SeasonNumber != season || event.EpisodeNumber != episode { + t.Fatalf("resolved episode numbers = S%dE%d, want S%dE%d", event.SeasonNumber, event.EpisodeNumber, season, episode) + } +} + +func TestResolveScrobbleIdentityDefaultsUnknownItemToMovie(t *testing.T) { + event := ResolveScrobbleIdentity(context.Background(), nil, ScrobbleEvent{MediaItemID: "movie-1"}) + if event.Kind != "movie" { + t.Fatalf("kind = %q, want movie", event.Kind) + } +} diff --git a/internal/watchsync/service.go b/internal/watchsync/service.go index c810c5f0..5586b4e1 100644 --- a/internal/watchsync/service.go +++ b/internal/watchsync/service.go @@ -2,6 +2,7 @@ package watchsync import ( "context" + "errors" "fmt" "log/slog" "sort" @@ -31,6 +32,12 @@ type scrobbleQueue struct { tail chan struct{} } +type confirmedScrobbleTarget struct { + provider Provider + scrobbler Scrobbler + connection Connection +} + type mediaMatcher interface { Match(ctx context.Context, record historyimport.Record) (*historyimport.Match, string, error) } @@ -42,8 +49,15 @@ type watchStateImporter interface { const ( manualSyncCooldown = time.Hour manualSyncTimeout = 10 * time.Minute + // Built-in providers bind requests to this context and use HTTP client + // timeouts of at most 20 seconds. Keep dispatch below the reclaim lease; + // the per-session queue remains occupied until the worker itself exits. + confirmedStopDispatchTimeout = 25 * time.Second + confirmedStopLease = time.Minute ) +var errConfirmedStopInProgress = errors.New("watch provider stop confirmation already in progress") + func NewService(repo Repository, registry *Registry) *Service { return &Service{ repo: repo, @@ -1670,18 +1684,25 @@ func parseInt(value string) int { } func (s *Service) ScrobbleStart(ctx context.Context, event ScrobbleEvent) error { - return s.scrobble(ctx, event, "start") + return s.scrobble(ctx, event, "start", false) } func (s *Service) ScrobblePause(ctx context.Context, event ScrobbleEvent) error { - return s.scrobble(ctx, event, "pause") + return s.scrobble(ctx, event, "pause", false) } func (s *Service) ScrobbleStop(ctx context.Context, event ScrobbleEvent) error { - return s.scrobble(ctx, event, "stop") + return s.scrobble(ctx, event, "stop", false) } -func (s *Service) scrobble(ctx context.Context, event ScrobbleEvent, action string) error { +// ScrobbleStopConfirmed waits for each provider dispatch and reports provider +// or persistence failures to the caller. It is used when the caller owns a +// durable terminal event that must not be acknowledged after a mere enqueue. +func (s *Service) ScrobbleStopConfirmed(ctx context.Context, event ScrobbleEvent) error { + return s.scrobble(ctx, event, "stop", true) +} + +func (s *Service) scrobble(ctx context.Context, event ScrobbleEvent, action string, confirm bool) error { if event.PlaybackSessionID == "" || event.UserID == 0 || event.ProfileID == "" { return nil } @@ -1689,24 +1710,55 @@ func (s *Service) scrobble(ctx context.Context, event ScrobbleEvent, action stri if err != nil { return err } + // An empty authoritative query means this profile has no enabled scrobble + // destinations. There is nothing to retain or retry in that case. + if confirm && len(conns) == 0 { + return nil + } + var dispatchErrors []error + var confirmedTargets []confirmedScrobbleTarget for _, conn := range conns { provider, ok := s.registry.Get(conn.Provider) - if !ok || !provider.Capabilities().ScrobblePlayback { + if !ok { + if confirm { + dispatchErrors = append(dispatchErrors, fmt.Errorf( + "watch provider %q is not registered", conn.Provider, + )) + } + continue + } + if !provider.Capabilities().ScrobblePlayback { + if confirm { + dispatchErrors = append(dispatchErrors, fmt.Errorf( + "watch provider %q does not support playback scrobbling", conn.Provider, + )) + } continue } scrobbler, ok := provider.(Scrobbler) if !ok { + if confirm { + dispatchErrors = append(dispatchErrors, fmt.Errorf( + "watch provider %q has no scrobble dispatcher", conn.Provider, + )) + } continue } if action == "start" { if err := s.repo.UpsertScrobbleSession(ctx, event, conn.ID, action); err != nil { return err } - } else { + } else if !confirm { if err := s.repo.UpdateScrobbleSession(ctx, event.PlaybackSessionID, conn.ID, action, event.PositionSeconds, event.HistoryID, "", nil); err != nil { return err } } + if confirm { + confirmedTargets = append(confirmedTargets, confirmedScrobbleTarget{ + provider: provider, scrobbler: scrobbler, connection: conn, + }) + continue + } cfg, err := s.serverConfig(ctx, conn.Provider) if err != nil { _ = s.repo.UpdateScrobbleSession(ctx, event.PlaybackSessionID, conn.ID, action, event.PositionSeconds, event.HistoryID, err.Error(), nil) @@ -1719,20 +1771,88 @@ func (s *Service) scrobble(ctx context.Context, event ScrobbleEvent, action stri } s.dispatchScrobbleAsync(scrobbler, cfg, conn, event, action) } - return nil + if len(confirmedTargets) > 0 { + results := make(chan error, len(confirmedTargets)) + for _, target := range confirmedTargets { + go func() { + providerCtx, cancel := context.WithTimeout(ctx, confirmedStopDispatchTimeout) + defer cancel() + results <- s.dispatchScrobbleConfirmed( + providerCtx, + target.provider, + target.scrobbler, + target.connection, + event, + ) + }() + } + for range confirmedTargets { + if err := <-results; err != nil { + dispatchErrors = append(dispatchErrors, err) + } + } + } + return errors.Join(dispatchErrors...) } func (s *Service) dispatchScrobbleAsync(scrobbler Scrobbler, cfg ServerConfig, conn Connection, event ScrobbleEvent, action string) { + s.enqueueOrderedScrobble(scrobbleDispatchKey(scrobbler, conn, event), func() { + _ = s.dispatchScrobble(context.Background(), scrobbler, cfg, conn, event, action, nil) + }) +} + +func (s *Service) dispatchScrobbleConfirmed(ctx context.Context, provider Provider, scrobbler Scrobbler, conn Connection, event ScrobbleEvent) error { + dispatch := func() error { + preparation, claimVersion, err := s.repo.PrepareConfirmedScrobbleStop( + ctx, event, conn.ID, s.now().Add(-confirmedStopLease), + ) + if err != nil { + return err + } + switch preparation { + case confirmedStopAlreadySent: + return nil + case confirmedStopInProgress: + return errConfirmedStopInProgress + } + + cfg, err := s.serverConfig(ctx, conn.Provider) + if err != nil { + _ = s.repo.FailConfirmedScrobbleStop( + ctx, event.PlaybackSessionID, conn.ID, + event.PositionSeconds, event.HistoryID, claimVersion, err.Error(), + ) + return err + } + refreshedConn, err := s.refreshConnectionIfNeeded(ctx, provider, cfg, conn) + if err != nil { + _ = s.repo.FailConfirmedScrobbleStop( + ctx, event.PlaybackSessionID, conn.ID, + event.PositionSeconds, event.HistoryID, claimVersion, err.Error(), + ) + return err + } + return s.dispatchScrobble(ctx, scrobbler, cfg, refreshedConn, event, "stop", &claimVersion) + } + result := make(chan error, 1) + s.enqueueOrderedScrobble(scrobbleDispatchKey(scrobbler, conn, event), func() { + result <- dispatch() + }) + select { + case err := <-result: + return err + case <-ctx.Done(): + return ctx.Err() + } +} + +func scrobbleDispatchKey(scrobbler Scrobbler, conn Connection, event ScrobbleEvent) string { if ordered, ok := scrobbler.(OrderedScrobbler); ok { - key := ordered.ScrobbleOrderingKey(conn, event) - if strings.TrimSpace(key) != "" { - s.enqueueOrderedScrobble(key, func() { - s.dispatchScrobble(context.Background(), scrobbler, cfg, conn, event, action) - }) - return + if key := strings.TrimSpace(ordered.ScrobbleOrderingKey(conn, event)); key != "" { + return key } } - go s.dispatchScrobble(context.Background(), scrobbler, cfg, conn, event, action) + return conn.ID + ":" + event.PlaybackSessionID } func (s *Service) enqueueOrderedScrobble(key string, dispatch func()) { @@ -1754,7 +1874,7 @@ func (s *Service) enqueueOrderedScrobble(key string, dispatch func()) { }() } -func (s *Service) dispatchScrobble(ctx context.Context, scrobbler Scrobbler, cfg ServerConfig, conn Connection, event ScrobbleEvent, action string) { +func (s *Service) dispatchScrobble(ctx context.Context, scrobbler Scrobbler, cfg ServerConfig, conn Connection, event ScrobbleEvent, action string, confirmedClaim *time.Time) error { var err error switch action { case "pause": @@ -1765,13 +1885,29 @@ func (s *Service) dispatchScrobble(ctx context.Context, scrobbler Scrobbler, cfg err = scrobbler.Start(ctx, cfg, conn, event) } if err != nil { - _ = s.repo.UpdateScrobbleSession(ctx, event.PlaybackSessionID, conn.ID, action, event.PositionSeconds, event.HistoryID, err.Error(), nil) - return + if confirmedClaim != nil { + _ = s.repo.FailConfirmedScrobbleStop( + ctx, event.PlaybackSessionID, conn.ID, + event.PositionSeconds, event.HistoryID, *confirmedClaim, err.Error(), + ) + } else { + _ = s.repo.UpdateScrobbleSession(ctx, event.PlaybackSessionID, conn.ID, action, event.PositionSeconds, event.HistoryID, err.Error(), nil) + } + return err } if action == "stop" { stopSentAt := s.now() - _ = s.repo.UpdateScrobbleSession(ctx, event.PlaybackSessionID, conn.ID, action, event.PositionSeconds, event.HistoryID, "", &stopSentAt) + if confirmedClaim != nil { + return s.repo.CompleteConfirmedScrobbleStop( + ctx, event.PlaybackSessionID, conn.ID, + event.PositionSeconds, event.HistoryID, *confirmedClaim, stopSentAt, + ) + } + if err := s.repo.UpdateScrobbleSession(ctx, event.PlaybackSessionID, conn.ID, action, event.PositionSeconds, event.HistoryID, "", &stopSentAt); err != nil { + return err + } } + return nil } func (s *Service) SweepOpenScrobbles(ctx context.Context) error { @@ -1805,7 +1941,7 @@ func (s *Service) SweepOpenScrobbles(ctx context.Context) error { _ = s.repo.UpdateScrobbleSession(ctx, session.PlaybackSessionID, session.ConnectionID, "stop", session.LastProgress, session.HistoryID, err.Error(), nil) continue } - s.dispatchScrobble(ctx, scrobbler, cfg, conn, scrobbleEventFromSession(session, conn, s.now()), "stop") + _ = s.dispatchScrobble(ctx, scrobbler, cfg, conn, scrobbleEventFromSession(session, conn, s.now()), "stop", nil) } return nil } diff --git a/internal/watchsync/service_test.go b/internal/watchsync/service_test.go index a476e1f6..3a293553 100644 --- a/internal/watchsync/service_test.go +++ b/internal/watchsync/service_test.go @@ -7,6 +7,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -26,6 +27,10 @@ type serviceFakeRepo struct { scrobbleConnections []Connection scrobbleSessions []ScrobbleSession scrobbleUpdates []scrobbleUpdate + reopenedScrobbles []scrobbleUpdate + confirmedScrobbles map[string]bool + confirmingScrobbles map[string]time.Time + scrobbleMu sync.Mutex } type scrobbleUpdate struct { @@ -40,8 +45,10 @@ type scrobbleUpdate struct { func newServiceFakeRepo() *serviceFakeRepo { return &serviceFakeRepo{ - connections: make(map[string]Connection), - sessions: make(map[string]DeviceAuthSession), + connections: make(map[string]Connection), + sessions: make(map[string]DeviceAuthSession), + confirmedScrobbles: make(map[string]bool), + confirmingScrobbles: make(map[string]time.Time), settings: map[string]string{ "watchsync.trakt.client_id": "client-id", "watchsync.trakt.client_secret": "client-secret", @@ -423,6 +430,67 @@ func (r *serviceFakeRepo) UpsertScrobbleSession(_ context.Context, _ ScrobbleEve return nil } +func (r *serviceFakeRepo) PrepareConfirmedScrobbleStop(_ context.Context, event ScrobbleEvent, connectionID string, _ time.Time) (confirmedStopPreparation, time.Time, error) { + r.scrobbleMu.Lock() + defer r.scrobbleMu.Unlock() + key := event.PlaybackSessionID + "|" + connectionID + if r.confirmedScrobbles[key] { + return confirmedStopAlreadySent, time.Time{}, nil + } + if !r.confirmingScrobbles[key].IsZero() { + return confirmedStopInProgress, time.Time{}, nil + } + claimVersion := time.Now() + r.confirmingScrobbles[key] = claimVersion + r.reopenedScrobbles = append(r.reopenedScrobbles, scrobbleUpdate{ + playbackSessionID: event.PlaybackSessionID, + connectionID: connectionID, + action: "stop", + positionSeconds: event.PositionSeconds, + historyID: event.HistoryID, + }) + return confirmedStopPrepared, claimVersion, nil +} + +func (r *serviceFakeRepo) CompleteConfirmedScrobbleStop(_ context.Context, playbackSessionID string, connectionID string, positionSeconds float64, historyID string, claimVersion time.Time, stopSentAt time.Time) error { + r.scrobbleMu.Lock() + defer r.scrobbleMu.Unlock() + key := playbackSessionID + "|" + connectionID + if r.confirmingScrobbles[key] != claimVersion { + return errConfirmedStopClaimLost + } + delete(r.confirmingScrobbles, key) + r.confirmedScrobbles[key] = true + r.scrobbleUpdates = append(r.scrobbleUpdates, scrobbleUpdate{ + playbackSessionID: playbackSessionID, + connectionID: connectionID, + action: "stop_confirmed", + positionSeconds: positionSeconds, + historyID: historyID, + stopSentAt: &stopSentAt, + }) + return nil +} + +func (r *serviceFakeRepo) FailConfirmedScrobbleStop(_ context.Context, playbackSessionID string, connectionID string, positionSeconds float64, historyID string, claimVersion time.Time, lastError string) error { + r.scrobbleMu.Lock() + defer r.scrobbleMu.Unlock() + key := playbackSessionID + "|" + connectionID + if r.confirmingScrobbles[key] != claimVersion { + return errConfirmedStopClaimLost + } + delete(r.confirmingScrobbles, key) + r.scrobbleUpdates = append(r.scrobbleUpdates, scrobbleUpdate{ + playbackSessionID: playbackSessionID, + connectionID: connectionID, + action: "stop_confirming", + positionSeconds: positionSeconds, + historyID: historyID, + lastError: lastError, + }) + return nil +} + func (r *serviceFakeRepo) UpdateScrobbleSession(_ context.Context, playbackSessionID string, connectionID string, action string, positionSeconds float64, historyID string, lastError string, stopSentAt *time.Time) error { r.scrobbleUpdates = append(r.scrobbleUpdates, scrobbleUpdate{ playbackSessionID: playbackSessionID, @@ -673,6 +741,22 @@ type scrobblerStub struct { refreshErr error stopConns chan Connection stopEvents chan ScrobbleEvent + stopStarted chan struct{} + stopRelease chan struct{} + stopFailures *atomic.Int32 +} + +type keyedScrobblerStub struct { + scrobblerStub + key string +} + +func (p keyedScrobblerStub) Key() string { + return p.key +} + +func (p keyedScrobblerStub) DisplayName() string { + return p.key } func (p scrobblerStub) Key() string { @@ -696,12 +780,21 @@ func (p scrobblerStub) Pause(context.Context, ServerConfig, Connection, Scrobble } func (p scrobblerStub) Stop(_ context.Context, _ ServerConfig, conn Connection, event ScrobbleEvent) error { + if p.stopStarted != nil { + p.stopStarted <- struct{}{} + } + if p.stopRelease != nil { + <-p.stopRelease + } if p.stopConns != nil { p.stopConns <- conn } if p.stopEvents != nil { p.stopEvents <- event } + if p.stopFailures != nil && p.stopFailures.Add(-1) >= 0 { + return errors.New("stop failed") + } return p.stopErr } @@ -1795,6 +1888,433 @@ func TestServiceScrobbleStopKeepsSessionOpenWhenProviderStopFails(t *testing.T) } } +func TestServiceConfirmedStopWaitsForProviderAndReopensFailedReplacement(t *testing.T) { + repo := newServiceFakeRepo() + repo.scrobbleConnections = []Connection{{ + ID: "conn-1", + Provider: "trakt", + UserID: 7, + ProfileID: "profile-1", + ScrobbleEnabled: true, + }} + provider := scrobblerStub{stopErr: errors.New("stop failed")} + reg := NewRegistry() + if err := reg.Register(provider); err != nil { + t.Fatalf("Register: %v", err) + } + service := NewService(repo, reg) + + err := service.ScrobbleStopConfirmed(context.Background(), ScrobbleEvent{ + PlaybackSessionID: "playback-1", + UserID: 7, + ProfileID: "profile-1", + MediaItemID: "movie-1", + HistoryID: "history-1", + PositionSeconds: 120, + }) + if err == nil || err.Error() != "stop failed" { + t.Fatalf("ScrobbleStopConfirmed error = %v, want stop failed", err) + } + if len(repo.reopenedScrobbles) != 1 { + t.Fatalf("reopened scrobbles = %+v, want one durable reopen", repo.reopenedScrobbles) + } + reopened := repo.reopenedScrobbles[0] + if reopened.positionSeconds != 120 || reopened.historyID != "history-1" { + t.Fatalf("reopened scrobble = %+v, want authoritative progress", reopened) + } + for _, update := range repo.scrobbleUpdates { + if update.stopSentAt != nil { + t.Fatalf("failed confirmed stop marked session closed: %+v", repo.scrobbleUpdates) + } + } +} + +func TestServiceConfirmedStopRetriesImmediatelyAfterProviderFailure(t *testing.T) { + repo := newServiceFakeRepo() + repo.scrobbleConnections = []Connection{{ + ID: "conn-1", + Provider: "trakt", + UserID: 7, + ProfileID: "profile-1", + ScrobbleEnabled: true, + }} + var failures atomic.Int32 + failures.Store(1) + provider := scrobblerStub{ + stopEvents: make(chan ScrobbleEvent, 2), + stopFailures: &failures, + } + reg := NewRegistry() + if err := reg.Register(provider); err != nil { + t.Fatalf("Register: %v", err) + } + service := NewService(repo, reg) + event := ScrobbleEvent{ + PlaybackSessionID: "session-1", + UserID: 7, + ProfileID: "profile-1", + MediaItemID: "movie-1", + } + + if err := service.ScrobbleStopConfirmed(context.Background(), event); err == nil { + t.Fatal("first ScrobbleStopConfirmed unexpectedly succeeded") + } + if err := service.ScrobbleStopConfirmed(context.Background(), event); err != nil { + t.Fatalf("retry ScrobbleStopConfirmed: %v", err) + } + if len(provider.stopEvents) != 2 { + t.Fatalf("provider stop attempts = %d, want immediate retry", len(provider.stopEvents)) + } +} + +func TestServiceConfirmedStopWaitsInProviderOrder(t *testing.T) { + repo := newServiceFakeRepo() + repo.scrobbleConnections = []Connection{{ + ID: "conn-1", + Provider: "simkl", + UserID: 7, + ProfileID: "profile-1", + ScrobbleEnabled: true, + }} + provider := newOrderedScrobblerStub() + reg := NewRegistry() + if err := reg.Register(provider); err != nil { + t.Fatalf("Register: %v", err) + } + service := NewService(repo, reg) + event := ScrobbleEvent{ + PlaybackSessionID: "session-1", + UserID: 7, + ProfileID: "profile-1", + MediaItemID: "movie-1", + } + if err := service.ScrobbleStart(context.Background(), event); err != nil { + t.Fatalf("ScrobbleStart: %v", err) + } + if action := <-provider.started; action != "start" { + t.Fatalf("first dispatch = %q, want start", action) + } + + result := make(chan error, 1) + go func() { + result <- service.ScrobbleStopConfirmed(context.Background(), event) + }() + select { + case err := <-result: + t.Fatalf("confirmed stop returned before queued start completed: %v", err) + case <-time.After(25 * time.Millisecond): + } + + provider.release <- struct{}{} + if action := <-provider.started; action != "stop" { + t.Fatalf("second dispatch = %q, want stop", action) + } + select { + case err := <-result: + t.Fatalf("confirmed stop returned before provider completed: %v", err) + case <-time.After(25 * time.Millisecond): + } + provider.release <- struct{}{} + if err := <-result; err != nil { + t.Fatalf("ScrobbleStopConfirmed: %v", err) + } +} + +func TestServiceConfirmedStopDispatchesProvidersIndependently(t *testing.T) { + repo := newServiceFakeRepo() + repo.settings["watchsync.slow.client_id"] = "client-id" + repo.settings["watchsync.slow.client_secret"] = "client-secret" + repo.settings["watchsync.healthy.client_id"] = "client-id" + repo.settings["watchsync.healthy.client_secret"] = "client-secret" + repo.scrobbleConnections = []Connection{ + { + ID: "conn-slow", + Provider: "slow", + UserID: 7, + ProfileID: "profile-1", + ScrobbleEnabled: true, + }, + { + ID: "conn-healthy", + Provider: "healthy", + UserID: 7, + ProfileID: "profile-1", + ScrobbleEnabled: true, + }, + } + slow := keyedScrobblerStub{ + key: "slow", + scrobblerStub: scrobblerStub{ + stopStarted: make(chan struct{}, 1), + stopRelease: make(chan struct{}), + }, + } + healthy := keyedScrobblerStub{ + key: "healthy", + scrobblerStub: scrobblerStub{ + stopEvents: make(chan ScrobbleEvent, 1), + }, + } + reg := NewRegistry() + if err := reg.Register(slow); err != nil { + t.Fatalf("Register slow provider: %v", err) + } + if err := reg.Register(healthy); err != nil { + t.Fatalf("Register healthy provider: %v", err) + } + service := NewService(repo, reg) + event := ScrobbleEvent{ + PlaybackSessionID: "session-1", + UserID: 7, + ProfileID: "profile-1", + MediaItemID: "movie-1", + } + + result := make(chan error, 1) + go func() { + result <- service.ScrobbleStopConfirmed(context.Background(), event) + }() + select { + case <-slow.stopStarted: + case <-time.After(time.Second): + t.Fatal("timed out waiting for slow provider") + } + select { + case <-healthy.stopEvents: + case <-time.After(time.Second): + t.Fatal("healthy provider was starved by slow provider") + } + close(slow.stopRelease) + if err := <-result; err != nil { + t.Fatalf("ScrobbleStopConfirmed: %v", err) + } +} + +func TestServiceConfirmedStopWaitsBehindFallbackWithoutProviderOrdering(t *testing.T) { + repo := newServiceFakeRepo() + repo.scrobbleConnections = []Connection{{ + ID: "conn-1", + Provider: "trakt", + UserID: 7, + ProfileID: "profile-1", + ScrobbleEnabled: true, + }} + provider := scrobblerStub{ + stopStarted: make(chan struct{}, 2), + stopRelease: make(chan struct{}), + } + reg := NewRegistry() + if err := reg.Register(provider); err != nil { + t.Fatalf("Register: %v", err) + } + service := NewService(repo, reg) + event := ScrobbleEvent{ + PlaybackSessionID: "session-1", + UserID: 7, + ProfileID: "profile-1", + MediaItemID: "movie-1", + } + + if err := service.ScrobbleStop(context.Background(), event); err != nil { + t.Fatalf("fallback ScrobbleStop: %v", err) + } + select { + case <-provider.stopStarted: + case <-time.After(time.Second): + t.Fatal("timed out waiting for fallback stop") + } + + result := make(chan error, 1) + go func() { + result <- service.ScrobbleStopConfirmed(context.Background(), event) + }() + select { + case <-provider.stopStarted: + t.Fatal("confirmed stop passed the in-flight fallback") + case <-time.After(25 * time.Millisecond): + } + + provider.stopRelease <- struct{}{} + select { + case <-provider.stopStarted: + case <-time.After(time.Second): + t.Fatal("confirmed stop did not follow the fallback") + } + provider.stopRelease <- struct{}{} + if err := <-result; err != nil { + t.Fatalf("ScrobbleStopConfirmed: %v", err) + } +} + +func TestServiceConfirmedStopDoesNotResendSuccessfulProvider(t *testing.T) { + repo := newServiceFakeRepo() + repo.scrobbleConnections = []Connection{{ + ID: "conn-1", + Provider: "trakt", + UserID: 7, + ProfileID: "profile-1", + ScrobbleEnabled: true, + }} + provider := scrobblerStub{stopEvents: make(chan ScrobbleEvent, 2)} + reg := NewRegistry() + if err := reg.Register(provider); err != nil { + t.Fatalf("Register: %v", err) + } + service := NewService(repo, reg) + event := ScrobbleEvent{ + PlaybackSessionID: "session-1", + UserID: 7, + ProfileID: "profile-1", + MediaItemID: "movie-1", + } + + if err := service.ScrobbleStopConfirmed(context.Background(), event); err != nil { + t.Fatalf("first ScrobbleStopConfirmed: %v", err) + } + if err := service.ScrobbleStopConfirmed(context.Background(), event); err != nil { + t.Fatalf("retry ScrobbleStopConfirmed: %v", err) + } + select { + case <-provider.stopEvents: + default: + t.Fatal("successful confirmed stop was not dispatched") + } + select { + case duplicate := <-provider.stopEvents: + t.Fatalf("successful provider received duplicate confirmed stop: %+v", duplicate) + default: + } + if len(repo.reopenedScrobbles) != 1 { + t.Fatalf("confirmed stop preparations = %+v, want one", repo.reopenedScrobbles) + } +} + +func TestServiceConfirmedStopSerializesConcurrentConfirmation(t *testing.T) { + repo := newServiceFakeRepo() + repo.scrobbleConnections = []Connection{{ + ID: "conn-1", + Provider: "trakt", + UserID: 7, + ProfileID: "profile-1", + ScrobbleEnabled: true, + }} + provider := scrobblerStub{ + stopStarted: make(chan struct{}, 1), + stopRelease: make(chan struct{}), + } + reg := NewRegistry() + if err := reg.Register(provider); err != nil { + t.Fatalf("Register: %v", err) + } + service := NewService(repo, reg) + event := ScrobbleEvent{ + PlaybackSessionID: "session-1", + UserID: 7, + ProfileID: "profile-1", + MediaItemID: "movie-1", + } + + firstResult := make(chan error, 1) + go func() { + firstResult <- service.ScrobbleStopConfirmed(context.Background(), event) + }() + select { + case <-provider.stopStarted: + case <-time.After(time.Second): + t.Fatal("timed out waiting for first provider stop") + } + + secondResult := make(chan error, 1) + go func() { + secondResult <- service.ScrobbleStopConfirmed(context.Background(), event) + }() + select { + case err := <-secondResult: + t.Fatalf("concurrent confirmation returned before the first completed: %v", err) + case <-time.After(25 * time.Millisecond): + } + close(provider.stopRelease) + if err := <-firstResult; err != nil { + t.Fatalf("first ScrobbleStopConfirmed: %v", err) + } + if err := <-secondResult; err != nil { + t.Fatalf("second ScrobbleStopConfirmed: %v", err) + } + select { + case <-provider.stopStarted: + t.Fatal("concurrent confirmation dispatched a duplicate provider stop") + default: + } +} + +func TestServiceConfirmedStopCannotCompleteReclaimedLease(t *testing.T) { + repo := newServiceFakeRepo() + repo.scrobbleConnections = []Connection{{ + ID: "conn-1", + Provider: "trakt", + UserID: 7, + ProfileID: "profile-1", + ScrobbleEnabled: true, + }} + provider := scrobblerStub{ + stopStarted: make(chan struct{}, 1), + stopRelease: make(chan struct{}), + } + reg := NewRegistry() + if err := reg.Register(provider); err != nil { + t.Fatalf("Register: %v", err) + } + service := NewService(repo, reg) + event := ScrobbleEvent{ + PlaybackSessionID: "session-1", + UserID: 7, + ProfileID: "profile-1", + MediaItemID: "movie-1", + } + + result := make(chan error, 1) + go func() { + result <- service.ScrobbleStopConfirmed(context.Background(), event) + }() + select { + case <-provider.stopStarted: + case <-time.After(time.Second): + t.Fatal("timed out waiting for provider stop") + } + + key := event.PlaybackSessionID + "|conn-1" + repo.confirmingScrobbles[key] = time.Now().Add(time.Second) + close(provider.stopRelease) + if err := <-result; !errors.Is(err, errConfirmedStopClaimLost) { + t.Fatalf("stale ScrobbleStopConfirmed error = %v, want claim lost", err) + } + if repo.confirmedScrobbles[key] { + t.Fatal("stale provider worker marked the reclaimed stop confirmed") + } +} + +func TestServiceConfirmedStopRejectsUnregisteredProvider(t *testing.T) { + repo := newServiceFakeRepo() + repo.scrobbleConnections = []Connection{{ + ID: "conn-1", + Provider: "temporarily-unavailable", + UserID: 7, + ProfileID: "profile-1", + ScrobbleEnabled: true, + }} + service := NewService(repo, NewRegistry()) + + err := service.ScrobbleStopConfirmed(context.Background(), ScrobbleEvent{ + PlaybackSessionID: "session-1", + UserID: 7, + ProfileID: "profile-1", + MediaItemID: "movie-1", + }) + if err == nil || !strings.Contains(err.Error(), "not registered") { + t.Fatalf("ScrobbleStopConfirmed error = %v, want unregistered provider", err) + } +} + func TestServiceScrobbleRefreshesExpiredTokenBeforeDispatch(t *testing.T) { repo := newServiceFakeRepo() now := time.Date(2026, 5, 4, 12, 0, 0, 0, time.UTC) diff --git a/migrations/playback_protocol_v3_test.go b/migrations/playback_protocol_v3_test.go new file mode 100644 index 00000000..e44710d0 --- /dev/null +++ b/migrations/playback_protocol_v3_test.go @@ -0,0 +1,21 @@ +package migrations + +import ( + "strings" + "testing" + + "github.com/Silo-Server/silo-server/internal/playback" +) + +func TestPlaybackProtocolV3MigrationAcceptsEveryRouteEvent(t *testing.T) { + migrationBytes, err := FS.ReadFile("sql/20260712103000_add_playback_protocol_v3.sql") + if err != nil { + t.Fatalf("read migration: %v", err) + } + migration := string(migrationBytes) + for _, event := range playback.RouteEventNamesV3() { + if !strings.Contains(migration, "'"+event+"'") { + t.Errorf("route event CHECK constraint is missing %q", event) + } + } +} diff --git a/migrations/sql/20260711114310_history_import_favorites_imported.sql b/migrations/sql/20260711114310_history_import_favorites_imported.sql new file mode 100644 index 00000000..6add3d94 --- /dev/null +++ b/migrations/sql/20260711114310_history_import_favorites_imported.sql @@ -0,0 +1,11 @@ +-- +goose Up +-- +goose StatementBegin +-- Count successfully applied Emby favorites during history import runs. +ALTER TABLE history_import_runs +ADD COLUMN favorites_imported integer NOT NULL DEFAULT 0; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +ALTER TABLE history_import_runs DROP COLUMN IF EXISTS favorites_imported; +-- +goose StatementEnd diff --git a/migrations/sql/20260712100125_builtin_host_providers.sql b/migrations/sql/20260712100125_builtin_host_providers.sql new file mode 100644 index 00000000..44d3f610 --- /dev/null +++ b/migrations/sql/20260712100125_builtin_host_providers.sql @@ -0,0 +1,52 @@ +-- +goose Up +-- Register built-in host metadata providers as data (#216, Phase A). +-- +-- Chain rows require a real plugin_installations id (FK + composite PK), so +-- built-ins are represented by one reserved installation row of kind +-- 'builtin' plus ordinary plugin_capabilities rows. The install_path is a +-- guaranteed-nonexistent sentinel: InstallationStore.Delete removes the +-- install directory on disk, so the sentinel must never resolve to a real +-- directory (the store additionally rejects deleting builtin rows). +-- +-- Statement order matters: the partial unique index must exist before any +-- ON CONFLICT targets it, and the installation insert uses +-- INSERT ... SELECT ... WHERE NOT EXISTS because plugin_id had no unique +-- constraint before this migration. +ALTER TABLE plugin_installations + ADD COLUMN kind TEXT NOT NULL DEFAULT 'plugin' + CONSTRAINT plugin_installations_kind_check CHECK (kind IN ('plugin', 'builtin')); + +CREATE UNIQUE INDEX idx_plugin_installations_builtin_plugin_id + ON plugin_installations (plugin_id) + WHERE kind = 'builtin'; + +INSERT INTO plugin_installations (plugin_id, version, install_path, enabled, update_policy, kind) +SELECT 'silo.builtin', '0', '/nonexistent/silo-builtin', true, 'manual', 'builtin' +WHERE NOT EXISTS ( + SELECT 1 FROM plugin_installations WHERE plugin_id = 'silo.builtin' +); + +-- The NFO provider: seeded disabled everywhere (default_enabled=false), at +-- declared priority 1 for movie/series when an admin enables it. +INSERT INTO plugin_capabilities (plugin_installation_id, capability_type, capability_id, metadata) +SELECT pi.id, 'metadata_provider.v1', 'nfo', + '{"display_name":"NFO Files","default_priority":{"movie":1,"series":1},"default_enabled":false}'::jsonb +FROM plugin_installations pi +WHERE pi.plugin_id = 'silo.builtin' AND pi.kind = 'builtin' +ON CONFLICT DO NOTHING; + +-- +goose Down +-- Rolling back deletes the reserved installation row; library_provider_chains +-- rows referencing it cascade, so any admin-configured NFO chain placement is +-- lost on rollback (accepted and documented). +DELETE FROM plugin_capabilities pc +USING plugin_installations pi +WHERE pc.plugin_installation_id = pi.id + AND pi.plugin_id = 'silo.builtin' AND pi.kind = 'builtin' + AND pc.capability_type = 'metadata_provider.v1' AND pc.capability_id = 'nfo'; + +DELETE FROM plugin_installations WHERE plugin_id = 'silo.builtin' AND kind = 'builtin'; + +DROP INDEX IF EXISTS idx_plugin_installations_builtin_plugin_id; + +ALTER TABLE plugin_installations DROP COLUMN kind; diff --git a/migrations/sql/20260712103000_add_playback_protocol_v3.sql b/migrations/sql/20260712103000_add_playback_protocol_v3.sql new file mode 100644 index 00000000..3930a059 --- /dev/null +++ b/migrations/sql/20260712103000_add_playback_protocol_v3.sql @@ -0,0 +1,92 @@ +-- +goose Up +-- +goose StatementBegin +INSERT INTO server_settings (key, value) +VALUES ('playback.protocol_v3_enabled', 'false') +ON CONFLICT (key) DO NOTHING; + +INSERT INTO server_settings (key, value) +VALUES ('playback.protocol_v3_shadow_enabled', 'false') +ON CONFLICT (key) DO NOTHING; + +CREATE TABLE playback_v3_attempts ( + playback_attempt_id TEXT PRIMARY KEY, + session_id UUID NOT NULL UNIQUE, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + profile_id TEXT NOT NULL, + requested_media_file_id BIGINT NOT NULL REFERENCES media_files(id) ON DELETE CASCADE, + effective_media_file_id BIGINT NOT NULL REFERENCES media_files(id) ON DELETE CASCADE, + current_plan_id TEXT NOT NULL, + current_plan JSONB NOT NULL, + normalized_request JSONB NOT NULL, + request_digest TEXT NOT NULL DEFAULT '', + expires_at TIMESTAMPTZ NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CHECK (jsonb_typeof(current_plan) = 'object'), + CHECK (jsonb_typeof(normalized_request) = 'object') +); + +CREATE INDEX playback_v3_attempts_expiry_idx + ON playback_v3_attempts (expires_at); + +CREATE TABLE playback_v3_replans ( + session_id UUID NOT NULL REFERENCES playback_v3_attempts(session_id) ON UPDATE CASCADE ON DELETE CASCADE, + replan_request_id TEXT NOT NULL, + request_digest TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'active', + lease_expires_at TIMESTAMPTZ NOT NULL, + response JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (session_id, replan_request_id), + CHECK (state IN ('active', 'completed')), + CHECK (response IS NULL OR jsonb_typeof(response) = 'object') +); + +CREATE INDEX playback_v3_replans_lease_idx + ON playback_v3_replans (state, lease_expires_at); + +CREATE TABLE playback_route_events ( + id BIGSERIAL PRIMARY KEY, + playback_attempt_id TEXT NOT NULL, + session_id UUID, + plan_id TEXT, + plan_attempt_id TEXT, + plan_attempt_key TEXT, + event TEXT NOT NULL, + failure_classification TEXT, + fallback_reason TEXT, + output_route_generation BIGINT NOT NULL DEFAULT 0, + diagnostics JSONB NOT NULL DEFAULT '{}'::jsonb, + user_id BIGINT NOT NULL REFERENCES users(id) ON DELETE CASCADE, + profile_id TEXT NOT NULL, + client_name TEXT, + client_version TEXT, + client_model TEXT, + received_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + CHECK (event IN ( + 'plan_selected', 'plan_invalidated', 'plan_failed', 'first_frame', + 'terminal', 'stopped', 'runtime_correction_applied', + 'runtime_correction_succeeded', 'runtime_correction_failed', + 'seek_reanchor_requested', 'seek_reanchored' + )), + CHECK (jsonb_typeof(diagnostics) = 'object') +); + +CREATE INDEX playback_route_events_attempt_idx + ON playback_route_events (playback_attempt_id, received_at); +CREATE INDEX playback_route_events_release_idx + ON playback_route_events (event, failure_classification, received_at); +-- The hourly retention delete scans by age alone; without this index it is a +-- sequential scan over up to 30 days of telemetry. +CREATE INDEX playback_route_events_received_idx + ON playback_route_events (received_at); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS playback_route_events; +DROP TABLE IF EXISTS playback_v3_replans; +DROP TABLE IF EXISTS playback_v3_attempts; +DELETE FROM server_settings WHERE key IN ('playback.protocol_v3_enabled', 'playback.protocol_v3_shadow_enabled'); +-- +goose StatementEnd diff --git a/migrations/sql/20260712124741_nfo_capability_season_episode_levels.sql b/migrations/sql/20260712124741_nfo_capability_season_episode_levels.sql new file mode 100644 index 00000000..31f3e5a5 --- /dev/null +++ b/migrations/sql/20260712124741_nfo_capability_season_episode_levels.sql @@ -0,0 +1,32 @@ +-- +goose Up +-- Declare season/episode support for the built-in NFO provider (#216, +-- Phase D). The capability's default_priority map enumerates the content +-- levels a provider participates in: without these keys the startup builtin +-- chain sync never appends the NFO provider to season/episode chains, so +-- season.nfo names and episode NFO titles could never resolve. Priority 1 +-- matches the movie/series declaration (local curation wins when enabled); +-- the capability stays default_enabled=false so nothing activates on its own. +UPDATE plugin_capabilities pc +SET metadata = jsonb_set( + pc.metadata, + '{default_priority}', + COALESCE(pc.metadata->'default_priority', '{}'::jsonb) + || '{"season": 1, "episode": 1}'::jsonb + ) +FROM plugin_installations pi +WHERE pc.plugin_installation_id = pi.id + AND pi.plugin_id = 'silo.builtin' AND pi.kind = 'builtin' + AND pc.capability_type = 'metadata_provider.v1' AND pc.capability_id = 'nfo'; + +-- +goose Down +UPDATE plugin_capabilities pc +SET metadata = jsonb_set( + pc.metadata, + '{default_priority}', + (pc.metadata->'default_priority') - 'season' - 'episode' + ) +FROM plugin_installations pi +WHERE pc.plugin_installation_id = pi.id + AND pi.plugin_id = 'silo.builtin' AND pi.kind = 'builtin' + AND pc.capability_type = 'metadata_provider.v1' AND pc.capability_id = 'nfo' + AND pc.metadata ? 'default_priority'; diff --git a/migrations/sql/20260713170000_add_playback_v3_attempt_revision.sql b/migrations/sql/20260713170000_add_playback_v3_attempt_revision.sql new file mode 100644 index 00000000..03928163 --- /dev/null +++ b/migrations/sql/20260713170000_add_playback_v3_attempt_revision.sql @@ -0,0 +1,93 @@ +-- +goose Up +ALTER TABLE playback_v3_attempts + ADD COLUMN current_replan_request_id TEXT NOT NULL DEFAULT ''; + +ALTER TABLE playback_v3_replans + ADD COLUMN base_replan_request_id TEXT NOT NULL DEFAULT ''; + +WITH latest_completed AS ( + SELECT DISTINCT ON (session_id) + session_id, + replan_request_id + FROM playback_v3_replans + WHERE state = 'completed' + ORDER BY session_id, updated_at DESC, created_at DESC, replan_request_id DESC +) +UPDATE playback_v3_attempts AS attempts +SET current_replan_request_id = latest_completed.replan_request_id +FROM latest_completed +WHERE attempts.session_id = latest_completed.session_id; + +UPDATE playback_v3_replans AS candidate +SET base_replan_request_id = COALESCE(( + SELECT previous.replan_request_id + FROM playback_v3_replans AS previous + WHERE previous.session_id = candidate.session_id + AND previous.state = 'completed' + AND previous.updated_at <= candidate.created_at + ORDER BY previous.updated_at DESC, previous.created_at DESC, previous.replan_request_id DESC + LIMIT 1 +), ''); + +-- Keep the revision correct during rolling deploys and rollbacks. Older V3 +-- binaries still complete playback_v3_replans in the same transaction, so +-- this trigger advances the attempt marker even when their attempt UPDATE +-- does not know about the new column. +-- +goose StatementBegin +CREATE FUNCTION sync_playback_v3_current_replan_request_id() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.state = 'completed' THEN + -- New binaries already write the marker in the same transaction; the + -- IS DISTINCT FROM guard skips a redundant second row version there + -- while still advancing the marker for older binaries. + UPDATE playback_v3_attempts + SET current_replan_request_id = NEW.replan_request_id + WHERE session_id = NEW.session_id + AND current_replan_request_id IS DISTINCT FROM NEW.replan_request_id; + END IF; + RETURN NEW; +END; +$$; +-- +goose StatementEnd + +CREATE TRIGGER playback_v3_replans_sync_attempt_revision +AFTER UPDATE OF state, response ON playback_v3_replans +FOR EACH ROW +EXECUTE FUNCTION sync_playback_v3_current_replan_request_id(); + +-- +goose StatementBegin +CREATE FUNCTION seed_playback_v3_replan_base_revision() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.base_replan_request_id = '' THEN + SELECT current_replan_request_id + INTO NEW.base_replan_request_id + FROM playback_v3_attempts + WHERE session_id = NEW.session_id; + END IF; + RETURN NEW; +END; +$$; +-- +goose StatementEnd + +CREATE TRIGGER playback_v3_replans_seed_base_revision +BEFORE INSERT ON playback_v3_replans +FOR EACH ROW +EXECUTE FUNCTION seed_playback_v3_replan_base_revision(); + +-- +goose Down +DROP TRIGGER IF EXISTS playback_v3_replans_seed_base_revision ON playback_v3_replans; +DROP FUNCTION IF EXISTS seed_playback_v3_replan_base_revision(); +DROP TRIGGER IF EXISTS playback_v3_replans_sync_attempt_revision ON playback_v3_replans; +DROP FUNCTION IF EXISTS sync_playback_v3_current_replan_request_id(); + +ALTER TABLE playback_v3_replans + DROP COLUMN IF EXISTS base_replan_request_id; + +ALTER TABLE playback_v3_attempts + DROP COLUMN IF EXISTS current_replan_request_id; diff --git a/migrations/sql/20260714120826_artwork_revision_gc.sql b/migrations/sql/20260714120826_artwork_revision_gc.sql new file mode 100644 index 00000000..429d76a7 --- /dev/null +++ b/migrations/sql/20260714120826_artwork_revision_gc.sql @@ -0,0 +1,195 @@ +-- +goose Up +CREATE TABLE public.artwork_revision_gc_candidates ( + id bigserial PRIMARY KEY, + original_path text NOT NULL UNIQUE, + -- Image type ("poster", "backdrop", ...) lets the collector expand the + -- expected object keys in Go (artworkkey.ObjectKeys) when no exact + -- manifest was tracked, so the variant ladder lives in one place. + image_type text NOT NULL DEFAULT '', + object_keys text[] NOT NULL DEFAULT '{}', + not_before timestamptz NOT NULL, + -- NULL means the revision is currently referenced and dormant. Artwork + -- displacement triggers or the collector's dormant sweep reactivate it. + next_attempt_at timestamptz, + -- Set once the objects have been deleted from storage. The row then only + -- survives until the post-delete reference heal succeeds, so a transient + -- heal failure keeps a durable retry instead of orphaning broken pointers. + deleted_at timestamptz, + attempt_count integer NOT NULL DEFAULT 0, + locked_at timestamptz, + locked_by text NOT NULL DEFAULT '', + last_error text NOT NULL DEFAULT '', + created_at timestamptz NOT NULL DEFAULT NOW(), + updated_at timestamptz NOT NULL DEFAULT NOW(), + CONSTRAINT artwork_revision_gc_original_path_check CHECK (BTRIM(original_path) <> ''), + CONSTRAINT artwork_revision_gc_manifest_check CHECK (cardinality(object_keys) > 0 OR BTRIM(image_type) <> ''), + CONSTRAINT artwork_revision_gc_attempt_count_check CHECK (attempt_count >= 0) +); + +CREATE INDEX artwork_revision_gc_due_idx + ON public.artwork_revision_gc_candidates (next_attempt_at, id) + WHERE locked_at IS NULL AND next_attempt_at IS NOT NULL; + +CREATE INDEX artwork_revision_gc_lease_idx + ON public.artwork_revision_gc_candidates (locked_at, id) + WHERE locked_at IS NOT NULL; + +-- The dormant sweep periodically re-verifies parked rows so a reference that +-- disappears through a surface without a displacement trigger degrades to +-- slower cleanup instead of a permanent leak. +CREATE INDEX artwork_revision_gc_dormant_idx + ON public.artwork_revision_gc_candidates (updated_at, id) + WHERE next_attempt_at IS NULL; + +-- Queue displaced immutable artwork at the database boundary so every writer +-- participates in the same lifecycle, including background refreshes, scanners, +-- localizations, and future code paths that do not use the admin publication API. +-- The trigger stores only the displaced path and its image type; the collector +-- expands object keys in Go so the variant ladder has a single source of truth. +-- +goose StatementBegin +CREATE OR REPLACE FUNCTION public.queue_displaced_artwork_revision() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + old_row jsonb := to_jsonb(OLD); + new_row jsonb; + arg_index integer := 0; + path_column text; + image_type text; + previous_path text; + replacement_path text; + cleanup_at timestamptz := NOW() + interval '24 hours'; +BEGIN + IF TG_OP = 'UPDATE' THEN + new_row := to_jsonb(NEW); + END IF; + + WHILE arg_index < TG_NARGS LOOP + path_column := TG_ARGV[arg_index]; + image_type := TG_ARGV[arg_index + 1]; + previous_path := old_row ->> path_column; + replacement_path := CASE WHEN new_row IS NULL THEN NULL ELSE new_row ->> path_column END; + + IF COALESCE(BTRIM(previous_path), '') <> '' + AND previous_path NOT LIKE '%://%' + AND previous_path ~ '/original\.[^/]+$' + AND (TG_OP = 'DELETE' OR previous_path IS DISTINCT FROM replacement_path) THEN + INSERT INTO public.artwork_revision_gc_candidates ( + original_path, image_type, object_keys, not_before, next_attempt_at + ) VALUES (previous_path, image_type, '{}', cleanup_at, cleanup_at) + ON CONFLICT (original_path) DO UPDATE SET + image_type = CASE + WHEN artwork_revision_gc_candidates.image_type = '' THEN EXCLUDED.image_type + ELSE artwork_revision_gc_candidates.image_type + END, + not_before = EXCLUDED.not_before, + next_attempt_at = EXCLUDED.next_attempt_at, + attempt_count = 0, + locked_at = NULL, + locked_by = '', + last_error = '', + updated_at = NOW(); + END IF; + + arg_index := arg_index + 2; + END LOOP; + + IF TG_OP = 'DELETE' THEN + RETURN OLD; + END IF; + RETURN NEW; +END; +$$; +-- +goose StatementEnd + +-- UPDATE triggers carry WHEN clauses so the function only runs when an artwork +-- column actually changes; catalog upserts assign these columns on every row +-- write. DELETE triggers cannot reference NEW and are declared separately. +CREATE TRIGGER media_items_artwork_revision_gc_update +AFTER UPDATE OF poster_path, backdrop_path, logo_path ON public.media_items +FOR EACH ROW +WHEN (OLD.poster_path IS DISTINCT FROM NEW.poster_path + OR OLD.backdrop_path IS DISTINCT FROM NEW.backdrop_path + OR OLD.logo_path IS DISTINCT FROM NEW.logo_path) +EXECUTE FUNCTION public.queue_displaced_artwork_revision( + 'poster_path', 'poster', 'backdrop_path', 'backdrop', 'logo_path', 'logo' +); + +CREATE TRIGGER media_items_artwork_revision_gc_delete +AFTER DELETE ON public.media_items +FOR EACH ROW EXECUTE FUNCTION public.queue_displaced_artwork_revision( + 'poster_path', 'poster', 'backdrop_path', 'backdrop', 'logo_path', 'logo' +); + +CREATE TRIGGER media_item_localizations_artwork_revision_gc_update +AFTER UPDATE OF poster_path, backdrop_path, logo_path ON public.media_item_localizations +FOR EACH ROW +WHEN (OLD.poster_path IS DISTINCT FROM NEW.poster_path + OR OLD.backdrop_path IS DISTINCT FROM NEW.backdrop_path + OR OLD.logo_path IS DISTINCT FROM NEW.logo_path) +EXECUTE FUNCTION public.queue_displaced_artwork_revision( + 'poster_path', 'poster', 'backdrop_path', 'backdrop', 'logo_path', 'logo' +); + +CREATE TRIGGER media_item_localizations_artwork_revision_gc_delete +AFTER DELETE ON public.media_item_localizations +FOR EACH ROW EXECUTE FUNCTION public.queue_displaced_artwork_revision( + 'poster_path', 'poster', 'backdrop_path', 'backdrop', 'logo_path', 'logo' +); + +CREATE TRIGGER seasons_artwork_revision_gc_update +AFTER UPDATE OF poster_path ON public.seasons +FOR EACH ROW +WHEN (OLD.poster_path IS DISTINCT FROM NEW.poster_path) +EXECUTE FUNCTION public.queue_displaced_artwork_revision('poster_path', 'poster'); + +CREATE TRIGGER seasons_artwork_revision_gc_delete +AFTER DELETE ON public.seasons +FOR EACH ROW EXECUTE FUNCTION public.queue_displaced_artwork_revision('poster_path', 'poster'); + +CREATE TRIGGER season_localizations_artwork_revision_gc_update +AFTER UPDATE OF poster_path ON public.season_localizations +FOR EACH ROW +WHEN (OLD.poster_path IS DISTINCT FROM NEW.poster_path) +EXECUTE FUNCTION public.queue_displaced_artwork_revision('poster_path', 'poster'); + +CREATE TRIGGER season_localizations_artwork_revision_gc_delete +AFTER DELETE ON public.season_localizations +FOR EACH ROW EXECUTE FUNCTION public.queue_displaced_artwork_revision('poster_path', 'poster'); + +CREATE TRIGGER episodes_artwork_revision_gc_update +AFTER UPDATE OF still_path ON public.episodes +FOR EACH ROW +WHEN (OLD.still_path IS DISTINCT FROM NEW.still_path) +EXECUTE FUNCTION public.queue_displaced_artwork_revision('still_path', 'still'); + +CREATE TRIGGER episodes_artwork_revision_gc_delete +AFTER DELETE ON public.episodes +FOR EACH ROW EXECUTE FUNCTION public.queue_displaced_artwork_revision('still_path', 'still'); + +CREATE TRIGGER people_artwork_revision_gc_update +AFTER UPDATE OF photo_path ON public.people +FOR EACH ROW +WHEN (OLD.photo_path IS DISTINCT FROM NEW.photo_path) +EXECUTE FUNCTION public.queue_displaced_artwork_revision('photo_path', 'profile'); + +CREATE TRIGGER people_artwork_revision_gc_delete +AFTER DELETE ON public.people +FOR EACH ROW EXECUTE FUNCTION public.queue_displaced_artwork_revision('photo_path', 'profile'); + +-- +goose Down +DROP TRIGGER IF EXISTS people_artwork_revision_gc_delete ON public.people; +DROP TRIGGER IF EXISTS people_artwork_revision_gc_update ON public.people; +DROP TRIGGER IF EXISTS episodes_artwork_revision_gc_delete ON public.episodes; +DROP TRIGGER IF EXISTS episodes_artwork_revision_gc_update ON public.episodes; +DROP TRIGGER IF EXISTS season_localizations_artwork_revision_gc_delete ON public.season_localizations; +DROP TRIGGER IF EXISTS season_localizations_artwork_revision_gc_update ON public.season_localizations; +DROP TRIGGER IF EXISTS seasons_artwork_revision_gc_delete ON public.seasons; +DROP TRIGGER IF EXISTS seasons_artwork_revision_gc_update ON public.seasons; +DROP TRIGGER IF EXISTS media_item_localizations_artwork_revision_gc_delete ON public.media_item_localizations; +DROP TRIGGER IF EXISTS media_item_localizations_artwork_revision_gc_update ON public.media_item_localizations; +DROP TRIGGER IF EXISTS media_items_artwork_revision_gc_delete ON public.media_items; +DROP TRIGGER IF EXISTS media_items_artwork_revision_gc_update ON public.media_items; +DROP FUNCTION IF EXISTS public.queue_displaced_artwork_revision(); +DROP TABLE IF EXISTS public.artwork_revision_gc_candidates; diff --git a/migrations/sql/20260714160000_push_devices_android.sql b/migrations/sql/20260714160000_push_devices_android.sql new file mode 100644 index 00000000..8f1d1937 --- /dev/null +++ b/migrations/sql/20260714160000_push_devices_android.sql @@ -0,0 +1,77 @@ +-- +goose Up +-- +goose StatementBegin +ALTER TABLE public.push_devices + ADD COLUMN fcm_token_ciphertext text, + ADD COLUMN fcm_token_hash text; + +ALTER TABLE public.push_devices + DROP CONSTRAINT push_devices_platform_check; +ALTER TABLE public.push_devices + ADD CONSTRAINT push_devices_platform_check CHECK (platform IN ('apple', 'android')); + +ALTER TABLE public.push_devices + DROP CONSTRAINT push_devices_apple_fields_check; +ALTER TABLE public.push_devices + ADD CONSTRAINT push_devices_platform_fields_check CHECK ( + ( + platform = 'apple' + AND apns_environment IS NOT NULL + AND apns_topic IS NOT NULL + AND apns_token_ciphertext IS NOT NULL + AND apns_token_hash IS NOT NULL + AND fcm_token_ciphertext IS NULL + AND fcm_token_hash IS NULL + ) + OR ( + platform = 'android' + AND fcm_token_ciphertext IS NOT NULL + AND fcm_token_hash IS NOT NULL + AND apns_environment IS NULL + AND apns_topic IS NULL + AND apns_token_ciphertext IS NULL + AND apns_token_hash IS NULL + ) + ); + +CREATE INDEX push_devices_fcm_token_hash_idx + ON public.push_devices (fcm_token_hash) + WHERE fcm_token_hash IS NOT NULL; + +ALTER TABLE public.push_delivery_attempts + DROP CONSTRAINT push_delivery_attempts_platform_check; +ALTER TABLE public.push_delivery_attempts + ADD CONSTRAINT push_delivery_attempts_platform_check CHECK (platform IN ('apple', 'android')); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DELETE FROM public.push_delivery_attempts WHERE platform = 'android'; +DELETE FROM public.push_devices WHERE platform = 'android'; + +ALTER TABLE public.push_delivery_attempts + DROP CONSTRAINT push_delivery_attempts_platform_check; +ALTER TABLE public.push_delivery_attempts + ADD CONSTRAINT push_delivery_attempts_platform_check CHECK (platform IN ('apple')); + +DROP INDEX IF EXISTS push_devices_fcm_token_hash_idx; + +ALTER TABLE public.push_devices + DROP CONSTRAINT push_devices_platform_fields_check; +ALTER TABLE public.push_devices + ADD CONSTRAINT push_devices_apple_fields_check CHECK ( + platform = 'apple' + AND apns_environment IS NOT NULL + AND apns_topic IS NOT NULL + AND apns_token_ciphertext IS NOT NULL + AND apns_token_hash IS NOT NULL + ); + +ALTER TABLE public.push_devices + DROP CONSTRAINT push_devices_platform_check; +ALTER TABLE public.push_devices + ADD CONSTRAINT push_devices_platform_check CHECK (platform IN ('apple')); + +ALTER TABLE public.push_devices + DROP COLUMN fcm_token_ciphertext, + DROP COLUMN fcm_token_hash; +-- +goose StatementEnd diff --git a/migrations/sql/20260714180000_enable_playback_protocol_v3_by_default.sql b/migrations/sql/20260714180000_enable_playback_protocol_v3_by_default.sql new file mode 100644 index 00000000..71c8387a --- /dev/null +++ b/migrations/sql/20260714180000_enable_playback_protocol_v3_by_default.sql @@ -0,0 +1,20 @@ +-- +goose Up +-- +goose StatementBegin + +-- Playback protocol v3 now defaults to enabled. The flag shipped seeded to +-- 'false' as a rollout safety valve, but it was never exposed in any settings +-- UI, so a stored 'false' means "never touched" rather than an explicit +-- opt-out — and the current Android clients are v3-only for video: against a +-- server with the flag off they refuse playback with a misleading "update +-- your server" error. Flip it everywhere. +INSERT INTO server_settings (key, value) +VALUES ('playback.protocol_v3_enabled', 'true') +ON CONFLICT (key) DO UPDATE SET value = 'true'; +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +UPDATE server_settings +SET value = 'false' +WHERE key = 'playback.protocol_v3_enabled'; +-- +goose StatementEnd diff --git a/migrations/sql/20260716193000_media_files_folder_path_prefix_index.sql b/migrations/sql/20260716193000_media_files_folder_path_prefix_index.sql new file mode 100644 index 00000000..1f7b9e98 --- /dev/null +++ b/migrations/sql/20260716193000_media_files_folder_path_prefix_index.sql @@ -0,0 +1,8 @@ +-- +goose NO TRANSACTION + +-- +goose Up +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_media_files_folder_file_path_pattern + ON media_files (media_folder_id, file_path text_pattern_ops); + +-- +goose Down +DROP INDEX CONCURRENTLY IF EXISTS idx_media_files_folder_file_path_pattern; diff --git a/migrations/sql/20260720084550_client_diagnostic_reports.sql b/migrations/sql/20260720084550_client_diagnostic_reports.sql new file mode 100644 index 00000000..84489231 --- /dev/null +++ b/migrations/sql/20260720084550_client_diagnostic_reports.sql @@ -0,0 +1,31 @@ +-- +goose Up +CREATE TABLE client_diagnostic_reports ( + id UUID PRIMARY KEY, + short_id TEXT NOT NULL, + user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE, + profile_id TEXT, + state TEXT NOT NULL CHECK (state IN ('receiving', 'ready', 'failed')), + captured_at TIMESTAMPTZ NOT NULL, + received_at TIMESTAMPTZ NOT NULL DEFAULT now(), + report_type TEXT NOT NULL CHECK (report_type IN ('crash', 'anr', 'native_crash', 'hang', 'abnormal_exit', 'manual')), + platform TEXT NOT NULL CHECK (platform IN ('android', 'android-tv', 'ios', 'tvos')), + app_version TEXT NOT NULL, + crash_summary TEXT, + manifest JSONB NOT NULL, + playback_session_ids TEXT[] NOT NULL DEFAULT '{}', + blob_bucket TEXT, + blob_key TEXT, + blob_bytes BIGINT, + uncompressed_bytes BIGINT, + blob_sha256 TEXT +); + +CREATE UNIQUE INDEX client_diagnostic_reports_short_id_lower_idx + ON client_diagnostic_reports (lower(short_id)); +CREATE INDEX client_diagnostic_reports_user_received_idx + ON client_diagnostic_reports (user_id, received_at DESC); +CREATE INDEX client_diagnostic_reports_received_idx + ON client_diagnostic_reports (received_at); + +-- +goose Down +DROP TABLE IF EXISTS client_diagnostic_reports; diff --git a/scripts/playback_v3_release_queries.sql b/scripts/playback_v3_release_queries.sql new file mode 100644 index 00000000..fb865f79 --- /dev/null +++ b/scripts/playback_v3_release_queries.sql @@ -0,0 +1,53 @@ +-- Playback protocol v3 release/validation queries. +-- Bind :since to the beginning of the validation window. + +-- Route outcomes and failures by client/device. +SELECT event, COALESCE(failure_classification, '') AS failure, + COALESCE(client_name, '') AS client, COALESCE(client_model, '') AS model, + COUNT(*) AS events +FROM playback_route_events +WHERE received_at >= :since +GROUP BY 1, 2, 3, 4 +ORDER BY events DESC; + +-- First-frame success rate and latency. Diagnostics are allowlisted at ingest. +SELECT + COUNT(*) FILTER (WHERE event = 'first_frame') AS first_frames, + COUNT(*) FILTER (WHERE event = 'plan_selected') AS selected_plans, + ROUND(100.0 * COUNT(*) FILTER (WHERE event = 'first_frame') / + NULLIF(COUNT(*) FILTER (WHERE event = 'plan_selected'), 0), 2) AS first_frame_pct, + percentile_cont(0.5) WITHIN GROUP + (ORDER BY CASE WHEN diagnostics->>'first_frame_ms' ~ '^[0-9]+([.][0-9]+)?$' + THEN (diagnostics->>'first_frame_ms')::double precision END) + FILTER (WHERE event = 'first_frame') AS first_frame_p50_ms, + percentile_cont(0.95) WITHIN GROUP + (ORDER BY CASE WHEN diagnostics->>'first_frame_ms' ~ '^[0-9]+([.][0-9]+)?$' + THEN (diagnostics->>'first_frame_ms')::double precision END) + FILTER (WHERE event = 'first_frame') AS first_frame_p95_ms +FROM playback_route_events +WHERE received_at >= :since; + +-- Replans, terminals, passthrough/PCM recovery and repeated recipe failures. +SELECT + COALESCE(failure_classification, fallback_reason, event) AS classification, + diagnostics->>'pcm_recovery' AS pcm_recovery, + diagnostics->>'retry_outcome' AS retry_outcome, + COUNT(*) AS occurrences, + COUNT(DISTINCT playback_attempt_id) AS attempts +FROM playback_route_events +WHERE received_at >= :since + AND event IN ('plan_invalidated', 'plan_failed', 'terminal') +GROUP BY 1, 2, 3 +ORDER BY occurrences DESC; + +-- Current recipes and declared degradation transformations. +SELECT + current_plan->>'delivery' AS delivery, + current_plan->'effective_recipe'->>'dynamic_range' AS dynamic_range, + transformation->>'name' AS transformation, + COUNT(*) AS attempts +FROM playback_v3_attempts +LEFT JOIN LATERAL jsonb_array_elements(current_plan->'transformations') transformation ON TRUE +WHERE created_at >= :since +GROUP BY 1, 2, 3 +ORDER BY attempts DESC; diff --git a/web/package.json b/web/package.json index b5e9362a..b5b4b23c 100644 --- a/web/package.json +++ b/web/package.json @@ -43,6 +43,7 @@ "pdfjs-dist": "^5.7.284", "radix-ui": "^1.4.3", "react": "^19.0.0", + "react-day-picker": "^10.0.1", "react-dom": "^19.0.0", "react-qr-code": "^2.0.18", "react-router": "^7.15.1", diff --git a/web/pnpm-lock.yaml b/web/pnpm-lock.yaml index 62c13ffb..ffc874fd 100644 --- a/web/pnpm-lock.yaml +++ b/web/pnpm-lock.yaml @@ -100,6 +100,9 @@ importers: react: specifier: ^19.0.0 version: 19.2.4 + react-day-picker: + specifier: ^10.0.1 + version: 10.0.1(@types/react@19.2.14)(react@19.2.4) react-dom: specifier: ^19.0.0 version: 19.2.4(react@19.2.4) @@ -375,6 +378,9 @@ packages: resolution: {integrity: sha512-QxULHAm7cNu72w97JUNCBFODFaXpbDg+dP8b/oWFAZ2MTRppA3U00Y2L1HqaS4J6yBqxwa/Y3nMBaxVKbB/NsA==} engines: {node: '>=20.19.0'} + '@date-fns/tz@1.5.0': + resolution: {integrity: sha512-lwYN/vDPeNRULcepoE/LO2Pgx+7/RV+S9ARfbc9lr2DtGkOD7pAiruHvbR1RX3Qyf6ja47EWJDMsNK5vK08DJg==} + '@dnd-kit/accessibility@3.1.1': resolution: {integrity: sha512-2P+YgaXF+gRsIihwwY1gCsQSYnu9Zyj2py8kY5fFvUM1qm2WA2u639R6YNVfU4GWr+ZM5mqEsfHZZLoRONbemw==} peerDependencies: @@ -2014,6 +2020,9 @@ packages: resolution: {integrity: sha512-23XHcCF+coGYevirZceTVD7NdJOqVn+49IHyxgszm+JIiHLoB2TkmPtsYkNWT1pvRSGkc35L6NHs0yHkN2SumA==} engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0} + date-fns@4.4.0: + resolution: {integrity: sha512-+1UMbeh68lH1SegH83CGWwpb6OHHbpSgr3+s5Eww5M4CAgswBpoWS0AjTOfEJ33HiYKz1hdj/KTFprzXHmq/6w==} + debug@4.4.3: resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} engines: {node: '>=6.0'} @@ -2704,6 +2713,16 @@ packages: '@types/react-dom': optional: true + react-day-picker@10.0.1: + resolution: {integrity: sha512-eNh6BlwcYInWaJtRv18mXQ06Ys/H6rdTZAnTaSdOYJuTpwP1JMCHNd1FDRadA+gbeinq+psdULN5Xnowy9mV8w==} + engines: {node: '>=18'} + peerDependencies: + '@types/react': '>=16.8.0' + react: '>=16.8.0' + peerDependenciesMeta: + '@types/react': + optional: true + react-dom@19.2.4: resolution: {integrity: sha512-AXJdLo8kgMbimY95O2aKQqsz2iWi9jMgKJhRBAxECE4IFxfcazB2LmzloIoibJI3C12IlY20+KFaLv+71bUJeQ==} peerDependencies: @@ -3335,6 +3354,8 @@ snapshots: '@csstools/css-tokenizer@4.0.0': {} + '@date-fns/tz@1.5.0': {} + '@dnd-kit/accessibility@3.1.1(react@19.2.4)': dependencies: react: 19.2.4 @@ -4890,6 +4911,8 @@ snapshots: transitivePeerDependencies: - '@noble/hashes' + date-fns@4.4.0: {} + debug@4.4.3: dependencies: ms: 2.1.3 @@ -5508,6 +5531,14 @@ snapshots: '@types/react': 19.2.14 '@types/react-dom': 19.2.3(@types/react@19.2.14) + react-day-picker@10.0.1(@types/react@19.2.14)(react@19.2.4): + dependencies: + '@date-fns/tz': 1.5.0 + date-fns: 4.4.0 + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + react-dom@19.2.4(react@19.2.4): dependencies: react: 19.2.4 diff --git a/web/src/App.tsx b/web/src/App.tsx index cc77f592..31e7ee8f 100644 --- a/web/src/App.tsx +++ b/web/src/App.tsx @@ -47,6 +47,7 @@ import RequestDetail from "@/pages/RequestDetail"; import AdminDashboard from "@/pages/AdminDashboard"; import AdminActivity from "@/pages/AdminActivity"; import AdminLogs from "@/pages/AdminLogs"; +import AdminDiagnostics from "@/pages/AdminDiagnostics"; import AdminAccessGroups from "@/pages/AdminAccessGroups"; import AdminUsers from "@/pages/AdminUsers"; import AdminRequests from "@/pages/AdminRequests"; @@ -412,6 +413,7 @@ function AppRoutes() { } /> } /> } /> + } /> } /> } /> } /> diff --git a/web/src/api/client.ts b/web/src/api/client.ts index a19ff289..77f93cf6 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -341,6 +341,21 @@ export async function restoreUserSession({ } export async function api(path: string, options: RequestInit = {}): Promise { + const res = await apiResponse(path, options); + + // Handle empty successful responses. + if (res.status === 204 || res.status === 205) { + return undefined as T; + } + const text = await res.text(); + if (text.trim() === "") { + return undefined as T; + } + return JSON.parse(text) as T; +} + +/** Performs an authenticated API request while leaving the successful body unread. */ +export async function apiResponse(path: string, options: RequestInit = {}): Promise { const headers = buildApiHeaders(options); let res = await fetch(`/api/v1${path}`, { ...options, headers }); @@ -354,8 +369,8 @@ export async function api(path: string, options: RequestInit = {}): Promise(path: string, options: RequestInit = {}): Promise { @@ -421,26 +427,7 @@ export async function apiDownload( filename: string, options: RequestInit = {}, ): Promise { - let headers = buildApiHeaders(options); - let res = await fetch(`/api/v1${path}`, { ...options, headers }); - - if (res.status === 401 && getRefreshToken()) { - if (!refreshPromise) { - refreshPromise = attemptRefresh().finally(() => { - refreshPromise = null; - }); - } - const refreshed = await refreshPromise; - if (refreshed) { - headers = buildApiHeaders(options); - headers["Authorization"] = `Bearer ${accessToken}`; - res = await fetch(`/api/v1${path}`, { ...options, headers }); - } - } - - if (!res.ok) { - throw apiClientErrorFrom(res.status, await parseApiError(res)); - } + const res = await apiResponse(path, options); const blob = await res.blob(); const url = URL.createObjectURL(blob); @@ -458,26 +445,7 @@ export async function apiDownload( export const API_BLOB_MAX_BYTES = 512 * 1024 * 1024; export async function apiBlob(path: string, options: RequestInit = {}): Promise { - let headers = buildApiHeaders(options); - let res = await fetch(`/api/v1${path}`, { ...options, headers }); - - if (res.status === 401 && getRefreshToken()) { - if (!refreshPromise) { - refreshPromise = attemptRefresh().finally(() => { - refreshPromise = null; - }); - } - const refreshed = await refreshPromise; - if (refreshed) { - headers = buildApiHeaders(options); - headers["Authorization"] = `Bearer ${accessToken}`; - res = await fetch(`/api/v1${path}`, { ...options, headers }); - } - } - - if (!res.ok) { - throw apiClientErrorFrom(res.status, await parseApiError(res)); - } + const res = await apiResponse(path, options); // Reject oversized bodies up front instead of crashing the tab while // buffering them. When the header is absent, proceed; streaming byte counts diff --git a/web/src/api/types.ts b/web/src/api/types.ts index bd4dfa3c..882446ea 100644 --- a/web/src/api/types.ts +++ b/web/src/api/types.ts @@ -422,6 +422,7 @@ export interface HistoryImportRun { progress_updated: number; history_created: number; watchlist_added: number; + favorites_imported: number; skipped: number; warnings: string[]; unmatched_samples: HistoryImportUnmatchedSample[]; @@ -969,6 +970,7 @@ export interface VersionVideoTrack { bitrate?: number; video_range?: string; video_range_type?: string; + color_range?: string; color_primaries?: string; color_space?: string; color_transfer?: string; @@ -1268,6 +1270,7 @@ export interface EpisodeListItem { still_thumbhash: string; user_data?: LeafItemUserData; files: EpisodeFile[]; + overlay_summary?: OverlaySummary | null; } export interface EpisodesResponse { @@ -2423,6 +2426,11 @@ export interface AdminSession { requested_video_resolution?: string; video_decision?: string; audio_decision?: string; + /** Server-computed activity bucket: direct | remux | transcode | audio. + * Absent when the per-stream decisions are unknown. */ + effective_play_method?: string; + /** Server-side identification of Jellyfin-ecosystem clients (the JF pill). */ + is_jellyfin_client?: boolean; } export interface OperationalLogEntry { @@ -2467,6 +2475,126 @@ export interface AuditLogListResponse { next_cursor?: string; } +export type DiagnosticAvailabilityStatus = "available" | "disabled" | "storage_unavailable"; + +export interface DiagnosticStatus { + status: DiagnosticAvailabilityStatus; + server_instance_id: string; + accepted_schema_versions: number[]; + max_bundle_bytes: number; + max_manifest_bytes: number; + retention_days: number; + consent_notice_version: number; +} + +export type DiagnosticReportState = "receiving" | "ready" | "failed"; +export type DiagnosticReportType = + | "crash" + | "anr" + | "native_crash" + | "hang" + | "abnormal_exit" + | "manual"; +export type DiagnosticPlatform = "android" | "android-tv" | "ios" | "tvos"; + +export interface ClientDiagnosticManifest { + schema_version: number; + report: { + type: DiagnosticReportType; + captured_at: string; + capture_session_id: string; + app_version: string; + app_build: string; + platform: DiagnosticPlatform; + os_version: string; + profile_id?: string; + [key: string]: unknown; + }; + destination: { + server_instance_id: string; + [key: string]: unknown; + }; + consent: { + mode: "prompt" | "always" | "manual"; + notice_version: number; + [key: string]: unknown; + }; + crash?: { + summary: string; + stack_excerpt?: string; + thread?: string; + foreground?: boolean; + source: "ueh" | "exit_info" | "metrickit" | "exit_sentinel"; + provenance: "pre_failure" | "post_restart" | "metric_reporting_period"; + occurred_at: string; + [key: string]: unknown; + }; + device_summary: { + manufacturer: string; + model: string; + os: string; + form_factor: string; + [key: string]: unknown; + }; + playback_session_ids: string[]; + log_summary: { + lines: number; + bytes_gz: number; + dropped_lines: number; + categories: string[]; + debug_logging: boolean; + [key: string]: unknown; + }; + archive: { + entries: string[]; + bytes: number; + uncompressed_bytes: number; + sha256: string; + [key: string]: unknown; + }; + [key: string]: unknown; +} + +// DiagnosticReportSummary is the shape returned by the admin list endpoint, +// which omits the full manifest JSONB for cheap paging. `app_build` is projected +// out of the manifest server-side so rows can still show the build number. +export interface DiagnosticReportSummary { + id: string; + short_id: string; + user_id: number; + profile_id?: string; + state: DiagnosticReportState; + captured_at: string; + received_at: string; + report_type: DiagnosticReportType; + platform: DiagnosticPlatform; + app_version: string; + app_build: string; + crash_summary?: string; + playback_session_ids: string[]; + blob_bucket?: string; + blob_key?: string; + blob_bytes?: number; + uncompressed_bytes?: number; + blob_sha256?: string; +} + +// DiagnosticReport is the full detail shape (GetByID), which additionally +// includes the parsed manifest for the detail pane. +export interface DiagnosticReport extends DiagnosticReportSummary { + manifest: ClientDiagnosticManifest; +} + +export interface DiagnosticReportListResponse { + reports: DiagnosticReportSummary[]; + next_cursor?: string; +} + +export interface DiagnosticDownloadResponse { + download_url: string; + expires_at: string; +} + export type AdminLogStream = "app" | "audit"; export interface AdminLogSnapshotMessage { @@ -2888,8 +3016,10 @@ export interface LibraryMountCheckRoot { | "not_directory" | "stat_failed" | "read_failed" + | "probe_timeout" | null; error_message: string | null; + suspect_empty: boolean; } export interface LibraryMountCheckResponse { @@ -3449,6 +3579,12 @@ export interface PluginSettingsSummary { user_config_schema: PluginConfigSchema[]; routes: PluginRoute[]; assets: PluginAsset[]; + /** + * Optional slash-delimited grouping path from the plugin manifest + * (e.g. "Tools/Utilities") that groups the plugin's entries in the + * Apps sidebar section. Absent when the manifest declares no category. + */ + category?: string; } export interface PluginSettingsListResponse { @@ -4328,6 +4464,8 @@ export interface ApplyItemImageResponse { content_id: string; stored_path: string; thumbhash: string; + image_url?: string; + revision?: string; } export interface UnmatchedLibraryItem { diff --git a/web/src/components/AdminSectionCommandDialog.test.tsx b/web/src/components/AdminSectionCommandDialog.test.tsx index aa100808..e18cc673 100644 --- a/web/src/components/AdminSectionCommandDialog.test.tsx +++ b/web/src/components/AdminSectionCommandDialog.test.tsx @@ -121,6 +121,16 @@ describe("AdminSectionCommandDialog", () => { expect(screen.queryByRole("searchbox", { name: "Search admin sections" })).toBeNull(); }); + it("opens client diagnostics from admin search", async () => { + renderDialog(); + + const searchBox = await openDialog(); + await userEvent.type(searchBox, "client diagnostics"); + await userEvent.click(screen.getByRole("option", { name: /Diagnostics/ })); + + expect(screen.getByLabelText("Current path")).toHaveTextContent("/admin/diagnostics"); + }); + it("closes with Escape", async () => { renderDialog(); diff --git a/web/src/components/AdminSidebar.test.tsx b/web/src/components/AdminSidebar.test.tsx index f65c2a66..3a6b380c 100644 --- a/web/src/components/AdminSidebar.test.tsx +++ b/web/src/components/AdminSidebar.test.tsx @@ -92,6 +92,13 @@ describe("AdminSidebar", () => { expect(markup).toContain(">Sections<"); }); + it("includes Diagnostics next to the operational overview links", () => { + const markup = renderSidebar(); + + expect(markup).toContain('href="/admin/diagnostics"'); + expect(markup).toContain(">Diagnostics<"); + }); + it("includes a Maintenance link in the system navigation", () => { const markup = renderSidebar(); diff --git a/web/src/components/AppSidebar.logic.ts b/web/src/components/AppSidebar.logic.ts index a18d0d7c..80c0cfa5 100644 --- a/web/src/components/AppSidebar.logic.ts +++ b/web/src/components/AppSidebar.logic.ts @@ -5,3 +5,64 @@ export function isSidebarExpanded(collapsed: boolean, hovered: boolean, profileM export function getProfileMenuSide(collapsed: boolean) { return collapsed ? "right" : "top"; } + +export interface AppNavLink { + id: string; + basePath: string; + label: string; + pluginId: string; + /** Slash-delimited category path from the plugin manifest, if any. */ + category?: string; +} + +export interface AppNavGroup { + /** Display label for the group ("Other" for uncategorized plugins). */ + category: string; + links: AppNavLink[]; +} + +/** Group label for plugins whose manifest declares no category. */ +export const UNCATEGORIZED_APP_GROUP = "Other"; + +/** + * Groups Apps sidebar entries by the FIRST segment of the plugin manifest's + * slash-delimited `category` path. + * + * SDK contract (silo-plugin-sdk proto/silo/plugin/v1/common.proto, + * PluginManifest.category): a slash-delimited path that groups plugins in + * the user-facing Apps section — e.g. "Tools/Utilities" lands in + * Apps → Tools → Utilities. Plugins without a category render under + * "Other". The host does not validate the value; the sidebar tolerates + * unknown segments. + * + * We currently render only ONE level of grouping, so deeper segments + * ("Utilities" in the example above) are intentionally ignored for now. + * + * Returns null when fewer than 2 distinct categories exist among the + * links; the caller should then keep the flat list under the single + * "Apps" header instead of rendering per-category sub-headers. + * + * Group ordering: categories alphabetically (locale-aware), with "Other" + * always last. Link order within each group preserves the input order. + */ +export function groupAppNavLinks(links: AppNavLink[]): AppNavGroup[] | null { + const groups = new Map(); + for (const link of links) { + const firstSegment = link.category?.split("/")[0]?.trim(); + const category = firstSegment || UNCATEGORIZED_APP_GROUP; + const bucket = groups.get(category); + if (bucket) { + bucket.push(link); + } else { + groups.set(category, [link]); + } + } + if (groups.size < 2) return null; + return [...groups.entries()] + .sort(([a], [b]) => { + if (a === UNCATEGORIZED_APP_GROUP) return 1; + if (b === UNCATEGORIZED_APP_GROUP) return -1; + return a.localeCompare(b); + }) + .map(([category, groupLinks]) => ({ category, links: groupLinks })); +} diff --git a/web/src/components/AppSidebar.test.tsx b/web/src/components/AppSidebar.test.tsx index c90a22e5..473d67c4 100644 --- a/web/src/components/AppSidebar.test.tsx +++ b/web/src/components/AppSidebar.test.tsx @@ -5,8 +5,15 @@ import { renderToStaticMarkup } from "react-dom/server"; import { MemoryRouter, Route, Routes } from "react-router"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import type { PluginSettingsSummary } from "@/api/types"; + import AppSidebar from "./AppSidebar"; -import { getProfileMenuSide, isSidebarExpanded } from "./AppSidebar.logic"; +import { + getProfileMenuSide, + groupAppNavLinks, + isSidebarExpanded, + type AppNavLink, +} from "./AppSidebar.logic"; const mockLogout = vi.fn(); const mockClearProfile = vi.fn(); @@ -44,12 +51,42 @@ vi.mock("@/hooks/queries/sidebarPins", () => ({ }), })); +let mockPluginInstallations: PluginSettingsSummary[] = []; + vi.mock("@/hooks/queries/pluginSettings", () => ({ usePluginSettingsList: () => ({ - data: { installations: [] }, + data: { installations: mockPluginInstallations }, }), })); +function pluginInstallation( + id: number, + pluginId: string, + label: string, + category?: string, +): PluginSettingsSummary { + return { + id, + plugin_id: pluginId, + version: "1.0.0", + user_config_schema: [], + routes: [ + { + id: "home", + method: "GET", + path: "/", + access: "user", + navigable: true, + navigation_label: label, + navigation_kind: "user", + static_asset: false, + }, + ], + assets: [], + category, + }; +} + vi.mock("@/hooks/queries/useRequests", () => ({ useRequestFeatureStatus: () => ({ data: { requests_enabled: false }, @@ -116,6 +153,7 @@ describe("AppSidebar", () => { mockLogout.mockReset(); mockClearProfile.mockReset(); mockTogglePin.mockReset(); + mockPluginInstallations = []; }); it("uses the cinema highlight text color for active catalog source links", () => { @@ -181,4 +219,104 @@ describe("AppSidebar", () => { expect(markup).toContain("mx-auto h-10 w-10 justify-center px-0"); }); + + it("keeps a flat Apps list when fewer than 2 distinct categories exist", () => { + mockPluginInstallations = [ + pluginInstallation(1, "alpha-app", "Alpha", "Tools/Utilities"), + pluginInstallation(2, "beta-app", "Beta", "Tools"), + ]; + + const markup = renderSidebar("/"); + + expect(markup).toContain(">Apps<"); + expect(markup).toContain(">Alpha<"); + expect(markup).toContain(">Beta<"); + // Both plugins share the first category segment "Tools", so no + // per-category sub-headers should render. + expect(markup).not.toContain(">Tools<"); + expect(markup).not.toContain(">Other<"); + }); + + it("groups Apps entries by first category segment with Other last when 2+ categories exist", () => { + mockPluginInstallations = [ + pluginInstallation(1, "alpha-app", "Alpha", "Tools/Utilities"), + pluginInstallation(2, "beta-app", "Beta", "Extras"), + pluginInstallation(3, "gamma-app", "Gamma"), + ]; + + const markup = renderSidebar("/"); + + expect(markup).toContain(">Apps<"); + expect(markup).toContain(">Extras<"); + expect(markup).toContain(">Tools<"); + expect(markup).toContain(">Other<"); + // Alphabetical category order with the uncategorized bucket last. + const extrasIndex = markup.indexOf(">Extras<"); + const toolsIndex = markup.indexOf(">Tools<"); + const otherIndex = markup.indexOf(">Other<"); + expect(extrasIndex).toBeGreaterThan(-1); + expect(toolsIndex).toBeGreaterThan(extrasIndex); + expect(otherIndex).toBeGreaterThan(toolsIndex); + }); + + it("hides Apps group headers the same way as other section headers when collapsed", () => { + mockPluginInstallations = [ + pluginInstallation(1, "alpha-app", "Alpha", "Tools"), + pluginInstallation(2, "beta-app", "Beta", "Extras"), + ]; + + const markup = renderSidebar("/", { collapsed: true }); + + // Group headers reuse SidebarSectionHeader, so the label slot stays in + // the layout (preventing shifts) but is visually hidden when collapsed. + expect(markup).toContain(">Tools<"); + expect(markup).toContain(">Extras<"); + const hiddenHeaderCount = (markup.match(/aria-hidden="true" class="[^"]*opacity-0/g) ?? []) + .length; + expect(hiddenHeaderCount).toBeGreaterThan(0); + }); +}); + +describe("groupAppNavLinks", () => { + const link = (id: string, category?: string): AppNavLink => ({ + id, + basePath: `/api/v1/plugins/${id}`, + label: id, + pluginId: id, + category, + }); + + it("returns null for an empty list", () => { + expect(groupAppNavLinks([])).toBeNull(); + }); + + it("returns null when all links are uncategorized", () => { + expect(groupAppNavLinks([link("a"), link("b")])).toBeNull(); + }); + + it("returns null when all links share the same first category segment", () => { + expect(groupAppNavLinks([link("a", "Tools/Utilities"), link("b", "Tools/Extras")])).toBeNull(); + }); + + it("groups by first segment, sorts alphabetically, and puts Other last", () => { + const groups = groupAppNavLinks([ + link("z", "Extras"), + link("a", "Tools/Utilities"), + link("m"), + link("b", "Tools"), + ]); + + expect(groups).not.toBeNull(); + expect(groups?.map((g) => g.category)).toEqual(["Extras", "Tools", "Other"]); + // Input order preserved within a group. + expect(groups?.[1]?.links.map((l) => l.id)).toEqual(["a", "b"]); + expect(groups?.[2]?.links.map((l) => l.id)).toEqual(["m"]); + }); + + it("treats blank or slash-only categories as uncategorized", () => { + const groups = groupAppNavLinks([link("a", " "), link("b", "/Tools"), link("c", "Extras")]); + + expect(groups?.map((g) => g.category)).toEqual(["Extras", "Other"]); + expect(groups?.[1]?.links.map((l) => l.id)).toEqual(["a", "b"]); + }); }); diff --git a/web/src/components/AppSidebar.tsx b/web/src/components/AppSidebar.tsx index e0d81fa6..3a85c505 100644 --- a/web/src/components/AppSidebar.tsx +++ b/web/src/components/AppSidebar.tsx @@ -2,7 +2,12 @@ import { useState, useCallback, useEffect, useMemo, useRef } from "react"; import type { ReactNode } from "react"; import { Link, useLocation, useParams } from "react-router"; import ViewTransitionLink from "@/components/ViewTransitionLink"; -import { getProfileMenuSide, isSidebarExpanded } from "@/components/AppSidebar.logic"; +import { + getProfileMenuSide, + groupAppNavLinks, + isSidebarExpanded, + type AppNavLink, +} from "@/components/AppSidebar.logic"; import { SiloBrand } from "@/components/SiloBrand"; import { useAuth } from "@/hooks/useAuth"; import { useCurrentProfile } from "@/hooks/useCurrentProfile"; @@ -190,7 +195,7 @@ export default function AppSidebar({ onNavigate, collapsed = false }: AppSidebar ); const pluginNavLinks = useMemo(() => { const installations = pluginSettings?.installations ?? []; - const links: { id: string; basePath: string; label: string; pluginId: string }[] = []; + const links: AppNavLink[] = []; for (const inst of installations) { for (const route of inst.routes) { if (!route.navigable || route.navigation_kind !== "user") continue; @@ -199,11 +204,15 @@ export default function AppSidebar({ onNavigate, collapsed = false }: AppSidebar basePath: pluginRouteHref(inst.id, route.path), label: route.navigation_label || inst.plugin_id, pluginId: inst.plugin_id, + category: inst.category, }); } } return links; }, [pluginSettings]); + // Grouped view of the Apps entries (null → keep the flat list under the + // single "Apps" header). See groupAppNavLinks for the SDK category contract. + const pluginNavGroups = useMemo(() => groupAppNavLinks(pluginNavLinks), [pluginNavLinks]); const catalogState = useMemo( () => @@ -304,6 +313,28 @@ export default function AppSidebar({ onNavigate, collapsed = false }: AppSidebar : "text-muted-foreground hover:text-sidebar-foreground hover:bg-sidebar-accent/70" }`; + const renderAppNavList = (links: AppNavLink[]) => ( + + ); + return (