diff --git a/cmd/silo/main.go b/cmd/silo/main.go index 89cc18c4..a5098b26 100644 --- a/cmd/silo/main.go +++ b/cmd/silo/main.go @@ -69,6 +69,7 @@ import ( "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/nodeconfig" "github.com/Silo-Server/silo-server/internal/nodepool" + "github.com/Silo-Server/silo-server/internal/noderecipe" "github.com/Silo-Server/silo-server/internal/nodesessions" "github.com/Silo-Server/silo-server/internal/notifications" "github.com/Silo-Server/silo-server/internal/opslog" @@ -616,6 +617,10 @@ func main() { } else { srv := transcodenode.NewServer(watcher, tracker) srv.SetFFmpegLogSink(playback.NewSlogFFmpegLogSink(slog.Default(), nodeID)) + // Read jellycompat reconstruction recipes central wrote at transcode + // start, so this node can rebuild a Jellyfin transcode after its own + // restart (the node hop token is recipe-less). Shares the offload Redis. + srv.SetRecipeStore(noderecipe.NewStore(redisClient, 0)) handler = srv.Handler() } @@ -1399,6 +1404,7 @@ func main() { ) userStoreProvider = notifications.WrapUserStoreProvider(userStoreProvider, notificationSystem) deps.Notifications = notificationSystem + if libraryIngestExecutor != nil { libraryIngestExecutor.SetAvailabilityDetector(notificationSystem.Detector) } @@ -2183,7 +2189,10 @@ func main() { JWTSecret: cfg.Auth.JWTSecret, RecWorker: recWorker, FrontendFS: deps.FrontendFS, - SessionSyncer: deps.SessionSyncer, + // Hand remote-transcode recipes to the shared recipe store so a dedicated + // transcode node that restarts can rebuild a jellycompat session. + RecipeNodeStore: noderecipe.NewStore(apiRedisClient, 0), + SessionSyncer: deps.SessionSyncer, } // Wire direct dependencies when DB is available. diff --git a/docs/architecture/restart-resilient-playback.md b/docs/architecture/restart-resilient-playback.md new file mode 100644 index 00000000..d4fbe9b8 --- /dev/null +++ b/docs/architecture/restart-resilient-playback.md @@ -0,0 +1,787 @@ +# Restart-resilient playback (TR-lease) + +How Silo keeps playback alive across a server restart, reconnect, or partial +stream — and how the design arrived at **TR-lease** (token-carried +reconstruction + a server-side session-deny marker for revocation). + +> **Status: the revocation half is deferred to a future PR.** Only the +> token-carried **reconstruction** core shipped in this PR. The **session-deny / +> stream-revocation** mechanism described throughout (the `silo:streamauth:` +> deny marker, the proxy's `Allowed()` enforcement, and the admin +> Stop/Terminate deny write) is **not present in the current implementation**. +> Admin Terminate and user Stop tear down the live in-memory session and the +> ffmpeg producer, but they do **not** prevent a still-valid stream token from +> reconstructing the session until its 24h TTL expires. Sections describing that +> mechanism are kept for design continuity and are individually flagged as +> deferred below. + +This document is the consolidated design record. It folds together four earlier +working notes so a future engineer can see the full evolution, the options +weighed, the issues raised across review rounds, the variables traded off, and +why the final shape was chosen. The superseded notes were: + +- *unified-playback-reconstruct* — the reconstruct core (PR #174), "RC". +- *token-carried-playback-reconstruct* — the storage evolution, "TR". +- *async-recipe-card-playback* — a monitoring-preserving middle path, "RC-async". +- *playback-reconstruct-options-comparison* — the goals/options matrix. + +Paths are repository-relative; assume the repository root is the cwd. + +--- + +## 1. The problem + +Before this work, a missing in-memory playback session was a `404`. A server +restart (deploy, crash, OOM) wiped the in-memory session map, so every in-flight +stream died: the client's next manifest/segment/byte-range request hit a session +that no longer existed and playback stopped hard. The same gap appeared on any +front-end that had never held the session (horizontal scale) and on a dedicated +transcode node reboot. + +Core priorities for the fix (from the repo guidelines): **performance first, +reliability first, predictable behavior under load and during failures**. The +hot path (per-segment serve) must stay off central/Postgres; bytes must stay +offloaded to nodes; correctness and robustness win over convenience. + +--- + +## 2. The core insight: reconstruct, don't rehydrate + +A missing session is a **reconstruct trigger, not a 404**. The client always +re-requests the next chunk after an outage, and its request already states where +it is (an HTTP `Range`, a `?seek=`, or a `seg_NNNNN` number). So the server +rebuilds the session just-in-time from two things: + +1. a small **durable descriptor** (identity, ownership, and the byte-affecting + encode parameters — the "recipe"), and +2. the **position the client supplies** on the triggering request. + +Nothing live is serialized (no ffmpeg process, context, channels, or log sink). +This is the contribution of PR #174 ("RC"), which also unified the native and +Jellyfin-compat paths behind one `playback.TranscodeManager` so the +card-lifetime rules, the reconstruct concurrency cap, and the node-affinity rule +live in exactly one place. The reconstruct skeleton — +`LoadOrReconstructSession`, `ReconstructSession` / `ReconstructTranscode`, the +single-flight + `reconstructSem` cap that paces the post-restart thundering +herd, `RegisterReconstructed`, the `CloseProcess`/`Close` split — is the +foundation every later option reused. **It was kept; only the descriptor storage +and the revocation model evolved.** + +The six routes that must survive a restart: + +| # | Route | Tier | How it is rebuilt | +|---|-------|:----:|-------------------| +| 1 | native direct | 1 | client `Range` re-serve; descriptor = identity | +| 2 | native remux | 1 | re-spawn ffmpeg at `?seek`; descriptor = id + audio | +| 3 | native transcode | 2 | rebuild `Session` + ffmpeg seeked to the requested segment; descriptor = full encode opts | +| 4 | jellycompat direct | 1 | compat store + `Range` re-serve | +| 5 | jellycompat remux | 1 | compat store + `?seek` | +| 6 | jellycompat transcode | 2 | compat store + shared-manager reconstruct | + +Tier 1 = stateless re-serve (client re-supplies position). Tier 2 = rebuild +ffmpeg. Routes 4–6 always retain a durable central compat store +(`jellycompat_playback_sessions`): third-party Jellyfin clients cannot +round-trip a native descriptor, so the compat layer keeps the full +`PlaybackSession` (route + media source resolved before native reconstruct +runs). + +--- + +## 3. The two axes the options differ on + +Every option is a point on a 2-D grid: **where the descriptor lives** (vertical) +× **how revocation is bounded** (horizontal). + +```text + AUTHORITY / REVOCATION MODEL -----------------------> + re-resolve at long token short TTL + + reconstruct (24h, expire) refresh checkpoint + D central +----------------+----------------+----------------+ + E Postgres row | RC (#174) | (n/a) | RC-async | + S +----------------+----------------+----------------+ + C client/node- | (n/a) | TR-24h | TR + re-mint | + R carried token | | | | + I +----------------+----------------+----------------+ + P + T top row = descriptor stays central (node serves bytes but cannot + | self-reconstruct → node-restart gap) + | bottom = recipe travels to the node (node self-sufficient, no central record) +``` + +**TR-lease is a fourth point off this grid:** bottom row (token-carried, 24h), +but the revocation column is replaced by a *server-side deny marker* — on an +admin kill, central would write a per-session deny to Redis that the node +enforces by withholding bytes. Revocation is decoupled from the token TTL and +needs no client refresh. (An earlier iteration polled and re-resolved every +stream on a timer; that was cut to the event-driven session-deny — see §7 and +the §8 design note.) + +> **Status: deferred to a future PR.** The deny-marker column described above is +> the design target, not the current shipped behavior. As shipped, the +> token-carried row offers no revocation tighter than the 24h token TTL on the +> node path; admin Terminate and user Stop only tear down the live session and +> ffmpeg producer. + +--- + +## 4. Goals weighed + +| ID | Goal | +|----|------| +| G1 | Hot path (per-segment serve) stays off central/Postgres | +| G2 | Bytes offloaded to nodes (served from disk, signature-verified, never central) | +| G3 | Restart resiliency across all 6 routes | +| G4 | Two-factor ownership (auth caller + ownership rebind; refuse `userID==0`/mismatch) | +| G5 | Revocation latency (how fast a banned user / pulled access stops playing) | +| G6 | Single-box: zero external store required for reconstruction | +| G7 | Restart runway (descriptor stays valid longer than the outage) | +| G8 | Multi-front-end / verify-anywhere (no integrated-transcode split-brain) | +| G9 | Observability — durable, identity-rich active-stream record | +| G10 | Low operational surface (tables, migrations, stores, endpoints) | +| G11 | Node-restart recovery | +| G12 | No new client coordination required | +| G13 | Small signing-key blast radius | +| G14 | Cleanup correctness (never wipe a live session's segment dir) | + +A note on G9 (observability): the *live* "who is streaming now" view does **not** +depend on the descriptor strategy. Every offload node calls the Redis-backed +node-session tracker (`internal/nodesessions/tracker.go`) on every serve, tied to +actual byte-serving (so a non-cooperative client or replayed token still shows +up) and surviving a central restart. G9 scores only the *durable, identity-rich* +(who + what + recipe) slice on top of that. + +--- + +## 5. The options, and the comparison + +Legend: ✓ meets · ✓\* minor asterisk · ~ partial · ✗ fails · ✓✓ best-in-class + +| Goal | `main` | RC #174 | TR-24h | TR + re-mint | RC-async | **TR-lease** | +|------|:------:|:-------:|:------:|:------------:|:--------:|:------------:| +| G1 hot path free of central DB | ✓ | ✓\* | ✓ | ✓ | ✓ | ✓ | +| G2 bytes offloaded | ✓ | ✓ | ✓✓ | ✓✓ | ✓✓ | ✓✓ | +| G3 restart resiliency (6/6) | ✗ | ✓ | ✓ | ✓ | ✓ | ✓ | +| G4 two-factor ownership | ✓ | ✓ | ~ | ~ | ~ | ✓~ | +| G5 revocation latency | ~ | ~ | ✗ (≤24h) | ✓ (≤TTL) | ✓✓ | ✓✓ | +| G6 single-box, no external store | ✓ | ✗ | ✓✓ | ~ | ~ | ~ | +| G7 restart runway | n/a | ✓ (30m) | ✓✓ (24h) | ~ | ~ | ✓✓ (24h) | +| G8 multi-front-end | ✗ | ~ | ✓~ | ✓~ | ✓~ | ✓~ | +| G9 durable identity-rich record | ✗ | ✓✓ | ✗ | ✗ | ✓ | ~ | +| G10 low operational surface | ✓✓ | ~ | ✓ | ~ | ✗ | ~ | +| G11 node-restart recovery | ✗ | ✗ | ✓ | ✓ | ✓ | ✓ | +| G12 no new client coordination | ✓ | ✓ | ✓ | ✗ | ✗ | ✓ | +| G13 small signing-key blast radius | ✓ | ✓ | ✗ | ✗ | ✗ | ✗ | +| G14 cleanup correctness | ✓ | ✓✓ | ~ | ~ | ~ | ~ | + +The TTL/revocation strategies head-to-head. **Note: the session-deny column is +the deferred design target — see §7; as shipped, the node path has no revocation +tighter than the 24h token TTL.** + +| Strategy | Revocation latency | Restart runway | Client refresh? | Central on refresh? | +|----------|--------------------|----------------|-----------------|---------------------| +| Re-resolve at reconstruct (RC) | on reconstruct; node hop ≤24h | 30 min | no | only on reconstruct | +| Leave 24h, expire only (TR-24h) | up to 24h | 24h | no | never | +| Short TTL + refresh + denylist (RC-async / TR item B) | ≤~5 min | ~90 s grace | **yes** | every ~3.5 min/session | +| 24h token + session-deny marker (TR-lease — **deferred**) | *design target:* admin kill cuts node bytes on next serve; passive ban = next-play / ≤24h. *As shipped:* node path bounded by ≤24h token only | 24h | **no** | nothing on the serve path; one write per admin-kill event | + +--- + +## 6. Why each non-final option was set aside + +- **RC (PR #174, the recipe card in Postgres).** Correct and the basis of + everything here, but it couples reconstruction to a shared per-session Postgres + store: a write on every start, per audio/quality change, and a ≤1/min refresh, + with resilience tied to the DB being reachable. It does not give the single-box + zero-dependency property, and the node hop's 24h token stays unrevocable + anyway. RC's reconstruct *core* was kept; its *storage* was the thing to evolve. + +- **TR-24h (token-carried, leave the token to expire).** Wins the single-box case + (zero external dependency) and removes the per-start DB write, but revocation is + bounded by *nothing tighter than the 24h token TTL* on the node path — a banned + user keeps node-served direct/remux for up to 24h. This is actually the + *status quo* for the node path even pre-branch (the proxy verifies by signature + alone, no DB), so TR-24h only adds identity claims to the token without + improving revocation. Not acceptable as the endpoint. + +- **TR + re-mint (short TTL + authenticated playlist refresh + denylist), and + RC-async (the same revocation idea but keeping an async-written Postgres mirror + for observability).** Both bound revocation to ~5 min by having the client + refetch a stable `RequireAuth` playlist that re-checks access and re-mints + short-lived segment URLs. The cost: a **hard client-coordination requirement** + (silo-android / silo-apple / Jellyfin clients must implement the refresh flow), + a shrunken restart runway (~90 s grace instead of 30 min — if central is down + past the grace, playback stalls), and a new authenticated re-mint transport + that does not exist on the supported proxy/node topology. RC-async additionally + keeps a write pipeline + denylist + the central row (the highest operational + surface of all). They solved revocation by pushing work onto the client and + shortening the runway — the opposite of the reliability priorities. + +The open item across TR and RC-async was always **revocation enforcement** ("item +B"): bounding revocation needs *either* a per-segment token check on the hot path +*or* an authenticated re-mint checkpoint the client drives. TR-lease removes that +dilemma. + +--- + +## 7. TR-lease: the chosen design + +TR-lease keeps TR's 24h token (so the restart runway is untouched and no client +refresh flow is needed) and adds a server-side **session-deny marker** for the +one revocation case the node path cannot otherwise cover — no client +participation, no steady-state cost. + +**Reconstruction (token-carried).** The signed stream token *is* the durable +descriptor. Its claims carry the full byte-affecting recipe plus `uid/pid/mfid` +ownership lookup keys. A front-end that lost its in-memory session rebuilds +ffmpeg from the token the client re-presents — no shared per-session store, zero +external dependency for reconstruction on a single box. `HWAccel`/`HWDevice` are +deliberately *not* carried; they are re-resolved from live config so an operator +config change applies to reconstructed sessions too. Ownership re-binds to the +live authenticated caller and refuses `userID==0` / mismatch — the token's `uid` +is never trusted alone (a leaked URL is useless without the owner's auth session +on the native path). + +**Revocation (session-deny marker).** + +> **Status: deferred to a future PR.** Everything in this subsection (the +> `silo:streamauth:` deny marker, the admin Stop/Terminate deny write, and +> the node-side `Allowed()` enforcement) describes the design target and is **not +> present in the current implementation**. As shipped, admin Terminate and user +> Stop tear down the live in-memory session and the ffmpeg producer, but a +> still-valid stream token can reconstruct the session until its 24h TTL expires; +> there is no node-side byte-withholding and no guaranteed/instant revocation on +> the node path. The design below is retained for when revocation lands. + +The revocation surface was deliberately +kept to the *one* case the offloaded topology cannot otherwise cover, rather than +a periodic re-check of every stream. An admin Stop/Terminate would write +`silo:streamauth: = deny` (TTL ≥ the 24h token lifetime); the offload node +would read it (a node-local Redis GET, sub-ms) before serving any bytes and +return `403` with no bytes on a hit. Enforcement is **byte-withholding, not +client cooperation** — a revoked client that ignores the 403 keeps hitting a +wall. + +Why this would be sufficient, and what it deliberately does *not* do: + +- **New playback is already blocked at central** — `/playback/start` is + `RequireAuth` + access check, so a ban/scope change always takes effect on the + next play, everywhere, with no marker needed. +- **Cooperative clients already stop** — central holds the realtime WebSocket and + can drop it + send a stop on a ban. +- **The only thing needing a hard node-side cut** is the *explicit, session-scoped + admin kill* of a *non-cooperative* client holding a valid token on a + node-served direct/remux stream (no producer to kill). That is exactly what the + (deferred) session-deny marker would cover; until it lands this case is bounded + by the ≤24h token like the passive bans below. +- **Not covered (accepted limitation):** a *passive* ban or *partial* access + change (lost one library, lowered rating) does NOT hard-revoke an + already-running, non-cooperative node stream. It is enforced at next play and + otherwise bounded by the ≤24h token. Closing this would require either a + periodic re-resolver (a poll over every active stream — rejected as constant + DB load for a rare case) or per-content deny enumeration; both were judged not + worth the machinery for a self-hosted server. A user-scoped deny was rejected + too: it over-blocks content the viewer still legitimately has access to. + +**Fail-open on absence (deferred).** In the design, the normal case is *no* +marker, which serves; a Redis error also serves; only a present deny withholds +bytes. This is a deliberate reliability-first choice: availability over +revocation latency. + +**Why this shape.** It keeps token-carried reconstruction's single-box and +verify-anywhere properties and would add revocation with no client coordination +(G12) and essentially no steady-state cost — the marker is written only on the +rare admin-kill event, never on a timer. It is, in effect, **TR-24 plus a single +session-scoped kill switch**. The kill switch is **deferred**; what shipped is +TR-24 (token-carried reconstruction, ≤24h token, no node-side revocation). + +--- + +## 8. What was implemented, and where + +This PR ships **Commit 1 only** (token-carried reconstruction). **Commit 2 (the +node session-deny marker) is deferred to a future PR** — the `internal/streamauth` +package, the proxy `Allowed()` guard, and the admin deny write described below +are **not present in the current implementation**. + +**Commit 1 — token-carried reconstruction; retire `transcode_recipes`. (shipped)** +- `streamtoken.Claims` gains the recipe + `uid/pid/mfid` + (`internal/streamtoken/token.go`). +- `playback.RecipeCard` projects to/from claims (`ToClaims` / + `RecipeCardFromClaims`); the `Reconstruct*` front door takes the decoded recipe + instead of reading Postgres (`internal/playback/recipecard.go`, + `transcode_manager.go`). +- Native serve URLs carry the token as `?st=`; the manifest rewriter already + appends `RawQuery` to segment URIs, so segments inherit it with no route change + (`internal/api/handlers/playback.go`, `stream.go`). The integrated server is + hit directly, so the query-stripping-proxy concern that motivates a path + segment does not apply there; the proxy/node path keeps the token in the URL + path as before. +- Jellycompat folds the recipe into its durable compat store + (`PlaybackSession.Recipe`) since Jellyfin clients cannot round-trip a native + token (`internal/jellycompat/...`). +- Segment-dir cleanup switches from the card index to in-memory liveness (live + map + in-flight reconstruct set + age past the max token TTL); a dedicated node + keeps its boot-time full wipe (`internal/playback/transcode_cleanup.go`, + `transcode_manager.go`). +- `PostgresRecipeStore`, the `RecipeStore` interface, and the unmerged + `transcode_recipes` migration are deleted. + +**Commit 2 — node session-deny marker. (DEFERRED — not in this PR)** + +> The items below describe the planned revocation commit. They were removed from +> this PR and deferred to a future one; none of these symbols exist in the +> current implementation. The admin Stop/Terminate path still tears down the live +> session and ffmpeg producer, but writes no deny marker, so a valid token can +> reconstruct until its 24h TTL expires. + +- `internal/streamauth`: a deny-only Redis `Store` — `Deny(sid)` (TTL ≥ token + lifetime) and a fail-open `Allowed(sid)` reader. No poller, no allow-leases, no + access re-resolver. +- The node guard in `internal/proxy/server.go` (`verifyToken` consults + `Allowed`). +- An admin Stop/Terminate writes the deny marker + (`internal/api/handlers/admin_playback_control.go` → `PlaybackHandler.LeaseDenier`). +- `nodesessions.SessionInfo` carries the numeric ownership keys + (`auth_user_id`/`profile_id`/`media_file_id`), populated by the node from the + verified token, to enrich the live admin "active streams" view (who/what, not + just session id) — see §10 monitoring. +- A single integrated box reads no marker and needs none — the producer-kill + + session removal on Terminate is the stop there. + +> Design note (about the deferred commit 2): an earlier iteration added a central +> *revalidator* that re-resolved access for every active stream every ~2 min and +> wrote allow/deny leases. It was dropped in favour of the event-driven +> session-deny above: the poll re-derived, at a constant all-streams DB cost, +> signals the +> system already emits (new playback is gated at `/playback/start`; admin kill is +> an explicit event), and its only unique coverage — sub-24h *passive* revocation +> of an in-flight node stream — was judged not worth the machinery (see §7). + +--- + +## 9. Issues raised across review, and their disposition + +From two adversarial review rounds on the reconstruct core and the storage +evolution: + +- **Integrated-transcode split-brain across front-ends (P-1 / A).** The + `SessionManager` + reconstruct single-flight are per-process and the descriptor + carries no owning-node id, so without sticky affinity two front-ends can run + divergent ffmpeg. **Disposition: deferred (owner-identity claim).** Supported + topology is dedicated `--mode=transcode` nodes (routed via `tnode`, no + split-brain) + single-box / LB-session-affinity integrated transcode. The + caveat is documented, not silently assumed. +- **24h node-path token is unrevocable (P-2).** The (deferred, see §7–§8) + session-deny marker would let the explicit admin kill cut node bytes on the next + serve. **Until that ships, this remains open:** the node-path token is + unrevocable, so admin kill and passive bans / partial access changes are all + bounded by next-play + the ≤24h token. +- **Dedicated transcode-node restart (P-3 / tr-10). Fixed for both paths.** + *Native:* the proxy forwards the verified stream token to the transcode node + (`X-Silo-Stream-Token`); on a manifest/segment miss the node re-verifies the + token, decodes the recipe, and self-reconstructs ffmpeg seeked to the requested + segment — single-flighted and concurrency-capped exactly like the integrated + reconstruct. *Jellycompat:* the node-hop token is server-minted and *could* carry the + recipe like native, but it deliberately does not — the recipe is mutated in + place under a stable session id (a Jellyfin audio/subtitle switch can arrive as + a `/Sessions/Playing/Progress` report that restarts ffmpeg without re-minting + the client's token), and a third-party Jellyfin client cannot be driven to + refresh a stale token, so a token *snapshot* could reconstruct a stale + rendition. Central instead writes the recipe to a shared Redis recipe store + (`internal/noderecipe`) keyed by upstream session id and overwrites it on every + switch, and the node (which cannot reach Postgres) reads the *current* recipe on + the reconstruct miss — over the same Redis the session tracker already uses (and + the deferred deny-lease would use), so no central URL is needed. The boot-time + full segment-dir wipe + stands; reconstruct re-transcodes from the requested segment. See §10 for the + full rationale (and why this is *not* a token-carryable case). +- **`userID==0` tolerance / "auth optional" (P-5).** Reconstruct hard-rejects + `userID==0` and a card whose session id does not match the URL. +- **Mutable recipe under a stable id (D).** A parameter change re-mints a new + manifest + token and the client reloads, so the common case self-heals. The + only residual is a restart *during* the brief switch transition with a prior + token in flight: it reconstructs the prior *rendition* (access is re-resolved, + only the rendition can be stale). TTL-bounded, self-healing on reload; minor, + accepted. +- **Cleanup liveness (F / tr-9).** Each process owns its `TranscodeDir`, so the + liveness signal is the in-process live map + the in-flight reconstruct set + + age-past-max-TTL; a dir is reaped only when absent from both and older than any + surviving token. No cross-process enumeration-failure mode for the supported + topology. +- **Signing-key blast radius (E / tr-7, G13).** A single `JWTSecret` signs native + auth, the stream token, and the node bearer, so a node-secret leak can forge + native logins. **Accepted risk** on the trusted-node precondition. **Deferred + hardening:** split the stream-token secret from the native-auth secret, then + asymmetric stream-token signing before any untrusted/edge node. + +--- + +## 10. Residuals and follow-ups + +- **Revocation of an in-flight node stream (all cases, until deny-lease ships).** + Because the session-deny marker is deferred (§7–§8), *no* ban — admin kill, + passive ban, or partial access change — hard-cuts an already-running, + non-cooperative node-served stream; all are enforced at next play, bounded by + the ≤24h token. The future deny-lease closes the *admin-kill* case; passive + bans would still need a per-content deny enumeration on the access-change event, + or accepting it. See §7. +- **Monitoring.** Two distinct questions: + - *Live "who is watching what right now"* — covered. Every node serve writes a + record to the Redis tracker (`silo:sessions:*`), now enriched with + `auth_user_id` / `media_file_id` from the verified token, so the admin + active-streams view answers who/what/where, tied to real byte-serving and + surviving a central restart. Integrated (no-node) sessions show in central's + in-memory session list. + - *Durable historical/audit ("who watched what last week")* — NOT provided by + this subsystem (there is no longer a durable per-session recipe record, by + design — G9 was the deliberate trade for token-carried reconstruction). That + history lives in the separate watch-progress / scrobble system. +- **Multiple viewers on one stream token.** The token is per playback session, so + a shared/leaked stream URL lets several clients pull bytes under one session id. + They **collapse to a single entry** in the tracker (one `sid`) — monitoring + cannot distinguish or count them — and the (deferred) session-deny would cut + them as a group, not individually. Detecting/limiting this needs a per-account + concurrency cap or + per-device session ids (device binding); deferred. Do not bind to client IP + (mobile / CGNAT). +- **Token in logs (tr-1).** Scrub the `?st=` token / path token from request + loggers (the jellycompat logger logs `RawQuery`; the native logger omits it). + Short TTL + signature bound the exposure. +- **Owner-identity (A).** Required before multi-front-end *integrated* transcode + without sticky affinity; defines owner election, self-URL discovery, a + peer-forward endpoint, and failover. +- **Node-side reconstruct (P-3). Done** for native (token-forwarded) and + jellycompat (Redis recipe handoff via `internal/noderecipe`: central writes the + recipe at remote-transcode start, the node reads it on a reconstruct miss). + Precondition: central and the nodes share the same Redis (already required for + the session tracker, and for the deferred deny-lease). Residual: the recipe + store is consulted + only for a jellycompat token, so a node restart still depends on the recipe key + surviving (24h TTL, ≥ token lifetime). + - *Why a server-side store and not the token here (the real rationale).* It is + tempting to delete `internal/noderecipe` and let the jellycompat node-hop + token carry the recipe like native — the token is server-minted and opaque to + the Jellyfin client (it follows it only as a redirect target), so it *could*. + That does **not** work, and the reason is **not** the often-stated "a Jellyfin + client can't round-trip a token." The reason is that the recipe is **mutated + in place under a stable session id**: a Jellyfin audio/subtitle switch can + arrive as a `/Sessions/Playing/Progress` report + (`restartCompatTranscodeForAudioSelection`) that restarts ffmpeg in place + **without re-minting the client's token**. A third-party Jellyfin client + cannot be driven to refresh the now-stale token — its VOD/`ENDLIST` manifest + is fetched once, and the compat WebSocket (`HandleSocket`) is a receive-only + keep-alive stub with no server→client command to force a manifest re-fetch. A + token is a snapshot at mint time, so after such a switch it would reconstruct + the **stale** rendition (old audio/subtitle) until the client happens to + re-fetch the manifest (e.g. on a seek), which for a VOD stream may be never. + Because the node also cannot reach Postgres (where the authoritative compat + recipe lives), the only way the node rebuilds the *current* recipe is a + node-reachable, server-authoritative store that central overwrites on every + switch — i.e. `internal/noderecipe`. Native has no equivalent path: every + native audio switch re-mints the manifest + token, so the native client's + token is never stale, which is exactly why native needs no such store. The + store is **load-bearing, not redundant**. +- **Key split / asymmetric signing (E).** Closes the account-takeover path; a + precondition for untrusted edge nodes. + +--- + +## 11. Preconditions + +- **Persistent `TranscodeDir` (recommended).** With persistence, reconstruct + serves surviving segments; with an ephemeral dir it re-transcodes from the + descriptor (which survives a wiped dir). A node *restart* still loses the + session regardless (P-3). +- **Re-transcode of one segment beats the buffer runway** on worst-case hardware + (software decode, subtitle burn-in). Measure. +- **Client retry/backoff exists.** Clients retry segment/manifest GETs on + 5xx/connection error and reconnect the WebSocket. The token rides the URL, so no + extra client logic is needed to carry it; a client that does not retry sees the + failed request as fatal. + +--- + +## 12. Diagrams + +The `mermaid` blocks render in GitHub / VS Code / most Markdown viewers; the +ASCII blocks render anywhere. + +### 12.1 Topology — where state lives + +> **Deferred:** the `silo:streamauth: = deny` marker, the "Admin +> Stop/Terminate → write session-deny marker" arrow, and the node's "deny GET" +> are the planned revocation path and are **not** in the current implementation. +> The tracker (`silo:sessions:*`) and the recipe handoff are shipped. + +```text + ┌─────────────────────────── CENTRAL (mode=server) ───────────────────────────┐ + Client │ RequireAuth /playback/start /playback/transcode/start (mint token) │ + (web / android / │ native serve: /playback/transcode/{sid}/master.m3u8?st= /stream/{sid}?st= │ + apple / jellyfin) │ Admin Stop/Terminate → write session-deny marker (event, not a timer) │ + └───────────┬──────────────────────────┬───────────────────────┬──────────────┘ + │ Postgres │ Redis │ + ▼ ▼ │ (multi-node only) + ┌───────────────────────┐ ┌──────────────────────────┐ ▼ + │ catalog / access │ │ silo:sessions:* (tracker)│ ┌────────────────────────┐ + │ jellycompat_playback_ │ │ silo:streamauth: │ │ Proxy node (mode=proxy) │ + │ sessions (compat │ │ = deny (session kill) │◀─│ verifyToken + deny GET │ + │ store, carries Recipe)│ └──────────────────────────┘ │ serves direct/remux │ + └───────────────────────┘ ▲ │ proxies transcode ──────┼──▶ Transcode node + ✗ NO transcode_recipes table (deleted this branch) │ node-local GET └────────────────────────┘ (mode=transcode) + └── sub-ms, off Postgres ffmpeg /transcode/{sid}/… +``` + +Descriptor location by route (the core change). The "lease (node)" revocation +column is the **deferred** design; as shipped, the node path has no revocation +tighter than the ≤24h token: + +| route family | descriptor (how it reconstructs) | revocation | +|--------------|----------------------------------|------------| +| native (1–3) | the signed token the client re-presents | *deferred:* lease (node) / shipped: producer-kill (integrated), ≤24h token (node) | +| jellycompat (4–6) | `jellycompat_playback_sessions.data.Recipe` (token can't round-trip) | *deferred:* lease (node) / shipped: producer-kill, ≤24h token (node) | +| ~~all~~ | ~~`transcode_recipes` Postgres row~~ — removed | — | + +### 12.2 Where the token rides + +```text +NATIVE (integrated server is hit directly → query param; segments inherit it) + manifest : GET /playback/transcode/{sid}/master.m3u8?st= + segment : GET /playback/transcode/{sid}/segment/seg_00042.m4s?st= ← RawQuery propagated by the manifest rewriter + direct : GET /stream/{sid}?st= + +PROXY/NODE (capability URL → path segment; survives a query-stripping hop) + GET {proxy}/stream/transcode/{JWT}/master.m3u8 → relative segment URIs inherit {JWT} + GET {proxy}/stream/direct/{JWT} GET {proxy}/stream/remux/{JWT} + +JELLYCOMPAT → recipe is NOT in the token; it lives in the compat store. + the path token is used only for the node hop. +``` + +### 12.3 Normal playback — native, integrated box (no nodes) + +```mermaid +sequenceDiagram + participant C as Client + participant S as Central server (integrated) + participant PG as Postgres (access) + Note over C,S: transcode start + C->>S: POST /playback/transcode/start (RequireAuth) + S->>PG: resolve file + access + S->>S: StartTranscode (ffmpeg) + RegisterTranscodeSession + S->>S: card = NewRecipeCard(opts); token = Sign(card.ToClaims(), 24h) + S-->>C: 202 manifestURL = /playback/transcode/{sid}/master.m3u8?st=TOKEN + Note over C,S: hot path — NO Postgres, NO lease (live session in memory) + C->>S: GET .../master.m3u8?st=TOKEN + S->>S: LoadOrReconstructSession → live hit → BuildPlaybackManifest (segments carry ?st) + S-->>C: manifest + loop every segment + C->>S: GET .../segment/seg_N.m4s?st=TOKEN + S->>S: live session → GetSegment → ServeFile + S-->>C: bytes + end +``` + +The hot path touches neither Postgres nor the lease — it is an in-memory map +hit. `?st` is verified only to decode a card if reconstruct is needed; a live +session ignores it. + +### 12.4 Normal playback — native, multi-node (offloaded) + +> **Deferred:** the `GET silo:streamauth:{sid}` lease guard and its `deny → 403` +> branch are the planned revocation path, **not** in the current implementation. +> As shipped the node verifies the token, tracks the session, and serves — there +> is no deny check. + +```mermaid +sequenceDiagram + participant C as Client + participant S as Central server + participant N as Proxy node + participant T as Transcode node + participant R as Redis + C->>S: POST /playback/transcode/start (RequireAuth) + S->>S: card(full recipe, tnode=T); token = Sign(card.ToClaims()) + S-->>C: manifestURL = {N}/stream/transcode/TOKEN/master.m3u8 + loop manifest + segments + C->>N: GET /stream/transcode/TOKEN/... + N->>N: verifyToken(TOKEN) + N->>R: GET silo:streamauth:{sid} (lease guard) + alt deny + N-->>C: 403 (no bytes) + else allow / absent + N->>R: Track silo:sessions:{node}:{sid} (uid, mfid from claims) + N->>T: proxy /transcode/{sid}/... + T-->>N: bytes + N-->>C: bytes + end + end +``` + +The node copies `uid`/`mfid` from the verified token into the Redis tracker — +that is what enriches the live admin "active streams" view (who/what per node). + +### 12.5 Restart → reconstruct (native transcode) + +```mermaid +sequenceDiagram + participant C as Client + participant S as Central server (just restarted — session map empty) + Note over S: in-memory session GONE; NO transcode_recipes table to read + C->>S: GET /playback/transcode/{sid}/segment/seg_42.m4s?st=TOKEN + S->>S: card = streamCardFromQuery(?st) → Verify + decode recipe + S->>S: LoadOrReconstructSession → GetSession = NotFound, card != nil + S->>S: ReconstructSession (re-bind: refuse userID==0 / != card.uid; card.sid == {sid}) + S->>S: ReconstructTranscode(seg=42) → single-flight, mark in-flight, respawn ffmpeg SEEKED to seg 42 + S-->>C: bytes (resumes near where the client was) +``` + +- direct/remux restart is the same but Tier-1: reconstruct the `Session` from the + identity token, then `Range` / `?seek` re-serve — no ffmpeg rebuild. +- multi-node restart: the integrated front-end reconstructs only the `Session` + (to learn `tnode`) and re-proxies; the transcode node keeps serving. If the + transcode *node* itself reboots, the proxy forwards the stream token + (`X-Silo-Stream-Token`) and the node self-reconstructs ffmpeg seeked to the + requested segment (native path, P-3). A jellycompat session on a rebooted node + reconstructs too: the node fetches the recipe central wrote to the shared Redis + recipe store (`internal/noderecipe`) at transcode start, since the Jellyfin + client's token cannot carry it. +- cleanup race: a segment dir is reaped only if absent from the live map AND the + in-flight set AND older than 24h (max token TTL), so a token that can still + reconstruct never gets its dir wiped. + +### 12.6 Auth deny — the session-deny marker + +> **Status: deferred to a future PR.** The entire "Admin Stop/Terminate → write +> deny marker" and "Offload node → GET deny → 403" flow below is the planned +> revocation design and is **not** in the current implementation. As shipped, +> admin Stop/Terminate does the "realtime WS stop + producer teardown" half only +> (the right branch); it writes no marker, and the node performs no deny check, so +> a valid token reconstructs until its ≤24h TTL. New playback is still gated at +> `/playback/start` (the top subgraph is accurate). + +```mermaid +flowchart TD + subgraph New["New playback (any topology)"] + P["/playback/start — RequireAuth + access check"] --> PB{allowed?} + PB -- no --> PR[refused — ban takes effect at next play] + PB -- yes --> PM[mint token] + end + subgraph Admin["Admin Stop/Terminate (event, not a timer)"] + K[denyStreamLease] --> WD[SET silo:streamauth:sid = deny
TTL = 24h] + K --> WS[realtime WS stop + producer teardown] + end + subgraph Node["Offload node — every serve"] + G[verifyToken OK] --> L{GET silo:streamauth:sid} + L -- present deny --> F[403, no bytes] + L -- absent / redis-err --> SV[serve bytes fail-open] + end +``` + +Invariants: +- No steady-state writes: the marker is set only on an admin kill, never on a + timer. The normal serve path finds *no* key and serves. +- Fail-open: absent key or Redis error → serve. The only hard stop is a present + deny. +- The integrated box reads no marker — the producer-kill + session removal on + Terminate is the stop there; the marker only matters where nodes serve. +- Not covered here (by design): a *passive* ban / partial access change of an + in-flight non-cooperative node stream — enforced at next play, bounded by the + ≤24h token (§7). + +### 12.7 Jellycompat — what differs + +```mermaid +sequenceDiagram + participant J as Jellyfin client + participant S as Central (jellycompat) + participant PG as Postgres compat store + J->>S: negotiate → PlaybackSession{UpstreamSessionID, MediaSources} + S->>PG: Put (jellycompat_playback_sessions) + Note over S: transcode start + S->>S: StartTranscode; recipe = NewRecipeCard(opts) + S->>PG: Update(ps): TranscodeStarted=true, Recipe=recipe (same write) + Note over J,S: restart → reconstruct from the COMPAT STORE, not a token + J->>S: GET HLS segment + S->>PG: Get(playSessionID) → ps.Recipe + S->>S: LoadOrReconstructSession(UpstreamSessionID, card = ps.Recipe) → ReconstructTranscode(*ps.Recipe) + S-->>J: bytes +``` + +Jellyfin clients cannot carry a native `?st` token, so the recipe is folded into +the durable compat row (`PlaybackSession.Recipe`, persisted in the `data` JSONB) +and written in the same `Update` that flips `TranscodeStarted`. The node hop +(multi-node jellycompat) still uses a path token exactly like native (the lease +guard is deferred along with the rest of the revocation path). + +### 12.8 Hot-path cost per request + +The "lease" / "SET deny" Redis costs below are **deferred** (the revocation path +is not shipped); as shipped the node does only the tracker write, and admin +Stop/Terminate writes no Redis key: + +| request | Postgres | Redis | central CPU | reconstruct | +|---------|----------|-------|-------------|-------------| +| native segment, live, integrated | — | — | map hit | — | +| native segment, live, node | — | 1 track (+ deferred: GET lease) | — | — | +| native segment, after restart, integrated | — | — | verify JWT + ffmpeg respawn (once, single-flight) | yes | +| jellycompat segment, live | — | track (+ deferred: lease) | map hit | — | +| jellycompat segment, after restart | 1 Get (compat row) | track (+ deferred: lease) | respawn | yes | +| admin Stop/Terminate (event, not per-request) | — | deferred: 1 SET deny | one write | — | + +--- + +## 13. Restart-resiliency behavior matrix (role × path) + +The observable reconstruction behavior of every playback path under a restart of +each server role. **✅** = the path reconstructs and resumes on the client's next +request; **—** = the path never executes on that role (nothing to reconstruct); +**⚠️** = reconstructs only while a stated precondition holds. + +| path | integrated restart | proxy-node restart | transcode-node restart | +|------|--------------------|--------------------|------------------------| +| native · direct | ✅ rebuild `Session` from `?st` token → `Range` re-serve | ✅ stateless re-serve from `claims.MediaPath` | — not served on a transcode node | +| native · remux | ✅ token + `?seek` → re-spawn pipe | ✅ stateless per-request pipe re-spawn | — | +| native · transcode | ✅ token (full recipe) → respawn ffmpeg seeked to segment | ✅ transparent re-route + token forward | ✅ self-contained — token is recipe-complete | +| jellycompat · direct | ✅ re-resolve from durable compat row → `Range` re-serve | ✅ stateless re-serve from token `MediaPath` | — | +| jellycompat · remux | ✅ compat row + `?seek` | ✅ stateless per-request pipe re-spawn | — | +| jellycompat · transcode | ✅ `ReconstructTranscode(*ps.Recipe)` from compat row | ✅ transparent re-route + token forward | ⚠️ reconstructs **iff** the shared Redis recipe (`internal/noderecipe`, ≤24h TTL) still exists; fails closed (404) otherwise | + +**What carries the recovery state, per role** + +| role | native carrier | jellycompat carrier | +|------|----------------|---------------------| +| integrated | `?st` stream token — identity for direct/remux, full recipe for transcode | durable compat row `PlaybackSession.Recipe` | +| proxy node | self-describing path token, re-derived per request — the proxy holds no per-session state | same path token (identity); the recipe is not needed at the proxy | +| transcode node | forwarded `X-Silo-Stream-Token`, recipe-complete | identity token + Redis `noderecipe` recipe (the Jellyfin token cannot carry it) | + +**Reading the matrix** + +- **Integrated and proxy-node restarts are unconditional** for every applicable + path. The proxy holds no authoritative per-session state — direct/remux are + re-served from the token's `MediaPath`, transcode is reverse-proxied — so a + proxy bounce, or a reroute to any in-group proxy (`pickProxy` is round-robin + with no session affinity), is transparent. +- **The four `—` cells are structural, not gaps.** Direct and remux never run on + a transcode node — the proxy (or the integrated box) serves them from source + media — so a transcode-node restart has nothing to reconstruct on those paths. +- **The one conditional (⚠️)** is jellycompat transcode after a *transcode-node* + restart. The node-hop token is identity-only — not because a Jellyfin client + cannot round-trip it (the token is server-minted and opaque to the client), but + because the recipe is mutated in place under a stable session id and a + third-party Jellyfin client cannot be driven to refresh a stale token, so a + token snapshot could reconstruct a stale rendition (see §10). The node therefore + rebuilds from the Redis `noderecipe` entry central overwrites on every switch + rather than from the token. Native transcode has no such dependency — every + native audio switch re-mints the token, so it is always recipe-complete and + current. This is the only native-vs-jellycompat asymmetry in reconstruction + (see §10). + +**Out of scope of this matrix** (accepted behaviors, not restart-reconstruction +holes — see §7, §10): + +- Passive ban / partial access change of an in-flight node stream — enforced at + next play, bounded by the ≤24h token, not by reconstruction. +- Integrated transcode behind a non-sticky load balancer can split-brain into + divergent segment dirs (P-1); safe single-front-end or with LB affinity. Remote + (node) sessions are immune — node affinity routes every front-end to the same + transcode node. diff --git a/internal/api/handlers/admin_playback_control.go b/internal/api/handlers/admin_playback_control.go index 69ba6f06..6ffb5e6e 100644 --- a/internal/api/handlers/admin_playback_control.go +++ b/internal/api/handlers/admin_playback_control.go @@ -179,7 +179,7 @@ func (h *AdminPlaybackControlHandler) handleSessionCommand(w http.ResponseWriter fallback := func() { h.playback.forgetRealtimeCommand(commandID) - _ = h.playback.stopPlaybackSessionByID(context.Background(), sessionID) + _ = h.playback.stopPlaybackSessionByID(context.Background(), sessionID, true) } h.playback.rememberRealtimeCommand(commandID, sessionID, name) diff --git a/internal/api/handlers/playback.go b/internal/api/handlers/playback.go index a54a281b..68a29b6d 100644 --- a/internal/api/handlers/playback.go +++ b/internal/api/handlers/playback.go @@ -137,10 +137,12 @@ type PlaybackHandler struct { // playbackConfig(), which falls back to defaults when unset. PlaybackConfig func() config.PlaybackConfig FFmpegLogSink playback.FFmpegLogSink - transcodeMu sync.RWMutex - transcodes map[string]*playback.TranscodeSession realtimeCommandMu sync.Mutex realtimeCommands map[string]playbackCommandRecord + // tm owns the transcode-session lifecycle (live map, recipe cards, and + // restart reconstruct) shared with the jellycompat handler. The handler + // delegates all transcode-session and recipe operations to it. + tm *playback.TranscodeManager } type PlaybackWatchScrobbler interface { @@ -159,18 +161,81 @@ type sessionExpirationHookSetter interface { func NewPlaybackHandler(sessionMgr SessionManagerInterface, opts ...FilePathResolver) *PlaybackHandler { h := &PlaybackHandler{ sessionMgr: sessionMgr, - transcodes: make(map[string]*playback.TranscodeSession), realtimeCommands: make(map[string]playbackCommandRecord), + tm: playback.NewTranscodeManager(), } if len(opts) > 0 { h.fileResolver = opts[0] } + // Wire the shared transcode manager with closures so it reads the handler's + // (often late-set) config/store/secret fields lazily at call time, avoiding a + // field-ordering hazard during router setup. + h.tm.JWTSecretFn = func() string { return h.JWTSecret } + h.tm.LogSinkFn = func() playback.FFmpegLogSink { return h.FFmpegLogSink } + h.tm.Config = func() playback.TranscodeRuntimeConfig { + c := h.playbackConfig() + return playback.TranscodeRuntimeConfig{ + TranscodeDir: c.TranscodeDir, + FFmpegPath: c.FFmpegPath, + HWAccel: c.HWAccel, + HWDevice: c.HWDevice, + } + } + h.tm.StartThrottler = func(ctx context.Context, ts *playback.TranscodeSession) { + h.maybeStartThrottler(ctx, ts) + } + h.tm.OnFFmpegCrash = func(ctx context.Context, sessionID string, dead *playback.TranscodeSession) { + // ffmpeg crash — tear the session down; a client holding a valid stream + // token can reconstruct it on the next request. + // + // Compare-and-delete the dead transcode first: between ffmpeg's error exit + // and this teardown a reconstruct may have registered a fresh successor + // under the same id. CloseTranscodeSessionIf only removes (and Close()s, which + // reaps the shared output dir) the entry when it is still the dead session; + // if a successor won, it leaves the live one untouched and we must NOT tear + // down the reconstructed playback session that now backs it. + var nodeURL string + if s, err := h.sessionMgr.GetSession(sessionID); err == nil { + nodeURL = s.TranscodeNodeURL + } + if successor := h.tm.GetTranscodeSession(sessionID); successor != nil && successor != dead { + // A reconstruct already replaced the crashed process; the live successor + // and its session stand. Cheap fast-path only — the authoritative gate is + // the compare-and-delete result below. + return + } + // CloseTranscodeSessionIf is the authoritative gate: a successor may register + // under the same id between the pre-check above and here. We only tear down the + // upstream playback session when the compare-and-delete actually matched the + // dead transcode. When it returns false a successor owns the session — do + // nothing further, or finalizeSessionStop's unconditional CloseTranscodeSession + // would reap the live successor's output dir mid-serve. + if !h.tm.CloseTranscodeSessionIf(sessionID, dead, nodeURL) { + return + } + if err := h.stopPlaybackSessionByID(ctx, sessionID, false); err != nil && !errors.Is(err, playback.ErrSessionNotFound) { + slog.Error("failed to stop playback after local transcode exit", "session", sessionID, "error", err, "playback_session_id", sessionID) + } + } + if reg, ok := sessionMgr.(interface { + RegisterReconstructed(s *playback.Session) *playback.Session + RegisterReconstructedWithLimits(ctx context.Context, s *playback.Session) (*playback.Session, error) + }); ok { + h.tm.Sessions = reg + } if setter, ok := sessionMgr.(sessionExpirationHookSetter); ok { setter.SetExpirationHook(h.handleExpiredSession) } return h } +// TranscodeManager returns the shared transcode/reconstruct manager so sibling +// handlers (e.g. StreamHandler) can reuse the same recipe-card store, live +// transcode map, and reconstruct front door rather than wiring a second one. +func (h *PlaybackHandler) TranscodeManager() *playback.TranscodeManager { + return h.tm +} + // SetProfileStaler configures an optional staleness trigger for taste profiles. func (h *PlaybackHandler) SetProfileStaler(ps ProfileStaler) { h.profileStaler = ps @@ -195,16 +260,10 @@ func (h *PlaybackHandler) playbackConfig() config.PlaybackConfig { } // CleanupOrphanedTranscodes removes stale per-session temp directories for -// transcodes that are no longer tracked in memory. +// transcodes that are no longer tracked in memory, sparing dirs whose recipe +// card still exists. Delegates to the shared transcode manager. func (h *PlaybackHandler) CleanupOrphanedTranscodes() (int, error) { - h.transcodeMu.RLock() - active := make(map[string]struct{}, len(h.transcodes)) - for sessionID := range h.transcodes { - active[sessionID] = struct{}{} - } - h.transcodeMu.RUnlock() - - return playback.CleanupOrphanedTranscodeDirs(h.playbackConfig().TranscodeDir, active) + return h.tm.CleanupOrphanedTranscodes() } // playbackThresholds reads the playback.watched_threshold and @@ -354,7 +413,7 @@ type transcodeStartResponse struct { } // toPlaybackSessionResponse converts a playback.Session to an API response. -func toPlaybackSessionResponse(s *playback.Session) playbackSessionResponse { +func (h *PlaybackHandler) toPlaybackSessionResponse(s *playback.Session) playbackSessionResponse { return playbackSessionResponse{ SessionID: s.ID, UserID: s.UserID, @@ -363,7 +422,7 @@ func toPlaybackSessionResponse(s *playback.Session) playbackSessionResponse { PlayMethod: string(semanticPlayMethod(s)), Position: s.Position, IsPaused: s.IsPaused, - StreamURL: playbackStreamURL(s), + StreamURL: h.playbackStreamURL(s), AudioTrackIndex: s.AudioTrackIndex, } } @@ -438,14 +497,126 @@ func (h *PlaybackHandler) ensurePlaybackProbe(ctx context.Context, file *models. return file } -func playbackStreamURL(s *playback.Session) string { +// streamTokenParam is the query parameter that carries the signed stream token +// on the native integrated serve path. The token is the durable reconstruction +// descriptor: a front-end that lost its in-memory session rebuilds from it. It +// rides a query parameter (not a path segment) because the integrated server is +// hit directly by the client — there is no query-stripping proxy hop in between, +// and the transcode manifest rewriter already appends the request RawQuery to +// every segment URI, so segment requests inherit the token for free. The +// proxy/node path keeps the token in the URL path (see the proxy server). +const streamTokenParam = "st" + +// signSessionToken mints a stream token carrying the session's full +// reconstruction recipe. Returns "" when no signing secret is configured +// (reconstruct effectively disabled, e.g. in tests). +func (h *PlaybackHandler) signSessionToken(card playback.RecipeCard) string { + if h.JWTSecret == "" { + return "" + } + token, err := streamtoken.Sign(card.ToClaims(), h.JWTSecret, playback.MaxTokenTTL) + if err != nil { + slog.Warn("sign stream token failed", "error", err, "session", card.SessionID, "playback_session_id", card.SessionID) + return "" + } + return token +} + +// streamCardFromQuery verifies the stream token in the request's ?st= parameter +// and returns the decoded reconstruction recipe, or nil when the token is +// absent, invalid/expired, or bound to a different session. A live session needs +// no token (the result is simply nil); the recipe is consumed only on +// reconstruct. +func (h *PlaybackHandler) streamCardFromQuery(r *http.Request, sessionID string) *playback.RecipeCard { + return streamCardFromToken(r.URL.Query().Get(streamTokenParam), sessionID, h.JWTSecret) +} + +// loadTranscodeServeSession resolves the playback Session for the transcode +// manifest/segment serve routes while keeping stream-token verification off the +// hot path. The overwhelmingly common case is a live in-memory session, which +// needs no token at all, so the cheap GetSession lookup runs first and the +// (HMAC + JSON) token decode is performed only on a not-found miss where a +// reconstruct is actually required. On that miss it delegates to the shared +// LoadOrReconstructSession front door so reconstruct/ownership semantics stay +// identical. The returned card (nil on the live-session path) is the decoded +// recipe the caller's own reconstruct branch consumes. +func (h *PlaybackHandler) loadTranscodeServeSession(r *http.Request, sessionID string) (*playback.Session, playback.SessionLoadStatus, *playback.RecipeCard) { + requestUserID := apimw.GetUserID(r.Context()) + session, err := h.sessionMgr.GetSession(sessionID) + if err == nil { + // Live session: enforce the same ownership rule as LoadOrReconstructSession + // (a zero caller is allowed; a non-zero mismatch is refused). No token + // verification on this hot path. + if requestUserID != 0 && session.UserID != requestUserID { + return nil, playback.SessionForbidden, nil + } + return session, playback.SessionLoaded, nil + } + if !errors.Is(err, playback.ErrSessionNotFound) { + return nil, playback.SessionLoadFailed, nil + } + // Genuine miss (e.g. after a restart): now — and only now — pay for the token + // decode so the recipe is available for reconstruction. + card := h.streamCardFromQuery(r, sessionID) + session, status := h.tm.LoadOrReconstructSession(r.Context(), h.sessionMgr.GetSession, sessionID, requestUserID, card) + return session, status, card +} + +// streamCardFromToken verifies a stream token and decodes its reconstruction +// recipe, returning nil when the token is absent, unparseable/expired, or bound +// to a different session id. Shared by the native serve handlers (PlaybackHandler +// and StreamHandler). +func streamCardFromToken(tokenStr, sessionID, secret string) *playback.RecipeCard { + if tokenStr == "" || secret == "" { + return nil + } + claims, err := streamtoken.Verify(tokenStr, secret) + if err != nil || claims.SessionID != sessionID { + return nil + } + card := playback.RecipeCardFromClaims(claims) + return &card +} + +// appendStreamToken adds the ?st= parameter to a native serve URL. +func appendStreamToken(rawURL, token string) string { + if token == "" { + return rawURL + } + sep := "?" + if strings.ContainsRune(rawURL, '?') { + sep = "&" + } + return rawURL + sep + streamTokenParam + "=" + token +} + +// playbackStreamURL builds the native serve URL for a session and appends an +// identity stream token so a direct-play/remux session survives a restart (the +// client re-supplies its byte position). Transcode sessions receive their +// full-recipe manifest URL from HandleStartTranscode instead; the URL here is an +// informational placeholder the client replaces with that manifest. +func (h *PlaybackHandler) playbackStreamURL(s *playback.Session) string { if s == nil { return "" } if s.PlayMethod == playback.PlayTranscode { return fmt.Sprintf("/playback/transcode/%s/master.m3u8", s.ID) } - return fmt.Sprintf("/stream/%s", s.ID) + card := identityRecipeCard(s) + return appendStreamToken(fmt.Sprintf("/stream/%s", s.ID), h.signSessionToken(card)) +} + +// identityRecipeCard builds the identity-only recipe for a direct-play or remux +// session: reconstruction needs only ownership plus the audio selection, since +// the bytes are served by HTTP Range / a re-spawned remux pipe at the +// client-supplied position. +func identityRecipeCard(s *playback.Session) playback.RecipeCard { + switch s.PlayMethod { + case playback.PlayRemux: + return playback.NewRemuxRecipeCard(s.ID, s.UserID, s.ProfileID, s.MediaFileID, s.TranscodeAudio, s.AudioTrackIndex) + default: + return playback.NewDirectRecipeCard(s.ID, s.UserID, s.ProfileID, s.MediaFileID) + } } func fileBitrateKbps(file *models.MediaFile) int { @@ -962,7 +1133,7 @@ func (h *PlaybackHandler) touchSessionActivity(sessionID string) { } } -func (h *PlaybackHandler) finalizeSessionStop(ctx context.Context, session *playback.Session, syncNow bool, syncReason string) { +func (h *PlaybackHandler) finalizeSessionStop(ctx context.Context, session *playback.Session, syncNow bool, syncReason string, userInitiated bool) { if h == nil || session == nil || session.ID == "" { return } @@ -999,7 +1170,7 @@ func (h *PlaybackHandler) finalizeSessionStop(ctx context.Context, session *play } } - h.closeTranscodeSession(session.ID, session.TranscodeNodeURL) + h.tm.CloseTranscodeSession(session.ID, session.TranscodeNodeURL) if syncNow { h.syncSessionsNow(ctx, syncReason) } @@ -1031,7 +1202,9 @@ func (h *PlaybackHandler) finalizeSessionAbort(ctx context.Context, session *pla } } - h.closeTranscodeSession(session.ID, session.TranscodeNodeURL) + // Abort is a connection drop / non-terminal teardown — keep the recipe card + // so the client can reconstruct on reconnect. + h.tm.CloseTranscodeSession(session.ID, session.TranscodeNodeURL) if syncNow { h.syncSessionsNow(ctx, syncReason) } @@ -1044,7 +1217,10 @@ func (h *PlaybackHandler) handleExpiredSession(session *playback.Session) { sessionCopy := *session go func() { slog.Info("expired inactive playback session", "session", sessionCopy.ID, "playback_session_id", sessionCopy.ID) - h.finalizeSessionStop(context.Background(), &sessionCopy, false, "") + // Expiry is a liveness reap, not a user stop — keep the recipe card so a + // resume reconstructs under the same id (the card's own TTL reaps it if + // the session is truly abandoned). + h.finalizeSessionStop(context.Background(), &sessionCopy, false, "", false) }() } @@ -1379,7 +1555,11 @@ func (h *PlaybackHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Req } h.maybeQueueLazyPlaybackMarkers(r.Context(), session, effectiveFile) - resp := toPlaybackSessionResponse(session) + // Direct-play and remux sessions reconstruct from the identity stream token + // carried on their serve URL (see playbackStreamURL); there is no server-side + // card to persist. Transcode sessions receive their full-recipe token from + // HandleStartTranscode. + resp := h.toPlaybackSessionResponse(session) resp.DurationSeconds = fileDurationSeconds(effectiveFile) resp.PlaybackInfo = buildPlaybackInfo(session, effectiveFile) @@ -1398,8 +1578,11 @@ func (h *PlaybackHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Req proxyNode := plan.ProxyNode if proxyNode != nil && (!needsTranscode || plan.TranscodeNode != nil) { tokenClaims := streamtoken.Claims{ - SessionID: session.ID, - PlayMethod: string(session.PlayMethod), + SessionID: session.ID, + PlayMethod: string(session.PlayMethod), + UserID: session.UserID, + ProfileID: session.ProfileID, + MediaFileID: session.MediaFileID, } // Resolve media path if possible. @@ -1415,7 +1598,7 @@ func (h *PlaybackHandler) HandleStartPlayback(w http.ResponseWriter, r *http.Req _ = h.sessionMgr.SetTranscodeNodeURL(session.ID, plan.TranscodeNode.URL) } - token, signErr := streamtoken.Sign(tokenClaims, h.JWTSecret, 24*time.Hour) + token, signErr := streamtoken.Sign(tokenClaims, h.JWTSecret, playback.MaxTokenTTL) if signErr == nil { switch session.PlayMethod { case playback.PlayDirect: @@ -1647,7 +1830,7 @@ func (h *PlaybackHandler) HandleStopPlayback(w http.ResponseWriter, r *http.Requ return } - err = h.stopPlaybackSession(r.Context(), session) + err = h.stopPlaybackSession(r.Context(), session, true) if err != nil { if errors.Is(err, playback.ErrSessionNotFound) { writePlaybackSessionNotFound(w) @@ -1767,6 +1950,12 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. TargetVideoCodec: targetVideoCodec, TargetAudioCodec: targetAudioCodec, TargetBitrateKbps: targetBitrateKbps, + // Carry the byte-affecting recipe forward: an audio switch changes only the + // audio selection, so subtitles and cadence must survive the state update + // (UpdateStreamState overwrites these fields unconditionally). + SubtitleTrackIndex: session.SubtitleTrackIndex, + SubtitleBurnIn: session.SubtitleBurnIn, + SegmentDuration: session.SegmentDuration, }); err != nil { writeError(w, http.StatusInternalServerError, "internal_error", "Failed to update stream state") return @@ -1774,12 +1963,12 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. // Handle transcode restart. if session.PlayMethod == playback.PlayTranscode { - if ts := h.getTranscodeSession(sessionID); ts != nil { + if ts := h.tm.GetTranscodeSession(sessionID); ts != nil { ts.SetAudioTrackIndex(req.AudioTrackIndex) seekSeconds := req.Position startSegment := computeStartSegment(seekSeconds, ts.Opts().SegmentDuration) // Throttler + exit monitor re-arm via the session's restart hook. - if restartErr := ts.Restart(context.WithoutCancel(r.Context()), seekSeconds, startSegment); restartErr != nil { + if restartErr := h.tm.RestartSessionLocked(context.WithoutCancel(r.Context()), sessionID, ts, seekSeconds, startSegment); restartErr != nil { slog.Error("failed to restart transcode for audio switch", "session", sessionID, "error", restartErr) } } @@ -1796,12 +1985,34 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. updatedSession.TargetVideoCodec = targetVideoCodec updatedSession.TargetAudioCodec = targetAudioCodec updatedSession.TargetBitrateKbps = targetBitrateKbps + + // The switched recipe travels in the freshly minted stream token on the new + // serve URL below, so a post-restart reconstruct resumes with the switched + // audio/method. For transcode the full-recipe manifest URL is rebuilt further + // down (proxy or local); for direct/remux the identity token on StreamURL + // carries the new audio selection. h.persistAudioPreference(r.Context(), userID, session.ProfileID, file, req.AudioTrackIndex) + // For a local transcode, playbackStreamURL returns the bare manifest URL + // without the full-recipe ?st= token, so a post-restart reconstruct would + // fall back to the stale pre-switch token. Rebuild the signed manifest URL + // from the live transcode opts, mirroring HandleStartTranscode. The proxy + // branch below overrides this when a node plan picks a proxy/transcode node. + streamURL := h.playbackStreamURL(&updatedSession) + if updatedSession.PlayMethod == playback.PlayTranscode { + if ts := h.tm.GetTranscodeSession(sessionID); ts != nil { + card := playback.NewRecipeCard(updatedSession.UserID, updatedSession.ProfileID, updatedSession.MediaFileID, updatedSession.TranscodeNodeURL, ts.Opts()) + streamURL = appendStreamToken( + fmt.Sprintf("/playback/transcode/%s/master.m3u8", sessionID), + h.signSessionToken(card), + ) + } + } + resp := changeAudioResponse{ AudioTrackIndex: req.AudioTrackIndex, PlayMethod: string(newMethod), - StreamURL: playbackStreamURL(&updatedSession), + StreamURL: streamURL, SwitchMode: "reload", PlaybackInfo: buildPlaybackInfo(&updatedSession, file), } @@ -1814,23 +2025,144 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http. } plan := h.NodePlanner.PlanSession(sessionID, session.TranscodeNodeURL, needsTranscode, estKbps) if proxyNode := plan.ProxyNode; proxyNode != nil && (!needsTranscode || plan.TranscodeNode != nil) { - tokenClaims := streamtoken.Claims{ - SessionID: sessionID, - PlayMethod: string(updatedSession.PlayMethod), - MediaPath: file.FilePath, - TranscodeAudio: updatedSession.TranscodeAudio, - AudioTrackIndex: req.AudioTrackIndex, - } - if plan.TranscodeNode != nil { - tokenClaims.TranscodeNode = plan.TranscodeNode.URL - _ = h.sessionMgr.SetTranscodeNodeURL(sessionID, plan.TranscodeNode.URL) - } - if token, signErr := streamtoken.Sign(tokenClaims, h.JWTSecret, 24*time.Hour); signErr == nil { - switch updatedSession.PlayMethod { - case playback.PlayRemux: - resp.StreamURL = proxyNode.URL + "/stream/remux/" + token - case playback.PlayTranscode: - resp.StreamURL = proxyNode.URL + "/stream/transcode/" + token + "/master.m3u8" + // Remote (offloaded) transcode: the API server owns no local + // TranscodeSession (the LOCAL restart block above was a no-op), so + // the node's ffmpeg is still serving the OLD audio track. POST a + // fresh /transcode/start with the new AudioTrackIndex — the node + // tears down the existing session for this ID and restarts ffmpeg + // (handleStart in internal/transcodenode/server.go), which IS the + // remote restart mechanism — then mint the replacement proxy URL + // from a FULL recipe card so a later node restart reconstructs with + // the switched audio (the lean identity-only claims used for remux + // below omit the byte-affecting encode fields and would 404). + isOffloaded := strings.TrimSpace(session.TranscodeNodeURL) != "" + if needsTranscode && plan.TranscodeNode != nil && isOffloaded { + nodeURL := plan.TranscodeNode.URL + _ = h.sessionMgr.SetTranscodeNodeURL(sessionID, nodeURL) + + seekSeconds := req.Position + // Restart from the FULL live recipe, not a partial re-derivation. + // An audio switch alters only audio selection — subtitle burn-in and + // the segment cadence must be preserved, or the node re-encodes a + // different byte stream (subtitles silently dropped, wrong cadence) + // and signs that altered recipe into the new token. The session + // retains these from the original start (finalizeTranscodeStart) or a + // post-restart reconstruct, so recover them here. Embed a concrete + // segment duration (not 0): the node's recipe token treats + // SegmentDuration<=0 as "incomplete" and would 404 on a node restart. + segmentDuration := session.SegmentDuration + if segmentDuration <= 0 { + segmentDuration = playback.DefaultSegmentDuration + } + subtitleTrackIndex := session.SubtitleTrackIndex + subtitleBurnIn := session.SubtitleBurnIn + startSegment := computeStartSegment(seekSeconds, segmentDuration) + + // Derive the encode recipe the same way HandleStartTranscode + // does — from the durable session target fields plus the file — + // changing only the audio track. SourceVideoCodec/TotalDuration + // come from the file; the resolution/codec/bitrate targets and + // hwaccel come from the session's persisted stream state. + nodeReq := transcodenode.TranscodeStartRequest{ + SessionID: sessionID, + InputPath: file.FilePath, + SourceVideoCodec: file.CodecVideo, + SeekSeconds: seekSeconds, + StartSegmentNumber: startSegment, + TargetResolution: updatedSession.TargetResolution, + TargetCodecVideo: updatedSession.TargetVideoCodec, + TargetCodecAudio: updatedSession.TargetAudioCodec, + TargetBitrateKbps: updatedSession.TargetBitrateKbps, + SegmentDuration: segmentDuration, + HWAccel: session.TranscodeHWAccel, + AudioTrackIndex: req.AudioTrackIndex, + SubtitleTrackIndex: subtitleTrackIndex, + SubtitleBurnIn: subtitleBurnIn, + TotalDuration: float64(file.Duration), + } + if strings.TrimSpace(nodeReq.HWAccel) == "" { + nodeReq.HWAccel = h.playbackConfig().HWAccel + } + + body, _ := json.Marshal(nodeReq) + ctx, cancel := context.WithTimeout(context.WithoutCancel(r.Context()), 10*time.Second) + defer cancel() + httpReq, reqErr := http.NewRequestWithContext(ctx, http.MethodPost, nodeURL+"/transcode/start", bytes.NewReader(body)) + if reqErr != nil { + slog.Error("failed to build remote transcode restart for audio switch", "session", sessionID, "node", nodeURL, "error", reqErr) + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to build transcode request") + return + } + httpReq.Header.Set("Content-Type", "application/json") + httpReq.Header.Set("Authorization", "Bearer "+h.JWTSecret) + + nodeResp, doErr := http.DefaultClient.Do(httpReq) + if doErr != nil { + slog.Error("remote transcode restart for audio switch failed", "session", sessionID, "node", nodeURL, "error", doErr) + writeError(w, http.StatusBadGateway, "transcode_node_unavailable", "Transcode node is unavailable") + return + } + defer nodeResp.Body.Close() + if nodeResp.StatusCode != http.StatusAccepted { + slog.Error("remote transcode restart for audio switch rejected", "session", sessionID, "node", nodeURL, "status", nodeResp.StatusCode) + writeError(w, http.StatusBadGateway, "transcode_start_failed", "Transcode node rejected the request") + return + } + + var startResp transcodenode.TranscodeStartResponse + if decErr := json.NewDecoder(nodeResp.Body).Decode(&startResp); decErr != nil { + slog.Warn("remote transcode restart response decode failed", "session", sessionID, "node", nodeURL, "error", decErr) + } + effectiveHWAccel := strings.TrimSpace(startResp.HWAccel) + if effectiveHWAccel == "" { + effectiveHWAccel = strings.TrimSpace(nodeReq.HWAccel) + } + + card := playback.NewRecipeCard(updatedSession.UserID, updatedSession.ProfileID, updatedSession.MediaFileID, nodeURL, playback.TranscodeOpts{ + InputPath: nodeReq.InputPath, + SessionID: nodeReq.SessionID, + SourceVideoCodec: nodeReq.SourceVideoCodec, + SeekSeconds: nodeReq.SeekSeconds, + StartSegmentNumber: nodeReq.StartSegmentNumber, + TargetResolution: nodeReq.TargetResolution, + TargetCodecVideo: nodeReq.TargetCodecVideo, + TargetCodecAudio: nodeReq.TargetCodecAudio, + TargetBitrateKbps: nodeReq.TargetBitrateKbps, + SegmentDuration: nodeReq.SegmentDuration, + HWAccel: effectiveHWAccel, + AudioTrackIndex: nodeReq.AudioTrackIndex, + SubtitleTrackIndex: nodeReq.SubtitleTrackIndex, + SubtitleBurnIn: nodeReq.SubtitleBurnIn, + TotalDuration: nodeReq.TotalDuration, + }) + resp.StreamURL = h.buildProxyManifestURL(card, proxyNode) + } else { + // Remux, or a non-offloaded (locally served) transcode: no remote + // node ffmpeg to restart, so carry the new audio selection on the + // identity claims of the proxy serve URL, exactly as before. A + // local transcode reconstructs from the API server's own state, so + // the lean token is sufficient here. + tokenClaims := streamtoken.Claims{ + SessionID: sessionID, + PlayMethod: string(updatedSession.PlayMethod), + MediaPath: file.FilePath, + TranscodeAudio: updatedSession.TranscodeAudio, + AudioTrackIndex: req.AudioTrackIndex, + UserID: updatedSession.UserID, + ProfileID: updatedSession.ProfileID, + MediaFileID: updatedSession.MediaFileID, + } + if plan.TranscodeNode != nil { + tokenClaims.TranscodeNode = plan.TranscodeNode.URL + _ = h.sessionMgr.SetTranscodeNodeURL(sessionID, plan.TranscodeNode.URL) + } + if token, signErr := streamtoken.Sign(tokenClaims, h.JWTSecret, playback.MaxTokenTTL); signErr == nil { + switch updatedSession.PlayMethod { + case playback.PlayRemux: + resp.StreamURL = proxyNode.URL + "/stream/remux/" + token + case playback.PlayTranscode: + resp.StreamURL = proxyNode.URL + "/stream/transcode/" + token + "/master.m3u8" + } } } } @@ -1934,18 +2266,30 @@ func (h *PlaybackHandler) finalizeTranscodeStart(r *http.Request, st transcodeSt "transcode_audio", transcodeAudio, ) + // Persist the byte-affecting recipe (subtitles + segment cadence) so a later + // offloaded audio switch can rebuild the exact same stream. The session is the + // only recovery source for offloaded transcodes (no local ts.Opts()). Normalize + // the cadence to a concrete value so the restart never falls back to 0. + segmentDuration := st.req.SegmentDuration + if segmentDuration <= 0 { + segmentDuration = playback.DefaultSegmentDuration + } + if err := h.sessionMgr.UpdateStreamState(st.req.SessionID, playback.SessionStreamState{ - PlayMethod: playback.PlayTranscode, - BasePlayMethod: baseMethod, - AudioTrackIndex: st.session.AudioTrackIndex, - TranscodeAudio: transcodeAudio, - ClientIP: st.session.ClientIP, - StreamBitrateKbps: streamBitrateKbps, - TargetResolution: st.req.TargetResolution, - TargetVideoCodec: st.req.TargetCodecVideo, - TargetAudioCodec: st.req.TargetCodecAudio, - TargetBitrateKbps: st.req.TargetBitrateKbps, - TranscodeHWAccel: st.hwAccel, + PlayMethod: playback.PlayTranscode, + BasePlayMethod: baseMethod, + AudioTrackIndex: st.session.AudioTrackIndex, + TranscodeAudio: transcodeAudio, + ClientIP: st.session.ClientIP, + StreamBitrateKbps: streamBitrateKbps, + TargetResolution: st.req.TargetResolution, + TargetVideoCodec: st.req.TargetCodecVideo, + TargetAudioCodec: st.req.TargetCodecAudio, + TargetBitrateKbps: st.req.TargetBitrateKbps, + TranscodeHWAccel: st.hwAccel, + SubtitleTrackIndex: st.req.SubtitleTrackIndex, + SubtitleBurnIn: st.req.SubtitleBurnIn, + SegmentDuration: segmentDuration, }); err != nil { slog.Error("failed to update transcode stream state", "session", st.req.SessionID, "error", err, "playback_session_id", st.req.SessionID) } @@ -1989,8 +2333,10 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re // Check both local sessions AND remote node assignments — without the // remote check, switching quality on a transcode node never sends DELETE, // leaving the old ffmpeg running and its segments on disk. - if h.getTranscodeSession(req.SessionID) != nil || session.TranscodeNodeURL != "" { - h.closeTranscodeSession(req.SessionID, session.TranscodeNodeURL) + if h.tm.GetTranscodeSession(req.SessionID) != nil || session.TranscodeNodeURL != "" { + // Restarting the transcode under the SAME session id (quality/seek + // change) — keep the card; it is re-saved with the new opts below. + h.tm.CloseTranscodeSession(req.SessionID, session.TranscodeNodeURL) } abortCurrentSession := func(reason string, cause error) { if abortErr := h.abortPlaybackSession(r.Context(), session); abortErr != nil && !errors.Is(abortErr, playback.ErrSessionNotFound) { @@ -2149,7 +2495,28 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re effectiveHWAccel = strings.TrimSpace(nodeReq.HWAccel) } - manifestURL := h.buildProxyManifestURL(req.SessionID, session, tcNode.URL, plan.ProxyNode) + // The remote transcode's full recipe rides the proxy manifest token so the + // integrated server can re-bind and re-proxy the session after a restart + // (and a node could someday self-reconstruct from it). Node-side segment + // reconstruction is a follow-up (see spec multi-node section). + card := playback.NewRecipeCard(session.UserID, session.ProfileID, session.MediaFileID, tcNode.URL, playback.TranscodeOpts{ + InputPath: nodeReq.InputPath, + SessionID: nodeReq.SessionID, + SourceVideoCodec: nodeReq.SourceVideoCodec, + SeekSeconds: nodeReq.SeekSeconds, + StartSegmentNumber: nodeReq.StartSegmentNumber, + TargetResolution: nodeReq.TargetResolution, + TargetCodecVideo: nodeReq.TargetCodecVideo, + TargetCodecAudio: nodeReq.TargetCodecAudio, + TargetBitrateKbps: nodeReq.TargetBitrateKbps, + SegmentDuration: nodeReq.SegmentDuration, + HWAccel: effectiveHWAccel, + AudioTrackIndex: nodeReq.AudioTrackIndex, + SubtitleTrackIndex: nodeReq.SubtitleTrackIndex, + SubtitleBurnIn: nodeReq.SubtitleBurnIn, + TotalDuration: nodeReq.TotalDuration, + }) + manifestURL := h.buildProxyManifestURL(card, plan.ProxyNode) h.finalizeTranscodeStart(r, transcodeStartState{ req: req, file: file, @@ -2177,6 +2544,13 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re return } + // Hold the per-session lifecycle lock across teardown → spawn → register so a + // concurrent reconstruct (or another fresh start) cannot run a second ffmpeg + // writer against this session's output directory. The in-lock close tears down + // any session a reconstruct rebuilt between the earlier close and here so the + // fresh ffmpeg is the sole writer. + unlock := h.tm.LockSessionLifecycle(req.SessionID) + h.tm.CloseTranscodeSession(req.SessionID, "") transcodeSession, err := playback.StartTranscode(context.WithoutCancel(r.Context()), playback.TranscodeOpts{ InputPath: file.FilePath, OutputDir: filepath.Join(playbackCfg.TranscodeDir, req.SessionID), @@ -2202,28 +2576,36 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re FFmpegLogSink: h.FFmpegLogSink, }) if err != nil { + unlock() writeError(w, http.StatusInternalServerError, "internal_error", "Failed to start transcode session") return } - h.transcodeMu.Lock() - h.transcodes[req.SessionID] = transcodeSession - h.transcodeMu.Unlock() + h.tm.RegisterTranscodeSession(req.SessionID, transcodeSession) + unlock() // Re-arm the throttler and exit monitor after every Restart of this // handler-created session, regardless of which code path triggers it // (web segment recovery or an audio switch). Sessions created by - // jellycompat's own StartTranscode path live in a separate registry and - // never had throttler/exit-monitor wiring, so they are unaffected. + // jellycompat's own StartTranscode path never had throttler/exit-monitor + // wiring, so they are unaffected. transcodeSession.SetRestartHook(func(ctx context.Context) { h.maybeStartThrottler(ctx, transcodeSession) - h.monitorLocalTranscodeExit(req.SessionID, transcodeSession) + h.tm.MonitorLocalTranscodeExit(req.SessionID, transcodeSession) }) h.maybeStartThrottler(r.Context(), transcodeSession) - h.monitorLocalTranscodeExit(req.SessionID, transcodeSession) + h.tm.MonitorLocalTranscodeExit(req.SessionID, transcodeSession) - manifestURL := fmt.Sprintf("/playback/transcode/%s/master.m3u8", req.SessionID) + // The full reconstruction recipe rides the manifest token so this local + // transcode can be rebuilt after a server restart (the client re-presents the + // token on its next manifest/segment request). The token is carried as a + // query parameter; the manifest rewriter propagates it onto every segment URI. + card := playback.NewRecipeCard(session.UserID, session.ProfileID, session.MediaFileID, "", transcodeSession.Opts()) + manifestURL := appendStreamToken( + fmt.Sprintf("/playback/transcode/%s/master.m3u8", req.SessionID), + h.signSessionToken(card), + ) h.finalizeTranscodeStart(r, transcodeStartState{ req: req, file: file, @@ -2244,21 +2626,20 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re // still uses the same synthetic VOD path when the session duration is known. func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") - session, err := h.sessionMgr.GetSession(sessionID) - if err != nil { - if errors.Is(err, playback.ErrSessionNotFound) { - writePlaybackSessionNotFound(w) - return - } + session, status, card := h.loadTranscodeServeSession(r, sessionID) + switch status { + case playback.SessionMissing: + writePlaybackSessionNotFound(w) + return + case playback.SessionLoadFailed: writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load playback session") return - } - if userID := apimw.GetUserID(r.Context()); userID != 0 && session.UserID != userID { + case playback.SessionForbidden: writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return } - transcodeSession := h.getTranscodeSession(sessionID) + transcodeSession := h.tm.GetTranscodeSession(sessionID) if transcodeSession == nil { // No local session — try proxying to remote transcode node. if session.TranscodeNodeURL != "" { @@ -2267,8 +2648,18 @@ func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *h "/transcode/"+sessionID+"/master.m3u8") return } - writeError(w, http.StatusNotFound, "not_found", "Transcode session not found") - return + // Local transcode whose process state was lost: reconstruct it from the + // token recipe. The manifest path has no segment context, so pass -1 (use + // the token's seek position). + if card == nil { + writeError(w, http.StatusNotFound, "not_found", "Transcode session not found") + return + } + transcodeSession = h.tm.ReconstructTranscode(r.Context(), sessionID, -1, *card) + if transcodeSession == nil { + writeError(w, http.StatusNotFound, "not_found", "Transcode session not found") + return + } } h.touchSessionActivity(sessionID) @@ -2290,21 +2681,20 @@ func (h *PlaybackHandler) HandleGetTranscodeManifest(w http.ResponseWriter, r *h // Auth is optional — the session UUID serves as an access token. func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") - session, err := h.sessionMgr.GetSession(sessionID) - if err != nil { - if errors.Is(err, playback.ErrSessionNotFound) { - writePlaybackSessionNotFound(w) - return - } + session, status, card := h.loadTranscodeServeSession(r, sessionID) + switch status { + case playback.SessionMissing: + writePlaybackSessionNotFound(w) + return + case playback.SessionLoadFailed: writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load playback session") return - } - if userID := apimw.GetUserID(r.Context()); userID != 0 && session.UserID != userID { + case playback.SessionForbidden: writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return } - transcodeSession := h.getTranscodeSession(sessionID) + transcodeSession := h.tm.GetTranscodeSession(sessionID) if transcodeSession == nil { if session.TranscodeNodeURL != "" { h.touchSessionActivity(sessionID) @@ -2313,8 +2703,22 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht "/transcode/"+sessionID+"/segment/"+segmentName) return } - writeError(w, http.StatusNotFound, "not_found", "Transcode session not found") - return + // Resume near the segment the client is fetching so reconstruct does not + // restart from the original seek point and stall. A non-segment name + // (e.g. init.mp4) parses as negative and falls back to the token position. + requestedSegment := -1 + if segNum, parseErr := playback.ParseSegmentNumber(chi.URLParam(r, "name")); parseErr == nil { + requestedSegment = segNum + } + if card == nil { + writeError(w, http.StatusNotFound, "not_found", "Transcode session not found") + return + } + transcodeSession = h.tm.ReconstructTranscode(r.Context(), sessionID, requestedSegment, *card) + if transcodeSession == nil { + writeError(w, http.StatusNotFound, "not_found", "Transcode session not found") + return + } } h.touchSessionActivity(sessionID) @@ -2384,6 +2788,16 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht slog.Error("resolve transcode seek target", "error", seekErr, "segment", segmentName, "session", sessionID, "playback_session_id", sessionID) } + // Copy-mode with an unresolved seek target (ok=false, no error) + // means the manifest can't place this segment yet. Don't restart + // at a fabricated position; surface ErrSegmentNotFound so the + // client retries while the session keeps producing manifest. + // Mirrors the transcode-node guard in + // internal/transcodenode/server.go. + if !ok && seekErr == nil && transcodeSession.IsCopyVideo() { + err = playback.ErrSegmentNotFound + } + if ok { slog.Info("transcode seek restart", "segment", segmentName, @@ -2399,8 +2813,10 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht "session", sessionID, "playback_session_id", sessionID, ) - if restartErr := transcodeSession.Restart( + if restartErr := h.tm.RestartSessionLocked( context.WithoutCancel(r.Context()), + sessionID, + transcodeSession, seekSeconds, segNum, ); restartErr == nil { @@ -2444,107 +2860,19 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht http.ServeFile(w, r, segmentPath) } -func (h *PlaybackHandler) getTranscodeSession(sessionID string) *playback.TranscodeSession { - h.transcodeMu.RLock() - defer h.transcodeMu.RUnlock() - return h.transcodes[sessionID] -} - -func (h *PlaybackHandler) monitorLocalTranscodeExit(sessionID string, session *playback.TranscodeSession) { - if h == nil || sessionID == "" || session == nil { - return - } - - done := session.Done() - if done == nil { - return - } - - go func() { - <-done - time.Sleep(2 * time.Second) - - h.transcodeMu.RLock() - current := h.transcodes[sessionID] - h.transcodeMu.RUnlock() - if current != session { - return - } - if session.IsRunning() { - return - } - - // When ffmpeg exits cleanly (no error), the segments are fully - // written and should remain servable until the client stops the - // session. This is critical for copy-mode where ffmpeg finishes - // writing all content much faster than real-time playback. Only - // tear down the session on error exits. - if session.WaitError() == nil { - return - } - - if err := h.stopPlaybackSessionByID(context.Background(), sessionID); err != nil && !errors.Is(err, playback.ErrSessionNotFound) { - slog.Error("failed to stop playback after local transcode exit", "session", sessionID, "error", err, "playback_session_id", sessionID) - } - }() -} - -// closeTranscodeSession stops a transcode session. If transcodeNodeURL is non-empty, -// sends DELETE to the remote transcode node. Otherwise closes the local session. -func (h *PlaybackHandler) closeTranscodeSession(sessionID, transcodeNodeURL string) { - // Clean up local session if one exists (defensive). - h.transcodeMu.Lock() - session := h.transcodes[sessionID] - delete(h.transcodes, sessionID) - h.transcodeMu.Unlock() - if session != nil { - _ = session.Close() - } - - // Send DELETE to remote transcode node if assigned (synchronous with timeout). - if transcodeNodeURL != "" { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - deleteURL := transcodeNodeURL + "/transcode/" + sessionID - req, err := http.NewRequestWithContext(ctx, http.MethodDelete, deleteURL, nil) - if err != nil { - slog.Error("remote transcode delete: build request", "error", err, "session", sessionID, "playback_session_id", sessionID) - return - } - req.Header.Set("Authorization", "Bearer "+h.JWTSecret) - - resp, err := http.DefaultClient.Do(req) - if err != nil { - slog.Warn("remote transcode delete failed", "error", err, "session", sessionID, "node", transcodeNodeURL, "playback_session_id", sessionID) - return - } - resp.Body.Close() - } -} - -// buildProxyManifestURL signs a stream token and builds the proxy-based -// manifest URL. proxyNode is the planner's pick for this session; when nil -// the URL falls back to the API-local path. -func (h *PlaybackHandler) buildProxyManifestURL(sessionID string, session *playback.Session, transcodeNodeURL string, proxyNode *nodepool.Node) string { +// buildProxyManifestURL signs a stream token carrying the session's full +// reconstruction recipe and builds the manifest URL. proxyNode is the planner's +// pick; when nil the URL falls back to the API-local path, where the token rides +// the ?st= query parameter so the integrated server can reconstruct from it. +func (h *PlaybackHandler) buildProxyManifestURL(card playback.RecipeCard, proxyNode *nodepool.Node) string { + token := h.signSessionToken(card) + localURL := fmt.Sprintf("/playback/transcode/%s/master.m3u8", card.SessionID) if proxyNode == nil { - return fmt.Sprintf("/playback/transcode/%s/master.m3u8", sessionID) + return appendStreamToken(localURL, token) } - - tokenClaims := streamtoken.Claims{ - SessionID: sessionID, - PlayMethod: string(playback.PlayTranscode), - TranscodeNode: transcodeNodeURL, + if token == "" { + return localURL } - if session != nil { - tokenClaims.AudioTrackIndex = session.AudioTrackIndex - } - - token, err := streamtoken.Sign(tokenClaims, h.JWTSecret, 24*time.Hour) - if err != nil { - return fmt.Sprintf("/playback/transcode/%s/master.m3u8", sessionID) - } - return proxyNode.URL + "/stream/transcode/" + token + "/master.m3u8" } @@ -2552,8 +2880,17 @@ func (h *PlaybackHandler) buildProxyManifestURL(sessionID string, session *playb func (h *PlaybackHandler) proxyToTranscodeNode(w http.ResponseWriter, r *http.Request, transcodeNodeURL, path string) { sessionID := chi.URLParam(r, "session_id") targetURL := transcodeNodeURL + path - if rawQuery := r.URL.RawQuery; rawQuery != "" { - targetURL += "?" + rawQuery + // Capture the signed stream token ("st") before stripping it from the URL. + // We forward it out-of-band as a header so the node can reconstruct after a + // self-restart, while keeping it out of the forwarded/logged URL. + stToken := r.URL.Query().Get("st") + // Strip the signed stream token ("st") before forwarding/logging: it is a + // 24h bearer reconstruction descriptor exposing media path + recipe claims. + // Other query params are preserved. + query := r.URL.Query() + query.Del("st") + if encoded := query.Encode(); encoded != "" { + targetURL += "?" + encoded } req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, targetURL, nil) @@ -2562,6 +2899,22 @@ func (h *PlaybackHandler) proxyToTranscodeNode(w http.ResponseWriter, r *http.Re return } req.Header.Set("Authorization", "Bearer "+h.JWTSecret) + // Best-effort forward of the stream token as a header so the node's + // reconstruct path (X-Silo-Stream-Token) can rebuild after a self-restart. + // Verify at the API boundary and confirm it belongs to this session; an + // invalid or missing token never blocks the live proxy. validToken is kept so + // the same verified token can be re-injected into the node's manifest segment + // URIs below. + var validToken string + if stToken != "" && h.JWTSecret != "" { + claims, verifyErr := streamtoken.Verify(stToken, h.JWTSecret) + if verifyErr == nil && claims.SessionID == sessionID { + req.Header.Set("X-Silo-Stream-Token", stToken) + validToken = stToken + } else if verifyErr != nil { + slog.Warn("stream token not forwarded to transcode node", "error", verifyErr, "playback_session_id", sessionID) + } + } resp, err := http.DefaultClient.Do(req) if err != nil { @@ -2571,6 +2924,34 @@ func (h *PlaybackHandler) proxyToTranscodeNode(w http.ResponseWriter, r *http.Re } defer resp.Body.Close() + // The node strips "st" from the request query (kept out of node URLs/logs), + // so the segment/init URIs in the manifest it builds carry no token. Without + // it, a segment fetched after a node or API restart cannot reconstruct the + // session and 404s. Re-inject the client-facing token into every URI at this + // boundary so the client's later segment requests carry "st" again. Only the + // manifest body is rewritten; segments stream through untouched. + if validToken != "" && resp.StatusCode == http.StatusOK && strings.HasSuffix(path, ".m3u8") { + body, readErr := io.ReadAll(resp.Body) + if readErr != nil { + slog.Error("read transcode node manifest", "error", readErr, "url", targetURL, "playback_session_id", sessionID) + http.Error(w, "transcode node unavailable", http.StatusBadGateway) + return + } + rewritten := playback.AppendManifestQueryParam(body, streamTokenParam, validToken) + for k, vv := range resp.Header { + if http.CanonicalHeaderKey(k) == "Content-Length" { + continue + } + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.Header().Set("Content-Length", strconv.Itoa(len(rewritten))) + w.WriteHeader(resp.StatusCode) + _, _ = w.Write(rewritten) + return + } + for k, vv := range resp.Header { for _, v := range vv { w.Header().Add(k, v) diff --git a/internal/api/handlers/playback_realtime.go b/internal/api/handlers/playback_realtime.go index c4a990b5..80cf0c3c 100644 --- a/internal/api/handlers/playback_realtime.go +++ b/internal/api/handlers/playback_realtime.go @@ -13,7 +13,11 @@ type playbackCommandRecord struct { Name playback.CommandName } -func (h *PlaybackHandler) stopPlaybackSession(ctx context.Context, session *playback.Session) error { +// stopPlaybackSession stops a session. userInitiated must be true only for a +// genuine user/admin stop (the recipe card is then deleted); false for system +// teardown such as an ffmpeg-exit cleanup, where the card is kept so the client +// can reconstruct. +func (h *PlaybackHandler) stopPlaybackSession(ctx context.Context, session *playback.Session, userInitiated bool) error { if h == nil || session == nil || session.ID == "" { return playback.ErrSessionNotFound } @@ -21,11 +25,11 @@ func (h *PlaybackHandler) stopPlaybackSession(ctx context.Context, session *play if err := h.sessionMgr.StopSession(session.ID); err != nil { return err } - h.finalizeSessionStop(ctx, session, true, "stop") + h.finalizeSessionStop(ctx, session, true, "stop", userInitiated) return nil } -func (h *PlaybackHandler) stopPlaybackSessionByID(ctx context.Context, sessionID string) error { +func (h *PlaybackHandler) stopPlaybackSessionByID(ctx context.Context, sessionID string, userInitiated bool) error { if h == nil || sessionID == "" { return playback.ErrSessionNotFound } @@ -33,7 +37,7 @@ func (h *PlaybackHandler) stopPlaybackSessionByID(ctx context.Context, sessionID if err != nil { return err } - return h.stopPlaybackSession(ctx, session) + return h.stopPlaybackSession(ctx, session, userInitiated) } func (h *PlaybackHandler) abortPlaybackSession(ctx context.Context, session *playback.Session) error { diff --git a/internal/api/handlers/playback_test.go b/internal/api/handlers/playback_test.go index 0045d36c..69d9881d 100644 --- a/internal/api/handlers/playback_test.go +++ b/internal/api/handlers/playback_test.go @@ -22,6 +22,7 @@ import ( "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/nodepool" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtoken" "github.com/Silo-Server/silo-server/internal/transcodenode" "github.com/Silo-Server/silo-server/internal/userdb" "github.com/Silo-Server/silo-server/internal/userstore" @@ -1162,6 +1163,244 @@ func TestHandleStartTranscode_PreservesRecomputedBaseMethodAfterFallback(t *test } } +// newRemoteAudioSwitchSession builds a session that is already an offloaded +// (remote) transcode with a populated target recipe, mirroring the post-start +// state HandleChangeAudioTrack sees for an offloaded transcode. +func newRemoteAudioSwitchSession(t *testing.T, sessionMgr *playback.SessionManager, file *models.MediaFile, nodeURL string) string { + t.Helper() + session, err := sessionMgr.StartSession(1, "profile-1", file.ID, playback.PlayTranscode, true) + if err != nil { + t.Fatalf("StartSession: %v", err) + } + if err := sessionMgr.UpdateStreamState(session.ID, playback.SessionStreamState{ + PlayMethod: playback.PlayTranscode, + BasePlayMethod: playback.PlayTranscode, + AudioTrackIndex: 0, + TranscodeAudio: true, + TargetResolution: "720p", + TargetVideoCodec: "h264", + TargetAudioCodec: "aac", + TargetBitrateKbps: 2000, + TranscodeHWAccel: "none", + }); err != nil { + t.Fatalf("UpdateStreamState: %v", err) + } + if err := sessionMgr.SetTranscodeNodeURL(session.ID, nodeURL); err != nil { + t.Fatalf("SetTranscodeNodeURL: %v", err) + } + return session.ID +} + +// TestHandleChangeAudioTrack_RemoteTranscodeRestartsNodeAndMintsFullRecipe +// verifies BUG A + BUG B: an audio switch on an offloaded transcode POSTs a +// fresh /transcode/start to the node carrying the NEW AudioTrackIndex (so the +// node's ffmpeg actually switches tracks), and the returned proxy URL carries a +// recipe-COMPLETE stream token (not identity-only) so a later node restart can +// reconstruct with the switched audio. +func TestHandleChangeAudioTrack_RemoteTranscodeRestartsNodeAndMintsFullRecipe(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie.mkv"), + Resolution: "1080p", + CodecVideo: "h264", + CodecAudio: "aac", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{ + {Codec: "aac", Default: true}, + {Codec: "ac3"}, + }, + } + + var remoteStartRequests int + var remoteStartReq transcodenode.TranscodeStartRequest + remote := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost || r.URL.Path != "/transcode/start" { + t.Errorf("unexpected remote request %s %s", r.Method, r.URL.Path) + http.Error(w, "unexpected", http.StatusBadRequest) + return + } + remoteStartRequests++ + if err := json.NewDecoder(r.Body).Decode(&remoteStartReq); err != nil { + t.Errorf("decode remote start request: %v", err) + } + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(transcodenode.TranscodeStartResponse{ + SessionID: remoteStartReq.SessionID, + Status: "accepted", + HWAccel: "none", + }) + })) + defer remote.Close() + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.JWTSecret = "test-secret" + + group := "rack-a" + proxies := nodepool.NewProxyPool() + proxies.SetNodes([]*nodepool.Node{{ + ID: 1, Name: "proxy-1", Type: nodepool.NodeTypeProxy, URL: "http://proxy-1", + Enabled: true, Healthy: true, Group: &group, + }}) + transcodes := nodepool.NewTranscodePool() + transcodes.SetNodes([]*nodepool.Node{{ + ID: 2, Name: "transcode-1", Type: nodepool.NodeTypeTranscode, URL: remote.URL, + Enabled: true, Healthy: true, Group: &group, + }}) + handler.NodePlanner = nodepool.NewPlanner(proxies, transcodes) + + sessionID := newRemoteAudioSwitchSession(t, sessionMgr, file, remote.URL) + + changeReq := httptest.NewRequest( + http.MethodPatch, + "/api/v1/playback/"+sessionID+"/audio", + strings.NewReader(`{"audio_track_index":1,"position":120}`), + ) + changeReq = changeReq.WithContext(newAuthorizedPlaybackContext()) + changeReq = withPlaybackRouteParam(changeReq, "session_id", sessionID) + + rr := httptest.NewRecorder() + handler.HandleChangeAudioTrack(rr, changeReq) + if rr.Code != http.StatusOK { + t.Fatalf("change status = %d, body = %s", rr.Code, rr.Body.String()) + } + + // BUG A: the node must have been told to restart on the new track. + if remoteStartRequests != 1 { + t.Fatalf("remote /transcode/start calls = %d, want 1", remoteStartRequests) + } + if remoteStartReq.AudioTrackIndex != 1 { + t.Fatalf("remote AudioTrackIndex = %d, want 1", remoteStartReq.AudioTrackIndex) + } + if remoteStartReq.SessionID != sessionID { + t.Fatalf("remote SessionID = %q, want %q", remoteStartReq.SessionID, sessionID) + } + // Seek 120s with the node-default 2s segments => start segment 60. + if remoteStartReq.StartSegmentNumber != 60 { + t.Fatalf("remote StartSegmentNumber = %d, want 60", remoteStartReq.StartSegmentNumber) + } + if remoteStartReq.TargetResolution != "720p" || remoteStartReq.TargetCodecVideo != "h264" { + t.Fatalf("remote target recipe = %q/%q, want 720p/h264", remoteStartReq.TargetResolution, remoteStartReq.TargetCodecVideo) + } + + var resp changeAudioResponse + if err := json.NewDecoder(rr.Body).Decode(&resp); err != nil { + t.Fatalf("decode change response: %v", err) + } + + // BUG B: the returned proxy URL token must be recipe-COMPLETE. + prefix := "http://proxy-1/stream/transcode/" + suffix := "/master.m3u8" + if !strings.HasPrefix(resp.StreamURL, prefix) || !strings.HasSuffix(resp.StreamURL, suffix) { + t.Fatalf("stream URL = %q, want proxy transcode manifest URL", resp.StreamURL) + } + token := strings.TrimSuffix(strings.TrimPrefix(resp.StreamURL, prefix), suffix) + claims, err := streamtoken.Verify(token, handler.JWTSecret) + if err != nil { + t.Fatalf("verify stream token: %v", err) + } + if claims.AudioTrackIndex != 1 { + t.Fatalf("token AudioTrackIndex = %d, want 1", claims.AudioTrackIndex) + } + // Identity-only claims would leave these zero; a full recipe carries them. + if claims.TargetCodec != "h264" { + t.Fatalf("token TargetCodec (video) = %q, want h264 (recipe-complete)", claims.TargetCodec) + } + if claims.TargetCodecAudio != "aac" { + t.Fatalf("token TargetCodecAudio = %q, want aac (recipe-complete)", claims.TargetCodecAudio) + } + if claims.TargetRes != "720p" { + t.Fatalf("token TargetRes = %q, want 720p (recipe-complete)", claims.TargetRes) + } + if claims.TargetBitrateKbps != 2000 { + t.Fatalf("token TargetBitrateKbps = %d, want 2000 (recipe-complete)", claims.TargetBitrateKbps) + } + if claims.SeekSeconds != 120 { + t.Fatalf("token SeekSeconds = %v, want 120", claims.SeekSeconds) + } + if claims.StartSegmentNumber != 60 { + t.Fatalf("token StartSegmentNumber = %d, want 60", claims.StartSegmentNumber) + } + // SegmentDuration must be > 0: the node treats a zero/omitted value as an + // incomplete token and falls back to a recipe store the native path never + // populates, which would 404 on a node restart. This guards that regression. + if claims.SegmentDuration != playback.DefaultSegmentDuration { + t.Fatalf("token SegmentDuration = %d, want %d (recipe-complete)", claims.SegmentDuration, playback.DefaultSegmentDuration) + } + if claims.MediaPath != file.FilePath { + t.Fatalf("token MediaPath = %q, want %q", claims.MediaPath, file.FilePath) + } + if claims.TranscodeNode != remote.URL { + t.Fatalf("token TranscodeNode = %q, want %q", claims.TranscodeNode, remote.URL) + } +} + +// TestHandleChangeAudioTrack_RemoteTranscodeNodeUnreachableSurfacesError +// verifies the chosen error posture: when the transcode node cannot be reached +// for the restart, the handler surfaces a 502 rather than returning a 200 that +// clients (silo-android / silo-apple) would trust as "audio switched". +func TestHandleChangeAudioTrack_RemoteTranscodeNodeUnreachableSurfacesError(t *testing.T) { + sessionMgr := playback.NewSessionManager(0, 0) + file := &models.MediaFile{ + ID: 42, + ContentID: "movie-1", + FilePath: writePlaybackTestMediaFile(t, "movie.mkv"), + Resolution: "1080p", + CodecVideo: "h264", + CodecAudio: "aac", + Container: "mkv", + Bitrate: 8000, + Duration: 3600, + AudioTracks: []models.AudioTrack{ + {Codec: "aac", Default: true}, + {Codec: "ac3"}, + }, + } + + // A node URL that nothing is listening on => connection refused. + deadNodeURL := "http://127.0.0.1:1" + + handler := NewPlaybackHandler(sessionMgr, testPlaybackFileResolver{file: file}) + handler.ItemAccess = allowAllPlaybackItemAccess{} + handler.JWTSecret = "test-secret" + + group := "rack-a" + proxies := nodepool.NewProxyPool() + proxies.SetNodes([]*nodepool.Node{{ + ID: 1, Name: "proxy-1", Type: nodepool.NodeTypeProxy, URL: "http://proxy-1", + Enabled: true, Healthy: true, Group: &group, + }}) + transcodes := nodepool.NewTranscodePool() + transcodes.SetNodes([]*nodepool.Node{{ + ID: 2, Name: "transcode-1", Type: nodepool.NodeTypeTranscode, URL: deadNodeURL, + Enabled: true, Healthy: true, Group: &group, + }}) + handler.NodePlanner = nodepool.NewPlanner(proxies, transcodes) + + sessionID := newRemoteAudioSwitchSession(t, sessionMgr, file, deadNodeURL) + + changeReq := httptest.NewRequest( + http.MethodPatch, + "/api/v1/playback/"+sessionID+"/audio", + strings.NewReader(`{"audio_track_index":1,"position":0}`), + ) + changeReq = changeReq.WithContext(newAuthorizedPlaybackContext()) + changeReq = withPlaybackRouteParam(changeReq, "session_id", sessionID) + + rr := httptest.NewRecorder() + handler.HandleChangeAudioTrack(rr, changeReq) + + // Chosen posture: surface the failure (mirror HandleStartTranscode) so a 200 + // never lies to the client about an audio switch that didn't happen. + if rr.Code != http.StatusBadGateway { + t.Fatalf("change status = %d, want %d, body = %s", rr.Code, http.StatusBadGateway, rr.Body.String()) + } +} + func TestHandleStartTranscode_LocalPathPropagatesSelectedAudioTrack(t *testing.T) { sessionMgr := playback.NewSessionManager(0, 0) filePath := writePlaybackTestMediaFile(t, "movie.mkv") @@ -1213,7 +1452,7 @@ func TestHandleStartTranscode_LocalPathPropagatesSelectedAudioTrack(t *testing.T t.Fatalf("transcode status = %d, body = %s", transcodeRR.Code, transcodeRR.Body.String()) } - transcodeSession := handler.getTranscodeSession(startResp.SessionID) + transcodeSession := handler.tm.GetTranscodeSession(startResp.SessionID) if transcodeSession == nil { t.Fatal("expected local transcode session") } @@ -1264,7 +1503,7 @@ func TestHandleStartTranscode_MPEG2SeekedCopyRemainsCopyVideo(t *testing.T) { t.Fatalf("transcode status = %d, body = %s", transcodeRR.Code, transcodeRR.Body.String()) } - transcodeSession := handler.getTranscodeSession(session.ID) + transcodeSession := handler.tm.GetTranscodeSession(session.ID) if transcodeSession == nil { t.Fatal("expected local transcode session") } diff --git a/internal/api/handlers/session_ws.go b/internal/api/handlers/session_ws.go index 26b6c9d7..f2836d21 100644 --- a/internal/api/handlers/session_ws.go +++ b/internal/api/handlers/session_ws.go @@ -182,7 +182,7 @@ func (h *PlaybackHandler) handleRealtimeClientMessage(sessionID string, data []b } switch record.Name { case playback.CommandStop, playback.CommandTerminate: - err := h.stopPlaybackSessionByID(context.Background(), sessionID) + err := h.stopPlaybackSessionByID(context.Background(), sessionID, true) if err != nil && !errors.Is(err, playback.ErrSessionNotFound) { slog.Error("failed to stop playback after realtime completion", "session", sessionID, "playback_session_id", sessionID, "error", err) } diff --git a/internal/api/handlers/stream.go b/internal/api/handlers/stream.go index 0c2a4264..a82f6613 100644 --- a/internal/api/handlers/stream.go +++ b/internal/api/handlers/stream.go @@ -32,6 +32,15 @@ type StreamHandler struct { EventsHub *evt.Hub AdminStore PlaybackAdminStore SessionSyncer PlaybackSessionSyncer + // TM is the shared transcode/reconstruct manager (same instance as the + // PlaybackHandler's). It lets a direct/remux stream rebuild its playback + // Session from the recipe card after a server restart instead of 404-ing. + // May be nil (tests / minimal setups) — reconstruct is then simply off. + TM *playback.TranscodeManager + // JWTSecret verifies the stream token carried on the serve URL (?st=), which + // is the reconstruction descriptor for direct/remux after a restart. Empty + // disables token-based reconstruct (tests / minimal setups). + JWTSecret string // PlaybackConfig returns the current playback config; read it through // ffmpegPath(). May be nil (tests). PlaybackConfig func() config.PlaybackConfig @@ -54,6 +63,10 @@ func NewStreamHandler(sessionMgr SessionManagerInterface, fileResolver FilePathR return &StreamHandler{ sessionMgr: sessionMgr, fileResolver: fileResolver, + // A bare manager (no recipe store) behaves as "no reconstruct" — plain + // GetSession + ownership — so HandleStream has a single code path. The + // router overwrites this with the shared manager to enable reconstruct. + TM: playback.NewTranscodeManager(), } } @@ -75,14 +88,22 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) { } setPlaybackSessionLogContext(r, sessionID) - session, err := h.sessionMgr.GetSession(sessionID) - if err != nil { + // Look up the session, reconstructing it from the recipe card on a not-found + // miss (e.g. after a server restart) so a direct/remux stream resumes instead + // of 404-ing. The client re-supplies its position (HTTP Range for direct, the + // ?seek= query for remux), so no runtime beyond the Session needs rebuilding. + // Without a token (or signing secret) reconstruct is off, collapsing to a + // plain GetSession + ownership check. + card := streamCardFromToken(r.URL.Query().Get(streamTokenParam), sessionID, h.JWTSecret) + session, status := h.TM.LoadOrReconstructSession(r.Context(), h.sessionMgr.GetSession, sessionID, userID, card) + switch status { + case playback.SessionMissing: writePlaybackSessionNotFound(w) return - } - - // Verify session ownership. - if session.UserID != userID { + case playback.SessionLoadFailed: + writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load playback session") + return + case playback.SessionForbidden: writeError(w, http.StatusForbidden, "forbidden", "Session belongs to another user") return } diff --git a/internal/api/router.go b/internal/api/router.go index c45a839c..1e517320 100644 --- a/internal/api/router.go +++ b/internal/api/router.go @@ -748,6 +748,13 @@ func NewRouter(deps Dependencies) chi.Router { playbackHandler.SessionSyncer = deps.SessionSyncer } if streamHandler != nil { + // Share the playback handler's transcode/reconstruct manager so a + // direct/remux stream can rebuild its session from the token recipe + // after a restart (same manager, same SessionManager). + streamHandler.TM = playbackHandler.TranscodeManager() + if deps.Config != nil { + streamHandler.JWTSecret = deps.Config.Auth.JWTSecret + } streamHandler.AdminStore = playbackAdminStore streamHandler.EventsHub = deps.EventsHub streamHandler.SessionSyncer = deps.SessionSyncer diff --git a/internal/jellycompat/audio_selection_test.go b/internal/jellycompat/audio_selection_test.go index f6141a76..0cafd106 100644 --- a/internal/jellycompat/audio_selection_test.go +++ b/internal/jellycompat/audio_selection_test.go @@ -239,7 +239,7 @@ func TestHandlePlaybackReport_UpdatesSelectedAudioStreamAndUpstreamTrack(t *test handler := &PlaybackHandler{ playbackStore: playbackStore, sessionMgr: sessionMgr, - transcodes: make(map[string]*playback.TranscodeSession), + tm: playback.NewTranscodeManager(), } req := httptest.NewRequest(http.MethodPost, "/Sessions/Playing/Progress", strings.NewReader(`{"PlaySessionId":"play-1","MediaSourceId":"`+source.ID+`","AudioStreamIndex":2,"PositionTicks":30000000}`)) @@ -293,7 +293,7 @@ func TestEnsureTranscodeSession_UsesSelectedAudioTrack(t *testing.T) { fileResolver: testCompatFileResolver{file: &models.MediaFile{ID: version.FileID, FilePath: filePath}}, TranscodeDir: t.TempDir(), FFmpegPath: writeCompatTestFFmpeg(t), - transcodes: make(map[string]*playback.TranscodeSession), + tm: playback.NewTranscodeManager(), } transcodeSession, err := handler.ensureTranscodeSession(context.Background(), "play-1", "upstream-1", source) @@ -327,9 +327,16 @@ func TestStartRemoteTranscode_IncludesSelectedAudioTrack(t *testing.T) { })) defer server.Close() - handler := &PlaybackHandler{JWTSecret: "secret"} + playbackStore := NewPlaybackSessionStore(time.Hour, nil) + playbackStore.Put(PlaybackSession{ID: "play-1", UpstreamSessionID: "upstream-1"}) + handler := &PlaybackHandler{ + JWTSecret: "secret", + playbackStore: playbackStore, + tm: playback.NewTranscodeManager(), + } if err := handler.startRemoteTranscode( context.Background(), + "play-1", "upstream-1", source, &models.MediaFile{ID: version.FileID, FilePath: filePath}, diff --git a/internal/jellycompat/auth.go b/internal/jellycompat/auth.go index 2cdbc5e4..7a250c27 100644 --- a/internal/jellycompat/auth.go +++ b/internal/jellycompat/auth.go @@ -219,7 +219,7 @@ func resolveCompatToken(ctx context.Context, sessions *SessionStore, keyAuth *Ad // PlaybackSessionAuth creates middleware that falls back to playback session // authentication for media stream endpoints where external players (e.g. libmpv) // don't forward auth headers or query parameters. -func PlaybackSessionAuth(sessions *SessionStore, playbackStore *PlaybackSessionStore, keyAuth *AdminAPIKeyAuthenticator) func(next http.Handler) http.Handler { +func PlaybackSessionAuth(sessions *SessionStore, playbackStore CompatPlaybackStore, keyAuth *AdminAPIKeyAuthenticator) func(next http.Handler) http.Handler { return func(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // Try standard token auth first — a compat session token or an sa_ diff --git a/internal/jellycompat/cleanup.go b/internal/jellycompat/cleanup.go index 937b50b8..d6202bd9 100644 --- a/internal/jellycompat/cleanup.go +++ b/internal/jellycompat/cleanup.go @@ -2,14 +2,47 @@ package jellycompat import ( "context" + "errors" "log/slog" "time" ) +type playbackSessionExpirer interface { + DeleteExpired(ctx context.Context) (int64, error) +} + +func cleanupExpiredCompatState(ctx context.Context, repo *SessionRepository, playbackExpirer playbackSessionExpirer, now time.Time) (int, int64, error) { + var ( + authDeleted int + playbackDeleted int64 + authErr error + playbackErr error + ) + if repo != nil { + authDeleted, authErr = repo.DeleteExpired(ctx, now) + } + if playbackExpirer != nil { + playbackDeleted, playbackErr = playbackExpirer.DeleteExpired(ctx) + } + return authDeleted, playbackDeleted, errors.Join(authErr, playbackErr) +} + // StartSessionCleanup runs a background goroutine that periodically removes // expired compat sessions from the database. It stops when ctx is cancelled. func StartSessionCleanup(ctx context.Context, repo *SessionRepository, interval time.Duration) { + StartSessionCleanupWithPlaybackStore(ctx, repo, nil, interval) +} + +// StartSessionCleanupWithPlaybackStore also sweeps expired durable playback +// negotiation rows when the configured playback store supports it. +func StartSessionCleanupWithPlaybackStore(ctx context.Context, repo *SessionRepository, playbackStore CompatPlaybackStore, interval time.Duration) { if repo == nil { + if _, ok := playbackStore.(playbackSessionExpirer); !ok { + return + } + } + playbackExpirer, _ := playbackStore.(playbackSessionExpirer) + if repo == nil && playbackExpirer == nil { return } go func() { @@ -20,13 +53,13 @@ func StartSessionCleanup(ctx context.Context, repo *SessionRepository, interval case <-ctx.Done(): return case <-ticker.C: - deleted, err := repo.DeleteExpired(ctx, time.Now()) + authDeleted, playbackDeleted, err := cleanupExpiredCompatState(ctx, repo, playbackExpirer, time.Now()) if err != nil { slog.Warn("jellycompat session cleanup failed", "error", err) continue } - if deleted > 0 { - slog.Debug("jellycompat session cleanup", "deleted", deleted) + if authDeleted > 0 || playbackDeleted > 0 { + slog.Debug("jellycompat session cleanup", "auth_sessions", authDeleted, "playback_sessions", playbackDeleted) } } } diff --git a/internal/jellycompat/cleanup_test.go b/internal/jellycompat/cleanup_test.go new file mode 100644 index 00000000..0a1461b5 --- /dev/null +++ b/internal/jellycompat/cleanup_test.go @@ -0,0 +1,41 @@ +package jellycompat + +import ( + "context" + "testing" + "time" +) + +type fakePlaybackSessionExpirer struct { + called bool + count int64 + err error +} + +func (f *fakePlaybackSessionExpirer) DeleteExpired(context.Context) (int64, error) { + f.called = true + return f.count, f.err +} + +func TestCleanupExpiredCompatStateIncludesPlaybackSessions(t *testing.T) { + expirer := &fakePlaybackSessionExpirer{count: 3} + + authDeleted, playbackDeleted, err := cleanupExpiredCompatState( + context.Background(), + nil, + expirer, + time.Unix(100, 0), + ) + if err != nil { + t.Fatalf("cleanupExpiredCompatState returned error: %v", err) + } + if authDeleted != 0 { + t.Fatalf("authDeleted = %d, want 0 with nil repo", authDeleted) + } + if playbackDeleted != 3 { + t.Fatalf("playbackDeleted = %d, want 3", playbackDeleted) + } + if !expirer.called { + t.Fatal("playback session expirer was not called") + } +} diff --git a/internal/jellycompat/handlers_playback.go b/internal/jellycompat/handlers_playback.go index c384c7ba..35fe49fb 100644 --- a/internal/jellycompat/handlers_playback.go +++ b/internal/jellycompat/handlers_playback.go @@ -15,7 +15,6 @@ import ( "path/filepath" "strconv" "strings" - "sync" "time" "github.com/go-chi/chi/v5" @@ -126,7 +125,7 @@ type PlaybackHandler struct { content ContentService codec *ResourceIDCodec deviceProfiles *DeviceProfileStore - playbackStore *PlaybackSessionStore + playbackStore CompatPlaybackStore sessionMgr SessionManagerInterface fileResolver FilePathResolver storeProvider userstore.UserStoreProvider @@ -137,13 +136,35 @@ type PlaybackHandler struct { FFmpegPath string HWAccel string TranscodeDir string - transcodeMu sync.RWMutex - transcodes map[string]*playback.TranscodeSession - SubtitleRepo subtitles.Repository // optional; enables downloaded subtitles - S3Client subtitles.S3Client // optional; for serving S3 subtitles - S3Bucket string // bucket for subtitle storage - SettingsRepo SettingsReader // optional; reads watched threshold setting - SessionSyncer PlaybackSessionSyncer // optional; enables immediate session sync to shared admin view + // tm is the shared transcode-session lifecycle (live map, reconstruct) — the + // same type the native handler uses, so jellycompat gets the reconstruct cap + // and node-affinity rule for free. The reconstruction recipe is carried in the + // compat playback store (PlaybackSession.Recipe), since Jellyfin clients cannot + // round-trip a native stream token. + tm *playback.TranscodeManager + SubtitleRepo subtitles.Repository // optional; enables downloaded subtitles + S3Client subtitles.S3Client // optional; for serving S3 subtitles + S3Bucket string // bucket for subtitle storage + SettingsRepo SettingsReader // optional; reads watched threshold setting + SessionSyncer PlaybackSessionSyncer // optional; enables immediate session sync to shared admin view + // RecipeNodeStore hands a remote transcode's reconstruction recipe to the + // control-plane recipe store (Redis) so a dedicated transcode node that + // restarts can rebuild ffmpeg from it. The node-hop token is server-minted and + // could carry the recipe, but it is mutated in place and the client can't be + // driven to refresh a stale token, so the node reconstructs from this + // server-authoritative store instead (see internal/noderecipe). Optional + // (nil disables it — integrated/no-node deployments need no handoff). + RecipeNodeStore recipeNodePutter +} + +// recipeNodePutter persists and removes a remote transcode's reconstruction +// recipe in a control-plane store keyed by upstream session id. *noderecipe.Store +// implements it. Delete is nil-safe and treats a missing key as a no-op success; +// it is called on deliberate teardown so a stopped session cannot be resurrected +// from a leaked recipe. +type recipeNodePutter interface { + Put(ctx context.Context, sessionID string, card playback.RecipeCard) error + Delete(ctx context.Context, sessionID string) error } // playbackThresholds reads the playback.watched_threshold and @@ -188,7 +209,7 @@ func NewPlaybackHandler( content ContentService, codec *ResourceIDCodec, deviceProfiles *DeviceProfileStore, - playbackStore *PlaybackSessionStore, + playbackStore CompatPlaybackStore, sessionMgr SessionManagerInterface, fileResolver FilePathResolver, storeProvider userstore.UserStoreProvider, @@ -216,16 +237,54 @@ func NewPlaybackHandler( FFmpegPath: ffmpegPath, HWAccel: hwAccel, TranscodeDir: transcodeDir, - transcodes: make(map[string]*playback.TranscodeSession), + tm: playback.NewTranscodeManager(), } - if cleaned, err := playback.CleanupOrphanedTranscodeDirs(h.TranscodeDir, nil); err != nil { - slog.Warn("jellycompat transcode cleanup failed", "dir", h.TranscodeDir, "error", err) - } else if cleaned > 0 { - slog.Info("jellycompat transcode cleanup removed orphaned dirs", "dir", h.TranscodeDir, "count", cleaned) + // Wire the shared transcode manager with closures so it reads the handler's + // (late-set) JWTSecret lazily, matching the native handler. + h.tm.JWTSecretFn = func() string { return h.JWTSecret } + h.tm.Config = func() playback.TranscodeRuntimeConfig { + return playback.TranscodeRuntimeConfig{ + TranscodeDir: h.TranscodeDir, + FFmpegPath: h.FFmpegPath, + HWAccel: h.HWAccel, + } + } + if reg, ok := sessionMgr.(interface { + GetSession(string) (*playback.Session, error) + RegisterReconstructed(*playback.Session) *playback.Session + RegisterReconstructedWithLimits(context.Context, *playback.Session) (*playback.Session, error) + }); ok { + h.tm.Sessions = reg + } + h.tm.OnFFmpegCrash = func(ctx context.Context, sessionID string, dead *playback.TranscodeSession) { + // ffmpeg crash: drop the dead transcode and stop the upstream native + // session. The recipe stays in the compat store so a resume reconstructs. + nodeURL := "" + if h.sessionMgr != nil { + if up, err := h.sessionMgr.GetSession(sessionID); err == nil && up != nil { + nodeURL = up.TranscodeNodeURL + } + } + // Guarded close is the authoritative gate: only tear down if the live entry + // is still the crashed one. We must NOT stop the upstream session before this + // check — if a successor reconstructed under the same id between ffmpeg's exit + // and here, an early StopSession would orphan its ffmpeg (live transcode, no + // session). Only stop the upstream session when the compare-and-delete matched + // the dead transcode. The recipe stays in the compat store either way so a + // resume reconstructs. + if h.sessionMgr != nil && h.tm.CloseTranscodeSessionIf(sessionID, dead, nodeURL) { + _ = h.sessionMgr.StopSession(sessionID) + } } return h } +// CleanupOrphanedTranscodes removes stale per-session transcode dirs, sparing +// those whose recipe card still exists. Delegates to the shared manager. +func (h *PlaybackHandler) CleanupOrphanedTranscodes() (int, error) { + return h.tm.CleanupOrphanedTranscodes() +} + // buildProxyRedirectURL signs a stream token and builds the redirect URL for // the given proxy node (the planner's pick for this session). func (h *PlaybackHandler) buildProxyRedirectURL( @@ -299,6 +358,7 @@ func clampSeekSeconds(seekSeconds float64, sources []PlaybackMediaSource) float6 func (h *PlaybackHandler) startRemoteTranscode( ctx context.Context, + playSessionID string, upstreamSessionID string, source PlaybackMediaSource, file *models.MediaFile, @@ -360,6 +420,92 @@ func (h *PlaybackHandler) startRemoteTranscode( if resp.StatusCode != http.StatusAccepted { return fmt.Errorf("remote transcode start rejected: status %d", resp.StatusCode) } + + // Mirror the byte-affecting opts sent to the node into a RecipeCard and persist + // it for restart resilience. The node-hop token is identity-only by design (see + // internal/noderecipe), and central serves Jellyfin clients that carry no native + // token of their own, so without a persisted recipe a node or central restart + // cannot rebuild ffmpeg and segment serves 404. + opts := playback.TranscodeOpts{ + SessionID: upstreamSessionID, + InputPath: reqBody.InputPath, + SeekSeconds: reqBody.SeekSeconds, + StartSegmentNumber: reqBody.StartSegmentNumber, + TargetCodecVideo: reqBody.TargetCodecVideo, + TargetCodecAudio: reqBody.TargetCodecAudio, + SegmentDuration: reqBody.SegmentDuration, + HWAccel: reqBody.HWAccel, + AudioTrackIndex: reqBody.AudioTrackIndex, + TotalDuration: reqBody.TotalDuration, + } + if source.TranscodeAudio { + opts.TargetCodecVideo = "copy" + } + + if err := h.persistTranscodeRecipe(ctx, playSessionID, upstreamSessionID, opts); err != nil { + // Roll back the already-started node ffmpeg so it isn't leaked. + h.tm.CloseTranscodeSession(upstreamSessionID, transcodeNodeURL) + return err + } + + return nil +} + +// persistTranscodeRecipe builds the reconstruction recipe from the upstream +// session's identity and persists it for restart resilience. It is shared by the +// local (ensureLocalTranscode) and remote (startRemoteTranscode) transcode paths +// so both stores stay in lock-step. +// +// The recipe is recorded in the compat store in the same Update that marks the +// transcode started — a failed write leaves neither set — and then best-effort +// handed to the node recipe store (Redis) for dedicated transcode nodes. The node +// URL is taken from the upstream session (bound before start on the remote path), +// so it is "" for integrated transcodes and the node-store write is skipped. +// A Jellyfin client carries no native token of its own and the node-hop token is +// deliberately identity-only (see internal/noderecipe), so the persisted recipe is +// the only way a node or central restart can rebuild ffmpeg. +// +// Returns an error only when the compat-store Update fails; the caller owns +// rolling back its (local or remote) transcode in that case. A missing upstream +// session (a start/build race, or no session manager in tests) is logged and the +// live transcode is left serving — only restart resilience is forfeited. +func (h *PlaybackHandler) persistTranscodeRecipe( + ctx context.Context, + playSessionID, upstreamSessionID string, + opts playback.TranscodeOpts, +) error { + var recipe *playback.RecipeCard + if h.sessionMgr != nil { + if upstream, err := h.sessionMgr.GetSession(upstreamSessionID); err == nil && upstream != nil { + card := playback.NewRecipeCard(upstream.UserID, upstream.ProfileID, upstream.MediaFileID, upstream.TranscodeNodeURL, opts) + recipe = &card + } + } + if recipe == nil { + slog.Warn("transcode recipe not persisted: upstream session unavailable", + "playback_session_id", upstreamSessionID) + } + + if err := h.playbackStore.Update(playSessionID, func(current *PlaybackSession) error { + current.TranscodeStarted = true + current.Recipe = recipe + return nil + }); err != nil { + return fmt.Errorf("update playback session: %w", err) + } + + // Hand the recipe to the control-plane store (Redis) so a dedicated transcode + // node that restarts can rebuild ffmpeg from it. Bounded and best effort: a + // stalled write must not hang the manifest request, and a failed write only + // forfeits node-restart resilience for this session, never the start. + if recipe != nil && recipe.TranscodeNodeURL != "" && h.RecipeNodeStore != nil { + putCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 2*time.Second) + defer cancel() + if err := h.RecipeNodeStore.Put(putCtx, upstreamSessionID, *recipe); err != nil { + slog.Warn("persist node transcode recipe failed", "error", err, + "playback_session_id", upstreamSessionID, "node", recipe.TranscodeNodeURL) + } + } return nil } diff --git a/internal/jellycompat/playback_4k_test.go b/internal/jellycompat/playback_4k_test.go index b7b75682..25923621 100644 --- a/internal/jellycompat/playback_4k_test.go +++ b/internal/jellycompat/playback_4k_test.go @@ -159,7 +159,7 @@ func TestStartRemoteTranscode4KGuard(t *testing.T) { FileID: 1, Version: catalog.FileVersion{FileID: 1, Resolution: "2160p", CodecVideo: "hevc"}, } - if err := h.startRemoteTranscode(context.Background(), "session", source, nil, 0, "http://node"); !errors.Is(err, errTranscode4KDisallowed) { + if err := h.startRemoteTranscode(context.Background(), "play", "session", source, nil, 0, "http://node"); !errors.Is(err, errTranscode4KDisallowed) { t.Errorf("startRemoteTranscode() error = %v, want errTranscode4KDisallowed", err) } } diff --git a/internal/jellycompat/playback_report_liveness_test.go b/internal/jellycompat/playback_report_liveness_test.go index b893152d..6707552b 100644 --- a/internal/jellycompat/playback_report_liveness_test.go +++ b/internal/jellycompat/playback_report_liveness_test.go @@ -61,7 +61,7 @@ func newReportLivenessHandler(upstreamID string, registerUpstream bool) (*Playba codec: codec, playbackStore: playbackStore, sessionMgr: sessionMgr, - transcodes: make(map[string]*playback.TranscodeSession), + tm: playback.NewTranscodeManager(), } return handler, sessionMgr, encodedItemID, source.ID } @@ -371,7 +371,9 @@ func TestHandlePlaybackReport_StopViaRouteMatchDoesNotTearDown(t *testing.T) { // a second ffmpeg would start alongside the orphaned one. func TestEnsureUpstreamPlayback_ReviveClosesStaleTranscode(t *testing.T) { handler, mgr, _, _ := newReportLivenessHandler("upstream-reaped", false) - handler.transcodes["upstream-reaped"] = nil // stale entry keyed by the reaped id + // Stale transcode still keyed by the reaped id (never-started session; Close is a no-op). + staleTranscode := &playback.TranscodeSession{} + handler.tm.RegisterTranscodeSession("upstream-reaped", staleTranscode) playSession, _ := handler.playbackStore.Get("play-1") source := playSession.MediaSources[0] @@ -384,7 +386,7 @@ func TestEnsureUpstreamPlayback_ReviveClosesStaleTranscode(t *testing.T) { if mgr.startCalls != 1 { t.Fatalf("StartSession calls = %d, want 1", mgr.startCalls) } - if _, stale := handler.transcodes["upstream-reaped"]; stale { + if handler.tm.GetTranscodeSession("upstream-reaped") != nil { t.Fatal("expected the stale transcode entry to be closed before recreating the upstream session") } } @@ -507,7 +509,7 @@ func TestHandlePlaybackReport_AliasContradictedByItemFallsBack(t *testing.T) { // play session is re-pointed at a different upstream id/method. type racingStartSessionManager struct { testCompatSessionManager - store *PlaybackSessionStore + store CompatPlaybackStore winnerMethod string } diff --git a/internal/jellycompat/playback_sessions.go b/internal/jellycompat/playback_sessions.go index e393d898..584f7086 100644 --- a/internal/jellycompat/playback_sessions.go +++ b/internal/jellycompat/playback_sessions.go @@ -5,6 +5,7 @@ import ( "time" "github.com/Silo-Server/silo-server/internal/catalog" + "github.com/Silo-Server/silo-server/internal/playback" ) // PlaybackSession stores compat-owned playback negotiation state before the @@ -20,14 +21,19 @@ type PlaybackSession struct { // that id resolve to this session directly instead of by ambiguous route. ClientPlaySessionID string UserID string - InitialSeekSeconds float64 - MediaSources []PlaybackMediaSource - UpstreamSessionID string - UpstreamPlayMethod string - TranscodeStarted bool - CreatedAt time.Time - UpdatedAt time.Time - ExpiresAt time.Time + InitialSeekSeconds float64 + MediaSources []PlaybackMediaSource + UpstreamSessionID string + UpstreamPlayMethod string + TranscodeStarted bool + // Recipe is the transcode reconstruction descriptor for this session. Jellyfin + // clients cannot round-trip a native stream token, so jellycompat carries the + // recipe in its own durable compat store (this struct, persisted as JSONB) + // rather than in the token. Nil until a transcode actually starts. + Recipe *playback.RecipeCard + CreatedAt time.Time + UpdatedAt time.Time + ExpiresAt time.Time } // PlaybackMediaSource stores one negotiated stream source within a compat play session. @@ -46,7 +52,31 @@ type PlaybackMediaSource struct { ETag string } -// PlaybackSessionStore keeps compat playback sessions in memory. +// CompatPlaybackStore persists compat playback negotiation sessions (the +// PlaySessionId → upstream-session mapping plus media sources, route, and seek). +// It is an interface so the backing store is swappable: the in-memory +// PlaybackSessionStore is the default, and a durable (Postgres/Redis) +// implementation lets the mapping survive a server restart so a Jellyfin client +// can resume — a Redis switch then touches only the constructor, nothing else. +type CompatPlaybackStore interface { + // Put stores or replaces a compat playback session. + Put(session PlaybackSession) + // Get returns a session when it exists and is not expired. + Get(id string) (*PlaybackSession, bool) + // Delete removes a session. + Delete(id string) + // Update modifies a session in place under the store's lock. + Update(id string, fn func(*PlaybackSession) error) error + // FindByRoute resolves a route item / media-source id to a session. + FindByRoute(compatToken, routeID string) (*PlaybackSession, *PlaybackMediaSource, bool) + // FindByClientPlaySessionID resolves the client-generated PlaySessionId + // alias recorded for plays that skipped PlaybackInfo. The alias must + // identify exactly one live session; ambiguity returns not-found. + FindByClientPlaySessionID(compatToken, clientPlaySessionID string) (*PlaybackSession, bool) +} + +// PlaybackSessionStore keeps compat playback sessions in memory. It is the +// default CompatPlaybackStore implementation. type PlaybackSessionStore struct { mu sync.RWMutex sessions map[string]PlaybackSession @@ -60,7 +90,11 @@ func NewPlaybackSessionStore(ttl time.Duration, now func() time.Time) *PlaybackS now = time.Now } if ttl <= 0 { - ttl = 6 * time.Hour + // Default the absolute session lifetime to the absolute stream-token TTL + // (playback.MaxTokenTTL, 24h) so a session never expires while its token + // is still valid. Absolute from creation, not sliding; mirrors the + // router default and is config-overridable. + ttl = playback.MaxTokenTTL } return &PlaybackSessionStore{ sessions: make(map[string]PlaybackSession), @@ -71,6 +105,15 @@ func NewPlaybackSessionStore(ttl time.Duration, now func() time.Time) *PlaybackS // Put stores or replaces a compat playback session. func (s *PlaybackSessionStore) Put(session PlaybackSession) { + s.putNormalized(session) +} + +// putNormalized stores or replaces a compat playback session and returns the +// stored copy with normalized timestamps (CreatedAt/UpdatedAt/ExpiresAt). The +// durable wrapper uses the return value to persist the same timestamps the cache +// just assigned without a second Get (extra lock + copy). Put keeps the +// no-return signature the CompatPlaybackStore interface requires. +func (s *PlaybackSessionStore) putNormalized(session PlaybackSession) PlaybackSession { s.mu.Lock() defer s.mu.Unlock() @@ -82,6 +125,7 @@ func (s *PlaybackSessionStore) Put(session PlaybackSession) { session.ExpiresAt = session.CreatedAt.Add(s.ttl) } s.sessions[session.ID] = session + return session } // Get returns a playback session when it exists and is not expired. diff --git a/internal/jellycompat/playback_sessions_postgres.go b/internal/jellycompat/playback_sessions_postgres.go new file mode 100644 index 00000000..75c2b598 --- /dev/null +++ b/internal/jellycompat/playback_sessions_postgres.go @@ -0,0 +1,341 @@ +package jellycompat + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgxpool" +) + +var ( + _ CompatPlaybackStore = (*PlaybackSessionStore)(nil) + _ CompatPlaybackStore = (*DurableCompatPlaybackStore)(nil) +) + +// DurableCompatPlaybackStore is a CompatPlaybackStore that persists compat +// playback sessions to Postgres so the PlaySessionId -> upstream-session mapping +// (and the negotiated media sources) survives a server restart. It wraps an +// in-memory PlaybackSessionStore as a write-through cache so the hot segment path +// (Get on every segment request) stays in-process; a cache miss falls back to a +// DB read and repopulates the cache. A Redis swap would reimplement this same +// interface, leaving every caller unchanged. +type DurableCompatPlaybackStore struct { + mem *PlaybackSessionStore + pool *pgxpool.Pool + ttl time.Duration + now func() time.Time +} + +// NewDurableCompatPlaybackStore returns a Postgres-backed compat store. pool must +// be non-nil (callers fall back to the in-memory store when there is no DB). +func NewDurableCompatPlaybackStore(pool *pgxpool.Pool, ttl time.Duration, now func() time.Time) *DurableCompatPlaybackStore { + if now == nil { + now = time.Now + } + if ttl <= 0 { + ttl = 6 * time.Hour + } + return &DurableCompatPlaybackStore{ + mem: NewPlaybackSessionStore(ttl, now), + pool: pool, + ttl: ttl, + now: now, + } +} + +// Put writes through to both the cache and Postgres. putNormalized returns the +// stored copy carrying the timestamps the cache just assigned (CreatedAt / +// UpdatedAt / ExpiresAt), so the persisted row matches the cache without a second +// Get (extra lock + full struct copy) to recover them. +// +// The DB upsert is intentionally kept synchronous: restart resilience depends on +// the just-negotiated session being durable before the client's next request +// (which may arrive after a restart), and the codebase relies on read-your-write +// across a fresh instance. The upsert itself is still best-effort (a DB failure +// is logged, not propagated); the cache holds the authoritative in-process state. +func (d *DurableCompatPlaybackStore) Put(session PlaybackSession) { + stored := d.mem.putNormalized(session) + if d.pool == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + d.upsert(ctx, stored) +} + +// Get returns the cached session, falling back to Postgres on a miss (e.g. after +// a restart) and repopulating the cache. +func (d *DurableCompatPlaybackStore) Get(id string) (*PlaybackSession, bool) { + if s, ok := d.mem.Get(id); ok { + return s, true + } + s, ok := d.load(id) + if !ok { + return nil, false + } + d.mem.Put(*s) + return d.mem.Get(id) +} + +// Delete removes the session from both the cache and Postgres. +func (d *DurableCompatPlaybackStore) Delete(id string) { + d.mem.Delete(id) + if d.pool == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + if _, err := d.pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id); err != nil { + slog.Warn("delete compat playback session failed", "error", err, "play_session_id", id) + } +} + +// Update modifies the session in place under the cache's lock (in-process +// atomicity), then persists the result. The session is loaded from Postgres into +// the cache first when absent so an update after a restart still applies. +// +// The DB persist is atomic against concurrent writers: it re-reads the +// authoritative row under SELECT ... FOR UPDATE inside a transaction, re-applies +// fn to that row, and upserts the result before committing. This stops two +// processes (or a cache-evicted writer racing another) from clobbering each +// other's JSON with a blind whole-document upsert — e.g. a transcode-recipe +// write being lost to a concurrent upstream-session write. The DB step is still +// best-effort for availability: a DB failure is logged and the in-memory mutation +// stands, but a successful DB read-modify-write is never silently lost. +func (d *DurableCompatPlaybackStore) Update(id string, fn func(*PlaybackSession) error) error { + if _, ok := d.mem.Get(id); !ok { + if s, ok := d.load(id); ok { + d.mem.Put(*s) + } + } + if err := d.mem.Update(id, fn); err != nil { + return err + } + committed, err := d.updateDB(id, fn) + if committed != nil { + // Refresh the cache from the DB-authoritative committed row so the cache + // reflects any concurrent writer's fields that fn merged on top of. + d.mem.Put(*committed) + } + if err != nil { + // The in-memory mutation stands (live state is correct), but the durable + // row was NOT updated: surface the failure so durability-sensitive callers + // (recipe/upstream-session writes that promise restart resilience) can roll + // back or fail rather than reporting a session as restart-safe when a later + // restart would reload a stale row or 404. A genuinely absent/expired row + // and a nil pool are not durability failures and return nil (best-effort, + // unchanged) — only real DB round-trip errors propagate. + return fmt.Errorf("durably persist compat playback session %s: %w", id, err) + } + return nil +} + +// updateDB applies fn to the DB-authoritative row inside a transaction using +// SELECT ... FOR UPDATE, then upserts and commits. It returns the committed +// session when the round-trip succeeded. A nil pool or a genuinely +// missing/expired row returns (nil, nil): there is no durable row to update, so +// this is treated as best-effort (the caller keeps the in-memory mutation), not a +// durability failure. A real DB round-trip error returns (nil, err) so the caller +// can learn the session was not durably persisted. fn is expected to be +// idempotent — it is applied to the cache copy and again to the DB-authoritative +// copy. +func (d *DurableCompatPlaybackStore) updateDB(id string, fn func(*PlaybackSession) error) (*PlaybackSession, error) { + if d.pool == nil { + return nil, nil + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + tx, err := d.pool.Begin(ctx) + if err != nil { + slog.Warn("begin compat playback session update tx failed", "error", err, "play_session_id", id) + return nil, err + } + defer func() { _ = tx.Rollback(ctx) }() + + var raw []byte + err = tx.QueryRow(ctx, + `SELECT data FROM jellycompat_playback_sessions WHERE id = $1 AND expires_at > $2 FOR UPDATE`, + id, d.now(), + ).Scan(&raw) + if err != nil { + if errors.Is(err, pgx.ErrNoRows) { + // No durable row to update (never persisted or already expired): not a + // durability failure — best-effort, the in-memory mutation stands. + return nil, nil + } + slog.Warn("load compat playback session for update failed", "error", err, "play_session_id", id) + return nil, err + } + + var session PlaybackSession + if err := json.Unmarshal(raw, &session); err != nil { + slog.Warn("unmarshal compat playback session for update failed", "error", err, "play_session_id", id) + return nil, err + } + if err := fn(&session); err != nil { + // The mutation itself rejected the authoritative row; the cache mutation + // (already applied) stands. This is a fn/data condition, not an + // infrastructure failure, so it is not surfaced as a durability error. + slog.Warn("apply compat playback session update failed", "error", err, "play_session_id", id) + return nil, nil + } + session.UpdatedAt = d.now() + + data, err := json.Marshal(session) + if err != nil { + slog.Warn("marshal compat playback session for update failed", "error", err, "play_session_id", id) + return nil, err + } + expiresAt := session.ExpiresAt + if expiresAt.IsZero() { + expiresAt = d.now().Add(d.ttl) + } + if _, err := tx.Exec(ctx, upsertSessionQuery, session.ID, session.CompatToken, session.UserID, data, expiresAt); err != nil { + slog.Warn("persist compat playback session update failed", "error", err, "play_session_id", id) + return nil, err + } + if err := tx.Commit(ctx); err != nil { + slog.Warn("commit compat playback session update failed", "error", err, "play_session_id", id) + return nil, err + } + return &session, nil +} + +// FindByRoute resolves a route id, checking the cache first and falling back to +// loading the matching compat-token rows from Postgres into the cache. +func (d *DurableCompatPlaybackStore) FindByRoute(compatToken, routeID string) (*PlaybackSession, *PlaybackMediaSource, bool) { + if s, src, ok := d.mem.FindByRoute(compatToken, routeID); ok { + return s, src, ok + } + // An empty compat token cannot be pushed into a bounded, indexed DB query, so + // the only DB fallback would be loading every live row and scanning it on this + // request goroutine — an O(table) cliff. The sole caller + // (resolvePlaybackRoute) always passes a non-empty compat token, so the + // empty-token DB fallback is never load-bearing for route resolution; return + // the in-memory result rather than incurring a full-table scan. + if compatToken == "" { + return nil, nil, false + } + d.loadByCompatToken(compatToken) + return d.mem.FindByRoute(compatToken, routeID) +} + +// FindByClientPlaySessionID resolves the client-generated PlaySessionId alias, +// checking the cache first and falling back to loading the compat token's live +// rows from Postgres into the cache (same bounded fallback as FindByRoute; the +// alias uniqueness check runs against the repopulated cache). +func (d *DurableCompatPlaybackStore) FindByClientPlaySessionID(compatToken, clientPlaySessionID string) (*PlaybackSession, bool) { + if s, ok := d.mem.FindByClientPlaySessionID(compatToken, clientPlaySessionID); ok { + return s, ok + } + if compatToken == "" { + return nil, false + } + d.loadByCompatToken(compatToken) + return d.mem.FindByClientPlaySessionID(compatToken, clientPlaySessionID) +} + +// DeleteExpired physically removes lapsed rows. Reads already filter on +// expires_at, so this only bounds table growth; run it on the janitor cadence. +func (d *DurableCompatPlaybackStore) DeleteExpired(ctx context.Context) (int64, error) { + if d.pool == nil { + return 0, nil + } + tag, err := d.pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE expires_at < $1`, d.now()) + if err != nil { + return 0, err + } + return tag.RowsAffected(), nil +} + +const upsertSessionQuery = ` + INSERT INTO jellycompat_playback_sessions (id, compat_token, user_id, data, expires_at) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (id) DO UPDATE SET + compat_token = EXCLUDED.compat_token, + user_id = EXCLUDED.user_id, + data = EXCLUDED.data, + expires_at = EXCLUDED.expires_at` + +// upsert persists a session on the given context. It is best-effort: a DB +// failure is logged and swallowed (the cache holds the authoritative in-process +// state). Callers own the context and its timeout. +func (d *DurableCompatPlaybackStore) upsert(ctx context.Context, session PlaybackSession) { + if d.pool == nil { + return + } + data, err := json.Marshal(session) + if err != nil { + slog.Warn("marshal compat playback session failed", "error", err, "play_session_id", session.ID) + return + } + expiresAt := session.ExpiresAt + if expiresAt.IsZero() { + expiresAt = d.now().Add(d.ttl) + } + if _, err := d.pool.Exec(ctx, upsertSessionQuery, session.ID, session.CompatToken, session.UserID, data, expiresAt); err != nil { + slog.Warn("persist compat playback session failed", "error", err, "play_session_id", session.ID) + } +} + +func (d *DurableCompatPlaybackStore) load(id string) (*PlaybackSession, bool) { + if d.pool == nil { + return nil, false + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + var raw []byte + err := d.pool.QueryRow(ctx, + `SELECT data FROM jellycompat_playback_sessions WHERE id = $1 AND expires_at > $2`, id, d.now(), + ).Scan(&raw) + if err != nil { + if !errors.Is(err, pgx.ErrNoRows) { + slog.Warn("load compat playback session failed", "error", err, "play_session_id", id) + } + return nil, false + } + var session PlaybackSession + if err := json.Unmarshal(raw, &session); err != nil { + slog.Warn("unmarshal compat playback session failed", "error", err, "play_session_id", id) + return nil, false + } + return &session, true +} + +// loadByCompatToken loads the live rows for a (non-empty) compat token into the +// cache so a subsequent cache scan can resolve the route. The query is bounded by +// the indexed compat_token predicate; FindByRoute never calls it with an empty +// token (that would be an unbounded full-table load). +func (d *DurableCompatPlaybackStore) loadByCompatToken(compatToken string) { + if d.pool == nil || compatToken == "" { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + rows, err := d.pool.Query(ctx, + `SELECT data FROM jellycompat_playback_sessions WHERE compat_token = $1 AND expires_at > $2`, + compatToken, d.now()) + if err != nil { + slog.Warn("load compat playback sessions by token failed", "error", err) + return + } + defer rows.Close() + for rows.Next() { + var raw []byte + if err := rows.Scan(&raw); err != nil { + slog.Warn("scan compat playback session failed", "error", err) + return + } + var session PlaybackSession + if err := json.Unmarshal(raw, &session); err != nil { + continue + } + d.mem.Put(session) + } +} diff --git a/internal/jellycompat/playback_sessions_postgres_test.go b/internal/jellycompat/playback_sessions_postgres_test.go new file mode 100644 index 00000000..37d1a970 --- /dev/null +++ b/internal/jellycompat/playback_sessions_postgres_test.go @@ -0,0 +1,218 @@ +package jellycompat + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/jackc/pgx/v5/pgxpool" +) + +func newCompatTestPool(t *testing.T) *pgxpool.Pool { + t.Helper() + dsn := os.Getenv("SILO_TEST_DATABASE_URL") + if dsn == "" { + t.Skip("SILO_TEST_DATABASE_URL is not set") + } + ctx := context.Background() + pool, err := pgxpool.New(ctx, dsn) + if err != nil { + t.Fatalf("connect test database: %v", err) + } + t.Cleanup(pool.Close) + var tableName *string + if err := pool.QueryRow(ctx, `SELECT to_regclass('public.jellycompat_playback_sessions')::text`).Scan(&tableName); err != nil { + t.Fatalf("check jellycompat_playback_sessions table: %v", err) + } + if tableName == nil || *tableName == "" { + t.Skip("test database has not applied jellycompat playback sessions migration") + } + return pool +} + +// A session written by one store instance must be reloadable by a fresh instance +// (empty cache) — i.e. it survived in Postgres, as it would across a restart. +func TestDurableCompatPlaybackStore_SurvivesRestart(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + id := fmt.Sprintf("compat-test-%d", time.Now().UnixNano()) + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id) }) + + store1 := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + store1.Put(PlaybackSession{ + ID: id, + CompatToken: "tok", + UserID: "u1", + RouteItemID: "route-1", + UpstreamSessionID: "up-1", + InitialSeekSeconds: 12.5, + MediaSources: []PlaybackMediaSource{{ID: "src-1", FileID: 7}}, + }) + + // Fresh instance => empty cache => must hit Postgres. + store2 := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + got, ok := store2.Get(id) + if !ok { + t.Fatal("session not reloaded from Postgres after restart") + } + if got.UpstreamSessionID != "up-1" || got.RouteItemID != "route-1" || got.InitialSeekSeconds != 12.5 { + t.Fatalf("reloaded session lost fields: %+v", got) + } + + // FindByRoute on the fresh instance resolves via a DB-backed scan. + if _, _, ok := store2.FindByRoute("tok", "route-1"); !ok { + t.Fatal("FindByRoute failed to resolve a persisted session") + } + + // Update persists; reload on yet another instance sees it. + if err := store2.Update(id, func(s *PlaybackSession) error { + s.TranscodeStarted = true + return nil + }); err != nil { + t.Fatalf("update: %v", err) + } + store3 := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + if got, ok := store3.Get(id); !ok || !got.TranscodeStarted { + t.Fatalf("update did not persist: ok=%v got=%+v", ok, got) + } + + store3.Delete(id) + store4 := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + if _, ok := store4.Get(id); ok { + t.Fatal("session still present after delete") + } +} + +// M5: an empty compat token must never trigger a DB scan — FindByRoute returns +// the in-memory result only. With a non-nil pool but no live DB, a scan attempt +// would block/error on the pool; instead the empty-token path returns cleanly +// from cache. We assert: (a) a cached empty-token route resolves, and (b) a +// cache-miss empty-token lookup returns false without consulting the DB. The +// nil-pool variant proves the early return independent of any pool. +func TestDurableCompatPlaybackStore_EmptyTokenFindByRouteNoDBScan(t *testing.T) { + store := NewDurableCompatPlaybackStore(nil, time.Hour, nil) + store.Put(PlaybackSession{ID: "ps-empty", CompatToken: "", RouteItemID: "route-x"}) + + // Cached empty-token route resolves from memory. + if _, _, ok := store.FindByRoute("", "route-x"); !ok { + t.Fatal("empty-token FindByRoute should resolve a cached route") + } + // Cache miss with an empty token returns false without a DB fallback. + if _, _, ok := store.FindByRoute("", "route-missing"); ok { + t.Fatal("empty-token FindByRoute should not resolve an unknown route") + } + + // loadByCompatToken must early-return for an empty token even with a non-nil + // pool, so it can never issue a full-table query. Use a closed pool so any + // query attempt would error; reaching the early return means no query ran. + if dsn := os.Getenv("SILO_TEST_DATABASE_URL"); dsn != "" { + pool := newCompatTestPool(t) + s := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + // Should be a no-op (no panic, no scan); cache stays empty. + s.loadByCompatToken("") + if _, _, ok := s.FindByRoute("", "anything"); ok { + t.Fatal("empty-token FindByRoute resolved unexpectedly against DB") + } + } +} + +// M6: an Update must not lose a concurrent writer's field. We simulate two +// interleaved writers that each mutate a different field; after both commit, the +// DB-authoritative row (reloaded on a fresh instance) must carry BOTH fields, +// proving the second writer merged onto the first's committed row rather than +// clobbering it with a stale whole-document upsert. +func TestDurableCompatPlaybackStore_UpdateAtomicNoLostField(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + id := fmt.Sprintf("compat-atomic-%d", time.Now().UnixNano()) + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id) }) + + seed := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + seed.Put(PlaybackSession{ID: id, CompatToken: "tok", UserID: "u1"}) + + // Writer A (its own cache) sets TranscodeStarted. + writerA := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + if err := writerA.Update(id, func(s *PlaybackSession) error { + s.TranscodeStarted = true + return nil + }); err != nil { + t.Fatalf("writerA update: %v", err) + } + + // Writer B started before A committed (its cache lacks A's field) sets a + // different field. Its DB step re-reads A's committed row FOR UPDATE and + // merges UpstreamPlayMethod on top, so A's TranscodeStarted survives. + writerB := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + if err := writerB.Update(id, func(s *PlaybackSession) error { + s.UpstreamPlayMethod = "Transcode" + return nil + }); err != nil { + t.Fatalf("writerB update: %v", err) + } + + fresh := NewDurableCompatPlaybackStore(pool, time.Hour, nil) + got, ok := fresh.Get(id) + if !ok { + t.Fatal("session missing after interleaved updates") + } + if !got.TranscodeStarted { + t.Fatalf("writerA's field was lost: %+v", got) + } + if got.UpstreamPlayMethod != "Transcode" { + t.Fatalf("writerB's field was lost: %+v", got) + } +} + +// M11: the DB expiry filter must honor the injected clock, not Postgres now(). +// A row written with a near-future expiry is visible while the fake clock is +// before it and invisible once the fake clock advances past it, even though +// Postgres wall-clock now() never moves enough to matter. +func TestDurableCompatPlaybackStore_InjectedClockExpiry(t *testing.T) { + pool := newCompatTestPool(t) + ctx := context.Background() + id := fmt.Sprintf("compat-clock-%d", time.Now().UnixNano()) + t.Cleanup(func() { _, _ = pool.Exec(ctx, `DELETE FROM jellycompat_playback_sessions WHERE id = $1`, id) }) + + base := time.Now() + fake := base + clock := func() time.Time { return fake } + + // TTL 1m; row expires at base+1m by the injected clock. + store := NewDurableCompatPlaybackStore(pool, time.Minute, clock) + store.Put(PlaybackSession{ID: id, CompatToken: "tok", UserID: "u1", RouteItemID: "r1"}) + + // Fresh instance, empty cache, same injected clock: load hits the DB and the + // row is live (fake clock still at base). + reader := NewDurableCompatPlaybackStore(pool, time.Minute, clock) + if _, ok := reader.Get(id); !ok { + t.Fatal("row should be live by injected clock before expiry") + } + + // Advance the fake clock past expiry. The DB filter uses d.now(), so a fresh + // instance must treat the row as expired even though wall-clock now() is still + // far before base+1m. + fake = base.Add(2 * time.Minute) + expired := NewDurableCompatPlaybackStore(pool, time.Minute, clock) + if _, ok := expired.Get(id); ok { + t.Fatal("row should be expired by injected clock past expiry") + } + if _, _, ok := expired.FindByRoute("tok", "r1"); ok { + t.Fatal("FindByRoute should not resolve an injected-clock-expired row") + } +} + +// With a nil pool the durable store degrades to the in-memory cache only, so it +// still satisfies the interface and basic operations work (no DB available). +func TestDurableCompatPlaybackStore_NilPoolInMemory(t *testing.T) { + store := NewDurableCompatPlaybackStore(nil, time.Hour, nil) + store.Put(PlaybackSession{ID: "x", UpstreamSessionID: "u"}) + if got, ok := store.Get("x"); !ok || got.UpstreamSessionID != "u" { + t.Fatalf("nil-pool Get failed: ok=%v got=%+v", ok, got) + } + store.Delete("x") + if _, ok := store.Get("x"); ok { + t.Fatal("nil-pool Delete failed") + } +} diff --git a/internal/jellycompat/remote_transcode_reconstruct_test.go b/internal/jellycompat/remote_transcode_reconstruct_test.go new file mode 100644 index 00000000..7ceab1d9 --- /dev/null +++ b/internal/jellycompat/remote_transcode_reconstruct_test.go @@ -0,0 +1,283 @@ +package jellycompat + +import ( + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/models" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/transcodenode" +) + +// stubRecipeNodeStore is an in-memory stand-in for the control-plane recipe +// store (*noderecipe.Store) so the round-trip tests can assert what central +// wrote and let the "node" read it back without Redis. +type stubRecipeNodeStore struct { + cards map[string]playback.RecipeCard +} + +func (s *stubRecipeNodeStore) Put(_ context.Context, sessionID string, card playback.RecipeCard) error { + if s.cards == nil { + s.cards = make(map[string]playback.RecipeCard) + } + s.cards[sessionID] = card + return nil +} + +func (s *stubRecipeNodeStore) Get(sessionID string) (playback.RecipeCard, bool) { + card, ok := s.cards[sessionID] + return card, ok +} + +func (s *stubRecipeNodeStore) Delete(_ context.Context, sessionID string) error { + delete(s.cards, sessionID) + return nil +} + +// localSessionRegistry is a GetSession + RegisterReconstructed double for +// exercising TranscodeManager reconstruction from the jellycompat package +// (the playback package's own fake is not exported across packages). +type localSessionRegistry struct { + sessions map[string]*playback.Session +} + +func (r *localSessionRegistry) GetSession(id string) (*playback.Session, error) { + if s, ok := r.sessions[id]; ok { + return s, nil + } + return nil, playback.ErrSessionNotFound +} + +func (r *localSessionRegistry) RegisterReconstructed(s *playback.Session) *playback.Session { + if r.sessions == nil { + r.sessions = map[string]*playback.Session{} + } + if existing, ok := r.sessions[s.ID]; ok { + return existing + } + r.sessions[s.ID] = s + return s +} + +func (r *localSessionRegistry) RegisterReconstructedWithLimits(_ context.Context, s *playback.Session) (*playback.Session, error) { + return r.RegisterReconstructed(s), nil +} + +// newRemoteTranscodeHandler builds a handler wired for the remote (offloaded) +// transcode path: a fake node that accepts /transcode/start, an upstream +// session carrying the native identity used to build the recipe, and a stub +// recipe store standing in for the control-plane Redis store. +func newRemoteTranscodeHandler(t *testing.T, nodeURL string, recipeStore *stubRecipeNodeStore) (*PlaybackHandler, *testCompatSessionManager, *PlaybackSessionStore) { + t.Helper() + + playbackStore := NewPlaybackSessionStore(time.Hour, nil) + sessionMgr := &testCompatSessionManager{ + sessions: map[string]*playback.Session{ + "upstream-1": { + ID: "upstream-1", + UserID: 7, + ProfileID: "profile-1", + MediaFileID: 42, + PlayMethod: playback.PlayTranscode, + BasePlayMethod: playback.PlayTranscode, + TranscodeNodeURL: nodeURL, + }, + }, + } + handler := &PlaybackHandler{ + playbackStore: playbackStore, + sessionMgr: sessionMgr, + fileResolver: testCompatFileResolver{file: &models.MediaFile{ID: 42, FilePath: "/media/movie.mkv"}}, + tm: playback.NewTranscodeManager(), + JWTSecret: "test-secret", + RecipeNodeStore: recipeStore, + } + return handler, sessionMgr, playbackStore +} + +// fakeTranscodeNode returns an httptest server that accepts /transcode/start +// with 202 Accepted (the only response the start path inspects) and records +// the request body it received. +func fakeTranscodeNode(t *testing.T, received *transcodenode.TranscodeStartRequest) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/transcode/start" { + w.WriteHeader(http.StatusNotFound) + return + } + if received != nil { + body, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(body, received) + } + w.WriteHeader(http.StatusAccepted) + })) + t.Cleanup(srv.Close) + return srv +} + +func testRemoteTranscodeSource() PlaybackMediaSource { + codec := NewResourceIDCodec() + version := testCompatVersion() + source := testCompatSource(codec, version) + return source +} + +// TestStartRemoteTranscode_NodeRestartReconstruct covers the node-restart leg: +// central persists the recipe to the control-plane store at start, the node +// loses all in-memory state, and a reconstruct fetch via the store's Get +// rebuilds the same recipe the node would re-spawn ffmpeg from (rather than a +// Redis miss → 404). +func TestStartRemoteTranscode_NodeRestartReconstruct(t *testing.T) { + recipeStore := &stubRecipeNodeStore{} + node := fakeTranscodeNode(t, nil) + handler, _, playbackStore := newRemoteTranscodeHandler(t, node.URL, recipeStore) + + playbackStore.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + UpstreamSessionID: "upstream-1", + UpstreamPlayMethod: "transcode", + MediaSources: []PlaybackMediaSource{testRemoteTranscodeSource()}, + }) + + source := testRemoteTranscodeSource() + err := handler.startRemoteTranscode( + context.Background(), + "play-1", + "upstream-1", + source, + &models.MediaFile{ID: 42, FilePath: "/media/movie.mkv"}, + 0, + node.URL, + ) + if err != nil { + t.Fatalf("startRemoteTranscode: %v", err) + } + + // Node side: it lost its session map; reconstruct reads the recipe central + // wrote keyed by the upstream session id. + card, ok := recipeStore.Get("upstream-1") + if !ok { + t.Fatal("expected recipe in control-plane store after remote start") + } + if card.SessionID != "upstream-1" { + t.Fatalf("recipe SessionID = %q, want upstream-1", card.SessionID) + } + if card.TranscodeNodeURL != node.URL { + t.Fatalf("recipe TranscodeNodeURL = %q, want %q", card.TranscodeNodeURL, node.URL) + } + // The recipe must be reconstruct-complete: the node refuses to rebuild from a + // recipe lacking encode parameters (server.go reconstructFromToken). + if card.SegmentDuration <= 0 || card.TargetCodecVideo == "" { + t.Fatalf("recipe not reconstruct-complete: seg=%d video=%q", card.SegmentDuration, card.TargetCodecVideo) + } + if card.UserID != 7 || card.ProfileID != "profile-1" || card.MediaFileID != 42 { + t.Fatalf("recipe identity wrong: user=%d profile=%q file=%d", card.UserID, card.ProfileID, card.MediaFileID) + } + if card.InputPath != "/media/movie.mkv" { + t.Fatalf("recipe InputPath = %q, want /media/movie.mkv", card.InputPath) + } +} + +// TestStartRemoteTranscode_CentralRestartReconstruct covers the central-restart +// leg: the recipe lives in the compat store (PlaybackSession.Recipe), the +// in-memory native session is gone, and LoadOrReconstructSession rebuilds the +// session from the stored recipe rather than returning SessionMissing (which a +// remote segment serve renders as a 404). +func TestStartRemoteTranscode_CentralRestartReconstruct(t *testing.T) { + recipeStore := &stubRecipeNodeStore{} + node := fakeTranscodeNode(t, nil) + handler, _, playbackStore := newRemoteTranscodeHandler(t, node.URL, recipeStore) + + playbackStore.Put(PlaybackSession{ + ID: "play-1", + CompatToken: "token-1", + UpstreamSessionID: "upstream-1", + UpstreamPlayMethod: "transcode", + MediaSources: []PlaybackMediaSource{testRemoteTranscodeSource()}, + }) + + source := testRemoteTranscodeSource() + if err := handler.startRemoteTranscode(context.Background(), "play-1", "upstream-1", source, &models.MediaFile{ID: 42, FilePath: "/media/movie.mkv"}, 0, node.URL); err != nil { + t.Fatalf("startRemoteTranscode: %v", err) + } + + // The compat store must now carry the recipe so central can reconstruct. + stored, ok := playbackStore.Get("play-1") + if !ok { + t.Fatal("expected compat session") + } + if !stored.TranscodeStarted { + t.Fatal("expected TranscodeStarted=true after remote start") + } + if stored.Recipe == nil { + t.Fatal("expected PlaybackSession.Recipe persisted after remote start") + } + + // Simulate a central restart: the in-memory native session map is empty, so + // GetSession misses. The recipe from the compat store must let + // LoadOrReconstructSession rebuild the session rather than 404. + reg := &localSessionRegistry{} + tm := playback.NewTranscodeManager() + tm.Sessions = reg + tm.JWTSecretFn = func() string { return "test-secret" } + + session, status := tm.LoadOrReconstructSession( + context.Background(), + reg.GetSession, + "upstream-1", + stored.Recipe.UserID, + stored.Recipe, + ) + if status != playback.SessionLoaded || session == nil { + t.Fatalf("LoadOrReconstructSession status=%v session=%v, want reconstructed", status, session) + } + if session.ID != "upstream-1" || session.UserID != 7 || session.MediaFileID != 42 { + t.Fatalf("reconstructed session wrong: %+v", session) + } + if session.TranscodeNodeURL != node.URL { + t.Fatalf("reconstructed TranscodeNodeURL = %q, want %q", session.TranscodeNodeURL, node.URL) + } +} + +// TestStartRemoteTranscode_UpdateFailureRollsBackNode asserts the local-path +// rollback is mirrored: a compat-store Update failure closes the already-started +// node ffmpeg (DELETE /transcode/{id}) so it isn't leaked. +func TestStartRemoteTranscode_UpdateFailureRollsBackNode(t *testing.T) { + recipeStore := &stubRecipeNodeStore{} + deleted := make(chan string, 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodPost && r.URL.Path == "/transcode/start": + w.WriteHeader(http.StatusAccepted) + case r.Method == http.MethodDelete: + deleted <- r.URL.Path + w.WriteHeader(http.StatusOK) + default: + w.WriteHeader(http.StatusNotFound) + } + })) + t.Cleanup(srv.Close) + + handler, _, _ := newRemoteTranscodeHandler(t, srv.URL, recipeStore) + // No play session put → Update("missing") fails, triggering rollback. + source := testRemoteTranscodeSource() + err := handler.startRemoteTranscode(context.Background(), "missing", "upstream-1", source, &models.MediaFile{ID: 42, FilePath: "/media/movie.mkv"}, 0, srv.URL) + if err == nil { + t.Fatal("expected error when compat Update fails") + } + + select { + case path := <-deleted: + if path != "/transcode/upstream-1" { + t.Fatalf("rollback DELETE path = %q, want /transcode/upstream-1", path) + } + case <-time.After(2 * time.Second): + t.Fatal("expected rollback DELETE to the node after Update failure") + } +} diff --git a/internal/jellycompat/router.go b/internal/jellycompat/router.go index f3875a49..de3718f2 100644 --- a/internal/jellycompat/router.go +++ b/internal/jellycompat/router.go @@ -15,6 +15,7 @@ import ( "github.com/Silo-Server/silo-server/internal/catalog" "github.com/Silo-Server/silo-server/internal/clientip" + "github.com/Silo-Server/silo-server/internal/playback" "github.com/Silo-Server/silo-server/internal/recommendations" "github.com/Silo-Server/silo-server/internal/sections" "github.com/Silo-Server/silo-server/internal/subtitles" @@ -107,8 +108,17 @@ func NewRouter(deps Dependencies) chi.Router { } playbackHandler.NodePlanner = deps.NodePlanner playbackHandler.JWTSecret = deps.JWTSecret + // Compat transcode reconstruct is driven by the recipe carried in the durable + // compat playback store (jellycompat_playback_sessions); no separate native + // recipe table is needed. + if cleaned, err := playbackHandler.CleanupOrphanedTranscodes(); err != nil { + slog.Warn("jellycompat transcode cleanup failed", "dir", playbackHandler.TranscodeDir, "error", err) + } else if cleaned > 0 { + slog.Info("jellycompat transcode cleanup removed orphaned dirs", "dir", playbackHandler.TranscodeDir, "count", cleaned) + } playbackHandler.profileRefreshRequester = deps.RecWorker playbackHandler.SettingsRepo = deps.SettingsRepo + playbackHandler.RecipeNodeStore = deps.RecipeNodeStore playbackHandler.SessionSyncer = deps.SessionSyncer if subtitleRepo != nil { playbackHandler.SubtitleRepo = subtitleRepo @@ -296,7 +306,13 @@ func withDefaults(deps Dependencies) Dependencies { } deps.ImageCache = NewImageCache(cacheTTL, deps.Now) } - playbackTTL := 6 * time.Hour + // Align the compat playback session's absolute lifetime with the absolute + // stream-token TTL (playback.MaxTokenTTL, 24h). This is an ABSOLUTE window + // from creation, not sliding/idle: the session need not outlive its token + // while token re-mint is unimplemented, and at 6h long content (audiobooks, + // movies) and paused-overnight sessions expired mid-playback even though the + // stream token was still valid. Overridable per-deployment via config. + playbackTTL := playback.MaxTokenTTL if deps.Config != nil && deps.Config.JellyfinCompat.PlaybackSessionTTL > 0 { playbackTTL = deps.Config.JellyfinCompat.PlaybackSessionTTL } @@ -304,7 +320,14 @@ func withDefaults(deps Dependencies) Dependencies { deps.DeviceProfiles = NewDeviceProfileStore(playbackTTL, deps.Now) } if deps.PlaybackStore == nil { - deps.PlaybackStore = NewPlaybackSessionStore(playbackTTL, deps.Now) + // Back the compat playback store with Postgres when a pool is available so + // the PlaySessionId -> upstream-session mapping survives a restart and a + // Jellyfin client can resume; fall back to in-memory otherwise. + if deps.DB != nil { + deps.PlaybackStore = NewDurableCompatPlaybackStore(deps.DB, playbackTTL, deps.Now) + } else { + deps.PlaybackStore = NewPlaybackSessionStore(playbackTTL, deps.Now) + } } if deps.HTTPClient == nil { deps.HTTPClient = &http.Client{Timeout: 30 * time.Second} diff --git a/internal/jellycompat/server.go b/internal/jellycompat/server.go index 67020bce..0e70a0ca 100644 --- a/internal/jellycompat/server.go +++ b/internal/jellycompat/server.go @@ -37,10 +37,14 @@ type Dependencies struct { IDCodec *ResourceIDCodec ImageCache *ImageCache DeviceProfiles *DeviceProfileStore - PlaybackStore *PlaybackSessionStore - LoginResolver loginResolver - Authenticator *Authenticator - WebFS fs.FS + PlaybackStore CompatPlaybackStore + // RecipeNodeStore hands remote-transcode reconstruction recipes to the + // control-plane recipe store (Redis) so a restarted transcode node can rebuild + // a jellycompat session. Optional; nil disables the handoff. + RecipeNodeStore recipeNodePutter + LoginResolver loginResolver + Authenticator *Authenticator + WebFS fs.FS // FrontendFS is the embedded Silo frontend asset filesystem (web/dist), // used to serve app-relative artwork such as bundled collection-template // posters that have no remote origin. Optional. @@ -163,7 +167,7 @@ func (s *Server) SessionStore() *SessionStore { func (s *Server) StartBackgroundTasks(ctx context.Context) { if s.deps.DB != nil { repo := NewSessionRepository(s.deps.DB, s.deps.SecretCipher) - StartSessionCleanup(ctx, repo, 1*time.Hour) + StartSessionCleanupWithPlaybackStore(ctx, repo, s.deps.PlaybackStore, 1*time.Hour) } } diff --git a/internal/jellycompat/streams.go b/internal/jellycompat/streams.go index 0139432a..1a3d7ce3 100644 --- a/internal/jellycompat/streams.go +++ b/internal/jellycompat/streams.go @@ -239,7 +239,7 @@ func (h *PlaybackHandler) HandleMasterManifest(w http.ResponseWriter, r *http.Re writeError(w, http.StatusInternalServerError, "ServerError", "Failed to bind transcode node") return } - if err := h.startRemoteTranscode(r.Context(), playSession.UpstreamSessionID, *source, file, playSession.InitialSeekSeconds, tcNode.URL); err != nil { + if err := h.startRemoteTranscode(r.Context(), playSession.ID, playSession.UpstreamSessionID, *source, file, playSession.InitialSeekSeconds, tcNode.URL); err != nil { if errors.Is(err, errTranscode4KDisallowed) { writeError(w, http.StatusForbidden, "Forbidden", "4K video transcoding is disabled on this server") return @@ -363,17 +363,39 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques name := chiURLParam(r, "segmentId") ext := chiURLParam(r, "segmentContainer") - upstreamSession, err := h.sessionMgr.GetSession(playSession.UpstreamSessionID) - if err != nil { + // Load the upstream native session, reconstructing it from the compat-stored + // recipe on a not-found miss (e.g. after a server restart). Ownership is + // re-bound to the Jellyfin caller's native user id (StreamAppUserID), matching + // the recipe owner. + upstreamSession, status := h.tm.LoadOrReconstructSession(r.Context(), h.sessionMgr.GetSession, playSession.UpstreamSessionID, session.StreamAppUserID, playSession.Recipe) + switch status { + case playback.SessionMissing: writeError(w, http.StatusNotFound, "NotFound", "Upstream session not found") return - } - _ = upstreamSession // session exists, serve the segment - - transcodeSession := h.getTranscodeSession(playSession.UpstreamSessionID) - if transcodeSession == nil { - writeError(w, http.StatusNotFound, "NotFound", "Transcode session not found") + case playback.SessionLoadFailed: + writeError(w, http.StatusInternalServerError, "ServerError", "Failed to load upstream session") return + case playback.SessionForbidden: + writeError(w, http.StatusForbidden, "Forbidden", "Session belongs to another user") + return + } + + transcodeSession := h.tm.GetTranscodeSession(playSession.UpstreamSessionID) + if transcodeSession == nil { + // Local transcode whose process state was lost (restart): reconstruct it + // seeked to the requested segment. Remote-node sessions are served by the + // proxy, not here, so only reconstruct an integrated (no node URL) session. + if upstreamSession.TranscodeNodeURL == "" && playSession.Recipe != nil { + requestedSegment := -1 + if segNum, parseErr := playback.ParseSegmentNumber(name); parseErr == nil { + requestedSegment = segNum + } + transcodeSession = h.tm.ReconstructTranscode(r.Context(), playSession.UpstreamSessionID, requestedSegment, *playSession.Recipe) + } + if transcodeSession == nil { + writeError(w, http.StatusNotFound, "NotFound", "Transcode session not found") + return + } } segmentFile := name + "." + ext @@ -444,6 +466,16 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques ) } + // Copy-mode with an unresolved seek target (ok=false, no error) + // means the manifest can't place this segment yet. Don't restart + // at a fabricated position; surface ErrSegmentNotFound so the + // client retries while the session keeps producing manifest. + // Mirrors the transcode-node guard in + // internal/transcodenode/server.go. + if !ok && seekErr == nil && transcodeSession.IsCopyVideo() { + err = playback.ErrSegmentNotFound + } + if ok { slog.Info("transcode seek restart", "segment", segmentFile, @@ -460,8 +492,10 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques "playback_session_id", playSession.UpstreamSessionID, ) - if restartErr := transcodeSession.Restart( + if restartErr := h.tm.RestartSessionLocked( context.WithoutCancel(r.Context()), + playSession.UpstreamSessionID, + transcodeSession, seekSeconds, segNum, ); restartErr == nil { @@ -760,10 +794,21 @@ func (h *PlaybackHandler) teardownPlaySession(ctx context.Context, playSession * transcodeNodeURL = upstreamSession.TranscodeNodeURL } } - h.closeTranscodeSession(playSession.UpstreamSessionID, transcodeNodeURL) + h.tm.CloseTranscodeSession(playSession.UpstreamSessionID, transcodeNodeURL) if h.sessionMgr != nil { _ = h.sessionMgr.StopSession(playSession.UpstreamSessionID) } + // Deliberate stop: drop the node recipe so a buffered/retrying request after + // a node restart cannot reconstruct a fresh ffmpeg for this stopped session. + // Best effort and bounded — never fail teardown on a recipe-store hiccup. + if h.RecipeNodeStore != nil { + delCtx, cancel := context.WithTimeout(context.WithoutCancel(context.Background()), 2*time.Second) + defer cancel() + if err := h.RecipeNodeStore.Delete(delCtx, playSession.UpstreamSessionID); err != nil { + slog.Warn("delete node transcode recipe failed", "error", err, + "playback_session_id", playSession.UpstreamSessionID) + } + } h.playbackStore.Delete(playSession.ID) // Clients often drop the connection right after reporting a stop, so detach // the sync from request cancellation to keep the admin view accurate. @@ -923,6 +968,20 @@ func (h *PlaybackHandler) handlePlaybackReport(w http.ResponseWriter, r *http.Re w.WriteHeader(http.StatusNoContent) } +// upstreamRecipeCard returns the reconstruction recipe for a compat upstream +// session. A transcode carries its full recipe in the compat store +// (PlaybackSession.Recipe); direct/remux need only identity, rebuilt here from +// the compat session and the negotiated source. +func (h *PlaybackHandler) upstreamRecipeCard(ps *PlaybackSession, cs *Session, source PlaybackMediaSource, method string) playback.RecipeCard { + if ps != nil && ps.Recipe != nil { + return *ps.Recipe + } + if method == "remux" { + return playback.NewRemuxRecipeCard(ps.UpstreamSessionID, cs.StreamAppUserID, cs.ProfileID, source.FileID, source.TranscodeAudio, compatAudioTrackIndexOrDefault(source)) + } + return playback.NewDirectRecipeCard(ps.UpstreamSessionID, cs.StreamAppUserID, cs.ProfileID, source.FileID) +} + // reportMatchesPlaySession rejects an alias-resolved session whose item or // media source contradicts the report, so a stale or reused client id cannot // route a report (or its teardown) to the wrong play. @@ -967,26 +1026,41 @@ func (h *PlaybackHandler) ensureUpstreamPlayback(ctx context.Context, compatSess if !ok { return nil, ErrSessionNotFound } + // Captured before any mutation: the CAS attach below verifies no concurrent + // request replaced the upstream session this request observed. observedUpstreamID := playSession.UpstreamSessionID + if h.sessionMgr == nil { + return nil, fmt.Errorf("session manager not available") + } if playSession.UpstreamSessionID != "" && playSession.UpstreamPlayMethod == method { - upstreamLive := true - if h.sessionMgr != nil { - _, err := h.sessionMgr.GetSession(playSession.UpstreamSessionID) - upstreamLive = err == nil - } - if upstreamLive { + // After a restart the durable play session survives but the in-memory + // native session is gone; rebuild it from the recipe card so ownership and + // accounting are restored before the transcode is (re)started. + if _, err := h.sessionMgr.GetSession(playSession.UpstreamSessionID); err != nil { + if !errors.Is(err, playback.ErrSessionNotFound) { + return nil, err + } + if h.tm != nil { + if reconstructed := h.tm.ReconstructSession(ctx, playSession.UpstreamSessionID, compatSession.StreamAppUserID, h.upstreamRecipeCard(playSession, compatSession, source, method)); reconstructed != nil { + _ = h.syncUpstreamAudioSelection(playSession, source) + return playSession, nil + } + } + // The durable compat row outlived the native session and no recipe card + // can rebuild it. Any transcode still keyed to the stale id must go + // first, or a second ffmpeg would start alongside it. Then fall through + // to create a fresh upstream session and persist the replacement + // instead of serving under a stale ID. + if h.tm != nil { + h.tm.CloseTranscodeSession(playSession.UpstreamSessionID, "") + } + playSession.UpstreamSessionID = "" + playSession.UpstreamPlayMethod = "" + playSession.TranscodeStarted = false + } else { _ = h.syncUpstreamAudioSelection(playSession, source) return playSession, nil } - // The upstream session was reaped as stale while the client kept - // playing; recreate it under the same play session. Any transcode - // still keyed to the stale id must go first, or a second ffmpeg - // would start alongside it. - h.closeTranscodeSession(playSession.UpstreamSessionID, "") - } - - if h.sessionMgr == nil { - return nil, fmt.Errorf("session manager not available") } var playMethod playback.PlayMethod @@ -1006,12 +1080,24 @@ func (h *PlaybackHandler) ensureUpstreamPlayback(ctx context.Context, compatSess } if playSession.UpstreamSessionID != "" && playSession.UpstreamPlayMethod != "" && playSession.UpstreamPlayMethod != method { + oldUpstreamSessionID := playSession.UpstreamSessionID transcodeNodeURL := "" - if current, err := h.sessionMgr.GetSession(playSession.UpstreamSessionID); err == nil { + if current, err := h.sessionMgr.GetSession(oldUpstreamSessionID); err == nil { transcodeNodeURL = current.TranscodeNodeURL } - _ = h.sessionMgr.StopSession(playSession.UpstreamSessionID) - h.closeTranscodeSession(playSession.UpstreamSessionID, transcodeNodeURL) + _ = h.sessionMgr.StopSession(oldUpstreamSessionID) + h.tm.CloseTranscodeSession(oldUpstreamSessionID, transcodeNodeURL) + // Method switch discards the old upstream session: drop its node recipe so + // the abandoned id cannot reconstruct ffmpeg after a node restart. Best + // effort and bounded — never block the new method's start on a store hiccup. + if h.RecipeNodeStore != nil { + delCtx, cancel := context.WithTimeout(context.WithoutCancel(context.Background()), 2*time.Second) + if err := h.RecipeNodeStore.Delete(delCtx, oldUpstreamSessionID); err != nil { + slog.Warn("delete node transcode recipe failed", "error", err, + "playback_session_id", oldUpstreamSessionID) + } + cancel() + } } var session *playback.Session @@ -1104,9 +1190,20 @@ func (h *PlaybackHandler) ensureTranscodeManifest(ctx context.Context, compatSes } func (h *PlaybackHandler) ensureTranscodeSession(ctx context.Context, playSessionID, upstreamSessionID string, source PlaybackMediaSource) (*playback.TranscodeSession, error) { - if existing := h.getTranscodeSession(upstreamSessionID); existing != nil { + if existing := h.tm.GetTranscodeSession(upstreamSessionID); existing != nil { return existing, nil } + // If a recipe survived in the compat store (e.g. a server restart), rebuild + // the transcode from it — at the recipe's position — rather than starting + // fresh at the original seek. On a first play there is no recipe yet, so this + // is a no-op and we fall through to the normal start below. + if h.playbackStore != nil { + if ps, ok := h.playbackStore.Get(playSessionID); ok && ps.Recipe != nil { + if reconstructed := h.tm.ReconstructTranscode(ctx, upstreamSessionID, -1, *ps.Recipe); reconstructed != nil { + return reconstructed, nil + } + } + } if !source.TranscodeAudio && is4KResolution(source.Version.Resolution) && !h.allow4KVideoTranscode(ctx) { return nil, errTranscode4KDisallowed } @@ -1122,12 +1219,6 @@ func (h *PlaybackHandler) ensureTranscodeSession(ctx context.Context, playSessio return nil, fmt.Errorf("prepare transcode dir: %w", err) } - h.transcodeMu.Lock() - if existing := h.transcodes[upstreamSessionID]; existing != nil { - h.transcodeMu.Unlock() - return existing, nil - } - initialSeekSeconds := 0.0 startSegmentNumber := 0 if playSession, ok := h.playbackStore.Get(playSessionID); ok { @@ -1156,26 +1247,33 @@ func (h *PlaybackHandler) ensureTranscodeSession(ctx context.Context, playSessio } opts.SegmentDuration = h.compatSegmentDuration() + // Hold the per-session lifecycle lock across "check existing → spawn → + // register" so a concurrent reconstruct (or another manifest request) cannot + // run a second ffmpeg writer against this session's output dir. Re-check under + // the lock and yield to any live session instead of spawning a duplicate. + unlock := h.tm.LockSessionLifecycle(upstreamSessionID) + if existing := h.tm.GetTranscodeSession(upstreamSessionID); existing != nil { + unlock() + return existing, nil + } transcodeSession, err := playback.StartTranscode(context.WithoutCancel(ctx), opts) if err != nil { - h.transcodeMu.Unlock() + unlock() return nil, err } - h.transcodes[upstreamSessionID] = transcodeSession - h.transcodeMu.Unlock() + // Safe under the lifecycle lock: the re-check above held, so no other path + // registered this session. + h.tm.RegisterTranscodeSession(upstreamSessionID, transcodeSession) + unlock() - if err := h.playbackStore.Update(playSessionID, func(current *PlaybackSession) error { - current.TranscodeStarted = true - return nil - }); err != nil { - closeErr := transcodeSession.Close() - h.transcodeMu.Lock() - delete(h.transcodes, upstreamSessionID) - h.transcodeMu.Unlock() - if closeErr != nil { - return nil, fmt.Errorf("update playback session: %w (cleanup: %v)", err, closeErr) - } - return nil, fmt.Errorf("update playback session: %w", err) + // Register the exit monitor and persist the reconstruction recipe (shared with + // the remote path). On a failed compat-store write roll back this abandoned + // transcode rather than leaking it. + h.tm.MonitorLocalTranscodeExit(upstreamSessionID, transcodeSession) + + if err := h.persistTranscodeRecipe(ctx, playSessionID, upstreamSessionID, opts); err != nil { + h.tm.CloseTranscodeSession(upstreamSessionID, "") + return nil, err } return transcodeSession, nil @@ -1266,15 +1364,25 @@ func (h *PlaybackHandler) restartCompatTranscodeForAudioSelection( return false, nil } - if transcodeSession := h.getTranscodeSession(playSession.UpstreamSessionID); transcodeSession != nil { + if transcodeSession := h.tm.GetTranscodeSession(playSession.UpstreamSessionID); transcodeSession != nil { transcodeSession.SetAudioTrackIndex(audioTrackIndex) startSegment := 0 if segmentDuration := transcodeSession.Opts().SegmentDuration; segmentDuration > 0 && positionSeconds > 0 { startSegment = int(positionSeconds / float64(segmentDuration)) } - if err := transcodeSession.Restart(context.WithoutCancel(ctx), positionSeconds, startSegment); err != nil { + if err := h.tm.RestartSessionLocked(context.WithoutCancel(ctx), playSession.UpstreamSessionID, transcodeSession, positionSeconds, startSegment); err != nil { return false, err } + // Re-persist the durable recipe so reconstruct after a central restart + // rebuilds ffmpeg from the newly selected audio track rather than the + // stale original. SetAudioTrackIndex mutated the live opts, so read them + // back. Best-effort: a stale recipe only costs node-restart resilience, + // not the live stream. + opts := transcodeSession.Opts() + if err := h.persistTranscodeRecipe(context.WithoutCancel(ctx), playSession.ID, playSession.UpstreamSessionID, opts); err != nil { + slog.Warn("persist audio-restarted transcode recipe", "error", err, + "playback_session_id", playSession.ID) + } return true, nil } @@ -1295,7 +1403,7 @@ func (h *PlaybackHandler) restartCompatTranscodeForAudioSelection( if err != nil { return false, err } - if err := h.startRemoteTranscode(context.WithoutCancel(ctx), playSession.UpstreamSessionID, source, file, positionSeconds, upstreamSession.TranscodeNodeURL); err != nil { + if err := h.startRemoteTranscode(context.WithoutCancel(ctx), playSession.ID, playSession.UpstreamSessionID, source, file, positionSeconds, upstreamSession.TranscodeNodeURL); err != nil { return false, err } return true, nil @@ -1479,35 +1587,6 @@ func chiURLParam(r *http.Request, key string) string { return chi.URLParam(r, key) } -func (h *PlaybackHandler) getTranscodeSession(sessionID string) *playback.TranscodeSession { - h.transcodeMu.RLock() - defer h.transcodeMu.RUnlock() - return h.transcodes[sessionID] -} - -func (h *PlaybackHandler) closeTranscodeSession(sessionID, transcodeNodeURL string) { - h.transcodeMu.Lock() - session := h.transcodes[sessionID] - delete(h.transcodes, sessionID) - h.transcodeMu.Unlock() - - if session != nil { - _ = session.Close() - } - if transcodeNodeURL != "" && h.JWTSecret != "" { - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - - req, err := http.NewRequestWithContext(ctx, http.MethodDelete, transcodeNodeURL+"/transcode/"+sessionID, nil) - if err == nil { - req.Header.Set("Authorization", "Bearer "+h.JWTSecret) - if resp, doErr := http.DefaultClient.Do(req); doErr == nil { - resp.Body.Close() - } - } - } -} - func seekSecondsFromTicks(seekStr string) float64 { if seekStr == "" { return 0 diff --git a/internal/jellycompat/streams_test.go b/internal/jellycompat/streams_test.go index c7eb31b9..8ce3cfd4 100644 --- a/internal/jellycompat/streams_test.go +++ b/internal/jellycompat/streams_test.go @@ -4,10 +4,13 @@ import ( "context" "net/http" "net/http/httptest" + "os" + "path/filepath" "strings" "testing" "time" + "github.com/Silo-Server/silo-server/internal/models" "github.com/Silo-Server/silo-server/internal/playback" ) @@ -103,16 +106,55 @@ func TestRewriteManifest_PreservesPlaybackAndMediaSourceIDs(t *testing.T) { } } +func TestEnsureUpstreamPlayback_ReplacesStaleUpstreamWhenRecipeMissing(t *testing.T) { + store := NewPlaybackSessionStore(time.Hour, nil) + store.Put(PlaybackSession{ + ID: "ps-1", + CompatToken: "tok", + UpstreamSessionID: "stale-upstream", + UpstreamPlayMethod: "direct", + }) + mgr := &testCompatSessionManager{sessions: map[string]*playback.Session{}} + h := &PlaybackHandler{ + playbackStore: store, + sessionMgr: mgr, + tm: playback.NewTranscodeManager(), + } + + got, err := h.ensureUpstreamPlayback( + context.Background(), + &Session{Token: "tok", StreamAppUserID: 7, ProfileID: "profile-1"}, + "ps-1", + PlaybackMediaSource{FileID: 42}, + "direct", + ) + if err != nil { + t.Fatalf("ensureUpstreamPlayback returned error: %v", err) + } + if got.UpstreamSessionID != "upstream-started" { + t.Fatalf("UpstreamSessionID = %q, want fresh upstream session", got.UpstreamSessionID) + } + if mgr.startCalls != 1 { + t.Fatalf("StartSession calls = %d, want 1", mgr.startCalls) + } + reloaded, ok := store.Get("ps-1") + if !ok { + t.Fatal("play session missing after upstream replacement") + } + if reloaded.UpstreamSessionID != "upstream-started" || reloaded.UpstreamPlayMethod != "direct" { + t.Fatalf("store not updated with fresh upstream session: %+v", reloaded) + } +} + // newActiveEncodingsHandler builds a PlaybackHandler literal directly (not -// NewPlaybackHandler, which touches the filesystem) with the transcodes map -// initialized — closeTranscodeSession writes/deletes it and would nil-map-panic -// otherwise. +// NewPlaybackHandler, which touches the filesystem) with a transcode manager +// wired — teardown calls tm.CloseTranscodeSession and would nil-panic otherwise. func newActiveEncodingsHandler(mgr *testCompatSessionManager) (*PlaybackHandler, *PlaybackSessionStore) { store := NewPlaybackSessionStore(time.Hour, nil) h := &PlaybackHandler{ playbackStore: store, sessionMgr: mgr, - transcodes: make(map[string]*playback.TranscodeSession), + tm: playback.NewTranscodeManager(), } return h, store } @@ -140,6 +182,32 @@ func TestHandleDeleteActiveEncodings_StopsTranscodeAndDeletesSession(t *testing. } } +// TestTeardownPlaySession_DeletesNodeRecipe verifies the deliberate stop path +// drops the node recipe keyed by the upstream session id, so a buffered/retrying +// request after a node restart cannot resurrect ffmpeg for the stopped session. +func TestTeardownPlaySession_DeletesNodeRecipe(t *testing.T) { + mgr := &testCompatSessionManager{sessions: map[string]*playback.Session{"upstream-1": {ID: "upstream-1"}}} + h, store := newActiveEncodingsHandler(mgr) + recipeStore := &stubRecipeNodeStore{cards: map[string]playback.RecipeCard{ + "upstream-1": {SessionID: "upstream-1"}, + }} + h.RecipeNodeStore = recipeStore + store.Put(PlaybackSession{ID: "ps-1", UpstreamSessionID: "upstream-1", CompatToken: "tok"}) + + playSession, ok := store.Get("ps-1") + if !ok { + t.Fatal("expected play session") + } + h.teardownPlaySession(context.Background(), playSession) + + if _, ok := recipeStore.Get("upstream-1"); ok { + t.Fatal("node recipe should be deleted on deliberate teardown") + } + if len(mgr.stopCalls) != 1 || mgr.stopCalls[0] != "upstream-1" { + t.Fatalf("expected StopSession(upstream-1); got %v", mgr.stopCalls) + } +} + // TestHandleDeleteActiveEncodings_MissingPlaySessionIdReturns204 verifies a // request with no PlaySessionId is a 204 no-op (no teardown). func TestHandleDeleteActiveEncodings_MissingPlaySessionIdReturns204(t *testing.T) { @@ -273,6 +341,115 @@ func TestHandleDeleteActiveEncodings_NotYetStartedNotTornDown(t *testing.T) { } } +// TestRestartCompatTranscodeForAudioSelection_LocalRePersistsRecipe covers the +// integrated/single-box leg of an audio switch: the live ffmpeg is restarted on +// the new track, and the durable PlaybackSession.Recipe must be re-persisted so +// that a reconstruct after a central restart rebuilds ffmpeg from the NEWLY +// selected audio track rather than the stale original. Without the re-persist, +// Recipe.AudioTrackIndex keeps the original value and the stream silently +// resumes on the wrong language after a restart. +func TestRestartCompatTranscodeForAudioSelection_LocalRePersistsRecipe(t *testing.T) { + codec := NewResourceIDCodec() + version := testCompatVersion() // 1 video track, 2 audio tracks. + + // Initial source selects the first (main) audio track -> AudioTrackIndex 0. + mainSource := testCompatSource(codec, version) + mainSource.SelectedAudioStreamIndex = intPtr(len(version.VideoTracks)) // stream index 1 -> track 0. + + // Switch target selects the second (commentary) audio track -> AudioTrackIndex 1. + commentarySource := testCompatSource(codec, version) + commentarySource.SelectedAudioStreamIndex = intPtr(len(version.VideoTracks) + 1) // stream index 2 -> track 1. + + filePath := filepath.Join(t.TempDir(), "movie.mkv") + if err := os.WriteFile(filePath, []byte("video"), 0o644); err != nil { + t.Fatalf("write media file: %v", err) + } + + playbackStore := NewPlaybackSessionStore(time.Hour, nil) + playbackStore.Put(PlaybackSession{ + ID: "play-1", + UpstreamSessionID: "upstream-1", + UpstreamPlayMethod: "transcode", + MediaSources: []PlaybackMediaSource{commentarySource}, + }) + + sessionMgr := &testCompatSessionManager{ + sessions: map[string]*playback.Session{ + "upstream-1": { + ID: "upstream-1", + UserID: 7, + ProfileID: "profile-1", + MediaFileID: version.FileID, + PlayMethod: playback.PlayTranscode, + BasePlayMethod: playback.PlayTranscode, + }, + }, + } + + handler := &PlaybackHandler{ + playbackStore: playbackStore, + sessionMgr: sessionMgr, + fileResolver: testCompatFileResolver{file: &models.MediaFile{ID: version.FileID, FilePath: filePath}}, + TranscodeDir: t.TempDir(), + FFmpegPath: writeCompatTestFFmpeg(t), + tm: playback.NewTranscodeManager(), + } + + // Start the live transcode on the main track and persist its initial recipe + // (AudioTrackIndex 0), mirroring a normal play start. + transcodeSession, err := handler.ensureTranscodeSession(context.Background(), "play-1", "upstream-1", mainSource) + if err != nil { + t.Fatalf("ensureTranscodeSession: %v", err) + } + t.Cleanup(func() { _ = transcodeSession.Close() }) + + if got := transcodeSession.Opts().AudioTrackIndex; got != 0 { + t.Fatalf("initial AudioTrackIndex = %d, want 0", got) + } + if initial, ok := playbackStore.Get("play-1"); !ok || initial.Recipe == nil { + t.Fatal("expected initial recipe persisted after ensureTranscodeSession") + } else if initial.Recipe.AudioTrackIndex != 0 { + t.Fatalf("initial Recipe.AudioTrackIndex = %d, want 0", initial.Recipe.AudioTrackIndex) + } + + playSession, ok := playbackStore.Get("play-1") + if !ok { + t.Fatal("expected play session") + } + + // Switch audio to the commentary track via the LOCAL branch. + restarted, err := handler.restartCompatTranscodeForAudioSelection( + context.Background(), + playSession, + commentarySource, + 0, + ) + if err != nil { + t.Fatalf("restartCompatTranscodeForAudioSelection: %v", err) + } + if !restarted { + t.Fatal("expected local transcode restart to report restarted=true") + } + + // The live ffmpeg opts must reflect the new track... + if got := transcodeSession.Opts().AudioTrackIndex; got != 1 { + t.Fatalf("live AudioTrackIndex after switch = %d, want 1", got) + } + + // ...and, crucially, the durable recipe must track it so a reconstruct after + // a central restart rebuilds ffmpeg on the commentary track. + updated, ok := playbackStore.Get("play-1") + if !ok { + t.Fatal("expected play session after audio switch") + } + if updated.Recipe == nil { + t.Fatal("expected Recipe to remain persisted after local audio switch") + } + if updated.Recipe.AudioTrackIndex != 1 { + t.Fatalf("Recipe.AudioTrackIndex = %d, want 1 (re-persisted to newly selected track)", updated.Recipe.AudioTrackIndex) + } +} + // recordingSessionSyncer counts SyncNow calls and records the context state at // call time, standing in for the reconciler's immediate-sync trigger. type recordingSessionSyncer struct { diff --git a/internal/noderecipe/store.go b/internal/noderecipe/store.go new file mode 100644 index 00000000..26d194a3 --- /dev/null +++ b/internal/noderecipe/store.go @@ -0,0 +1,103 @@ +// Package noderecipe is the recipe-handoff half of restart-resilient playback +// for jellycompat on dedicated transcode nodes. A native transcode carries its +// reconstruction recipe in the stream token, so a transcode node that restarts +// can rebuild ffmpeg from the token the client re-presents. The jellycompat +// node-hop token is server-minted and could carry the recipe too, but it +// deliberately does not: the recipe is mutated in place under a stable session id +// (a Jellyfin audio/subtitle switch restarts ffmpeg without re-minting the +// client's token), and a third-party Jellyfin client cannot be driven to refresh +// a stale token — so a token snapshot could reconstruct a stale rendition. The +// authoritative, mutable recipe therefore lives server-side (the central compat +// store), out of a restarted node's reach (Postgres). +// +// This store bridges that gap over the same shared Redis the offload topology +// already relies on (the node-session tracker): central writes the recipe keyed +// by the upstream session id when it starts a remote transcode, and the +// transcode node reads it on a reconstruct miss. It is +// off the hot path — written once at start, read only after a node restart. +package noderecipe + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "time" + + "github.com/redis/go-redis/v9" + + "github.com/Silo-Server/silo-server/internal/playback" +) + +// KeyPrefix namespaces per-session recipe keys: silo:noderecipe:. +const KeyPrefix = "silo:noderecipe:" + +// DefaultTTL bounds how long a stored recipe survives. It matches the stream +// token lifetime (playback.MaxTokenTTL, 24h): past it no surviving token could +// still drive a reconstruct, so the recipe is safe to lapse. +const DefaultTTL = playback.MaxTokenTTL + +// Store is the Redis-backed recipe store shared by central (writer, at remote +// transcode start) and the transcode nodes (reader, on a reconstruct miss). +type Store struct { + rdb *redis.Client + ttl time.Duration +} + +// NewStore wraps a Redis client. A nil client yields a disabled store whose +// writes no-op and whose reads miss, so a single integrated box (no Redis, no +// remote node) needs no special-casing. +func NewStore(rdb *redis.Client, ttl time.Duration) *Store { + if ttl <= 0 { + ttl = DefaultTTL + } + return &Store{rdb: rdb, ttl: ttl} +} + +func key(sessionID string) string { return KeyPrefix + sessionID } + +// Put stores the reconstruction recipe for a remote transcode session. Best +// effort: a write error is returned for the caller to log, never fatal. +func (s *Store) Put(ctx context.Context, sessionID string, card playback.RecipeCard) error { + if s == nil || s.rdb == nil || sessionID == "" { + return nil + } + data, err := json.Marshal(card) + if err != nil { + return err + } + return s.rdb.Set(ctx, key(sessionID), data, s.ttl).Err() +} + +// Get returns the stored recipe for sessionID. It fails CLOSED — a miss or any +// error yields (nil, false) — because an absent recipe legitimately means the +// node cannot reconstruct and should 404, never rebuild from a bad recipe. +func (s *Store) Get(ctx context.Context, sessionID string) (*playback.RecipeCard, bool) { + if s == nil || s.rdb == nil || sessionID == "" { + return nil, false + } + data, err := s.rdb.Get(ctx, key(sessionID)).Bytes() + if err != nil { + if !errors.Is(err, redis.Nil) { + slog.Warn("load node recipe failed", "error", err, "playback_session_id", sessionID) + } + return nil, false + } + var card playback.RecipeCard + if err := json.Unmarshal(data, &card); err != nil { + slog.Warn("decode node recipe failed", "error", err, "playback_session_id", sessionID) + return nil, false + } + return &card, true +} + +// Delete removes the stored recipe for sessionID so an explicitly-stopped +// session cannot be resurrected from a leftover recipe after a node restart. +// Safe on a nil/disabled store and on a missing key (a missing key is not an +// error). A delete error is returned for the caller to log, never fatal. +func (s *Store) Delete(ctx context.Context, sessionID string) error { + if s == nil || s.rdb == nil || sessionID == "" { + return nil + } + return s.rdb.Del(ctx, key(sessionID)).Err() +} diff --git a/internal/noderecipe/store_test.go b/internal/noderecipe/store_test.go new file mode 100644 index 00000000..c85917df --- /dev/null +++ b/internal/noderecipe/store_test.go @@ -0,0 +1,66 @@ +package noderecipe + +import ( + "context" + "testing" + + "github.com/Silo-Server/silo-server/internal/playback" +) + +// A nil-backed store (single integrated box, no Redis) must be safe: writes +// no-op and reads miss, so callers need no nil-guarding. +func TestNilStore_PutNoopGetMiss(t *testing.T) { + var s *Store // nil receiver + if err := s.Put(context.Background(), "sid", playback.RecipeCard{}); err != nil { + t.Fatalf("nil store Put returned error: %v", err) + } + if card, ok := s.Get(context.Background(), "sid"); ok || card != nil { + t.Fatalf("nil store Get = (%v, %v), want (nil, false)", card, ok) + } + + disabled := NewStore(nil, 0) + if err := disabled.Put(context.Background(), "sid", playback.RecipeCard{}); err != nil { + t.Fatalf("disabled store Put returned error: %v", err) + } + if _, ok := disabled.Get(context.Background(), "sid"); ok { + t.Fatalf("disabled store Get returned a hit, want miss") + } +} + +// Delete on a nil/disabled store must be a safe no-op (callers in teardown +// paths need no nil-guarding), and on either store a subsequent Get still +// misses, matching the delete-then-get-not-found and delete-missing-is-no-op +// contract. +func TestNilStore_DeleteNoop(t *testing.T) { + var s *Store // nil receiver + if err := s.Delete(context.Background(), "sid"); err != nil { + t.Fatalf("nil store Delete returned error: %v", err) + } + if card, ok := s.Get(context.Background(), "sid"); ok || card != nil { + t.Fatalf("nil store Get after Delete = (%v, %v), want (nil, false)", card, ok) + } + + disabled := NewStore(nil, 0) + // Delete of a missing key is a no-op success. + if err := disabled.Delete(context.Background(), "sid"); err != nil { + t.Fatalf("disabled store Delete returned error: %v", err) + } + if _, ok := disabled.Get(context.Background(), "sid"); ok { + t.Fatalf("disabled store Get after Delete returned a hit, want miss") + } +} + +func TestKeyNamespacing(t *testing.T) { + if got := key("abc"); got != "silo:noderecipe:abc" { + t.Fatalf("key(abc) = %q, want silo:noderecipe:abc", got) + } +} + +func TestDefaultTTLMatchesTokenLifetime(t *testing.T) { + if DefaultTTL != playback.MaxTokenTTL { + t.Fatalf("DefaultTTL = %v, want playback.MaxTokenTTL %v", DefaultTTL, playback.MaxTokenTTL) + } + if NewStore(nil, 0).ttl != DefaultTTL { + t.Fatal("NewStore with ttl<=0 did not default to DefaultTTL") + } +} diff --git a/internal/nodesessions/tracker.go b/internal/nodesessions/tracker.go index 4ded00eb..773cf124 100644 --- a/internal/nodesessions/tracker.go +++ b/internal/nodesessions/tracker.go @@ -32,6 +32,15 @@ type SessionInfo struct { Resolution string `json:"resolution,omitempty"` HWAccel string `json:"hw_accel,omitempty"` StartedAt string `json:"started_at"` + + // AuthUserID / ProfileID / MediaFileID are the numeric ownership keys the + // node copies from the verified stream token. They enrich the live admin + // "active streams" view (served by SCANning these records) so it can answer + // *who* is watching *what* on each node, not just session id + node + type; + // the string UserID/MediaItemID/MediaTitle fields remain the display labels. + AuthUserID int `json:"auth_user_id,omitempty"` + ProfileID string `json:"profile_id,omitempty"` + MediaFileID int `json:"media_file_id,omitempty"` } // Tracker manages session lifecycle in Redis for a single node. diff --git a/internal/playback/errors.go b/internal/playback/errors.go index e62164be..b6253c53 100644 --- a/internal/playback/errors.go +++ b/internal/playback/errors.go @@ -12,4 +12,16 @@ var ( ErrTranscodeFailed = errors.New("transcode process failed") ErrSegmentNotFound = errors.New("segment not found") ErrManifestNotReady = errors.New("manifest not ready") + // ErrSessionSuperseded means the session a restart was about to re-spawn is + // no longer the live mapped session (a concurrent teardown or reconstruct + // replaced it while the restart waited for the per-session lifecycle lock). + // The caller must not re-spawn ffmpeg for the stale handle. + ErrSessionSuperseded = errors.New("transcode session superseded") + // ErrLimitProviderUnavailable wraps a failure to load a user's admission + // limits from the limit provider (e.g. a transient Postgres error during a + // post-restart reconstruct wave). It is distinct from the genuine over-cap + // sentinels (ErrTooManyStreams / ErrTooManyTranscodes): a provider failure + // means limits could not be evaluated at all, so callers may choose to fail + // open rather than treat the session as over its (unknown) cap. + ErrLimitProviderUnavailable = errors.New("session limit provider unavailable") ) diff --git a/internal/playback/recipecard.go b/internal/playback/recipecard.go new file mode 100644 index 00000000..980da82b --- /dev/null +++ b/internal/playback/recipecard.go @@ -0,0 +1,218 @@ +package playback + +import ( + "time" + + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +// RecipeCard is the small, durable "recipe" needed to reconstruct a transcode +// session after the server forgets its in-memory state (e.g. a restart). It +// captures session identity, ownership, and the full set of encode parameters +// that affect output bytes — everything required to re-spawn an equivalent +// ffmpeg seeked to any requested segment. +// +// It deliberately omits non-serializable runtime fields (the ffmpeg process, +// context, channels, log sink). Those are re-wired on reconstruct from the +// live config and request. +type RecipeCard struct { + SessionID string `json:"session_id"` + UserID int `json:"user_id"` + ProfileID string `json:"profile_id"` + MediaFileID int `json:"media_file_id"` + TranscodeNodeURL string `json:"transcode_node_url,omitempty"` + + // PlayMethod discriminates which serve path reconstructs this session + // (direct / remux / transcode). Empty decodes as PlayTranscode for + // back-compat with cards written before direct/remux were reconstructable. + PlayMethod PlayMethod `json:"play_method,omitempty"` + // TranscodeAudio mirrors Session.TranscodeAudio; used by the remux path to + // re-spawn ffmpeg with the same audio handling on reconstruct. + TranscodeAudio bool `json:"transcode_audio,omitempty"` + + // Encode parameters — mirror of the byte-affecting TranscodeOpts fields. + // Unused (zero) for direct/remux cards, which carry no segment-based encode. + InputPath string `json:"input_path"` + SourceVideoCodec string `json:"source_video_codec,omitempty"` + SeekSeconds float64 `json:"seek_seconds"` + TargetResolution string `json:"target_resolution,omitempty"` + TargetCodecVideo string `json:"target_codec_video,omitempty"` + TargetCodecAudio string `json:"target_codec_audio,omitempty"` + SegmentDuration int `json:"segment_duration"` + StartSegmentNumber int `json:"start_segment_number"` + HWAccel string `json:"hw_accel,omitempty"` + HWDevice string `json:"hw_device,omitempty"` + SubtitleTrackIndex int `json:"subtitle_track_index"` + SubtitleBurnIn bool `json:"subtitle_burn_in,omitempty"` + AudioTrackIndex int `json:"audio_track_index"` + TargetBitrateKbps int `json:"target_bitrate_kbps,omitempty"` + TotalDuration float64 `json:"total_duration"` + FastStart bool `json:"fast_start,omitempty"` +} + +// NewRecipeCard builds a RecipeCard from the durable identity fields plus the +// TranscodeOpts used to start the session. The non-serializable opts fields +// (FFmpegLogSink) are dropped; FFmpegPath/HWAccel/HWDevice are intentionally +// re-resolved from live config on reconstruct rather than pinned here, so an +// operator's config change applies to reconstructed sessions too. +func NewRecipeCard(userID int, profileID string, mediaFileID int, transcodeNodeURL string, opts TranscodeOpts) RecipeCard { + return RecipeCard{ + SessionID: opts.SessionID, + UserID: userID, + ProfileID: profileID, + MediaFileID: mediaFileID, + TranscodeNodeURL: transcodeNodeURL, + PlayMethod: PlayTranscode, + InputPath: opts.InputPath, + SourceVideoCodec: opts.SourceVideoCodec, + SeekSeconds: opts.SeekSeconds, + TargetResolution: opts.TargetResolution, + TargetCodecVideo: opts.TargetCodecVideo, + TargetCodecAudio: opts.TargetCodecAudio, + SegmentDuration: opts.SegmentDuration, + StartSegmentNumber: opts.StartSegmentNumber, + HWAccel: opts.HWAccel, + HWDevice: opts.HWDevice, + SubtitleTrackIndex: opts.SubtitleTrackIndex, + SubtitleBurnIn: opts.SubtitleBurnIn, + AudioTrackIndex: opts.AudioTrackIndex, + TargetBitrateKbps: opts.TargetBitrateKbps, + TotalDuration: opts.TotalDuration, + FastStart: opts.FastStart, + } +} + +// NewDirectRecipeCard builds a card for a direct-play session. Only identity is +// needed to rebuild the Session: the file is served by HTTP byte range and the +// client re-supplies its position, so there are no encode parameters and no +// runtime to reconstruct beyond the Session itself. +func NewDirectRecipeCard(sessionID string, userID int, profileID string, mediaFileID int) RecipeCard { + return RecipeCard{ + SessionID: sessionID, + UserID: userID, + ProfileID: profileID, + MediaFileID: mediaFileID, + PlayMethod: PlayDirect, + } +} + +// NewRemuxRecipeCard builds a card for a remux session: identity plus the audio +// selection. The remux ffmpeg is a single pipe re-spawned at the client-supplied +// ?seek= on the next request, so no segment/encode parameters are pinned. +func NewRemuxRecipeCard(sessionID string, userID int, profileID string, mediaFileID int, transcodeAudio bool, audioTrackIndex int) RecipeCard { + return RecipeCard{ + SessionID: sessionID, + UserID: userID, + ProfileID: profileID, + MediaFileID: mediaFileID, + PlayMethod: PlayRemux, + TranscodeAudio: transcodeAudio, + AudioTrackIndex: audioTrackIndex, + } +} + +// TranscodeOpts rebuilds the encode parameters for a reconstruct. outputDir, +// ffmpegPath and logSink are supplied by the caller from live config because +// they are environment-specific and not pinned in the card. +func (c RecipeCard) TranscodeOpts(outputDir, ffmpegPath string, logSink FFmpegLogSink) TranscodeOpts { + return TranscodeOpts{ + InputPath: c.InputPath, + OutputDir: outputDir, + SessionID: c.SessionID, + SourceVideoCodec: c.SourceVideoCodec, + SeekSeconds: c.SeekSeconds, + TargetResolution: c.TargetResolution, + TargetCodecVideo: c.TargetCodecVideo, + TargetCodecAudio: c.TargetCodecAudio, + SegmentDuration: c.SegmentDuration, + StartSegmentNumber: c.StartSegmentNumber, + FFmpegPath: ffmpegPath, + HWAccel: c.HWAccel, + HWDevice: c.HWDevice, + SubtitleTrackIndex: c.SubtitleTrackIndex, + SubtitleBurnIn: c.SubtitleBurnIn, + AudioTrackIndex: c.AudioTrackIndex, + TargetBitrateKbps: c.TargetBitrateKbps, + TotalDuration: c.TotalDuration, + FastStart: c.FastStart, + NodeType: "integrated", + ExecutionMode: "integrated", + FFmpegLogSink: logSink, + } +} + +// MaxTokenTTL is the absolute lifetime of a stream token, and therefore the +// longest a session can remain reconstructable. Under token-carried +// reconstruction there is no durable server-side card index, so segment-dir +// cleanup spares a dir that is not live in memory only until this age elapses: +// past it, no surviving token could still reconstruct the session, so the dir is +// safe to reap. It must comfortably outlast any realistic restart outage. +const MaxTokenTTL = 24 * time.Hour + +// ToClaims projects the reconstruction recipe into stream-token claims so the +// card can travel with the client instead of a shared per-session store. The +// environment-specific knobs (HWAccel/HWDevice) are intentionally NOT carried — +// they are re-resolved from live config on reconstruct, so an operator's config +// change applies to reconstructed sessions too. +func (c RecipeCard) ToClaims() streamtoken.Claims { + return streamtoken.Claims{ + SessionID: c.SessionID, + MediaPath: c.InputPath, + PlayMethod: string(c.PlayMethod), + TranscodeAudio: c.TranscodeAudio, + TranscodeNode: c.TranscodeNodeURL, + TargetCodec: c.TargetCodecVideo, + TargetRes: c.TargetResolution, + AudioTrackIndex: c.AudioTrackIndex, + UserID: c.UserID, + ProfileID: c.ProfileID, + MediaFileID: c.MediaFileID, + SourceVideoCodec: c.SourceVideoCodec, + SeekSeconds: c.SeekSeconds, + SegmentDuration: c.SegmentDuration, + StartSegmentNumber: c.StartSegmentNumber, + SubtitleTrackIndex: c.SubtitleTrackIndex, + SubtitleBurnIn: c.SubtitleBurnIn, + TargetBitrateKbps: c.TargetBitrateKbps, + TotalDuration: c.TotalDuration, + FastStart: c.FastStart, + TargetCodecAudio: c.TargetCodecAudio, + } +} + +// RecipeCardFromClaims rebuilds the reconstruction recipe from verified +// stream-token claims. HWAccel/HWDevice are deliberately absent (re-resolved +// from live config by the reconstruct path). An empty PlayMethod decodes to +// PlayTranscode for back-compat with any token minted before the discriminator. +func RecipeCardFromClaims(c *streamtoken.Claims) RecipeCard { + if c == nil { + return RecipeCard{} + } + method := PlayMethod(c.PlayMethod) + if method == "" { + method = PlayTranscode + } + return RecipeCard{ + SessionID: c.SessionID, + UserID: c.UserID, + ProfileID: c.ProfileID, + MediaFileID: c.MediaFileID, + TranscodeNodeURL: c.TranscodeNode, + PlayMethod: method, + TranscodeAudio: c.TranscodeAudio, + InputPath: c.MediaPath, + SourceVideoCodec: c.SourceVideoCodec, + SeekSeconds: c.SeekSeconds, + TargetResolution: c.TargetRes, + TargetCodecVideo: c.TargetCodec, + TargetCodecAudio: c.TargetCodecAudio, + SegmentDuration: c.SegmentDuration, + StartSegmentNumber: c.StartSegmentNumber, + SubtitleTrackIndex: c.SubtitleTrackIndex, + SubtitleBurnIn: c.SubtitleBurnIn, + AudioTrackIndex: c.AudioTrackIndex, + TargetBitrateKbps: c.TargetBitrateKbps, + TotalDuration: c.TotalDuration, + FastStart: c.FastStart, + } +} diff --git a/internal/playback/recipecard_test.go b/internal/playback/recipecard_test.go new file mode 100644 index 00000000..88979b47 --- /dev/null +++ b/internal/playback/recipecard_test.go @@ -0,0 +1,141 @@ +package playback + +import ( + "encoding/json" + "testing" + + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +func TestRecipeCardRoundTripOpts(t *testing.T) { + opts := TranscodeOpts{ + InputPath: "/media/movie.mkv", + OutputDir: "/tmp/silo-transcode/abc", + SessionID: "abc", + SourceVideoCodec: "hevc", + SeekSeconds: 900, + TargetResolution: "1080p", + TargetCodecVideo: "h264", + TargetCodecAudio: "aac", + SegmentDuration: 2, + StartSegmentNumber: 450, + HWAccel: "qsv", + HWDevice: "/dev/dri/renderD128", + SubtitleTrackIndex: 3, + SubtitleBurnIn: true, + AudioTrackIndex: 1, + TargetBitrateKbps: 8000, + TotalDuration: 7200, + FastStart: true, + } + + card := NewRecipeCard(42, "profile-1", 77, "", opts) + if card.SessionID != "abc" || card.UserID != 42 || card.ProfileID != "profile-1" || card.MediaFileID != 77 { + t.Fatalf("identity not captured: %+v", card) + } + + // Rebuild opts; environment-specific fields are re-supplied by the caller. + got := card.TranscodeOpts("/tmp/silo-transcode/abc", "/usr/bin/ffmpeg", nil) + if got.StartSegmentNumber != 450 { + t.Errorf("StartSegmentNumber = %d, want 450", got.StartSegmentNumber) + } + if got.SeekSeconds != 900 { + t.Errorf("SeekSeconds = %v, want 900", got.SeekSeconds) + } + if !got.SubtitleBurnIn { + t.Errorf("SubtitleBurnIn lost in round trip") + } + if got.AudioTrackIndex != 1 || got.SubtitleTrackIndex != 3 { + t.Errorf("track indices wrong: audio=%d sub=%d", got.AudioTrackIndex, got.SubtitleTrackIndex) + } + if got.TargetCodecVideo != "h264" || got.TargetBitrateKbps != 8000 { + t.Errorf("encode params wrong: %+v", got) + } + if got.FFmpegPath != "/usr/bin/ffmpeg" { + t.Errorf("FFmpegPath not re-supplied: %q", got.FFmpegPath) + } +} + +func TestRecipeCardPlayMethodConstructors(t *testing.T) { + if c := NewRecipeCard(1, "p", 2, "", TranscodeOpts{SessionID: "t"}); c.PlayMethod != PlayTranscode { + t.Errorf("transcode card PlayMethod = %q, want transcode", c.PlayMethod) + } + d := NewDirectRecipeCard("d", 1, "p", 2) + if d.PlayMethod != PlayDirect || d.SessionID != "d" || d.MediaFileID != 2 { + t.Errorf("direct card wrong: %+v", d) + } + r := NewRemuxRecipeCard("r", 1, "p", 2, true, 3) + if r.PlayMethod != PlayRemux || !r.TranscodeAudio || r.AudioTrackIndex != 3 { + t.Errorf("remux card wrong: %+v", r) + } +} + +// A card persisted before the play_method discriminator existed must decode with +// an empty PlayMethod so reconstruct can treat it as a transcode (back-compat). +func TestRecipeCardLegacyDecodeHasEmptyPlayMethod(t *testing.T) { + legacy := []byte(`{"session_id":"old","user_id":7,"media_file_id":9,"segment_duration":2,"start_segment_number":10}`) + var card RecipeCard + if err := json.Unmarshal(legacy, &card); err != nil { + t.Fatalf("decode legacy card: %v", err) + } + if card.PlayMethod != "" { + t.Fatalf("legacy card PlayMethod = %q, want empty (decodes as transcode)", card.PlayMethod) + } + if card.SessionID != "old" || card.UserID != 7 || card.StartSegmentNumber != 10 { + t.Fatalf("legacy fields lost: %+v", card) + } +} + +// A transcode recipe must survive a full round trip through stream-token claims: +// the token IS the durable descriptor under token-carried reconstruction, so any +// dropped byte-affecting field would reconstruct a divergent encode. HWAccel and +// HWDevice are deliberately excluded (re-resolved from live config), so they are +// not asserted here. +func TestRecipeCardClaimsRoundTrip(t *testing.T) { + card := NewRecipeCard(42, "profile-1", 77, "http://node:9000", TranscodeOpts{ + InputPath: "/media/movie.mkv", + SessionID: "abc", + SourceVideoCodec: "hevc", + SeekSeconds: 900, + TargetResolution: "1080p", + TargetCodecVideo: "h264", + TargetCodecAudio: "aac", + SegmentDuration: 2, + StartSegmentNumber: 450, + SubtitleTrackIndex: 3, + SubtitleBurnIn: true, + AudioTrackIndex: 1, + TargetBitrateKbps: 8000, + TotalDuration: 7200, + FastStart: true, + }) + + claims := card.ToClaims() + got := RecipeCardFromClaims(&claims) + + // Identity + routing. + if got.SessionID != card.SessionID || got.UserID != card.UserID || + got.ProfileID != card.ProfileID || got.MediaFileID != card.MediaFileID || + got.TranscodeNodeURL != card.TranscodeNodeURL || got.PlayMethod != card.PlayMethod { + t.Fatalf("identity/routing lost: %+v", got) + } + // Byte-affecting encode parameters. + if got.InputPath != card.InputPath || got.SourceVideoCodec != card.SourceVideoCodec || + got.SeekSeconds != card.SeekSeconds || got.TargetResolution != card.TargetResolution || + got.TargetCodecVideo != card.TargetCodecVideo || got.TargetCodecAudio != card.TargetCodecAudio || + got.SegmentDuration != card.SegmentDuration || got.StartSegmentNumber != card.StartSegmentNumber || + got.SubtitleTrackIndex != card.SubtitleTrackIndex || got.SubtitleBurnIn != card.SubtitleBurnIn || + got.AudioTrackIndex != card.AudioTrackIndex || got.TargetBitrateKbps != card.TargetBitrateKbps || + got.TotalDuration != card.TotalDuration || got.FastStart != card.FastStart { + t.Fatalf("encode parameters lost in round trip:\n have %+v\n want %+v", got, card) + } +} + +// An empty-method token (direct/remux carry only identity) decodes to a usable +// card; a transcode discriminator is restored for a token with no method. +func TestRecipeCardFromClaimsEmptyMethodIsTranscode(t *testing.T) { + claims := &streamtoken.Claims{SessionID: "x", UserID: 1, MediaFileID: 2} + if got := RecipeCardFromClaims(claims); got.PlayMethod != PlayTranscode { + t.Fatalf("empty method should decode as transcode, got %q", got.PlayMethod) + } +} diff --git a/internal/playback/reconstruct_test.go b/internal/playback/reconstruct_test.go new file mode 100644 index 00000000..e74bb45b --- /dev/null +++ b/internal/playback/reconstruct_test.go @@ -0,0 +1,60 @@ +package playback + +import ( + "sync" + "testing" +) + +func TestRegisterReconstructedInsertsUnderExistingID(t *testing.T) { + m := NewSessionManager(0, 0) + s := &Session{ID: "sess-1", UserID: 7, MediaFileID: 3, PlayMethod: PlayTranscode} + + got := m.RegisterReconstructed(s) + if got != s { + t.Fatal("expected the registered session to be returned") + } + if got.StartedAt.IsZero() || got.LastActivityAt.IsZero() { + t.Error("timestamps should be armed on register") + } + + // The session is now retrievable under its original ID (no new UUID minted). + back, err := m.GetSession("sess-1") + if err != nil { + t.Fatalf("GetSession after reconstruct: %v", err) + } + if back.UserID != 7 { + t.Errorf("UserID = %d, want 7", back.UserID) + } +} + +func TestRegisterReconstructedYieldsToExisting(t *testing.T) { + m := NewSessionManager(0, 0) + first := &Session{ID: "dup", UserID: 1} + m.RegisterReconstructed(first) + + // A concurrent reconstruct with the same ID must return the existing one. + second := &Session{ID: "dup", UserID: 999} + got := m.RegisterReconstructed(second) + if got != first { + t.Fatal("expected existing session to win the race") + } + if got.UserID != 1 { + t.Errorf("UserID = %d, want 1 (existing preserved)", got.UserID) + } +} + +func TestRegisterReconstructedConcurrent(t *testing.T) { + m := NewSessionManager(0, 0) + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + m.RegisterReconstructed(&Session{ID: "race", UserID: 5}) + }() + } + wg.Wait() + if _, err := m.GetSession("race"); err != nil { + t.Fatalf("session should exist after concurrent registers: %v", err) + } +} diff --git a/internal/playback/restart_locked_test.go b/internal/playback/restart_locked_test.go new file mode 100644 index 00000000..6e35ca63 --- /dev/null +++ b/internal/playback/restart_locked_test.go @@ -0,0 +1,145 @@ +package playback + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" +) + +// newFakeFFmpegSession builds a minimal TranscodeSession whose ffmpeg binary is +// a stub that sleeps, so Restart spawns a real (harmless) child process without +// a full transcode pipeline. Only the fields Restart touches are populated. +func newFakeFFmpegSession(t *testing.T) *TranscodeSession { + t.Helper() + dir := t.TempDir() + bin := filepath.Join(dir, "fake-ffmpeg.sh") + if err := os.WriteFile(bin, []byte("#!/bin/sh\nsleep 30\n"), 0o755); err != nil { + t.Fatalf("write fake ffmpeg: %v", err) + } + return &TranscodeSession{ + outputDir: dir, + opts: TranscodeOpts{ + SessionID: "sess-x", + FFmpegPath: bin, + OutputDir: dir, + }, + } +} + +// A restart must not spawn ffmpeg while another lifecycle holder (a fresh start, +// reconstruct, or another restart) owns the per-session lock — otherwise two +// ffmpeg processes write the same output directory. RestartSessionLocked must +// block on the lifecycle lock and only spawn once it is released. +func TestRestartSessionLocked_WaitsForLifecycleLock(t *testing.T) { + m := NewTranscodeManager() + s := newFakeFFmpegSession(t) + const sid = "sess-x" + m.RegisterTranscodeSession(sid, s) + t.Cleanup(func() { _ = s.Close() }) + + // Simulate a concurrent start/reconstruct holding the lifecycle lock. + release := m.LockSessionLifecycle(sid) + + done := make(chan error, 1) + go func() { + done <- m.RestartSessionLocked(context.Background(), sid, s, 0, 0) + }() + + // While the lock is held the restart must neither return nor spawn ffmpeg. + select { + case <-done: + t.Fatal("RestartSessionLocked returned while the lifecycle lock was held") + case <-time.After(150 * time.Millisecond): + } + if s.IsRunning() { + t.Fatal("ffmpeg spawned while another holder owned the lifecycle lock") + } + + release() + + select { + case err := <-done: + if err != nil { + t.Fatalf("RestartSessionLocked after release: %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("RestartSessionLocked did not complete after lock release") + } + if !s.IsRunning() { + t.Fatal("expected the session to be running after a successful restart") + } +} + +// Concurrent restarts on the same session (e.g. audio-switch racing +// segment-recovery) must serialize: the winner spawns exactly one live ffmpeg +// and neither call reports failure. +func TestRestartSessionLocked_ConcurrentRestartsSerialize(t *testing.T) { + m := NewTranscodeManager() + s := newFakeFFmpegSession(t) + const sid = "sess-x" + m.RegisterTranscodeSession(sid, s) + t.Cleanup(func() { _ = s.Close() }) + + const n = 8 + errs := make(chan error, n) + start := make(chan struct{}) + for i := 0; i < n; i++ { + go func() { + <-start + errs <- m.RestartSessionLocked(context.Background(), sid, s, 0, 0) + }() + } + close(start) + + for i := 0; i < n; i++ { + select { + case err := <-errs: + if err != nil { + t.Fatalf("concurrent RestartSessionLocked returned error: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("concurrent RestartSessionLocked did not complete") + } + } + if !s.IsRunning() { + t.Fatal("expected exactly one live ffmpeg after concurrent restarts settle") + } +} + +// A restart whose session was torn down (or replaced by a reconstruct winner) +// while it waited must not re-spawn ffmpeg for the stale handle. +func TestRestartSessionLocked_SupersededWhenUnregistered(t *testing.T) { + m := NewTranscodeManager() + s := newFakeFFmpegSession(t) + + err := m.RestartSessionLocked(context.Background(), "sess-x", s, 0, 0) + if !errors.Is(err, ErrSessionSuperseded) { + t.Fatalf("want ErrSessionSuperseded for an unmapped session, got %v", err) + } + if s.IsRunning() { + t.Fatal("a superseded restart must not spawn ffmpeg") + } +} + +func TestRestartSessionLocked_SupersededWhenReplaced(t *testing.T) { + m := NewTranscodeManager() + stale := newFakeFFmpegSession(t) + live := newFakeFFmpegSession(t) + const sid = "sess-x" + m.RegisterTranscodeSession(sid, live) // a different session now owns the id + t.Cleanup(func() { _ = live.Close() }) + + err := m.RestartSessionLocked(context.Background(), sid, stale, 0, 0) + if !errors.Is(err, ErrSessionSuperseded) { + t.Fatalf("want ErrSessionSuperseded for a replaced session, got %v", err) + } + if stale.IsRunning() { + t.Fatal("a superseded restart must not spawn ffmpeg for the stale handle") + } + if live.IsRunning() { + t.Fatal("the live session must be untouched by a stale restart") + } +} diff --git a/internal/playback/session.go b/internal/playback/session.go index 1925dd3e..ac021316 100644 --- a/internal/playback/session.go +++ b/internal/playback/session.go @@ -2,6 +2,7 @@ package playback import ( "context" + "errors" "fmt" "strings" "sync" @@ -35,6 +36,14 @@ type Session struct { TargetBitrateKbps int // requested output bitrate cap for transcodes TranscodeHWAccel string // effective hardware acceleration mode for transcodes + // Byte-affecting transcode recipe fields the offloaded restart path needs to + // rebuild the exact same stream after an audio switch. Local transcodes read + // these from the live ts.Opts(); offloaded transcodes own no local runtime, so + // the session is the only place to recover them (see HandleChangeAudioTrack). + SubtitleTrackIndex int // -1 = no subtitles + SubtitleBurnIn bool + SegmentDuration int // HLS segment length in seconds (cadence) + Position float64 IsPaused bool HasWebSocket bool @@ -64,6 +73,15 @@ type SessionStreamState struct { TargetAudioCodec string TargetBitrateKbps int TranscodeHWAccel string + + // Byte-affecting transcode recipe fields preserved so an offloaded restart + // (e.g. audio switch) can rebuild the exact same stream. SubtitleTrackIndex + // defaults to 0 on a zero-value state; callers that manage subtitles must set + // it explicitly (-1 for none) — burn-in is additionally gated by + // SubtitleBurnIn so a zero index never burns track 0 by accident. + SubtitleTrackIndex int + SubtitleBurnIn bool + SegmentDuration int } type clientInfoContextKey struct{} @@ -272,6 +290,89 @@ func (m *SessionManager) StartSessionWithFilesContext( return s, nil } +// RegisterReconstructed re-inserts a session under an existing ID after the +// in-memory state was lost (e.g. a server restart). Unlike StartSession* it +// does NOT mint a new UUID and does NOT run admission/limit accounting: the +// session already existed and was admitted before the restart, so counting it +// again would be wrong. If a live session with the same ID already exists +// (a concurrent reconstruct won the race), the existing one is returned and +// the caller's copy is discarded. +// +// The caller is responsible for having re-bound s.UserID to the live +// authenticated request before calling this — RegisterReconstructed performs +// no authorization itself. +func (m *SessionManager) RegisterReconstructed(s *Session) *Session { + if s == nil || s.ID == "" { + return s + } + m.mu.Lock() + defer m.mu.Unlock() + if existing, ok := m.sessions[s.ID]; ok { + return existing + } + now := time.Now() + if s.StartedAt.IsZero() { + s.StartedAt = now + } + s.UpdatedAt = now + s.LastActivityAt = now + m.sessions[s.ID] = s + return s +} + +// RegisterReconstructedWithLimits is RegisterReconstructed plus the same per-user +// admission caps StartSession enforces. Token-carried reconstruct replays a +// signed recipe to rebuild a session lost to a restart; without a cap check a +// client could replay one token repeatedly (or after legitimately reaching its +// limit) and reconstruct past the per-user concurrent stream/transcode caps, +// since RegisterReconstructed skips admission accounting. +// +// Legitimately reconstructing a user's own surviving sessions still succeeds: +// the cap counts the user's *currently-live* sessions, and the one being rebuilt +// is not yet in the map, so the first MaxStreams reconstructs admit. Only the +// over-cap replay is refused (ErrTooManyStreams / ErrTooManyTranscodes). If an +// identical session id is already live (a concurrent reconstruct won), it is +// returned without re-counting. Caps are looked up via the same limit provider +// as StartSession. +func (m *SessionManager) RegisterReconstructedWithLimits(ctx context.Context, s *Session) (*Session, error) { + if s == nil || s.ID == "" { + return s, nil + } + if ctx == nil { + ctx = context.Background() + } + limits, err := m.limitsForUser(ctx, s.UserID) + if err != nil { + return nil, err + } + + m.mu.Lock() + defer m.mu.Unlock() + + if existing, ok := m.sessions[s.ID]; ok { + return existing, nil + } + + // The session being reconstructed is not yet in the map, so the live counts + // reflect the user's *other* sessions; admitting one more must stay within cap. + if limits.MaxStreams > 0 && m.activeCountLocked(s.UserID) >= limits.MaxStreams { + return nil, ErrTooManyStreams + } + if s.PlayMethod == PlayTranscode && limits.MaxTranscodes > 0 && + m.transcodeCountLocked(s.UserID) >= limits.MaxTranscodes { + return nil, ErrTooManyTranscodes + } + + now := time.Now() + if s.StartedAt.IsZero() { + s.StartedAt = now + } + s.UpdatedAt = now + s.LastActivityAt = now + m.sessions[s.ID] = s + return s, nil +} + func (m *SessionManager) limitsForUser(ctx context.Context, userID int) (SessionLimits, error) { m.mu.RLock() provider := m.limitProvider @@ -286,7 +387,11 @@ func (m *SessionManager) limitsForUser(ctx context.Context, userID int) (Session } limits, err := provider(ctx, userID) if err != nil { - return SessionLimits{}, fmt.Errorf("load session limits for user %d: %w", userID, err) + // Tag provider failures with ErrLimitProviderUnavailable so the + // reconstruct admission path can distinguish a transient limit-lookup + // failure (which it may fail open on) from a genuine over-cap rejection. + return SessionLimits{}, fmt.Errorf("load session limits for user %d: %w", + userID, errors.Join(ErrLimitProviderUnavailable, err)) } return limits, nil } @@ -363,6 +468,9 @@ func (m *SessionManager) UpdateStreamState(sessionID string, state SessionStream s.TargetAudioCodec = state.TargetAudioCodec s.TargetBitrateKbps = state.TargetBitrateKbps s.TranscodeHWAccel = state.TranscodeHWAccel + s.SubtitleTrackIndex = state.SubtitleTrackIndex + s.SubtitleBurnIn = state.SubtitleBurnIn + s.SegmentDuration = state.SegmentDuration m.touchSessionLocked(s) return nil } diff --git a/internal/playback/session_test.go b/internal/playback/session_test.go index c7e5d4b8..21377f94 100644 --- a/internal/playback/session_test.go +++ b/internal/playback/session_test.go @@ -506,10 +506,13 @@ func TestUpdateStreamState(t *testing.T) { TranscodeAudio: true, ClientIP: "10.0.0.10", StreamBitrateKbps: 4200, - TargetResolution: "1080p", - TargetVideoCodec: "h264", - TargetAudioCodec: "aac", - TargetBitrateKbps: 4000, + TargetResolution: "1080p", + TargetVideoCodec: "h264", + TargetAudioCodec: "aac", + TargetBitrateKbps: 4000, + SubtitleTrackIndex: 3, + SubtitleBurnIn: true, + SegmentDuration: 4, }) if err != nil { t.Fatalf("UpdateStreamState: %v", err) @@ -549,6 +552,15 @@ func TestUpdateStreamState(t *testing.T) { if got.TargetBitrateKbps != 4000 { t.Errorf("TargetBitrateKbps = %d, want 4000", got.TargetBitrateKbps) } + if got.SubtitleTrackIndex != 3 { + t.Errorf("SubtitleTrackIndex = %d, want 3", got.SubtitleTrackIndex) + } + if !got.SubtitleBurnIn { + t.Error("SubtitleBurnIn = false, want true") + } + if got.SegmentDuration != 4 { + t.Errorf("SegmentDuration = %d, want 4", got.SegmentDuration) + } } func TestUpdateAudioTrack_PreservesTranscodeTransportForRemuxBase(t *testing.T) { diff --git a/internal/playback/transcode.go b/internal/playback/transcode.go index 9c61026d..68208555 100644 --- a/internal/playback/transcode.go +++ b/internal/playback/transcode.go @@ -115,6 +115,13 @@ type SegmentRecoveryDecision struct { // segments (2s) allow the player to start quickly while still maintaining // efficient HTTP delivery. This matches the approach used by Plex. const defaultSegmentDuration = 2 + +// DefaultSegmentDuration is the exported segment length used when a transcode +// request does not specify one. Callers minting a reconstruct recipe must embed +// a concrete (>0) value so the token passes the node's completeness gate and the +// embedded length matches what the node actually produces. +const DefaultSegmentDuration = defaultSegmentDuration + const maxPersistedFFmpegLines = 2000 const maxPersistedFFmpegBytes = 256 * 1024 const maxPersistedFFmpegChars = 2000 @@ -1299,6 +1306,20 @@ func (s *TranscodeSession) GetSegment(name string) (string, error) { // Close terminates the ffmpeg process and removes the temporary output directory. func (s *TranscodeSession) Close() error { + return s.shutdown(true) +} + +// CloseProcess terminates the ffmpeg process but leaves the output directory in +// place. It is used when another session owns the same output directory (e.g. a +// concurrent reconstruct race loser): tearing down this duplicate must not wipe +// the segments and init.mp4 the winning session is actively serving. +func (s *TranscodeSession) CloseProcess() error { + return s.shutdown(false) +} + +// shutdown kills the ffmpeg process and, when removeOutput is true, removes the +// temporary output directory. +func (s *TranscodeSession) shutdown(removeOutput bool) error { s.StopThrottler() // Cancel the context to kill the process (no mutex needed for cancel). if s.cancel != nil { @@ -1319,7 +1340,7 @@ func (s *TranscodeSession) Close() error { s.running = false // Clean up temporary directory. - if s.outputDir != "" { + if removeOutput && s.outputDir != "" { if err := os.RemoveAll(s.outputDir); err != nil { return fmt.Errorf("remove output dir: %w", err) } @@ -1625,6 +1646,82 @@ func rewriteMapURI(line []byte, segPrefix, suffix string) ([]byte, error) { return result, nil } +// AppendManifestQueryParam appends a single "key=value" query parameter to every +// segment and #EXT-X-MAP init URI in an HLS media playlist, preserving any query +// the URI already carries (using "?" or "&" as appropriate). The value is +// appended verbatim — callers must supply a URL-safe value (a signed stream token +// is base64url and safe). A manifest without a valid #EXTM3U header is returned +// unchanged so a non-manifest body is never corrupted. +// +// It exists for the API proxy boundary: a transcode node builds its manifest from +// the forwarded request query, which deliberately omits the signed stream token +// ("st") so the token never reaches the node URL or its logs. That leaves the +// node-built segment URIs token-less, so a later segment fetch after a node/API +// restart cannot reconstruct the session. Re-injecting the client-facing token +// here keeps reconstruction working without ever exposing it to the node. +func AppendManifestQueryParam(manifest []byte, key, value string) []byte { + if key == "" || validateManifestHeader(manifest) != nil { + return manifest + } + param := key + "=" + value + lines := bytes.Split(manifest, []byte("\n")) + for i, line := range lines { + trimmed := bytes.TrimSpace(line) + if len(trimmed) == 0 { + continue + } + if bytes.HasPrefix(trimmed, []byte("#EXT-X-MAP:")) { + lines[i] = appendMapURIQueryParam(trimmed, param) + continue + } + if trimmed[0] == '#' { + continue + } + lines[i] = appendURIQueryParam(trimmed, param) + } + return bytes.Join(lines, []byte("\n")) +} + +// appendURIQueryParam appends a "key=value" param to a bare URI line. +func appendURIQueryParam(uri []byte, param string) []byte { + sep := []byte("?") + if bytes.IndexByte(uri, '?') >= 0 { + sep = []byte("&") + } + out := make([]byte, 0, len(uri)+len(sep)+len(param)) + out = append(out, uri...) + out = append(out, sep...) + out = append(out, param...) + return out +} + +// appendMapURIQueryParam appends a "key=value" param to the URI inside an +// #EXT-X-MAP tag. A line without a well-formed URI attribute is returned +// unchanged. +func appendMapURIQueryParam(line []byte, param string) []byte { + uriStart := bytes.Index(line, []byte(`URI="`)) + if uriStart < 0 { + return line + } + uriStart += 5 // skip past URI=" + uriEnd := bytes.IndexByte(line[uriStart:], '"') + if uriEnd < 0 { + return line + } + uriEnd += uriStart + + sep := []byte("?") + if bytes.IndexByte(line[uriStart:uriEnd], '?') >= 0 { + sep = []byte("&") + } + result := make([]byte, 0, len(line)+len(sep)+len(param)) + result = append(result, line[:uriEnd]...) + result = append(result, sep...) + result = append(result, param...) + result = append(result, line[uriEnd:]...) + return result +} + func (s *TranscodeSession) manifestTimeoutError(timeout time.Duration) error { s.mu.Lock() running := s.running @@ -1705,6 +1802,15 @@ func (s *TranscodeSession) RestartSeekTarget(segNum int) (float64, bool, error) case err != nil && !errors.Is(err, ErrManifestNotReady): return 0, false, err } + // Copy-mode fragments have variable durations, so the encoded + // `seg×dur` math would seek FFmpeg to the wrong source time and + // desync A/V after restart. When the manifest can't resolve this + // segment yet (ok=false with no error, including ErrManifestNotReady + // in a freshly reconstructed window), report the seek target as + // unresolved (0, false, nil) rather than guessing. The caller treats + // this as a retryable miss so the session keeps producing manifest + // until real timing is available. + return 0, false, nil } segDuration := defaultSegmentDuration diff --git a/internal/playback/transcode_cleanup.go b/internal/playback/transcode_cleanup.go index 2ffaa304..1d9b3487 100644 --- a/internal/playback/transcode_cleanup.go +++ b/internal/playback/transcode_cleanup.go @@ -4,11 +4,21 @@ import ( "fmt" "os" "path/filepath" + "time" ) // CleanupOrphanedTranscodeDirs removes per-session transcode directories that // are not associated with any currently active session IDs. -func CleanupOrphanedTranscodeDirs(root string, activeSessionIDs map[string]struct{}) (int, error) { +// +// minAge spares a dir whose most recent modification is younger than the given +// age even when it is absent from activeSessionIDs. Under token-carried +// reconstruction there is no durable card index, so an in-memory miss does not +// prove a session is dead — a client holding a still-valid token may yet +// reconstruct it. Sparing dirs younger than the maximum token lifetime closes +// that race; once a dir is older than any surviving token, it is safe to reap. +// Pass 0 to disable age-sparing (e.g. a dedicated node's boot-time full wipe, +// where node restart is an accepted session loss). +func CleanupOrphanedTranscodeDirs(root string, activeSessionIDs map[string]struct{}, minAge time.Duration) (int, error) { if root == "" { return 0, nil } @@ -22,6 +32,7 @@ func CleanupOrphanedTranscodeDirs(root string, activeSessionIDs map[string]struc } removed := 0 + cutoff := time.Now().Add(-minAge) for _, entry := range entries { if !entry.IsDir() { continue @@ -32,6 +43,18 @@ func CleanupOrphanedTranscodeDirs(root string, activeSessionIDs map[string]struc } dir := filepath.Join(root, entry.Name()) + if minAge > 0 { + info, statErr := entry.Info() + if statErr != nil { + // Can't verify age: conservatively retain rather than risk + // reaping a dir a surviving token could still reconstruct. + continue + } + if info.ModTime().After(cutoff) { + // Recently active: a surviving token could still reconstruct it. + continue + } + } if err := os.RemoveAll(dir); err != nil { return removed, fmt.Errorf("remove orphaned transcode dir %q: %w", dir, err) } diff --git a/internal/playback/transcode_close_test.go b/internal/playback/transcode_close_test.go new file mode 100644 index 00000000..ad40edde --- /dev/null +++ b/internal/playback/transcode_close_test.go @@ -0,0 +1,33 @@ +package playback + +import ( + "os" + "path/filepath" + "testing" +) + +// CloseProcess must leave the output directory intact so a session sharing the +// same directory (e.g. a concurrent reconstruct winner) keeps serving its +// segments and init.mp4. Close, by contrast, removes the directory. +func TestCloseProcessKeepsOutputDir(t *testing.T) { + dir := t.TempDir() + seg := filepath.Join(dir, "init.mp4") + if err := os.WriteFile(seg, []byte("data"), 0o644); err != nil { + t.Fatalf("seed segment: %v", err) + } + + s := &TranscodeSession{outputDir: dir} + if err := s.CloseProcess(); err != nil { + t.Fatalf("CloseProcess: %v", err) + } + if _, err := os.Stat(seg); err != nil { + t.Fatalf("CloseProcess must not remove output dir, but segment is gone: %v", err) + } + + if err := s.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if _, err := os.Stat(dir); !os.IsNotExist(err) { + t.Fatalf("Close must remove output dir, stat err = %v", err) + } +} diff --git a/internal/playback/transcode_manager.go b/internal/playback/transcode_manager.go new file mode 100644 index 00000000..12b8ef00 --- /dev/null +++ b/internal/playback/transcode_manager.go @@ -0,0 +1,751 @@ +package playback + +import ( + "context" + "errors" + "log/slog" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + "golang.org/x/sync/singleflight" +) + +// TranscodeRuntimeConfig is the subset of playback configuration the transcode +// manager needs to (re)start ffmpeg. It is a small, config-package-free struct so +// internal/playback does not import internal/config (avoiding an import cycle); +// each embedding handler adapts its own config snapshot into this shape. +type TranscodeRuntimeConfig struct { + TranscodeDir string + FFmpegPath string + HWAccel string + HWDevice string +} + +// sessionReconstructor is the SessionManager capability used to re-register a +// session under an existing ID during reconstruct. *SessionManager implements it. +// RegisterReconstructedWithLimits additionally enforces the per-user admission +// caps so replaying a token cannot reconstruct past the concurrent stream / +// transcode limits a fresh StartSession would reject. +type sessionReconstructor interface { + RegisterReconstructed(s *Session) *Session + RegisterReconstructedWithLimits(ctx context.Context, s *Session) (*Session, error) +} + +// TranscodeManager owns the transcode-session lifecycle shared by every playback +// front end (native API and jellycompat): the live in-memory transcode map, the +// recipe-card persistence used to reconstruct a session after a server restart, +// and the reconstruct machinery (single-flight + concurrency cap) that rebuilds a +// lost ffmpeg from a card. Both PlaybackHandlers embed one and delegate to it so +// the card lifetime rules, the reconstruct cap, and the node-affinity constraint +// live in exactly one place. +// +// Dependencies are injected as function fields so an embedding handler can wire +// them lazily from its own (often late-set) fields without an ordering hazard. +type TranscodeManager struct { + // Sessions re-registers a reconstructed session under its existing id. + Sessions sessionReconstructor + // Config returns the current transcode runtime config (ffmpeg path, dir, + // hwaccel) so operator changes apply to newly (re)started transcodes. + Config func() TranscodeRuntimeConfig + // LogSinkFn returns the ffmpeg log sink for reconstructed processes. + LogSinkFn func() FFmpegLogSink + // JWTSecretFn returns the bearer used for remote transcode-node DELETEs. + JWTSecretFn func() string + // OnFFmpegCrash is invoked when a reconstructed/local ffmpeg exits with an + // error so the embedding handler can tear down the playback session (keeping + // the card, so a resume can respawn). dead is the exact session that crashed; + // the handler passes it back through CloseTranscodeSessionIf so a successor + // reconstructed under the same id between the exit and teardown is not killed. + // No-op when nil. + OnFFmpegCrash func(ctx context.Context, sessionID string, dead *TranscodeSession) + // StartThrottler optionally starts the segment throttler for a (re)started + // transcode, reading the embedding handler's settings. No-op when nil. + StartThrottler func(ctx context.Context, ts *TranscodeSession) + + transcodeMu sync.RWMutex + transcodes map[string]*TranscodeSession + + // inFlightMu guards reconstructInFlight, the set of session ids whose ffmpeg + // is mid-reconstruct. Cleanup unions it with the live map so a dir being + // rebuilt right now is never reaped (token-carried reconstruction has no + // durable card index to consult instead). + inFlightMu sync.Mutex + reconstructInFlight map[string]struct{} + + // reconstructGroup single-flights transcode reconstruction per session id so + // concurrent manifest/segment requests for a lost session spawn exactly one + // ffmpeg writing to the shared output directory, never a racing duplicate. + reconstructGroup singleflight.Group + // reconstructSem bounds how many transcodes may be reconstructed (ffmpeg + // re-spawned) at once. After a restart, every buffered client re-requests at + // once; without a cap that is a thundering herd of simultaneous cold-start + // ffmpeg launches. The semaphore paces the burst — sessions still all + // reconstruct, just not all in the same instant. Lazily sized on first use. + reconstructSemOnce sync.Once + reconstructSem chan struct{} + + // lifecycleMu guards lifecycleLocks, the per-session mutexes that serialize + // every path which spawns ffmpeg into a session's output directory (fresh + // start, quality/audio restart, and reconstruct). reconstructGroup only + // single-flights reconstructs against each other; without this a reconstruct + // racing a fresh start could run two ffmpeg writers against the same dir. + lifecycleMu sync.Mutex + lifecycleLocks map[string]*lifecycleLock +} + +// lifecycleLock is a refcounted per-session mutex. The refcount lets the manager +// drop the map entry once no path holds or waits on it, so the map does not grow +// unbounded across the lifetime of a long-running server. +type lifecycleLock struct { + mu sync.Mutex + refs int +} + +// NewTranscodeManager returns a manager with its internal maps initialized. The +// caller wires the dependency function fields before use. +func NewTranscodeManager() *TranscodeManager { + return &TranscodeManager{ + transcodes: make(map[string]*TranscodeSession), + reconstructInFlight: make(map[string]struct{}), + } +} + +func (m *TranscodeManager) jwtSecret() string { + if m.JWTSecretFn == nil { + return "" + } + return m.JWTSecretFn() +} + +func (m *TranscodeManager) logSink() FFmpegLogSink { + if m.LogSinkFn == nil { + return nil + } + return m.LogSinkFn() +} + +func (m *TranscodeManager) runtimeConfig() TranscodeRuntimeConfig { + if m.Config == nil { + return TranscodeRuntimeConfig{TranscodeDir: filepath.Join(os.TempDir(), "silo-transcode")} + } + return m.Config() +} + +// defaultReconstructConcurrency caps simultaneous transcode reconstructs when no +// explicit limit is configured. One in-flight ffmpeg launch per CPU paces the +// post-restart spawn burst without starving a host that genuinely ran many +// concurrent transcodes before the restart. +func defaultReconstructConcurrency() int { + if n := runtime.NumCPU(); n > 0 { + return n + } + return 4 +} + +// acquireReconstructSlot blocks until a reconstruct slot is free or the request +// context is canceled. It returns a release func and true on success, or a nil +// func and false if the caller gave up (so the burst does not queue work no one +// is waiting for). The semaphore is lazily initialized so struct-literal-built +// managers (tests) work without a constructor. +func (m *TranscodeManager) acquireReconstructSlot(ctx context.Context) (func(), bool) { + m.reconstructSemOnce.Do(func() { + if m.reconstructSem == nil { + m.reconstructSem = make(chan struct{}, defaultReconstructConcurrency()) + } + }) + select { + case m.reconstructSem <- struct{}{}: + return func() { <-m.reconstructSem }, true + case <-ctx.Done(): + return nil, false + } +} + +// GetTranscodeSession returns the live in-memory transcode session for sessionID, +// or nil if none is registered. +func (m *TranscodeManager) GetTranscodeSession(sessionID string) *TranscodeSession { + if m == nil { + return nil + } + m.transcodeMu.RLock() + defer m.transcodeMu.RUnlock() + return m.transcodes[sessionID] +} + +// RegisterTranscodeSession inserts a freshly started transcode session into the +// live map. Used by the normal (non-reconstruct) start paths. +func (m *TranscodeManager) RegisterTranscodeSession(sessionID string, ts *TranscodeSession) { + m.transcodeMu.Lock() + m.transcodes[sessionID] = ts + m.transcodeMu.Unlock() +} + +// LockSessionLifecycle acquires the per-session lifecycle mutex and returns a +// release func. Every path that spawns ffmpeg into a session's output directory +// (fresh start, restart, reconstruct) must hold it across "check existing → spawn +// → register" so two paths never run concurrent writers against the same dir. The +// lock is refcounted: the map entry is dropped once the last holder/waiter +// releases, so the map stays bounded. +func (m *TranscodeManager) LockSessionLifecycle(sessionID string) func() { + m.lifecycleMu.Lock() + if m.lifecycleLocks == nil { + m.lifecycleLocks = make(map[string]*lifecycleLock) + } + lk := m.lifecycleLocks[sessionID] + if lk == nil { + lk = &lifecycleLock{} + m.lifecycleLocks[sessionID] = lk + } + lk.refs++ + m.lifecycleMu.Unlock() + + lk.mu.Lock() + return func() { + lk.mu.Unlock() + m.lifecycleMu.Lock() + lk.refs-- + if lk.refs == 0 { + delete(m.lifecycleLocks, sessionID) + } + m.lifecycleMu.Unlock() + } +} + +// RestartSessionLocked re-spawns ts under the per-session lifecycle lock so a +// restart (audio-switch or segment-recovery) can never race a fresh start, +// reconstruct, or another restart into the same output directory — the +// concurrent-writer corruption the lifecycle lock exists to prevent. It holds +// the lock only across the cancel→respawn transition inside Restart and +// releases it before the caller waits on segments. Under the lock it confirms +// ts is still the live mapped session; if a concurrent teardown or reconstruct +// replaced it, the stale handle is not re-spawned and ErrSessionSuperseded is +// returned. +func (m *TranscodeManager) RestartSessionLocked(ctx context.Context, sessionID string, ts *TranscodeSession, seekSeconds float64, startSegment int) error { + unlock := m.LockSessionLifecycle(sessionID) + defer unlock() + if live := m.GetTranscodeSession(sessionID); live != ts { + return ErrSessionSuperseded + } + return ts.Restart(ctx, seekSeconds, startSegment) +} + +// markReconstructing records that sessionID's ffmpeg is mid-reconstruct and +// returns a release func to clear it. Cleanup unions this set with the live map +// so a dir being rebuilt is never reaped before it registers. +func (m *TranscodeManager) markReconstructing(sessionID string) func() { + if m == nil || sessionID == "" { + return func() {} + } + m.inFlightMu.Lock() + if m.reconstructInFlight == nil { + m.reconstructInFlight = make(map[string]struct{}) + } + m.reconstructInFlight[sessionID] = struct{}{} + m.inFlightMu.Unlock() + return func() { + m.inFlightMu.Lock() + delete(m.reconstructInFlight, sessionID) + m.inFlightMu.Unlock() + } +} + +// SessionLoadStatus is the outcome of LoadOrReconstructSession, letting each +// handler render its own error shape (native vs jellycompat) without the manager +// touching the http response. +type SessionLoadStatus int + +const ( + // SessionLoaded: a live or reconstructed session is returned, ownership ok. + SessionLoaded SessionLoadStatus = iota + // SessionMissing: no live session and no usable card (genuine not-found). + SessionMissing + // SessionLoadFailed: the session backend errored (not a clean miss). + SessionLoadFailed + // SessionForbidden: a live session exists but belongs to another user. + SessionForbidden +) + +// LoadOrReconstructSession is the single front door every serve handler uses to +// obtain a playback Session: it looks the session up via getSession and, on a +// not-found miss (e.g. after a restart), reconstructs it from the recipe card, +// re-binding ownership to the live caller. The two-factor ownership rule is +// preserved exactly — a live session with a non-zero, mismatched caller is +// refused; reconstruct itself refuses a zero/mismatched caller — so this widens +// no access. getSession is supplied by the caller (its SessionManager.GetSession) +// so the manager needs no direct handle on the manager type. +// +// card is the reconstruction recipe the caller decoded from the verified stream +// token the client presented (nil when the request carried no usable token). +// Under token-carried reconstruction it is the sole descriptor source — there is +// no shared per-session store to fall back on — so a not-found session with a nil +// card is a genuine miss. +func (m *TranscodeManager) LoadOrReconstructSession(ctx context.Context, getSession func(string) (*Session, error), sessionID string, requestUserID int, card *RecipeCard) (*Session, SessionLoadStatus) { + session, err := getSession(sessionID) + if err != nil { + if !errors.Is(err, ErrSessionNotFound) { + return nil, SessionLoadFailed + } + // A nil manager (documented optional on StreamHandler) cannot reconstruct, + // so a missing session is simply not-found rather than a panic. + if m == nil || card == nil { + return nil, SessionMissing + } + // Lost the in-memory session (e.g. restart): rebuild it from the token's + // recipe. ReconstructSession re-binds the session to the card owner and + // refuses a non-zero caller that mismatches it (a zero caller is allowed for + // the authless bearer routes), so a nil result here is a genuine not-found. + session = m.ReconstructSession(ctx, sessionID, requestUserID, *card) + if session == nil { + return nil, SessionMissing + } + return session, SessionLoaded + } + // Live session: enforce the existing ownership check. A zero caller is + // allowed (these routes treat the session UUID as a bearer when auth is + // optional); a non-zero mismatch is refused. + if requestUserID != 0 && session.UserID != requestUserID { + return nil, SessionForbidden + } + return session, SessionLoaded +} + +// ReconstructSession rebuilds the in-memory playback Session from a persisted +// recipe card after the server lost its state (restart). It re-binds the session +// to the live authenticated caller and refuses if ownership cannot be confirmed. +// Returns the (re)registered session, or nil if reconstruct is not possible (no +// card, ownership mismatch, or unsupported session manager). +func (m *TranscodeManager) ReconstructSession(ctx context.Context, sessionID string, requestUserID int, card RecipeCard) *Session { + if m == nil || m.Sessions == nil { + return nil + } + if card.SessionID == "" || card.SessionID != sessionID { + // The token's recipe must be for the session id in the URL; a mismatch is + // a forged or stale request. + return nil + } + // Re-bind ownership to the card owner. A zero caller is allowed (the authless + // transcode delivery routes — HLS master.m3u8 / segment — treat the session + // UUID as the bearer credential when auth is optional); a non-zero caller that + // mismatches the card owner is refused. Either way the reconstructed session is + // bound to card.UserID, never to the request's user. + if requestUserID != 0 && requestUserID != card.UserID { + slog.Warn("transcode reconstruct ownership rejected", + "session", sessionID, "playback_session_id", sessionID, + "request_user", requestUserID, "card_user", card.UserID) + return nil + } + + // An empty PlayMethod is a card written before direct/remux were + // reconstructable; treat it as a transcode (the only kind then persisted). + method := card.PlayMethod + if method == "" { + method = PlayTranscode + } + + s := &Session{ + ID: card.SessionID, + UserID: card.UserID, + ProfileID: card.ProfileID, + MediaFileID: card.MediaFileID, + PlayMethod: method, + BasePlayMethod: method, + TranscodeNodeURL: card.TranscodeNodeURL, + AudioTrackIndex: card.AudioTrackIndex, + TranscodeAudio: card.TranscodeAudio, + TargetResolution: card.TargetResolution, + TargetVideoCodec: card.TargetCodecVideo, + TargetAudioCodec: card.TargetCodecAudio, + TargetBitrateKbps: card.TargetBitrateKbps, + TranscodeHWAccel: card.HWAccel, + // Preserve the byte-affecting recipe so an audio switch after a restart + // rebuilds the same stream (subtitles/cadence) instead of dropping them. + SubtitleTrackIndex: card.SubtitleTrackIndex, + SubtitleBurnIn: card.SubtitleBurnIn, + SegmentDuration: card.SegmentDuration, + } + // Enforce the same per-user concurrency caps a fresh StartSession would, so a + // replayed token cannot reconstruct past the user's limit. Reconstructing the + // user's own surviving sessions still succeeds up to the cap; only the over-cap + // replay is rejected. + session, err := m.Sessions.RegisterReconstructedWithLimits(ctx, s) + if err != nil { + // A genuine over-cap rejection (the user is at their concurrent stream / + // transcode limit) must still refuse: a replayed token cannot reconstruct + // past the cap a fresh StartSession would enforce. + if errors.Is(err, ErrTooManyStreams) || errors.Is(err, ErrTooManyTranscodes) { + slog.Warn("playback session reconstruct refused by admission cap", + "session", sessionID, "playback_session_id", sessionID, + "user", card.UserID, "method", method, "error", err) + return nil + } + // Otherwise the limit provider itself could not be evaluated (e.g. a + // transient Postgres error during a post-restart reconstruct wave). Fail + // open and admit the session WITHOUT the limit gate: denying here would + // collapse a recoverable dependency error into a permanent 404 and stop + // playback for a user who is within their limits. The cap will re-apply on + // the next fresh StartSession once the provider recovers. + slog.Warn("playback session reconstruct admitting despite unevaluated limits (degraded; limit provider unavailable)", + "session", sessionID, "playback_session_id", sessionID, + "user", card.UserID, "method", method, "error", err) + session = m.Sessions.RegisterReconstructed(s) + } + slog.Info("playback session reconstructed from recipe card", + "session", sessionID, "playback_session_id", sessionID, "user", card.UserID, "method", method) + return session +} + +// ReconstructTranscode rebuilds the in-memory TranscodeSession (and, if +// necessary, the ffmpeg process) for a session whose card survived a restart. It +// is only used for local/integrated transcodes (no transcode node URL). +// +// requestedSegment is the segment number the caller is fetching, or a negative +// value when there is no segment context (manifest path). When the client has +// advanced past the card's original start position, the rebuilt ffmpeg is spawned +// at that position so playback resumes near the requested segment instead of +// restarting from the original seek point and stalling while the segment-recovery +// machinery seeks forward. +// +// Reconstruction is single-flighted per session id: concurrent manifest and +// segment requests for the same lost session share one ffmpeg process rather than +// racing to spawn duplicates against the shared output directory. Spawns are +// additionally bounded by reconstructSem so a post-restart wave of buffered +// clients paces its ffmpeg launches instead of stampeding the host. +// +// NODE AFFINITY CONSTRAINT: this re-spawns ffmpeg on the LOCAL host. The playback +// SessionManager is per-process and not shared across API front-ends, but recipe +// cards are shared (Postgres). For an integrated transcode (empty +// TranscodeNodeURL) the card carries no owning-node identity, so if requests for +// one session are spread across multiple API front-ends WITHOUT sticky session +// affinity, each front-end that misses the in-memory session will reconstruct its +// OWN local ffmpeg — a split-brain with divergent segment dirs. Integrated +// transcode is therefore only safe single-front-end or with session affinity at +// the load balancer. Remote transcode-node sessions are unaffected: their +// non-empty TranscodeNodeURL routes every front-end to the same ffmpeg via the +// proxy path, so ReconstructTranscode is never reached for them. +// +// This constraint is currently documented, not enforced: a robust fix needs a +// per-session owning-instance claim in a store shared across front-ends (e.g. +// a shared Redis or the recipe store), so a front-end refuses to +// reconstruct an integrated session it does not own. The TranscodeManager has no +// such shared handle wired today — only per-process config/secret closures and +// in-memory maps — so the claim cannot be made cheaply here. Until a topology +// signal reaches the manager, deploy integrated transcode single-front-end or +// behind sticky session affinity. See M8. +// card is the reconstruction recipe decoded from the client's verified stream +// token; it carries the encode parameters formerly read from the Postgres store. +// Returns the live session, or nil if reconstruct was not possible. +func (m *TranscodeManager) ReconstructTranscode(ctx context.Context, sessionID string, requestedSegment int, card RecipeCard) *TranscodeSession { + if m == nil { + return nil + } + if card.SessionID == "" || card.SessionID != sessionID { + return nil + } + + // A concurrent reconstruct may already have registered the session; serve it + // directly so we never enter single-flight only to discard a duplicate. + if existing := m.GetTranscodeSession(sessionID); existing != nil { + return existing + } + + v, err, _ := m.reconstructGroup.Do(sessionID, func() (interface{}, error) { + return m.doReconstructTranscode(ctx, sessionID, requestedSegment, card), nil + }) + if err != nil || v == nil { + return nil + } + session, _ := v.(*TranscodeSession) + return session +} + +// fastResumeSeek decides whether a reconstructed ffmpeg should be spawned at the +// segment the client is actually requesting instead of the card's original +// start. Resuming near requestedSegment avoids a wait-then-seek-restart stall +// when the client has already played past the card position. +// +// The returned (segment, seekSeconds) maps via seg×SegmentDuration, which is +// ONLY valid for ENCODED transcodes: their forced keyframes make every segment +// exactly SegmentDuration long. COPY-mode segments inherit the source's variable +// GOP boundaries, so seg×dur lands on the wrong source time and desyncs A/V after +// a restart — so for copy-mode cards this returns ok=false and the caller keeps +// the card's original start, letting the manifest-driven segment recovery +// (RestartSeekTarget) seek forward once the rebuilt manifest exposes the real +// per-segment timing. A negative requestedSegment (manifest path, no segment +// context) and a non-advanced client also return ok=false. +func fastResumeSeek(card RecipeCard, requestedSegment int) (segment int, seekSeconds float64, ok bool) { + if strings.EqualFold(card.TargetCodecVideo, "copy") { + return 0, 0, false + } + if requestedSegment > card.StartSegmentNumber && card.SegmentDuration > 0 { + return requestedSegment, float64(requestedSegment * card.SegmentDuration), true + } + return 0, 0, false +} + +// doReconstructTranscode performs the actual rebuild for a single reconstruct +// leader. It is only ever invoked inside reconstructGroup.Do, so it is the sole +// writer racing to register sessionID for this session. +func (m *TranscodeManager) doReconstructTranscode(ctx context.Context, sessionID string, requestedSegment int, card RecipeCard) *TranscodeSession { + // Only transcode cards drive ffmpeg reconstruction. Direct/remux sessions + // reconstruct without a runtime and must never reach here; guard so a + // direct/remux card ID cannot accidentally spawn an encode. An empty + // PlayMethod is back-compat for a token minted before the discriminator + // (transcode). + if card.PlayMethod != "" && card.PlayMethod != PlayTranscode { + return nil + } + + // Mark in-flight for the whole rebuild so a concurrent cleanup never reaps the + // output dir between spawn and map registration. + release := m.markReconstructing(sessionID) + defer release() + + cfg := m.runtimeConfig() + outputDir := filepath.Join(cfg.TranscodeDir, sessionID) + opts := card.TranscodeOpts(outputDir, cfg.FFmpegPath, m.logSink()) + // Re-resolve environment-specific encode knobs from current config so an + // operator config change applies to reconstructed sessions too. + opts.HWAccel = cfg.HWAccel + opts.HWDevice = cfg.HWDevice + + // Resume near the segment the client is actually requesting. The card records + // the original start; if the client has played past it, spawning ffmpeg at the + // old position forces a wait-then-seek-restart cycle (a visible stall). Seeking + // straight to requestedSegment avoids it. A negative requestedSegment (manifest + // path) carries no segment context, so the card position stands. + // + if seg, seek, ok := fastResumeSeek(card, requestedSegment); ok { + opts.StartSegmentNumber = seg + opts.SeekSeconds = seek + } + + // Pace the spawn so a post-restart wave of reconstructs does not launch a + // thousand cold-start ffmpeg processes at once. A client that disconnects while + // waiting releases its place rather than queueing dead work. + slotRelease, ok := m.acquireReconstructSlot(ctx) + if !ok { + return nil + } + + // Serialize against every other spawn path (fresh start, restart) for this + // session so a reconstruct and a fresh start never run two ffmpeg writers + // against the same output dir. reconstructGroup only single-flights reconstructs + // against each other, not against starts. + unlock := m.LockSessionLifecycle(sessionID) + defer unlock() + + // Re-check under the lifecycle lock: a fresh start (or a reconstruct that ran + // just before us) may already have a live session. Yield to it instead of + // spawning a duplicate writer. + if existing := m.GetTranscodeSession(sessionID); existing != nil { + slotRelease() + return existing + } + + transcodeSession, err := StartTranscode(context.WithoutCancel(ctx), opts) + slotRelease() + if err != nil { + slog.Error("reconstruct transcode start failed", "error", err, "session", sessionID, "playback_session_id", sessionID) + return nil + } + + // Register under the map lock. The lifecycle lock guarantees no other path + // registered since the re-check above; the existing-check is kept as defensive + // belt-and-braces, closing only the duplicate ffmpeg process (never the shared + // output dir the winner serves) on the should-be-impossible race. + m.transcodeMu.Lock() + if existing := m.transcodes[sessionID]; existing != nil { + m.transcodeMu.Unlock() + _ = transcodeSession.CloseProcess() + return existing + } + m.transcodes[sessionID] = transcodeSession + m.transcodeMu.Unlock() + + // Mirror the handler's start path: re-arm the throttler and exit monitor + // after every Restart of this reconstructed session, so seek/audio-switch + // restarts keep the same wiring as a freshly started transcode. + transcodeSession.SetRestartHook(func(ctx context.Context) { + if m.StartThrottler != nil { + m.StartThrottler(ctx, transcodeSession) + } + m.MonitorLocalTranscodeExit(sessionID, transcodeSession) + }) + + if m.StartThrottler != nil { + m.StartThrottler(ctx, transcodeSession) + } + m.MonitorLocalTranscodeExit(sessionID, transcodeSession) + slog.Info("transcode process reconstructed from recipe card", + "session", sessionID, "playback_session_id", sessionID, + "requested_segment", requestedSegment, "start_segment_number", opts.StartSegmentNumber) + return transcodeSession +} + +// MonitorLocalTranscodeExit watches a local ffmpeg process and, on an error exit, +// invokes OnFFmpegCrash so the embedding handler tears down the playback session. +// A clean exit (no error) leaves the segments servable until the client stops. +func (m *TranscodeManager) MonitorLocalTranscodeExit(sessionID string, session *TranscodeSession) { + if m == nil || sessionID == "" || session == nil { + return + } + + done := session.Done() + if done == nil { + return + } + + go func() { + <-done + time.Sleep(2 * time.Second) + + m.transcodeMu.RLock() + current := m.transcodes[sessionID] + m.transcodeMu.RUnlock() + if current != session { + return + } + if session.IsRunning() { + return + } + + // When ffmpeg exits cleanly (no error), the segments are fully written and + // should remain servable until the client stops the session. This is + // critical for copy-mode where ffmpeg finishes writing all content much + // faster than real-time playback. Only tear down the session on error exits. + if session.WaitError() == nil { + return + } + + // ffmpeg crash — tear the session down; a client holding a valid token can + // reconstruct it on the next request. Pass the dead session so teardown is a + // compare-and-delete: a reconstruct that registered a successor under this id + // between the current!=session check above and teardown must not be killed. + if m.OnFFmpegCrash != nil { + m.OnFFmpegCrash(context.Background(), sessionID, session) + } + }() +} + +// CloseTranscodeSession stops a transcode session. If transcodeNodeURL is +// non-empty, sends DELETE to the remote transcode node. Otherwise closes the +// local session. +// +// Under token-carried reconstruction there is no durable card to drop: a stopped +// session simply stops being served, and its segment dir is reaped by the +// in-memory-liveness + age cleanup once no live token could still reconstruct it +// (see CleanupOrphanedTranscodes). A sub-TTL hard cut of an abusive stream +// before the token expires depends on a node-side revocation mechanism that is +// deferred to a future PR; today a stopped session can be reconstructed by a +// still-valid token until it expires. +func (m *TranscodeManager) CloseTranscodeSession(sessionID, transcodeNodeURL string) { + // Clean up local session if one exists (defensive). + m.transcodeMu.Lock() + session := m.transcodes[sessionID] + delete(m.transcodes, sessionID) + m.transcodeMu.Unlock() + if session != nil { + _ = session.Close() + } + + m.deleteRemoteTranscode(sessionID, transcodeNodeURL) +} + +// CloseTranscodeSessionIf tears down a transcode session only when the live map +// still holds the exact session the caller observed dying (expected). This is +// the crash path: between a local ffmpeg's error exit and this teardown, a +// concurrent reconstruct can register a fresh successor under the same id. An +// unconditional close would delete+Close() that live successor — and Close() +// removes the shared output dir out from under it. Comparing under the same lock +// that reconstruct registers through makes the swap atomic: a non-matching entry +// is left untouched. The remote-DELETE still fires for the matched case (and is +// skipped entirely when the local successor already won, since there is nothing +// of ours to stop). +// +// Returns true iff the live entry still matched expected and was torn down; +// false iff a different (successor) or nil session held the slot and was left +// untouched. Callers MUST treat this return as the authoritative gate for any +// further teardown (e.g. stopping the upstream playback session): when it is +// false, a successor owns the id and must not be disturbed. +func (m *TranscodeManager) CloseTranscodeSessionIf(sessionID string, expected *TranscodeSession, transcodeNodeURL string) bool { + m.transcodeMu.Lock() + current := m.transcodes[sessionID] + if current != expected { + // A successor (or an already-completed close) holds the slot; leave it. + m.transcodeMu.Unlock() + return false + } + delete(m.transcodes, sessionID) + m.transcodeMu.Unlock() + if expected != nil { + _ = expected.Close() + } + + m.deleteRemoteTranscode(sessionID, transcodeNodeURL) + return true +} + +// deleteRemoteTranscode sends DELETE to the assigned transcode node if any +// (synchronous with timeout). A no-op for local/integrated sessions. +func (m *TranscodeManager) deleteRemoteTranscode(sessionID, transcodeNodeURL string) { + if transcodeNodeURL != "" { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + deleteURL := transcodeNodeURL + "/transcode/" + sessionID + req, err := http.NewRequestWithContext(ctx, http.MethodDelete, deleteURL, nil) + if err != nil { + slog.Error("remote transcode delete: build request", "error", err, "session", sessionID, "playback_session_id", sessionID) + return + } + req.Header.Set("Authorization", "Bearer "+m.jwtSecret()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + slog.Warn("remote transcode delete failed", "error", err, "session", sessionID, "node", transcodeNodeURL, "playback_session_id", sessionID) + return + } + _ = resp.Body.Close() + if resp.StatusCode >= http.StatusMultipleChoices { + slog.Warn("remote transcode delete returned non-success status", + "status", resp.StatusCode, "session", sessionID, "node", transcodeNodeURL, "playback_session_id", sessionID) + } + } +} + +// CleanupOrphanedTranscodes removes stale per-session temp directories for +// transcodes that are no longer reconstructable. Under token-carried +// reconstruction there is no durable card index to consult, so the liveness +// signal is: the in-process live transcode map, the set of sessions currently +// mid-reconstruct, and directory age. A dir is reaped only when it is absent from +// both sets AND older than the maximum token lifetime — past which no surviving +// token could reconstruct it. Each process owns its own TranscodeDir, so there is +// no cross-process enumeration-failure mode to fail safe against. +func (m *TranscodeManager) CleanupOrphanedTranscodes() (int, error) { + // Snapshot the live map and the in-flight set under both locks held at once. + // A reconstruct registers into m.transcodes and clears m.reconstructInFlight + // at different moments; snapshotting the two sets separately could miss a + // session that migrated between them, leaving its live dir absent from active + // and exposed to reaping. inFlightMu is taken first to match the only other + // site that holds both (none nests the reverse order). + m.inFlightMu.Lock() + m.transcodeMu.RLock() + active := make(map[string]struct{}, len(m.transcodes)+len(m.reconstructInFlight)) + for sessionID := range m.transcodes { + active[sessionID] = struct{}{} + } + // Spare sessions mid-reconstruct: their dir is being written right now but is + // not yet registered in the live map. + for sessionID := range m.reconstructInFlight { + active[sessionID] = struct{}{} + } + m.transcodeMu.RUnlock() + m.inFlightMu.Unlock() + + return CleanupOrphanedTranscodeDirs(m.runtimeConfig().TranscodeDir, active, MaxTokenTTL) +} diff --git a/internal/playback/transcode_manager_test.go b/internal/playback/transcode_manager_test.go new file mode 100644 index 00000000..825ce4e8 --- /dev/null +++ b/internal/playback/transcode_manager_test.go @@ -0,0 +1,460 @@ +package playback + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" +) + +// fakeSessionRegistry is a GetSession + RegisterReconstructed double. +type fakeSessionRegistry struct { + sessions map[string]*Session + // maxPerUser, when > 0, caps reconstructs per user via + // RegisterReconstructedWithLimits so the admission path can be tested. + maxPerUser int + // limitsErr, when non-nil, is returned by RegisterReconstructedWithLimits to + // simulate a limit-provider failure (e.g. a transient Postgres error). It + // takes precedence over the over-cap check so the fail-open admission path + // can be exercised. A real SessionManager surfaces such failures wrapped with + // ErrLimitProviderUnavailable. + limitsErr error +} + +func (f *fakeSessionRegistry) GetSession(id string) (*Session, error) { + if s, ok := f.sessions[id]; ok { + return s, nil + } + return nil, ErrSessionNotFound +} + +func (f *fakeSessionRegistry) RegisterReconstructed(s *Session) *Session { + if f.sessions == nil { + f.sessions = map[string]*Session{} + } + if existing, ok := f.sessions[s.ID]; ok { + return existing + } + f.sessions[s.ID] = s + return s +} + +// RegisterReconstructedWithLimits mirrors RegisterReconstructed for the tests. +// maxPerUser, when > 0, caps how many sessions a single user may reconstruct so +// the admission-rejection path can be exercised without a real SessionManager. +func (f *fakeSessionRegistry) RegisterReconstructedWithLimits(_ context.Context, s *Session) (*Session, error) { + if f.sessions == nil { + f.sessions = map[string]*Session{} + } + if existing, ok := f.sessions[s.ID]; ok { + return existing, nil + } + if f.limitsErr != nil { + return nil, f.limitsErr + } + if f.maxPerUser > 0 { + live := 0 + for _, existing := range f.sessions { + if existing.UserID == s.UserID { + live++ + } + } + if live >= f.maxPerUser { + return nil, ErrTooManyStreams + } + } + f.sessions[s.ID] = s + return s, nil +} + +// CloseTranscodeSession stops the live session and drops it from the transcode +// map. Under token-carried reconstruction there is no durable card to delete; the +// segment dir is reaped later by liveness+age cleanup. +func TestCloseTranscodeSession_DropsLiveSession(t *testing.T) { + m := NewTranscodeManager() + m.RegisterTranscodeSession("s1", &TranscodeSession{}) + m.CloseTranscodeSession("s1", "") + if got := m.GetTranscodeSession("s1"); got != nil { + t.Fatal("session must be removed from the live map on close") + } +} + +func TestLoadOrReconstructSession(t *testing.T) { + ctx := context.Background() + + newMgr := func(reg *fakeSessionRegistry) *TranscodeManager { + m := NewTranscodeManager() + m.Sessions = reg + return m + } + cardPtr := func(c RecipeCard) *RecipeCard { return &c } + + t.Run("live session, matching owner -> loaded", func(t *testing.T) { + reg := &fakeSessionRegistry{sessions: map[string]*Session{"s": {ID: "s", UserID: 5}}} + m := newMgr(reg) + got, status := m.LoadOrReconstructSession(ctx, reg.GetSession, "s", 5, nil) + if status != SessionLoaded || got == nil || got.ID != "s" { + t.Fatalf("got status=%v session=%+v", status, got) + } + }) + + t.Run("live session, mismatched owner -> forbidden", func(t *testing.T) { + reg := &fakeSessionRegistry{sessions: map[string]*Session{"s": {ID: "s", UserID: 5}}} + m := newMgr(reg) + if _, status := m.LoadOrReconstructSession(ctx, reg.GetSession, "s", 9, nil); status != SessionForbidden { + t.Fatalf("status = %v, want forbidden", status) + } + }) + + t.Run("live session, zero caller -> loaded (UUID as bearer)", func(t *testing.T) { + reg := &fakeSessionRegistry{sessions: map[string]*Session{"s": {ID: "s", UserID: 5}}} + m := newMgr(reg) + if _, status := m.LoadOrReconstructSession(ctx, reg.GetSession, "s", 0, nil); status != SessionLoaded { + t.Fatalf("status = %v, want loaded", status) + } + }) + + t.Run("miss + remux token + matching owner -> reconstructed with method", func(t *testing.T) { + reg := &fakeSessionRegistry{} + m := newMgr(reg) + card := NewRemuxRecipeCard("s", 5, "p", 77, true, 2) + got, status := m.LoadOrReconstructSession(ctx, reg.GetSession, "s", 5, cardPtr(card)) + if status != SessionLoaded || got == nil { + t.Fatalf("status=%v session=%+v", status, got) + } + if got.PlayMethod != PlayRemux || got.MediaFileID != 77 || !got.TranscodeAudio || got.AudioTrackIndex != 2 { + t.Fatalf("reconstructed remux session wrong: %+v", got) + } + if _, err := reg.GetSession("s"); err != nil { + t.Fatalf("reconstructed session not registered: %v", err) + } + }) + + t.Run("miss + token + mismatched owner -> missing (reconstruct refuses)", func(t *testing.T) { + reg := &fakeSessionRegistry{} + m := newMgr(reg) + card := NewDirectRecipeCard("s", 5, "p", 77) + if _, status := m.LoadOrReconstructSession(ctx, reg.GetSession, "s", 9, cardPtr(card)); status != SessionMissing { + t.Fatalf("status = %v, want missing", status) + } + }) + + t.Run("miss + token for a different session id -> missing", func(t *testing.T) { + reg := &fakeSessionRegistry{} + m := newMgr(reg) + card := NewDirectRecipeCard("other", 5, "p", 77) + if _, status := m.LoadOrReconstructSession(ctx, reg.GetSession, "s", 5, cardPtr(card)); status != SessionMissing { + t.Fatalf("status = %v, want missing (card session id mismatch)", status) + } + }) + + t.Run("miss + no token -> missing", func(t *testing.T) { + reg := &fakeSessionRegistry{} + m := newMgr(reg) + if _, status := m.LoadOrReconstructSession(ctx, reg.GetSession, "nope", 5, nil); status != SessionMissing { + t.Fatalf("status = %v, want missing", status) + } + }) +} + +// CloseTranscodeSessionIf must leave a successor registered under the same id +// untouched: a reconstruct that replaced the crashed ffmpeg between exit and +// teardown must not have its live session (and shared output dir) torn down. +func TestCloseTranscodeSessionIf_LeavesSuccessor(t *testing.T) { + m := NewTranscodeManager() + dead := &TranscodeSession{} + successor := &TranscodeSession{} + + // The map now holds the successor (the reconstruct won the race), not dead. + m.RegisterTranscodeSession("s1", successor) + + if matched := m.CloseTranscodeSessionIf("s1", dead, ""); matched { + t.Fatalf("CloseTranscodeSessionIf must report false when a successor holds the slot") + } + + if got := m.GetTranscodeSession("s1"); got != successor { + t.Fatalf("successor must survive a crash teardown for the dead session, got %v", got) + } +} + +// CloseTranscodeSessionIf must remove the entry when it is still the exact +// session that died (the ordinary crash case with no successor). +func TestCloseTranscodeSessionIf_RemovesMatching(t *testing.T) { + m := NewTranscodeManager() + dead := &TranscodeSession{} + m.RegisterTranscodeSession("s1", dead) + + if matched := m.CloseTranscodeSessionIf("s1", dead, ""); !matched { + t.Fatalf("CloseTranscodeSessionIf must report true when the dead session still holds the slot") + } + + if got := m.GetTranscodeSession("s1"); got != nil { + t.Fatalf("matching dead session must be removed, got %v", got) + } +} + +// The crash closures use the matched return of CloseTranscodeSessionIf as the +// authoritative gate for tearing down the upstream playback session. This test +// proves that contract end-to-end for the successor case: when a successor is +// present, the call returns false (so the closure returns early and never stops +// the successor's session), and a second call with the successor as expected +// returns true. The closures (handlers.OnFFmpegCrash / jellycompat +// OnFFmpegCrash) wire `matched` directly to the StopSession/stopPlaybackSessionByID +// decision, so a false gate guarantees the live session is left intact. +func TestCloseTranscodeSessionIf_GateContract(t *testing.T) { + m := NewTranscodeManager() + dead := &TranscodeSession{} + successor := &TranscodeSession{} + + // Reconstruct won the race: successor sits in the slot under the same id. + m.RegisterTranscodeSession("s1", successor) + + // Crash teardown for the dead session must not match -> closure must NOT + // proceed to stop the (successor's) playback session. + if m.CloseTranscodeSessionIf("s1", dead, "") { + t.Fatalf("gate must be false while a successor owns the id") + } + if got := m.GetTranscodeSession("s1"); got != successor { + t.Fatalf("successor transcode must survive, got %v", got) + } + + // Tearing down the successor itself (the ordinary later stop) must match. + if !m.CloseTranscodeSessionIf("s1", successor, "") { + t.Fatalf("gate must be true when expected matches the live entry") + } + if got := m.GetTranscodeSession("s1"); got != nil { + t.Fatalf("successor must be removed once it is the expected session, got %v", got) + } +} + +// fastResumeSeek must apply the seg×dur fast resume only for encoded transcodes; +// copy-mode cards have variable-duration segments so the fast seek is unsafe. +func TestFastResumeSeek(t *testing.T) { + encoded := RecipeCard{TargetCodecVideo: "h264", SegmentDuration: 4, StartSegmentNumber: 0} + if seg, seek, ok := fastResumeSeek(encoded, 10); !ok || seg != 10 || seek != 40 { + t.Fatalf("encoded fast resume = (%d, %v, %v), want (10, 40, true)", seg, seek, ok) + } + + // Copy-mode: never apply the seg×dur seek regardless of how far the client advanced. + copyCard := RecipeCard{TargetCodecVideo: "copy", SegmentDuration: 4, StartSegmentNumber: 0} + if _, _, ok := fastResumeSeek(copyCard, 10); ok { + t.Fatal("copy-mode must not apply the fast seg×dur resume seek") + } + // Case-insensitive guard. + if _, _, ok := fastResumeSeek(RecipeCard{TargetCodecVideo: "COPY", SegmentDuration: 4}, 10); ok { + t.Fatal("copy-mode detection must be case-insensitive") + } + + // Manifest path (negative segment) and a non-advanced client take no fast seek. + if _, _, ok := fastResumeSeek(encoded, -1); ok { + t.Fatal("manifest path (negative segment) must not fast-seek") + } + if _, _, ok := fastResumeSeek(encoded, 0); ok { + t.Fatal("non-advanced client must not fast-seek") + } +} + +// ReconstructSession must refuse to rebuild a session when the user is already at +// their per-user concurrency cap (token replay over-cap), while still allowing +// reconstructs up to the cap. +func TestReconstructSession_AdmissionCap(t *testing.T) { + ctx := context.Background() + reg := &fakeSessionRegistry{maxPerUser: 1} + m := NewTranscodeManager() + m.Sessions = reg + + // First reconstruct for the user admits. + card1 := NewDirectRecipeCard("a", 7, "p", 100) + if got := m.ReconstructSession(ctx, "a", 7, card1); got == nil { + t.Fatal("first reconstruct within cap should succeed") + } + // Second distinct session for the same user is over cap -> refused. + card2 := NewDirectRecipeCard("b", 7, "p", 101) + if got := m.ReconstructSession(ctx, "b", 7, card2); got != nil { + t.Fatal("over-cap reconstruct must be refused") + } +} + +// A transient limit-PROVIDER failure during reconstruct (e.g. a Postgres error +// in the post-restart wave) must NOT collapse into a permanent 404. The session +// must be admitted (fail open) so a user within their limits keeps playing. +func TestReconstructSession_ProviderErrorFailsOpen(t *testing.T) { + ctx := context.Background() + reg := &fakeSessionRegistry{ + limitsErr: fmt.Errorf("load session limits for user 7: %w", + errors.Join(ErrLimitProviderUnavailable, errors.New("db timeout"))), + } + m := NewTranscodeManager() + m.Sessions = reg + + card := NewDirectRecipeCard("a", 7, "p", 100) + got := m.ReconstructSession(ctx, "a", 7, card) + if got == nil { + t.Fatal("limit-provider error must fail open and admit the reconstructed session, not refuse") + } + if got.ID != "a" || got.UserID != 7 { + t.Fatalf("admitted session wrong: %+v", got) + } + // The fail-open path must register the session so LoadOrReconstructSession + // yields SessionLoaded, not SessionMissing. + if _, err := reg.GetSession("a"); err != nil { + t.Fatalf("failed-open session not registered: %v", err) + } +} + +// LoadOrReconstructSession must surface the fail-open admission as SessionLoaded +// (not SessionMissing -> 404) when the limit provider is transiently unavailable. +func TestLoadOrReconstructSession_ProviderErrorFailsOpen(t *testing.T) { + ctx := context.Background() + reg := &fakeSessionRegistry{ + limitsErr: errors.Join(ErrLimitProviderUnavailable, errors.New("db timeout")), + } + m := NewTranscodeManager() + m.Sessions = reg + + card := NewDirectRecipeCard("s", 5, "p", 77) + got, status := m.LoadOrReconstructSession(ctx, reg.GetSession, "s", 5, &card) + if status != SessionLoaded || got == nil { + t.Fatalf("provider error must yield SessionLoaded, got status=%v session=%+v", status, got) + } +} + +// A genuine over-cap rejection must STILL refuse even after the fail-open change: +// the ErrTooManyStreams / ErrTooManyTranscodes sentinels are not provider errors. +func TestReconstructSession_OverCapStillRefused(t *testing.T) { + ctx := context.Background() + + t.Run("too many streams", func(t *testing.T) { + reg := &fakeSessionRegistry{limitsErr: ErrTooManyStreams} + m := NewTranscodeManager() + m.Sessions = reg + card := NewDirectRecipeCard("a", 7, "p", 100) + if got := m.ReconstructSession(ctx, "a", 7, card); got != nil { + t.Fatal("ErrTooManyStreams over-cap must still be refused (nil)") + } + if _, err := reg.GetSession("a"); err == nil { + t.Fatal("over-cap reconstruct must not register the session") + } + }) + + t.Run("too many transcodes", func(t *testing.T) { + reg := &fakeSessionRegistry{limitsErr: ErrTooManyTranscodes} + m := NewTranscodeManager() + m.Sessions = reg + card := NewDirectRecipeCard("a", 7, "p", 100) + if got := m.ReconstructSession(ctx, "a", 7, card); got != nil { + t.Fatal("ErrTooManyTranscodes over-cap must still be refused (nil)") + } + }) +} + +// ReconstructSession ownership contract: the authless transcode delivery routes +// (HLS master.m3u8 / segment) present no userID, so a zero caller must be allowed +// and the rebuilt session bound to the card owner. A non-zero caller that does not +// match the card owner is still refused. +func TestReconstructSession_Ownership(t *testing.T) { + ctx := context.Background() + + t.Run("zero caller -> reconstructed, bound to card owner", func(t *testing.T) { + reg := &fakeSessionRegistry{} + m := NewTranscodeManager() + m.Sessions = reg + + card := NewDirectRecipeCard("s", 5, "p", 77) + got := m.ReconstructSession(ctx, "s", 0, card) + if got == nil { + t.Fatal("zero caller (UUID-as-bearer route) must reconstruct") + } + if got.UserID != 5 { + t.Fatalf("reconstructed session UserID = %d, want 5 (card owner)", got.UserID) + } + if _, err := reg.GetSession("s"); err != nil { + t.Fatalf("reconstructed session not registered: %v", err) + } + }) + + t.Run("non-zero mismatched caller -> refused", func(t *testing.T) { + reg := &fakeSessionRegistry{} + m := NewTranscodeManager() + m.Sessions = reg + + card := NewDirectRecipeCard("s", 5, "p", 77) + if got := m.ReconstructSession(ctx, "s", 9, card); got != nil { + t.Fatal("non-zero caller mismatching the card owner must be refused") + } + }) +} + +// acquireReconstructSlot must bound concurrent reconstructs and let a caller +// whose request is cancelled give up its place instead of queueing dead work. +func TestAcquireReconstructSlot(t *testing.T) { + m := &TranscodeManager{reconstructSem: make(chan struct{}, 1)} + + release, ok := m.acquireReconstructSlot(context.Background()) + if !ok { + t.Fatal("first acquire should succeed") + } + + // Cap is full: a cancelled request must back off rather than block forever. + cancelled, cancel := context.WithCancel(context.Background()) + cancel() + if _, ok := m.acquireReconstructSlot(cancelled); ok { + t.Fatal("acquire on a full semaphore with a cancelled context must fail") + } + + // Releasing frees the slot for the next reconstruct. + release() + release2, ok := m.acquireReconstructSlot(context.Background()) + if !ok { + t.Fatal("acquire should succeed after the slot is released") + } + release2() +} + +func TestLockSessionLifecycle_MutualExclusionAndCleanup(t *testing.T) { + m := NewTranscodeManager() + + // Two holders of the same key must be mutually exclusive. + var counter, maxConcurrent int32 + var wg sync.WaitGroup + for i := 0; i < 50; i++ { + wg.Add(1) + go func() { + defer wg.Done() + unlock := m.LockSessionLifecycle("sess-a") + defer unlock() + c := atomic.AddInt32(&counter, 1) + if c > atomic.LoadInt32(&maxConcurrent) { + atomic.StoreInt32(&maxConcurrent, c) + } + time.Sleep(time.Millisecond) + atomic.AddInt32(&counter, -1) + }() + } + wg.Wait() + if maxConcurrent != 1 { + t.Fatalf("lifecycle lock allowed %d concurrent holders, want 1", maxConcurrent) + } + + // Different keys do not block each other and the map drops entries once + // released. + u1 := m.LockSessionLifecycle("k1") + u2 := m.LockSessionLifecycle("k2") + m.lifecycleMu.Lock() + n := len(m.lifecycleLocks) + m.lifecycleMu.Unlock() + if n != 2 { + t.Fatalf("expected 2 live lifecycle locks, got %d", n) + } + u1() + u2() + m.lifecycleMu.Lock() + n = len(m.lifecycleLocks) + m.lifecycleMu.Unlock() + if n != 0 { + t.Fatalf("expected lifecycle lock map to drain to 0, got %d", n) + } +} diff --git a/internal/playback/transcode_manifest_test.go b/internal/playback/transcode_manifest_test.go index 6b7f54d4..8b7f11b0 100644 --- a/internal/playback/transcode_manifest_test.go +++ b/internal/playback/transcode_manifest_test.go @@ -318,7 +318,11 @@ func TestRestartSeekTarget_CopyModeUsesManifestTimelineWhenAvailable(t *testing. } } -func TestRestartSeekTarget_CopyModeFallsBackToFixedDurationWhenSegmentOutsideManifest(t *testing.T) { +func TestRestartSeekTarget_CopyModeReportsUnresolvedWhenSegmentOutsideManifest(t *testing.T) { + // Copy-mode fragments have variable durations, so a segment the manifest + // can't yet place must NOT fall back to fixed-duration seg×dur math (which + // would seek FFmpeg to the wrong source time and desync A/V). Instead it + // reports the seek target as unresolved (0, false, nil). tempDir := t.TempDir() manifest := strings.Join([]string{ "#EXTM3U", @@ -349,11 +353,11 @@ func TestRestartSeekTarget_CopyModeFallsBackToFixedDurationWhenSegmentOutsideMan if err != nil { t.Fatalf("RestartSeekTarget: %v", err) } - if !ok { - t.Fatal("RestartSeekTarget returned ok=false") + if ok { + t.Fatalf("RestartSeekTarget(50) returned ok=true (got %f); copy-mode should report unresolved, not seg×dur", got) } - if got != 100 { - t.Fatalf("RestartSeekTarget(50) = %f, want 100", got) + if got != 0 { + t.Fatalf("RestartSeekTarget(50) = %f, want 0", got) } } @@ -377,8 +381,13 @@ func TestRestartSeekTarget_EncodedUsesFixedDurationMath(t *testing.T) { } } -func TestRestartSeekTarget_CopyModeFallsBackWhenManifestNotReady(t *testing.T) { +func TestRestartSeekTarget_CopyModeReportsUnresolvedWhenManifestNotReady(t *testing.T) { + // A freshly reconstructed copy-mode window has a near-empty/absent manifest + // (ErrManifestNotReady). The seek target must be reported unresolved + // (0, false, nil) so the caller retries instead of seeking to a fabricated + // seg×dur position. session := &TranscodeSession{ + outputDir: t.TempDir(), // no stream.m3u8 written -> ErrManifestNotReady opts: TranscodeOpts{ TargetCodecVideo: "copy", SegmentDuration: 2, @@ -389,11 +398,11 @@ func TestRestartSeekTarget_CopyModeFallsBackWhenManifestNotReady(t *testing.T) { if err != nil { t.Fatalf("RestartSeekTarget: %v", err) } - if !ok { - t.Fatal("RestartSeekTarget returned ok=false") + if ok { + t.Fatalf("RestartSeekTarget(10) returned ok=true (got %f); copy-mode should report unresolved when manifest not ready", got) } - if got != 20 { - t.Fatalf("RestartSeekTarget(10) = %f, want 20", got) + if got != 0 { + t.Fatalf("RestartSeekTarget(10) = %f, want 0", got) } } @@ -737,3 +746,42 @@ func TestCleanStaleSegments(t *testing.T) { } } } + +func TestAppendManifestQueryParam(t *testing.T) { + manifest := strings.Join([]string{ + "#EXTM3U", + "#EXT-X-VERSION:7", + "#EXT-X-TARGETDURATION:2", + "#EXT-X-MAP:URI=\"segment/init.mp4\"", + "#EXTINF:2.000000,", + "segment/seg_00000.ts", + "#EXTINF:2.000000,", + "segment/seg_00001.ts?existing=1", + "", + }, "\n") + + got := string(AppendManifestQueryParam([]byte(manifest), "st", "TOKEN")) + for _, want := range []string{ + "#EXT-X-MAP:URI=\"segment/init.mp4?st=TOKEN\"", + "segment/seg_00000.ts?st=TOKEN", + "segment/seg_00001.ts?existing=1&st=TOKEN", + } { + if !strings.Contains(got, want) { + t.Fatalf("rewritten manifest missing %q:\n%s", want, got) + } + } + // Tags and EXTINF lines must be untouched. + if !strings.Contains(got, "#EXT-X-TARGETDURATION:2") || strings.Contains(got, "#EXT-X-TARGETDURATION:2?st") { + t.Fatalf("non-URI tag was rewritten:\n%s", got) + } +} + +func TestAppendManifestQueryParam_NonManifestUnchanged(t *testing.T) { + body := []byte("not a manifest\nsegment/seg_00000.ts\n") + if got := AppendManifestQueryParam(body, "st", "TOKEN"); string(got) != string(body) { + t.Fatalf("non-manifest body should be unchanged, got:\n%s", got) + } + if got := AppendManifestQueryParam([]byte("#EXTM3U\nseg.ts\n"), "", "TOKEN"); !strings.Contains(string(got), "seg.ts\n") || strings.Contains(string(got), "seg.ts?") { + t.Fatalf("empty key should be a no-op, got:\n%s", got) + } +} diff --git a/internal/proxy/server.go b/internal/proxy/server.go index c3d6565f..01a49a41 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -27,7 +27,8 @@ type Server struct { egress *egressMeter } -// NewServer creates a new proxy server backed by a config watcher and session tracker. +// NewServer creates a new proxy server backed by a config watcher and session +// tracker. func NewServer(watcher *nodeconfig.Watcher, tracker *nodesessions.Tracker) *Server { return &Server{ watcher: watcher, @@ -139,13 +140,7 @@ func (s *Server) handleDirectPlay(w http.ResponseWriter, r *http.Request) { return } - info := nodesessions.SessionInfo{ - SessionID: claims.SessionID, - NodeURL: s.tracker.NodeURL(), - NodeName: s.tracker.NodeName(), - Type: "direct_play", - StartedAt: time.Now().UTC().Format(time.RFC3339), - } + info := sessionInfo(s.tracker, claims, "direct_play") s.tracker.Track(r.Context(), info) defer s.tracker.Remove(r.Context(), claims.SessionID) @@ -158,13 +153,7 @@ func (s *Server) handleRemux(w http.ResponseWriter, r *http.Request) { return } - info := nodesessions.SessionInfo{ - SessionID: claims.SessionID, - NodeURL: s.tracker.NodeURL(), - NodeName: s.tracker.NodeName(), - Type: "remux", - StartedAt: time.Now().UTC().Format(time.RFC3339), - } + info := sessionInfo(s.tracker, claims, "remux") s.tracker.Track(r.Context(), info) defer s.tracker.Remove(r.Context(), claims.SessionID) @@ -201,13 +190,22 @@ func (s *Server) handleTranscodeSegment(w http.ResponseWriter, r *http.Request) // short manifest/segment requests, so the session is tracked by recent // activity instead of request lifetime. func (s *Server) touchTranscodeSession(r *http.Request, claims *streamtoken.Claims) { - s.tracker.Touch(r.Context(), nodesessions.SessionInfo{ - SessionID: claims.SessionID, - NodeURL: s.tracker.NodeURL(), - NodeName: s.tracker.NodeName(), - Type: "transcode", - StartedAt: time.Now().UTC().Format(time.RFC3339), - }) + s.tracker.Touch(r.Context(), sessionInfo(s.tracker, claims, "transcode")) +} + +// sessionInfo builds the node-session tracker record for a verified token, +// copying the numeric ownership keys the node-session tracker needs. +func sessionInfo(tr *nodesessions.Tracker, claims *streamtoken.Claims, kind string) nodesessions.SessionInfo { + return nodesessions.SessionInfo{ + SessionID: claims.SessionID, + NodeURL: tr.NodeURL(), + NodeName: tr.NodeName(), + Type: kind, + StartedAt: time.Now().UTC().Format(time.RFC3339), + AuthUserID: claims.UserID, + ProfileID: claims.ProfileID, + MediaFileID: claims.MediaFileID, + } } func (s *Server) handleSubtitle(w http.ResponseWriter, r *http.Request) { @@ -324,6 +322,14 @@ func (s *Server) proxyToTranscodeNode(w http.ResponseWriter, r *http.Request, cl return } req.Header.Set("Authorization", "Bearer "+cfg.Auth.JWTSecret) + // Forward the verified stream token so the transcode node can self-reconstruct + // a lost session after its OWN restart: the token carries the full byte-affecting + // recipe, so the node can re-spawn ffmpeg seeked to the requested segment instead + // of 404ing (the integrated server already does this from the same token). The + // node re-verifies the token independently before trusting it. + if token := chi.URLParam(r, "token"); token != "" { + req.Header.Set("X-Silo-Stream-Token", token) + } resp, err := s.httpClient.Do(req) if err != nil { diff --git a/internal/streamtoken/token.go b/internal/streamtoken/token.go index 2683310f..fdef16fe 100644 --- a/internal/streamtoken/token.go +++ b/internal/streamtoken/token.go @@ -9,6 +9,14 @@ import ( // Claims holds everything a stateless proxy or transcode node needs // to serve a streaming session without database access. +// +// Under token-carried reconstruction (TR-lease) the token is also the durable +// reconstruction descriptor: its claims carry the full set of byte-affecting +// encode parameters (the former Postgres "recipe card"), so a front-end that has +// lost its in-memory session can rebuild ffmpeg from the token the client +// re-presents — no shared per-session store. The ownership claims (uid/pid/mfid) +// are lookup keys re-resolved against the authority on reconstruct; they are +// never trusted on their own. type Claims struct { SessionID string `json:"sid"` MediaPath string `json:"path"` @@ -20,6 +28,31 @@ type Claims struct { AudioCodec string `json:"ac,omitempty"` AudioChannels int `json:"ach,omitempty"` AudioTrackIndex int `json:"ati,omitempty"` + + // Ownership / authorization lookup keys (re-resolved at reconstruct). + // Not trust assertions. + UserID int `json:"uid,omitempty"` + ProfileID string `json:"pid,omitempty"` + MediaFileID int `json:"mfid,omitempty"` + + // Reconstruction recipe — the byte-affecting encode parameters, mirroring the + // former playback.RecipeCard. Zero for direct/remux tokens, which reconstruct + // from identity alone plus the client-supplied position. + SourceVideoCodec string `json:"svc,omitempty"` + SeekSeconds float64 `json:"seek,omitempty"` + SegmentDuration int `json:"segd,omitempty"` + StartSegmentNumber int `json:"ssn,omitempty"` + SubtitleTrackIndex int `json:"sti,omitempty"` + SubtitleBurnIn bool `json:"sbi,omitempty"` + TargetBitrateKbps int `json:"tbr,omitempty"` + TotalDuration float64 `json:"dur,omitempty"` + FastStart bool `json:"fs,omitempty"` + TargetCodecAudio string `json:"tca,omitempty"` + + // Recipe staleness hint, bumped on each re-mint after a recipe mutation + // (audio/quality/seek switch). An optional client-side hint only. + Version int `json:"ver,omitempty"` + jwt.RegisteredClaims } diff --git a/internal/transcodenode/restart_locked_test.go b/internal/transcodenode/restart_locked_test.go new file mode 100644 index 00000000..a4e65ccb --- /dev/null +++ b/internal/transcodenode/restart_locked_test.go @@ -0,0 +1,78 @@ +package transcodenode + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/Silo-Server/silo-server/internal/playback" +) + +// The node's segment-recovery restart must also run under the per-session +// lifecycle lock so it cannot race a fresh start or reconstruct into the same +// output directory. These tests exercise the wrapper's gating and liveness +// re-check without spawning ffmpeg (the playback package covers the real spawn). + +// restartSessionLocked must block while another lifecycle holder owns the lock, +// and re-check session liveness once it acquires it — a session torn down while +// the restart waited must yield ErrSessionSuperseded rather than a stale spawn. +func TestRestartSessionLocked_WaitsForLockThenRechecks(t *testing.T) { + s := &Server{sessions: map[string]*playback.TranscodeSession{}} + const sid = "sess-x" + sess := &playback.TranscodeSession{} + s.mu.Lock() + s.sessions[sid] = sess + s.mu.Unlock() + + // A concurrent start/reconstruct holds the lifecycle lock. + release := s.lockSessionLifecycle(sid) + + done := make(chan error, 1) + go func() { + done <- s.restartSessionLocked(context.Background(), sid, sess, 0, 0) + }() + + select { + case <-done: + t.Fatal("restartSessionLocked returned while the lifecycle lock was held") + case <-time.After(150 * time.Millisecond): + } + + // Simulate a teardown removing the session while the restart is blocked, so + // the post-lock re-check fails and no ffmpeg is spawned. + s.mu.Lock() + delete(s.sessions, sid) + s.mu.Unlock() + release() + + select { + case err := <-done: + if !errors.Is(err, playback.ErrSessionSuperseded) { + t.Fatalf("want ErrSessionSuperseded after teardown, got %v", err) + } + case <-time.After(2 * time.Second): + t.Fatal("restartSessionLocked did not complete after lock release") + } +} + +func TestRestartSessionLocked_SupersededWhenUnregistered(t *testing.T) { + s := &Server{sessions: map[string]*playback.TranscodeSession{}} + err := s.restartSessionLocked(context.Background(), "sess-x", &playback.TranscodeSession{}, 0, 0) + if !errors.Is(err, playback.ErrSessionSuperseded) { + t.Fatalf("want ErrSessionSuperseded for an unmapped session, got %v", err) + } +} + +func TestRestartSessionLocked_SupersededWhenReplaced(t *testing.T) { + s := &Server{sessions: map[string]*playback.TranscodeSession{}} + const sid = "sess-x" + s.mu.Lock() + s.sessions[sid] = &playback.TranscodeSession{} // a different session owns the id + s.mu.Unlock() + + err := s.restartSessionLocked(context.Background(), sid, &playback.TranscodeSession{}, 0, 0) + if !errors.Is(err, playback.ErrSessionSuperseded) { + t.Fatalf("want ErrSessionSuperseded for a replaced session, got %v", err) + } +} diff --git a/internal/transcodenode/server.go b/internal/transcodenode/server.go index b3aa1122..6c925430 100644 --- a/internal/transcodenode/server.go +++ b/internal/transcodenode/server.go @@ -8,6 +8,7 @@ import ( "net/http" "os" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -15,11 +16,13 @@ import ( "time" "github.com/go-chi/chi/v5" + "golang.org/x/sync/singleflight" "github.com/Silo-Server/silo-server/internal/chapterthumbs" "github.com/Silo-Server/silo-server/internal/nodeconfig" "github.com/Silo-Server/silo-server/internal/nodesessions" "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtoken" ) // TranscodeStartRequest is the JSON body for POST /transcode/start. @@ -62,6 +65,84 @@ type Server struct { sessions map[string]*playback.TranscodeSession mu sync.RWMutex activeJobs atomic.Int32 + + // reconstructGroup single-flights node-side session reconstruction per session + // id so a post-restart wave of concurrent manifest/segment requests for the same + // lost session spawns exactly one ffmpeg, never racing duplicates into the shared + // output directory. + reconstructGroup singleflight.Group + // reconstructSem bounds how many sessions may be reconstructed (ffmpeg + // re-spawned) at once after a node restart, pacing the cold-start burst instead + // of stampeding the host. Lazily sized to NumCPU on first use. + reconstructSemOnce sync.Once + reconstructSem chan struct{} + + // lifecycleMu guards lifecycleLocks, the per-session mutexes that serialize + // every path which spawns ffmpeg into a session's output dir (fresh start and + // reconstruct). reconstructGroup only single-flights reconstructs against each + // other; without this a reconstruct racing a fresh /transcode/start could run + // two ffmpeg writers against the same dir. + lifecycleMu sync.Mutex + lifecycleLocks map[string]*sessionLifecycleLock + + // recipeStore is the control-plane recipe store consulted when a forwarded + // token carries no recipe (the jellycompat node hop). Nil disables that path. + recipeStore recipeStore +} + +// sessionLifecycleLock is a refcounted per-session mutex; the refcount lets the +// node drop the map entry once no path holds or waits on it so the map stays +// bounded over the node's lifetime. +type sessionLifecycleLock struct { + mu sync.Mutex + refs int +} + +// lockSessionLifecycle acquires the per-session lifecycle mutex and returns a +// release func. Held across "check existing → spawn → register" so a fresh start +// and a reconstruct never run concurrent ffmpeg writers for one session's dir. +func (s *Server) lockSessionLifecycle(sessionID string) func() { + s.lifecycleMu.Lock() + if s.lifecycleLocks == nil { + s.lifecycleLocks = make(map[string]*sessionLifecycleLock) + } + lk := s.lifecycleLocks[sessionID] + if lk == nil { + lk = &sessionLifecycleLock{} + s.lifecycleLocks[sessionID] = lk + } + lk.refs++ + s.lifecycleMu.Unlock() + + lk.mu.Lock() + return func() { + lk.mu.Unlock() + s.lifecycleMu.Lock() + lk.refs-- + if lk.refs == 0 { + delete(s.lifecycleLocks, sessionID) + } + s.lifecycleMu.Unlock() + } +} + +// restartSessionLocked re-spawns session under the per-session lifecycle lock so +// a segment-recovery restart can never race a fresh start, reconstruct, or +// another restart into the same output directory. It holds the lock only across +// the cancel→respawn transition inside Restart and releases it before the caller +// waits on segments. Under the lock it confirms session is still the live mapped +// session; a concurrent teardown or reconstruct that replaced it yields +// ErrSessionSuperseded rather than re-spawning the stale handle. +func (s *Server) restartSessionLocked(ctx context.Context, sessionID string, session *playback.TranscodeSession, seekSeconds float64, startSegment int) error { + unlock := s.lockSessionLifecycle(sessionID) + defer unlock() + s.mu.RLock() + live, ok := s.sessions[sessionID] + s.mu.RUnlock() + if !ok || live != session { + return playback.ErrSessionSuperseded + } + return session.Restart(ctx, seekSeconds, startSegment) } // NewServer creates a new transcode server. @@ -72,7 +153,7 @@ func NewServer(watcher *nodeconfig.Watcher, tracker *nodesessions.Tracker) *Serv sessions: make(map[string]*playback.TranscodeSession), } if cfg := watcher.Config(); cfg != nil { - if cleaned, err := playback.CleanupOrphanedTranscodeDirs(cfg.Playback.TranscodeDir, nil); err != nil { + if cleaned, err := playback.CleanupOrphanedTranscodeDirs(cfg.Playback.TranscodeDir, nil, 0); err != nil { slog.Warn("transcode node cleanup failed", "dir", cfg.Playback.TranscodeDir, "error", err) } else if cleaned > 0 { slog.Info("transcode node cleanup removed orphaned dirs", "dir", cfg.Playback.TranscodeDir, "count", cleaned) @@ -85,6 +166,27 @@ func (s *Server) SetFFmpegLogSink(sink playback.FFmpegLogSink) { s.ffmpegSink = sink } +// recipeStore reads a remote transcode's reconstruction recipe written by central +// at transcode start. The jellycompat node-hop token is identity-only by design — +// not because a Jellyfin client can't round-trip it, but because the recipe is +// mutated in place and the client can't be driven to refresh a stale token, so the +// authoritative recipe lives server-side (see internal/noderecipe). On a node +// restart the node fetches it here instead of 404ing. *noderecipe.Store implements it. +type recipeStore interface { + Get(ctx context.Context, sessionID string) (*playback.RecipeCard, bool) + // Delete drops a session's recipe so a buffered/retrying request after a node + // restart cannot reconstruct a brand-new ffmpeg for an already-stopped session. + // Called only on deliberate teardown; nil-safe and a missing key is a no-op. + Delete(ctx context.Context, sessionID string) error +} + +// SetRecipeStore wires the control-plane recipe store so this node can rebuild a +// jellycompat transcode after its own restart. Optional; without it a recipe-less +// (jellycompat) token cannot reconstruct and the request 404s as before. +func (s *Server) SetRecipeStore(store recipeStore) { + s.recipeStore = store +} + // Handler returns the chi.Router with all transcode routes. func (s *Server) Handler() http.Handler { r := chi.NewRouter() @@ -218,6 +320,11 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { opts.HWAccel = cfg.Playback.HWAccel } + // Hold the per-session lifecycle lock across teardown → spawn → register so a + // concurrent reconstruct cannot run a second ffmpeg writer against this + // session's output dir while we replace it. + unlock := s.lockSessionLifecycle(req.SessionID) + // Defensively close any existing session for this ID so that a quality // switch doesn't orphan the old ffmpeg process or leave stale segments. s.mu.Lock() @@ -242,6 +349,7 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { session, err := playback.StartTranscode(context.WithoutCancel(r.Context()), opts) if err != nil { + unlock() slog.Error("start transcode", "error", err, "session", req.SessionID, "playback_session_id", req.SessionID) http.Error(w, "failed to start transcode", http.StatusInternalServerError) return @@ -250,6 +358,7 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { s.mu.Lock() s.sessions[req.SessionID] = session s.mu.Unlock() + unlock() s.activeJobs.Add(1) // Track session in Redis off the request path — the API server (and @@ -277,6 +386,190 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) { }) } +// reconstructFromToken rebuilds a transcode session this node lost to its own +// restart. The proxy forwards the client's verified stream token in the +// X-Silo-Stream-Token header; the token carries the full byte-affecting recipe +// (the former Postgres "recipe card"), so the node can re-spawn ffmpeg seeked to +// the requested segment rather than 404ing — mirroring the integrated server's +// token-carried reconstruct. Returns nil when the request carries no usable +// transcode token, which the caller renders as a genuine not-found. +// +// requestedSegment is the segment the client is fetching, or negative on the +// manifest path. Reconstruction is single-flighted per session id so concurrent +// manifest and segment requests for the same lost session share one ffmpeg. +func (s *Server) reconstructFromToken(r *http.Request, sessionID string, requestedSegment int) *playback.TranscodeSession { + tokenStr := r.Header.Get("X-Silo-Stream-Token") + if tokenStr == "" { + return nil + } + cfg := s.watcher.Config() + claims, err := streamtoken.Verify(tokenStr, cfg.Auth.JWTSecret) + if err != nil { + slog.Warn("transcode node reconstruct: invalid stream token", "error", err, + "session", sessionID, "playback_session_id", sessionID) + return nil + } + card := playback.RecipeCardFromClaims(claims) + // The token's recipe must be a transcode card for the session id in the URL: a + // mismatch is a forged or stale request, and direct/remux cards carry no encode + // parameters to rebuild. An empty PlayMethod is a transcode card (back-compat). + if card.SessionID != sessionID || (card.PlayMethod != "" && card.PlayMethod != playback.PlayTranscode) { + return nil + } + // A native token carries the full byte-affecting recipe. The jellycompat node + // hop signs an identity-only token by design (see internal/noderecipe for why), + // so its card decodes with no encode parameters. For the jellycompat case the + // recipe is fetched from the control-plane recipe store below; without that + // store there is nothing to rebuild from, so 404. + tokenComplete := card.SegmentDuration > 0 && card.TargetCodecVideo != "" + if !tokenComplete && s.recipeStore == nil { + return nil + } + + v, _, _ := s.reconstructGroup.Do(sessionID, func() (interface{}, error) { + // A concurrent reconstruct (or a fresh start) may already have registered the + // session; serve it rather than spawning a duplicate ffmpeg. + s.mu.RLock() + existing, ok := s.sessions[sessionID] + s.mu.RUnlock() + if ok { + return existing, nil + } + resolved := card + if !tokenComplete { + // Recipe-less (jellycompat) token: fetch the recipe central wrote to the + // control-plane store at transcode start. A miss / incomplete recipe is a + // genuine not-found (404), never a spawn from a bad recipe. + fetched, ok := s.recipeStore.Get(r.Context(), sessionID) + if !ok || fetched == nil || fetched.SessionID != sessionID || + fetched.SegmentDuration <= 0 || fetched.TargetCodecVideo == "" { + return (*playback.TranscodeSession)(nil), nil + } + resolved = *fetched + } + return s.spawnReconstruct(r, sessionID, requestedSegment, resolved), nil + }) + if session, _ := v.(*playback.TranscodeSession); session != nil { + return session + } + return nil +} + +// spawnReconstruct re-spawns ffmpeg for a lost session from its recipe card and +// registers it in the live map. It is only ever called inside the per-session +// single-flight in reconstructFromToken, so it is the sole writer racing to +// register sessionID. Returns nil if the spawn fails or the slot wait is canceled. +func (s *Server) spawnReconstruct(r *http.Request, sessionID string, requestedSegment int, card playback.RecipeCard) *playback.TranscodeSession { + // Pace the cold-start burst so a node restart that loses many sessions does not + // launch every ffmpeg at once. A client that disconnects while waiting releases + // its slot rather than queueing dead work. + release, ok := s.acquireReconstructSlot(r.Context()) + if !ok { + return nil + } + defer release() + + // Serialize against a concurrent fresh /transcode/start for this session so the + // two never run ffmpeg writers against the same dir. Re-check under the lock and + // yield to any live session rather than spawning a duplicate. + unlock := s.lockSessionLifecycle(sessionID) + defer unlock() + s.mu.RLock() + existing, ok := s.sessions[sessionID] + s.mu.RUnlock() + if ok { + return existing + } + + cfg := s.watcher.Config() + outputDir := filepath.Join(cfg.Playback.TranscodeDir, sessionID) + opts := card.TranscodeOpts(outputDir, cfg.Playback.FFmpegPath, s.ffmpegSink) + // Re-resolve environment-specific encode knobs from this node's live config; the + // token deliberately omits HWAccel/HWDevice so an operator change applies on + // rebuild. Run as a transcode node, not integrated (card.TranscodeOpts defaults). + opts.HWAccel = cfg.Playback.HWAccel + opts.HWDevice = cfg.Playback.HWDevice + opts.NodeType = "transcode" + opts.ExecutionMode = "transcode_node" + + // Resume near the segment the client is actually requesting. The card records + // the original start; if the client has played past it, spawning at the old + // position forces a wait-then-seek stall. A negative requestedSegment (manifest + // path) carries no segment context, so the card position stands. + // + // The fast seg×dur mapping is only valid for ENCODED transcodes, whose forced + // keyframes make every segment exactly SegmentDuration long. Copy-mode segments + // have variable durations, so seg×dur points at the wrong source time and causes + // multi-second A/V desync after a restart. For copy-mode cards leave the card's + // original start untouched and let the segment-recovery machinery seek forward + // once the manifest is rebuilt. This mirrors doReconstructTranscode in + // internal/playback/transcode_manager.go so both reconstruct paths stay consistent. + if requestedSegment > card.StartSegmentNumber && card.SegmentDuration > 0 && + !strings.EqualFold(card.TargetCodecVideo, "copy") { + opts.StartSegmentNumber = requestedSegment + opts.SeekSeconds = float64(requestedSegment * card.SegmentDuration) + } + + session, err := playback.StartTranscode(context.WithoutCancel(r.Context()), opts) + if err != nil { + slog.Error("transcode node reconstruct start failed", "error", err, + "session", sessionID, "playback_session_id", sessionID) + return nil + } + + // Yield to a winner registered by another path; close only the duplicate ffmpeg, + // never the shared output directory the winner is actively serving. + s.mu.Lock() + if existing, ok := s.sessions[sessionID]; ok { + s.mu.Unlock() + _ = session.CloseProcess() + return existing + } + s.sessions[sessionID] = session + s.mu.Unlock() + s.activeJobs.Add(1) + + trackCtx := context.WithoutCancel(r.Context()) + go s.tracker.Track(trackCtx, nodesessions.SessionInfo{ + SessionID: sessionID, + NodeURL: s.tracker.NodeURL(), + NodeName: s.tracker.NodeName(), + Type: "transcode", + CodecVideo: card.TargetCodecVideo, + CodecAudio: card.TargetCodecAudio, + Resolution: card.TargetResolution, + HWAccel: session.Opts().HWAccel, + StartedAt: time.Now().UTC().Format(time.RFC3339), + AuthUserID: card.UserID, + ProfileID: card.ProfileID, + MediaFileID: card.MediaFileID, + }) + + slog.Info("transcode node session reconstructed from token", + "session", sessionID, "playback_session_id", sessionID, + "requested_segment", requestedSegment, "start_segment_number", opts.StartSegmentNumber) + return session +} + +// acquireReconstructSlot blocks until a reconstruct slot is free or the request +// context is canceled, returning a release func and true on success. The semaphore +// is lazily sized to NumCPU so a node restart paces its ffmpeg cold starts. +func (s *Server) acquireReconstructSlot(ctx context.Context) (func(), bool) { + s.reconstructSemOnce.Do(func() { + n := runtime.NumCPU() + if n < 1 { + n = 4 + } + s.reconstructSem = make(chan struct{}, n) + }) + select { + case s.reconstructSem <- struct{}{}: + return func() { <-s.reconstructSem }, true + case <-ctx.Done(): + return nil, false + } +} + func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { sessionID := chi.URLParam(r, "session_id") @@ -299,6 +592,15 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) { outputDir := filepath.Join(cfg.Playback.TranscodeDir, sessionID) os.RemoveAll(outputDir) + // Drop the recipe so a buffered/retrying request after a node restart cannot + // reconstruct a new ffmpeg for this now-stopped session. Best-effort: a stop + // must still succeed even if the recipe store is briefly unavailable. + if s.recipeStore != nil { + if err := s.recipeStore.Delete(r.Context(), sessionID); err != nil { + slog.Warn("delete transcode recipe on stop", "error", err, "session", sessionID, "playback_session_id", sessionID) + } + } + s.tracker.Remove(r.Context(), sessionID) w.WriteHeader(http.StatusNoContent) @@ -312,8 +614,14 @@ func (s *Server) handleManifest(w http.ResponseWriter, r *http.Request) { s.mu.RUnlock() if !ok { - http.Error(w, "session not found", http.StatusNotFound) - return + // Lost the in-memory session (this node restarted): rebuild it from the + // stream token the proxy forwarded. The manifest path carries no segment + // context, so reconstruct at the recipe's original start position. + session = s.reconstructFromToken(r, sessionID, -1) + if session == nil { + http.Error(w, "session not found", http.StatusNotFound) + return + } } manifest, err := session.BuildPlaybackManifest("segment/", r.URL.RawQuery) @@ -338,8 +646,18 @@ func (s *Server) handleSegment(w http.ResponseWriter, r *http.Request) { s.mu.RUnlock() if !ok { - http.Error(w, "session not found", http.StatusNotFound) - return + // Lost the in-memory session (this node restarted): rebuild it from the + // forwarded stream token, seeked to the segment the client is requesting so + // playback resumes near its position instead of restarting from the start. + requestedSegment := -1 + if n, parseErr := playback.ParseSegmentNumber(name); parseErr == nil { + requestedSegment = n + } + session = s.reconstructFromToken(r, sessionID, requestedSegment) + if session == nil { + http.Error(w, "session not found", http.StatusNotFound) + return + } } segPath, err := session.GetSegment(name) @@ -415,8 +733,10 @@ func (s *Server) handleSegment(w http.ResponseWriter, r *http.Request) { "playback_session_id", sessionID, ) - if restartErr := session.Restart( + if restartErr := s.restartSessionLocked( context.WithoutCancel(r.Context()), + sessionID, + session, seekSeconds, segNum, ); restartErr == nil { @@ -450,14 +770,27 @@ func (s *Server) handleForceReload(w http.ResponseWriter, r *http.Request) { } cfg := s.watcher.Config() s.mu.Lock() + stopped := make([]string, 0, len(s.sessions)) for id, session := range s.sessions { session.Close() os.RemoveAll(filepath.Join(cfg.Playback.TranscodeDir, id)) delete(s.sessions, id) + stopped = append(stopped, id) } s.activeJobs.Store(0) s.mu.Unlock() + // A force-reload tears every session down for good, so drop their recipes too: + // otherwise a buffered/retrying request could reconstruct a session this reload + // deliberately killed. Best-effort, done outside the map lock. + if s.recipeStore != nil { + for _, id := range stopped { + if err := s.recipeStore.Delete(r.Context(), id); err != nil { + slog.Warn("delete transcode recipe on force reload", "error", err, "session", id, "playback_session_id", id) + } + } + } + s.tracker.Cleanup(r.Context()) slog.Info("transcode force reload completed") diff --git a/internal/transcodenode/server_test.go b/internal/transcodenode/server_test.go new file mode 100644 index 00000000..5ee91b48 --- /dev/null +++ b/internal/transcodenode/server_test.go @@ -0,0 +1,242 @@ +package transcodenode + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/go-chi/chi/v5" + + "github.com/Silo-Server/silo-server/internal/config" + "github.com/Silo-Server/silo-server/internal/nodeconfig" + "github.com/Silo-Server/silo-server/internal/nodesessions" + "github.com/Silo-Server/silo-server/internal/playback" + "github.com/Silo-Server/silo-server/internal/streamtoken" +) + +const testSecret = "node-reconstruct-test-secret" + +// newTestServer builds a transcode Server whose config carries a known JWT secret +// so reconstructFromToken can verify forwarded stream tokens. The tracker is left +// nil: the guard-rejection cases never reach the spawn/track path. +func newTestServer(t *testing.T) *Server { + t.Helper() + w := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{}) + cfg := &config.Config{} + cfg.Auth.JWTSecret = testSecret + cfg.Playback.TranscodeDir = t.TempDir() + w.SetConfigForTest(cfg) + return &Server{ + watcher: w, + sessions: make(map[string]*playback.TranscodeSession), + } +} + +func signCard(t *testing.T, card playback.RecipeCard) string { + t.Helper() + tok, err := streamtoken.Sign(card.ToClaims(), testSecret, time.Hour) + if err != nil { + t.Fatalf("sign card: %v", err) + } + return tok +} + +func requestWithToken(sessionID, token string) *http.Request { + r := httptest.NewRequest(http.MethodGet, "/transcode/"+sessionID+"/master.m3u8", nil) + if token != "" { + r.Header.Set("X-Silo-Stream-Token", token) + } + return r +} + +func transcodeCard(sessionID string) playback.RecipeCard { + return playback.NewRecipeCard(7, "profile-1", 42, "", playback.TranscodeOpts{ + SessionID: sessionID, + InputPath: "/media/movie.mkv", + TargetCodecVideo: "h264", + SegmentDuration: 6, + }) +} + +// reconstructFromToken must refuse — without spawning ffmpeg — every request that +// does not carry a valid, matching transcode token. These guards run before any +// StartTranscode, so they are safe to assert without ffmpeg or a media file. +func TestReconstructFromToken_RejectsUnusableTokens(t *testing.T) { + const sid = "sess-123" + s := newTestServer(t) + + t.Run("missing token header", func(t *testing.T) { + if got := s.reconstructFromToken(requestWithToken(sid, ""), sid, -1); got != nil { + t.Fatalf("expected nil for missing token, got %v", got) + } + }) + + t.Run("invalid signature", func(t *testing.T) { + bad, err := streamtoken.Sign(transcodeCard(sid).ToClaims(), "wrong-secret", time.Hour) + if err != nil { + t.Fatalf("sign: %v", err) + } + if got := s.reconstructFromToken(requestWithToken(sid, bad), sid, -1); got != nil { + t.Fatalf("expected nil for bad signature, got %v", got) + } + }) + + t.Run("session id mismatch", func(t *testing.T) { + tok := signCard(t, transcodeCard("other-session")) + if got := s.reconstructFromToken(requestWithToken(sid, tok), sid, -1); got != nil { + t.Fatalf("expected nil for session id mismatch, got %v", got) + } + }) + + t.Run("non-transcode card", func(t *testing.T) { + tok := signCard(t, playback.NewDirectRecipeCard(sid, 7, "profile-1", 42)) + if got := s.reconstructFromToken(requestWithToken(sid, tok), sid, -1); got != nil { + t.Fatalf("expected nil for direct-play card, got %v", got) + } + }) + + // The jellycompat node hop signs an identity-only transcode token (the recipe + // lives in the central compat store). Its card decodes as PlayTranscode for the + // right session id but with no encode parameters; with no recipe store wired the + // node must refuse it rather than spawn a malformed ffmpeg. + t.Run("recipe-less transcode token, no recipe store", func(t *testing.T) { + tok := signCard(t, playback.RecipeCard{ + SessionID: sid, + UserID: 7, + PlayMethod: playback.PlayTranscode, + InputPath: "/media/movie.mkv", + }) + if got := s.reconstructFromToken(requestWithToken(sid, tok), sid, 5); got != nil { + t.Fatalf("expected nil for recipe-less transcode token, got %v", got) + } + }) +} + +// stubRecipeStore is a recipeStore for the jellycompat node-restart fetch path. +type stubRecipeStore struct { + card *playback.RecipeCard + ok bool + hits int + deletes []string + delErr error +} + +func (s *stubRecipeStore) Get(context.Context, string) (*playback.RecipeCard, bool) { + s.hits++ + return s.card, s.ok +} + +func (s *stubRecipeStore) Delete(_ context.Context, sessionID string) error { + s.deletes = append(s.deletes, sessionID) + return s.delErr +} + +// When the forwarded token is recipe-less (jellycompat), the node consults the +// recipe store. A miss or an incomplete recipe must yield a clean nil (404) with +// no ffmpeg spawn — these assert the resolve guards without needing ffmpeg. +func TestReconstructFromToken_JellycompatRecipeFetch(t *testing.T) { + const sid = "compat-sess-1" + recipeLessToken := func(t *testing.T) string { + return signCard(t, playback.RecipeCard{ + SessionID: sid, + UserID: 7, + PlayMethod: playback.PlayTranscode, + InputPath: "/media/movie.mkv", + }) + } + + t.Run("store miss -> nil", func(t *testing.T) { + s := newTestServer(t) + store := &stubRecipeStore{ok: false} + s.SetRecipeStore(store) + if got := s.reconstructFromToken(requestWithToken(sid, recipeLessToken(t)), sid, 5); got != nil { + t.Fatalf("expected nil on store miss, got %v", got) + } + if store.hits != 1 { + t.Fatalf("recipe store consulted %d times, want 1", store.hits) + } + }) + + t.Run("incomplete fetched recipe -> nil", func(t *testing.T) { + s := newTestServer(t) + // Right session id but missing encode params: must not spawn. + s.SetRecipeStore(&stubRecipeStore{ok: true, card: &playback.RecipeCard{SessionID: sid, PlayMethod: playback.PlayTranscode}}) + if got := s.reconstructFromToken(requestWithToken(sid, recipeLessToken(t)), sid, 5); got != nil { + t.Fatalf("expected nil for incomplete fetched recipe, got %v", got) + } + }) + + t.Run("fetched recipe for wrong session -> nil", func(t *testing.T) { + s := newTestServer(t) + s.SetRecipeStore(&stubRecipeStore{ok: true, card: &playback.RecipeCard{ + SessionID: "other", PlayMethod: playback.PlayTranscode, SegmentDuration: 6, TargetCodecVideo: "h264", + }}) + if got := s.reconstructFromToken(requestWithToken(sid, recipeLessToken(t)), sid, 5); got != nil { + t.Fatalf("expected nil for wrong-session recipe, got %v", got) + } + }) +} + +// handleStop is a deliberate teardown, so it must drop the session's recipe to +// stop a buffered/retrying post-restart request from reconstructing a brand-new +// ffmpeg for an already-stopped session. A zero-value TranscodeSession needs no +// ffmpeg or media file to Close, so this asserts the wiring without a real spawn. +func TestHandleStop_DeletesRecipe(t *testing.T) { + const sid = "stop-sess-1" + s := newTestServer(t) + s.tracker = nodesessions.NewTracker(nil, "node-url", "node-name", "transcode") + store := &stubRecipeStore{} + s.SetRecipeStore(store) + + s.sessions[sid] = &playback.TranscodeSession{} + s.activeJobs.Store(1) + + r := httptest.NewRequest(http.MethodDelete, "/transcode/"+sid, nil) + rctx := chi.NewRouteContext() + rctx.URLParams.Add("session_id", sid) + r = r.WithContext(context.WithValue(r.Context(), chi.RouteCtxKey, rctx)) + rec := httptest.NewRecorder() + s.handleStop(rec, r) + + if rec.Code != http.StatusNoContent { + t.Fatalf("handleStop status = %d, want %d", rec.Code, http.StatusNoContent) + } + if len(store.deletes) != 1 || store.deletes[0] != sid { + t.Fatalf("recipe deletes = %v, want [%q]", store.deletes, sid) + } + if _, ok := s.sessions[sid]; ok { + t.Fatalf("session %q still registered after stop", sid) + } +} + +// spawnReconstruct must NOT apply the fast seg×dur resume seek for copy-mode +// cards: copy-mode segments have variable durations, so seg×dur points at the +// wrong source time. The card's original start must stand. Asserting opts off a +// real spawn would need ffmpeg, so this checks the gating condition directly. +func TestCopyModeReconstruct_SkipsFastSeek(t *testing.T) { + const dur = 6 + card := playback.RecipeCard{ + SessionID: "copy-sess-1", + PlayMethod: playback.PlayTranscode, + TargetCodecVideo: "copy", + SegmentDuration: dur, + StartSegmentNumber: 0, + } + const requestedSegment = 10 + applyFastSeek := requestedSegment > card.StartSegmentNumber && card.SegmentDuration > 0 && + !strings.EqualFold(card.TargetCodecVideo, "copy") + if applyFastSeek { + t.Fatalf("copy-mode card must not apply the seg×dur fast seek") + } + + // Same shape but ENCODED: the fast seek must apply. + card.TargetCodecVideo = "h264" + applyFastSeek = requestedSegment > card.StartSegmentNumber && card.SegmentDuration > 0 && + !strings.EqualFold(card.TargetCodecVideo, "copy") + if !applyFastSeek { + t.Fatalf("encoded card must apply the seg×dur fast seek") + } +} diff --git a/migrations/sql/20260618053223_add_jellycompat_playback_sessions.sql b/migrations/sql/20260618053223_add_jellycompat_playback_sessions.sql new file mode 100644 index 00000000..bcfb1ee4 --- /dev/null +++ b/migrations/sql/20260618053223_add_jellycompat_playback_sessions.sql @@ -0,0 +1,34 @@ +-- +goose Up +-- +goose StatementBegin +-- jellycompat_playback_sessions durably stores the compat (Jellyfin) playback +-- negotiation session so a Jellyfin client can resume across a server restart. +-- It holds the load-bearing PlaySessionId -> upstream-session mapping plus the +-- negotiated media sources, route item id, and seek (the full PlaybackSession in +-- the data JSONB). Without it, after a restart the in-memory map is empty and the +-- compat segment/manifest handlers 404 at their first lookup before any transcode +-- reconstruct can run. One row per active compat play session; rows expire via +-- expires_at (the negotiation TTL, re-armed on update) and are swept by the +-- reconciler janitor. compat_token is denormalized for lookup; user_id is +-- denormalized for auditing. The canonical state is the data column. +CREATE TABLE IF NOT EXISTS public.jellycompat_playback_sessions ( + id TEXT PRIMARY KEY, + compat_token TEXT NOT NULL DEFAULT '', + user_id TEXT NOT NULL DEFAULT '', + data JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + expires_at TIMESTAMPTZ NOT NULL +); + +-- FindByRoute scans live (non-expired) rows for a matching compat token; the +-- expiry sweep filters on expires_at. Index both access paths. +CREATE INDEX IF NOT EXISTS idx_jellycompat_playback_sessions_expires_at +ON public.jellycompat_playback_sessions USING btree (expires_at); + +CREATE INDEX IF NOT EXISTS idx_jellycompat_playback_sessions_compat_token +ON public.jellycompat_playback_sessions USING btree (compat_token); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE IF EXISTS public.jellycompat_playback_sessions; +-- +goose StatementEnd diff --git a/web/src/player/stream-url.test.ts b/web/src/player/stream-url.test.ts new file mode 100644 index 00000000..e38a1e8c --- /dev/null +++ b/web/src/player/stream-url.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { buildPlayerStreamUrl } from "./stream-url"; + +describe("buildPlayerStreamUrl", () => { + it("joins the access token with `&` when the stream path already has `?st=`", () => { + const url = buildPlayerStreamUrl( + "https://api.example.com", + "/api/v1/playback/stream/abc.m3u8?st=streamtoken123", + "jwt-access-token", + "direct", + 0, + ); + + const parsed = new URL(url); + // Both params must survive as separate query keys. + expect(parsed.searchParams.get("st")).toBe("streamtoken123"); + expect(parsed.searchParams.get("token")).toBe("jwt-access-token"); + }); + + it("uses `?` when the stream path has no existing query string", () => { + const url = buildPlayerStreamUrl( + "https://api.example.com", + "/api/v1/playback/stream/abc.m3u8", + "jwt-access-token", + "direct", + 0, + ); + + expect(url).toBe( + "https://api.example.com/api/v1/playback/stream/abc.m3u8?token=jwt-access-token", + ); + const parsed = new URL(url); + expect(parsed.searchParams.get("token")).toBe("jwt-access-token"); + }); + + it("adds a seek param for remux without clobbering an existing `?st=`", () => { + const url = buildPlayerStreamUrl( + "https://api.example.com", + "/api/v1/playback/stream/abc.m3u8?st=streamtoken123", + "jwt-access-token", + "remux", + 12.5, + ); + + const parsed = new URL(url); + expect(parsed.searchParams.get("st")).toBe("streamtoken123"); + expect(parsed.searchParams.get("token")).toBe("jwt-access-token"); + expect(parsed.searchParams.get("seek")).toBe("12.500"); + }); + + it("returns the path unchanged when there is no token or extra params", () => { + const url = buildPlayerStreamUrl( + "https://api.example.com", + "/api/v1/playback/proxy/sometoken/abc.m3u8", + null, + "direct", + 0, + ); + + expect(url).toBe("https://api.example.com/api/v1/playback/proxy/sometoken/abc.m3u8"); + }); +}); diff --git a/web/src/player/stream-url.ts b/web/src/player/stream-url.ts index 1a7c505e..7178fc4a 100644 --- a/web/src/player/stream-url.ts +++ b/web/src/player/stream-url.ts @@ -51,5 +51,12 @@ export function buildPlayerStreamUrl( streamPath.startsWith("http://") || streamPath.startsWith("https://") ? streamPath : `${apiBaseUrl}${streamPath}`; - return `${base}${query ? `?${query}` : ""}`; + if (!query) { + return base; + } + // The backend stream URL may already carry its own query string (e.g. the + // `?st=` reconstruct token for integrated-mode direct/remux). + // Join with `&` in that case so we don't clobber it into `st=X?token=Y`. + const separator = base.includes("?") ? "&" : "?"; + return `${base}${separator}${query}`; }