Compare commits

..
Author SHA1 Message Date
Quick104 fd3d737a58 fix(playback): trust completed segment writes 2026-08-11 15:02:38 -04:00
Quick104 86a02a6f61 fix(playback): bound sparse seek cleanup 2026-08-11 14:55:27 -04:00
Quick104 1cbb59d0fa fix(playback): finish segment acknowledgements 2026-08-11 14:45:33 -04:00
Quick104 874a26de0f fix(playback): confirm proxy downstream delivery 2026-08-11 14:30:55 -04:00
Quick104 0544213730 fix(playback): verify segment completion before pruning 2026-08-11 14:21:33 -04:00
Quick104 e05c30c637 fix(playback): keep restart buffers prunable 2026-08-11 13:42:43 -04:00
Quick104 7f6225bb11 fix(playback): remove redundant retention floor assignment 2026-08-11 13:21:53 -04:00
Quick104 977104eac5 Merge remote-tracking branch 'origin/main' into t3code/pr565-review 2026-08-11 13:14:44 -04:00
Quick104 dffa6ae34e Merge origin/main into codex/bound-transcode-segments 2026-08-11 13:05:12 -04:00
Quick104 e2899ff619 fix(playback): preserve retained segments across restarts 2026-08-11 12:59:25 -04:00
Quick104 f524c48d19 fix(playback): close retention gaps across ranges and seeks 2026-08-08 18:49:27 -04:00
Quick104 c26ab22809 test(playback): strengthen retention regression coverage 2026-08-08 16:08:31 -04:00
Quick104 0a3adf6552 fix(playback): harden segment retention lifecycle 2026-08-08 15:59:56 -04:00
Quick104 35de68b589 fix(playback): synchronize prune option snapshot 2026-08-08 15:29:02 -04:00
Quick104 6a3fcef165 fix(playback): prune downloaded transcode segments 2026-08-08 15:23:16 -04:00
237 changed files with 3274 additions and 17286 deletions
+1 -17
View File
@@ -135,22 +135,6 @@ transcoding.
Multi-host operators can use those examples as a starting point for a dedicated worker Compose
file connected to the deployment's shared PostgreSQL and Redis services.
Proxy nodes serve source downloads from the same absolute media paths used by direct playback.
Prepared-download work can also run on transcode nodes. Each selected transcode node retains its
result on node-local storage and exposes it only through Silo's authenticated internal artifact API;
the paired proxy relays those bytes, so no shared artifact mount is required. Dedicated transcode
nodes default to retaining prepared downloads in a protected directory inside the transcode volume
captured at process startup. `download.artifact_dir` overrides that location for both dedicated
transcode nodes and the integrated/API-local fallback, so mount the configured path on every process
that prepares downloads. Changing either artifact-path setting requires a restart. Downloads with a
configured server-wide or per-user bandwidth limit remain API-local so those aggregate limits stay exact.
Clients discover distributed delivery through `proxy_delivery` on the download capability response.
When it is true, they may opt into `GET` or `HEAD /api/v1/downloads/{id}/file-proxy` and
`/api/v1/direct-download-proxy`; those routes may return a temporary redirect to a proxy node. The
established `/file` and `/direct-download` routes keep serving bytes directly with their existing
status-code contract; for a node-local prepared artifact, the API itself performs the authenticated
relay on that fallback route.
### Deployment Notes
The default compose stack intentionally bundles PostgreSQL and Redis for ease of setup and assumes a fresh install without those services already available. If you already operate PostgreSQL and Redis, omit those examples from compose and point Silo at your existing infrastructure instead. For serious installs, PostgreSQL is better on a separate VM or a managed service so upgrades, tuning, and backups are isolated from the app host. Redis can stay local for many installs, but externalizing it is also reasonable if you already operate shared infrastructure.
@@ -174,7 +158,7 @@ through the admin UI after first launch.
| `integrated` | Full server: API + frontend + scanner + transcode (default) |
| `api` | API server only, no local transcoding |
| `proxy` | Stream proxy node that connects to the shared deployment database and Redis |
| `transcode` | HLS and prepared-download worker node that connects to the shared deployment database and Redis |
| `transcode` | HLS transcode worker node that connects to the shared deployment database and Redis |
### PostgreSQL Auto-Tuning
BIN
View File
Binary file not shown.

Before

Width:  |  Height:  |  Size: 20 KiB

After

Width:  |  Height:  |  Size: 62 KiB

-5
View File
@@ -662,10 +662,6 @@ func goldenConformanceMatrix() playback.ConformanceMatrixV3 {
qualityChange.ReplanRequestID = "replan-quality-change-0001"
qualityChange.Failure = playback.FailureV3{}
qualityChange.QualityPreference = resolutionHD
outputChange := goldenReplanRequest()
outputChange.Operation = playback.ReplanOperationOutputChangeV3
outputChange.ReplanRequestID = "replan-output-change-0001"
outputChange.Failure = playback.FailureV3{}
seekReanchor := goldenReplanRequest()
seekReanchor.Operation = playback.ReplanOperationSeekReanchorV3
seekReanchor.ReplanRequestID = "replan-seek-reanchor-0001"
@@ -689,7 +685,6 @@ func goldenConformanceMatrix() playback.ConformanceMatrixV3 {
replans := []playback.ReplanScenarioV3{
{Name: "track_change", Category: "track_change_replan", Request: trackChange, Expected: playback.ReplanExpectationV3{HTTPStatus: http.StatusOK, PreserveUnmodifiedTracks: true}},
{Name: "quality_change", Category: "quality_change_replan", Request: qualityChange, Expected: playback.ReplanExpectationV3{HTTPStatus: http.StatusOK, SelectedQuality: resolutionHD}},
{Name: "output_change", Category: "output_change_replan", Request: outputChange, Expected: playback.ReplanExpectationV3{HTTPStatus: http.StatusOK, PreserveUnmodifiedTracks: true}},
{Name: "track_change_idempotent_duplicate", Category: "idempotent_replan", Request: trackDuplicate, Expected: playback.ReplanExpectationV3{SameRequestAndBodyStatus: http.StatusOK, ResponseReplayedVerbatim: true, ChangedBodyStatus: http.StatusConflict, ChangedBodyError: "idempotency_key_reused"}},
{Name: "quality_change_idempotent_duplicate", Category: "idempotent_replan", Request: qualityDuplicate, Expected: playback.ReplanExpectationV3{SameRequestAndBodyStatus: http.StatusOK, ResponseReplayedVerbatim: true, ChangedBodyStatus: http.StatusConflict, ChangedBodyError: "idempotency_key_reused"}},
{Name: "track_change_concurrent_duplicate", Category: "concurrent_replan", Request: trackConcurrent, Expected: playback.ReplanExpectationV3{WhileFirstLeaseActiveStatus: http.StatusConflict, ConcurrentError: "replan_in_progress", AfterCompletionStatus: http.StatusOK, ResponseReplayedVerbatim: true}},
+11 -36
View File
@@ -718,19 +718,9 @@ func main() {
var handler http.Handler
if mode == "proxy" {
srv := proxy.NewServer(watcher, tracker)
srv.SetRemoteArtifactMissReporter(downloads.NewArtifactManager(
downloads.NewArtifactRepository(pool),
downloads.NewRepository(pool),
nil,
downloads.NewPlaybackPreparer(),
nodeID,
watcher.Config,
nil,
))
handler = srv.Handler()
} else {
srv := transcodenode.NewServer(watcher, tracker)
srv.SetInputPathAuthorizer(transcodenode.NewCatalogPathAuthorizer(scanner.NewFileRepository(pool)))
srv.SetFFmpegLogSink(playback.NewSlogFFmpegLogSink(slog.Default(), nodeID))
// Read jellycompat reconstruction recipes central wrote at transcode
// start, so this node can rebuild a Jellyfin transcode after its own
@@ -930,14 +920,8 @@ func main() {
proxyPool := nodepool.NewProxyPool()
transcodePool := nodepool.NewTranscodePool()
proxyNodes, err := nodeRepo.ListEnabled(appCtx, nodepool.NodeTypeProxy)
if err != nil {
log.Fatalf("load enabled proxy nodes: %v", err)
}
transcodeNodes, err := nodeRepo.ListEnabled(appCtx, nodepool.NodeTypeTranscode)
if err != nil {
log.Fatalf("load enabled transcode nodes: %v", err)
}
proxyNodes, _ := nodeRepo.ListEnabled(context.Background(), nodepool.NodeTypeProxy)
transcodeNodes, _ := nodeRepo.ListEnabled(context.Background(), nodepool.NodeTypeTranscode)
proxyPool.SetNodes(proxyNodes)
transcodePool.SetNodes(transcodeNodes)
@@ -2076,29 +2060,20 @@ func main() {
// Download prepare-to-file pipeline (Phase 3): a durable, leased encode
// queue hosted on the task manager. Built here (before Start) and shared
// with the API via deps so the download service can enqueue jobs.
liveDownloadConfig := func() *config.Config {
if deps.LiveConfig != nil {
if c := deps.LiveConfig(); c != nil {
return c
}
}
return deps.Config
}
var downloadWorkPlanner nodepool.TranscodeWorkPlanner
if deps.NodePlanner != nil {
downloadWorkPlanner = deps.NodePlanner
}
preparer := downloads.NewNodeAwarePreparer(downloads.NewPlaybackPreparer(), downloadWorkPlanner, liveDownloadConfig)
if deps.NodeRepo != nil {
preparer.SetOriginLookup(deps.NodeRepo)
}
artifactMgr := downloads.NewArtifactManager(
downloads.NewArtifactRepository(deps.DB),
downloads.NewRepository(deps.DB),
deps.FileRepo,
preparer,
downloads.NewPlaybackPreparer(),
deps.NodeID,
liveDownloadConfig,
func() *config.Config {
if deps.LiveConfig != nil {
if c := deps.LiveConfig(); c != nil {
return c
}
}
return deps.Config
},
func(ctx context.Context, d *downloads.Download) {
if deps.EventsHub == nil {
return
-2
View File
@@ -25,8 +25,6 @@ func buildLiveSessionSync(s *playback.Session, reportingNode string) worker.Sess
ClientIP: s.ClientIP,
ClientName: s.ClientName,
ClientVersion: s.ClientVersion,
ClientBuild: s.ClientBuild,
ClientChannel: s.ClientChannel,
ClientUserAgent: s.ClientUserAgent,
AudioTrackIndex: s.AudioTrackIndex,
TranscodeAudio: s.TranscodeAudio,
+30 -58
View File
@@ -97,24 +97,23 @@ the document is always the full one:
"enabled": true,
"protocol_versions": [3],
"features": ["playback_plan_v3", "neutral_playback_v3_contract_v1", "layout_aware_passthrough", "playback_route_diagnostics",
"device_quirks_v1", "seek_reanchor_v1", "output_change_v1", "direct_stream_resume_v1",
"device_quirks_v1", "seek_reanchor_v1", "direct_stream_resume_v1",
"plan_source_duration_v1"],
"deliveries": ["original_http", "server_remux_progressive", "server_remux_hls", "server_transcode_hls"],
"transformations": [{"name": "audio_to_aac", "executor": "server", "recipe_version": "1", "validated_claims": ["audio_decode"]}]
}
```
The nine feature strings above are the full set this server version advertises:
The eight feature strings above are the full set this server version advertises:
| Feature | What it promises |
| --- | --- |
| `playback_plan_v3` | The three plan endpoints exist and behave as specified here |
| `neutral_playback_v3_contract_v1` | The server mints opaque `plan_attempt_key` values that clients only echo, and exposes track/quality intent replans distinct from failure recovery |
| `neutral_playback_v3_contract_v1` | The server mints opaque `plan_attempt_key` values that clients only echo, and exposes `track_change` / `quality_change` as intent replans distinct from failure recovery |
| `layout_aware_passthrough` | Audio passthrough is decided from channel *layouts*, not just channel counts (§3) |
| `playback_route_diagnostics` | `POST /playback/route-events` is accepted |
| `device_quirks_v1` | Plans may carry `applied_quirks` and `runtime_corrections` (§9) |
| `seek_reanchor_v1` | The `seek_reanchor` replan operation is available (§6) |
| `output_change_v1` | The `output_change` intent replan is available; clients must keep the active route when this feature is absent |
| `direct_stream_resume_v1` | A direct route may resume mid-file rather than restarting |
| `plan_source_duration_v1` | `source.duration_seconds` is populated when known, so its absence means *unknown* rather than *unsupported* (§5) |
@@ -381,25 +380,6 @@ HDR10, and the plan carries the `hdr_range_assumed_hdr10` degradation warning.
Refusing to play those outright would be worse than an assumption the client is
told about.
The web client does not promote the generic high-dynamic-range media query to a
format claim, and it does not gate format claims on it either. Decoder capability
and active-output HDR are separate facts — browsers tone-map HDR content onto SDR
outputs, and Safari 26 reports `dynamic-range: standard` even on an XDR display —
so the media query survives only as the best-effort `hdr` output boolean. Format
claims come from exact shape probes matched to the bytes the remux delivers:
HDR10 requires Media Capabilities support for Silo's progressive 2160p HEVC
Main10, Rec. 2020, PQ, SMPTE ST 2086 shape, probed under exactly the `hvc1`
sample entry because the explicit v3 HDR10 strip remux labels its output `hvc1`
(legacy and automatic strip paths retain FFmpeg's default `hev1`). Dolby Vision
requires a definitive media-element answer for exactly `dvh1.05.06` or
`dvh1.08.06`, because the preserve remux tags its output `dvh1`. An answer only
for the other spelling (`hev1`/`dvhe`) is evidence for a file Silo never sends
and earns no claim. Both claims are scoped to `progressive`: they
are cleared from `original_http` and `hls` because those delivery paths were not
tested by the same probe. An HDR10 claim can carry `hdr10_max_width`, `hdr10_max_height`,
`hdr10_max_frame_rate`, and `hdr10_max_bitrate_kbps`; these ceilings keep a
successful format probe from admitting an untested stream class.
---
## 4. Deliveries
@@ -471,19 +451,17 @@ position, `can_seek_anywhere` is true when the runtime is known, and
`seek_restoration` is `player_position` — the client seeks locally.
**Copy remux over HLS** is served from FFmpeg's live, still-growing playlist,
which starts at the preceding keyframe selected by FFmpeg's input seek.
`stream_origin` and `timeline_offset` both equal that resolved source position,
while `player_start` is the requested position minus the resolved origin so the
client advances past copied pre-roll. `seek_window_start_seconds` is the resolved
origin, and **`seek_window_end_seconds` is deliberately absent**. An open end
marks the window as incomplete; combined with `can_seek_anywhere: false` it
routes every seek back through the server as a reanchor (§6), which is correct
because the playlist has no bytes for positions FFmpeg has not reached yet.
`seek_restoration` is `source_position`.
which starts at the requested position. So `player_start` is `0`,
`stream_origin` and `timeline_offset` both equal the seek position,
`seek_window_start_seconds` is that position, and **`seek_window_end_seconds` is
deliberately absent**. An open end marks the window as incomplete; combined with
`can_seek_anywhere: false` it routes every seek back through the server as a
reanchor (§6), which is correct because the playlist has no bytes for positions
FFmpeg has not reached yet. `seek_restoration` is `source_position`.
**Progressive remux** is a freshly generated chunked response with no byte-range
support, so it uses the same resolved keyframe origin and player pre-roll offset;
the window is open-ended and seeks go through the server.
support, so it behaves the same way: player-zero is stream-zero is the seek
position, the window is open-ended, and seeks go through the server.
### `source.duration_seconds`
@@ -505,7 +483,7 @@ The media's full runtime, and nothing else. Specifically:
Replan is the only way a plan changes. It covers both "that didn't work" and
"the user asked for something else," and the distinction matters to the server.
`operation` is one of six:
`operation` is one of five:
| Operation | Meaning | Requires |
| --- | --- | --- |
@@ -514,7 +492,6 @@ Replan is the only way a plan changes. It covers both "that didn't work" and
| `seek_reanchor` | Move a server-anchored timeline to a new position | — |
| `track_change` | The user picked a different audio or subtitle track | — |
| `quality_change` | The user picked a rung from `available_qualities` | non-empty `quality_preference` |
| `output_change` | The active display/output capabilities changed | — |
The asymmetry is intentional. A seek reanchor is a timeline operation, not a
failed recipe — a classification is still accepted from older callers but never
@@ -525,25 +502,23 @@ normalizes to `auto`, which is a different user intent than the menu selection
the operation models, so the server rejects it rather than silently doing
something else.
**Intent operations behave differently from failure recovery.**
`track_change`, `quality_change`, and `output_change` keep the previous route
eligible because nothing established that its recipe failed: neither attempted-
key history nor the failed-plan exclusion applies. The first two replace what
were separate v2 endpoints (an audio PATCH and a client-posted transcode start).
A client may therefore be handed back a plan it has already tried — that is
correct here, and a client must not treat a repeated `plan_attempt_key` as a
loop.
**User-intent operations behave differently from failure recovery.**
`track_change` and `quality_change` replace what were separate v2 endpoints (an
audio PATCH and a client-posted transcode start). Nothing failed, so the previous
route stays eligible: neither the attempted-key history nor the failed-plan
exclusion applies. A client may therefore be handed back a plan it has already
tried — that is correct here, and a client must not treat a repeated
`plan_attempt_key` as a loop.
When such an operation actually changes something — the request's tracks,
quality, or output capabilities differ from what the session currently has —
the server also tries to return to the *requested* edition rather than staying
on whatever alternate version a previous fallback landed on, since the newly
requested intent may fit the original file again. That is a preference, not a
guarantee: if the requested edition no longer resolves or fails its preflight,
the healthy active alternate is kept. Track identities are remapped only when
the edition really changes, because remapping within one file would degrade an
exact selection to a best-match lookup and could silently move a listener off a
commentary track.
When such an operation actually changes something — the request's tracks or
quality differ from what the session currently has — the server also tries to
return to the *requested* edition rather than staying on whatever alternate
version a previous fallback landed on, since a user switching tracks may well
want the original file back. That is a preference, not a guarantee: if the
requested edition no longer resolves or fails its preflight, the healthy active
alternate is kept. Track identities are remapped only when the edition really
changes, because remapping within one file would degrade an exact selection to a
best-match lookup and could silently move a listener off a commentary track.
Omitting `quality_preference` on any replan preserves the session's current
preference; sending it replaces that preference. A track change therefore does
@@ -620,10 +595,7 @@ help. Delivered inside a `201` (start) or `200` (replan), never a 4xx.
`source_metadata_incomplete`, `source_unavailable`,
`audio_conversion_unsupported`, `video_conversion_unsupported`,
`dv_conversion_unsupported`, `transcoding_disabled`,
`subtitle_conversion_unsupported`. When a video adaptation is forced solely by a
subtitle burn-in requirement and cannot execute, the terminal is
`subtitle_conversion_unsupported` naming the subtitle rather than the underlying
HDR, 4K, or transcode-policy reason — deselecting the subtitle restores playback.
`subtitle_conversion_unsupported`.
*Subtitle policy:* `subtitle_burn_in_source_unsupported`,
`subtitle_codec_unsupported`, `subtitle_track_invalid`,
@@ -39,22 +39,6 @@
"audio_underruns": {
"type": "integer",
"description": "Audio underrun count."
},
"session_id": {
"type": "string",
"description": "Playback session identifier for correlation with operational logs."
},
"play_method": {
"type": "string",
"description": "Selected playback delivery method such as direct, remux, or transcode."
},
"reason": {
"type": "string",
"description": "Playback route, interruption, or lifecycle reason."
},
"position_ms": {
"type": "integer",
"description": "Current playback position in milliseconds."
}
},
"focus": {
@@ -83,44 +67,12 @@
"duration_ms": {
"type": "integer",
"description": "Request duration in milliseconds."
},
"outcome": {
"type": "string",
"description": "Request outcome such as success, http_error, transport_error, or cancelled."
},
"error_code": {
"type": "string",
"description": "Stable error classification for a failed request."
},
"attempt": {
"type": "integer",
"description": "Attempt ordinal for a retried request."
}
},
"lifecycle": {
"state": {
"type": "string",
"description": "App lifecycle state."
},
"phase": {
"type": "string",
"description": "Startup or lifecycle phase identifier."
},
"duration_ms": {
"type": "integer",
"description": "Elapsed duration for the phase in milliseconds."
},
"outcome": {
"type": "string",
"description": "Result of the phase, such as success, failure, or skipped."
},
"reason": {
"type": "string",
"description": "Classification for the phase outcome."
},
"launch_type": {
"type": "string",
"description": "Launch classification, such as cold or warm."
}
},
"crash": {
@@ -761,11 +761,6 @@
"dolby_vision_profile": {
"type": "integer"
},
"dolby_vision_level": {
"type": "integer",
"minimum": 1,
"maximum": 13
},
"dv_bl_compat_id": {
"type": "integer"
},
@@ -10,7 +10,6 @@
"playback_route_diagnostics",
"device_quirks_v1",
"seek_reanchor_v1",
"output_change_v1",
"direct_stream_resume_v1",
"plan_source_duration_v1"
],
@@ -7,7 +7,6 @@
"playback_route_diagnostics",
"device_quirks_v1",
"seek_reanchor_v1",
"output_change_v1",
"direct_stream_resume_v1",
"plan_source_duration_v1"
],
@@ -31,8 +31,7 @@
"seek_reanchor",
"seek_failure_recovery",
"track_change",
"quality_change",
"output_change"
"quality_change"
],
"maxLength": 32
},
@@ -153,7 +152,7 @@
},
"failure": {
"type": "object",
"description": "Required for failure_recovery and seek_failure_recovery: both need a non-empty classification. Intent operations carry no failure.",
"description": "Required for failure_recovery and seek_failure_recovery: both need a non-empty classification. A track_change or quality_change carries no failure.",
"properties": {
"classification": {
"type": "string",
@@ -226,79 +225,14 @@
"hlg": {
"type": "boolean"
},
"hdr10_max_width": {
"type": "integer",
"minimum": 0
},
"hdr10_max_height": {
"type": "integer",
"minimum": 0
},
"hdr10_max_frame_rate": {
"type": "number",
"minimum": 0
},
"hdr10_max_bitrate_kbps": {
"type": "integer",
"minimum": 0
},
"dolby_vision_profiles": {
"type": "array",
"maxItems": 16,
"items": {
"type": "integer"
}
},
"dolby_vision_profile_levels": {
"type": "array",
"maxItems": 16,
"items": {
"type": "object",
"required": [
"profile",
"max_level"
],
"properties": {
"profile": {
"type": "integer",
"minimum": 1
},
"max_level": {
"type": "integer",
"minimum": 1,
"maximum": 13
},
"bl_compatibility_ids": {
"type": "array",
"maxItems": 16,
"uniqueItems": true,
"items": {
"type": "integer",
"minimum": 0,
"maximum": 15
}
}
},
"additionalProperties": true
}
}
},
"allOf": [
{
"if": {
"anyOf": [
{ "required": ["hdr10_max_width"] },
{ "required": ["hdr10_max_height"] },
{ "required": ["hdr10_max_frame_rate"] },
{ "required": ["hdr10_max_bitrate_kbps"] }
]
},
"then": {
"required": ["hdr10"],
"properties": { "hdr10": { "const": true } }
}
}
],
"additionalProperties": true
},
"audio_passthrough": {
@@ -580,14 +514,6 @@
"type": "string",
"maxLength": 64
},
"app_build": {
"type": "string",
"maxLength": 64
},
"app_channel": {
"type": "string",
"maxLength": 32
},
"device": {
"type": "object",
"description": "Manufacturer and model stay first-class because the device-quirk registry matches on them. Everything platform-specific travels in platform_details as opaque bounded strings.",
@@ -731,30 +657,6 @@
}
}
}
},
{
"$comment": "Intent-only replans describe a user or output change and must not carry route-failure evidence.",
"if": {
"properties": {
"operation": {
"enum": [
"track_change",
"quality_change",
"output_change"
]
}
},
"required": [
"operation"
]
},
"then": {
"not": {
"required": [
"failure"
]
}
}
}
],
"additionalProperties": true
@@ -177,79 +177,14 @@
"hlg": {
"type": "boolean"
},
"hdr10_max_width": {
"type": "integer",
"minimum": 0
},
"hdr10_max_height": {
"type": "integer",
"minimum": 0
},
"hdr10_max_frame_rate": {
"type": "number",
"minimum": 0
},
"hdr10_max_bitrate_kbps": {
"type": "integer",
"minimum": 0
},
"dolby_vision_profiles": {
"type": "array",
"maxItems": 16,
"items": {
"type": "integer"
}
},
"dolby_vision_profile_levels": {
"type": "array",
"maxItems": 16,
"items": {
"type": "object",
"required": [
"profile",
"max_level"
],
"properties": {
"profile": {
"type": "integer",
"minimum": 1
},
"max_level": {
"type": "integer",
"minimum": 1,
"maximum": 13
},
"bl_compatibility_ids": {
"type": "array",
"maxItems": 16,
"uniqueItems": true,
"items": {
"type": "integer",
"minimum": 0,
"maximum": 15
}
}
},
"additionalProperties": true
}
}
},
"allOf": [
{
"if": {
"anyOf": [
{ "required": ["hdr10_max_width"] },
{ "required": ["hdr10_max_height"] },
{ "required": ["hdr10_max_frame_rate"] },
{ "required": ["hdr10_max_bitrate_kbps"] }
]
},
"then": {
"required": ["hdr10"],
"properties": { "hdr10": { "const": true } }
}
}
],
"additionalProperties": true
},
"audio_passthrough": {
@@ -531,14 +466,6 @@
"type": "string",
"maxLength": 64
},
"app_build": {
"type": "string",
"maxLength": 64
},
"app_channel": {
"type": "string",
"maxLength": 32
},
"device": {
"type": "object",
"description": "Manufacturer and model stay first-class because the device-quirk registry matches on them. Everything platform-specific travels in platform_details as opaque bounded strings.",
-32
View File
@@ -48,38 +48,6 @@ the other. Send the exact lower-case family:
| macOS | `desktop` |
| Browser | `web` |
### App identity headers
Separately from the family, every first-party client should send its own app
identity on playback requests. These are server-wide contextual headers, not
settings-specific: the playback session stores them, the admin Activity page
renders them ("Silo Android TV 1.0.0 (build 5)"), and playback decision logs
carry them so a report can be tied to an exact build.
| Header | Clamp | Meaning |
| ----------------------- | ----- | ------------------------------------------------------------------------------------------------------------- |
| `X-Silo-Client` | 128 | Product name, e.g. `Silo Android TV`, `Silo iOS`. |
| `X-Silo-Client-Version` | 64 | Marketing version, e.g. `1.0.0`. Sent verbatim and displayed verbatim — do not pre-shorten it. |
| `X-Silo-Client-Build` | 64 | Opaque per-platform build identifier (Android `versionCode`, Apple `CFBundleVersion`). Never parsed or compared. |
| `X-Silo-Client-Channel` | 32 | Opaque distribution channel: `release`, `beta`, `sideload`, `dev`. Stored verbatim; `release` is not displayed. |
Values are trimmed and truncated to the clamp above — never rejected, on either
route, because an identity label must not be able to fail a playback start. The
clamp counts characters, not bytes, matching `maxLength` in the v3 request
schemas, and is applied where the request is read rather than where the session
is created so the decision logs and `playback_route_events` observe it too.
Nothing is validated against an enum either, so a client may introduce a new
channel without a server change.
Protocol-v3 `POST /playback/start` accepts `client_playback_context.app_version`,
`.app_build`, and `.app_channel` as a body-level fallback for clients that cannot
set the headers on every request. The headers win field by field when both are
present, and the fallback applies **only to a client that sent `X-Silo-Client`**:
`client_playback_context` carries no app name, so nothing in the body can
identify a client that did not name itself — such a session is labeled from its
user agent, and its `app_version` is a free-form platform string rather than the
marketing version `client_version` promises.
## Remote scopes
Every stored value has exactly one identity. Context fields not named by the
@@ -235,9 +235,7 @@ DROP TABLE IF EXISTS public.download_artifacts;
`params_hash = sha256(format | container | codec_video | codec_audio | resolution | audio_track_index | subtitle_burn_in)`.
Two devices (or a web user and a phone) requesting the same 1080p H.264/AAC of the same source share one
artifact. `last_used_at` drives LRU cleanup. `output_path` remains the deterministic API-local
fallback target; distributed preparation additionally stores an opaque artifact id plus its
transcode-node origin and group, allowing an authenticated proxy relay without a shared filesystem. The
artifact. `last_used_at` drives LRU cleanup; `output_path` is on a configured artifact volume. The
`attempts`/`lease_*`/`next_retry_at` columns make the table a **durable, recoverable job queue** (see the
"Durable artifact queue" subsection) so a crash mid-encode cannot strand a download in `preparing`.
@@ -246,8 +244,7 @@ transcode-node origin and group, allowing an authenticated proxy relay without a
`server_settings` is key/value, so new keys need no migration — written when an admin saves them, parsed
in `internal/config/db_loader.go`:
- `download.transcode_enabled` (bool, default **false**) — server gate for transcode-to-file.
- `download.artifact_dir` (string) — local artifact output volume; dedicated transcode nodes default
to a protected directory inside their existing transcode volume.
- `download.artifact_dir` (string) — artifact output volume; defaults under the existing transcode/temp root.
- `download.max_concurrent_prepares` (int, default 2) — encode/remux worker-pool size.
- `download.artifact_max_bytes` (int64, default 0 = unlimited) — LRU eviction budget for artifacts.
-9
View File
@@ -278,15 +278,6 @@ func (s stubStore) GetSeriesPlaybackPreference(context.Context, string, string)
func (s stubStore) DeleteSeriesPlaybackPreference(context.Context, string, string) error {
panic("unused")
}
func (s stubStore) SetCollectionSortPreference(context.Context, userstore.CollectionSortPreference) error {
panic("unused")
}
func (s stubStore) GetCollectionSortPreference(context.Context, string, string, string) (*userstore.CollectionSortPreference, error) {
panic("unused")
}
func (s stubStore) ClearCollectionSortPreference(context.Context, string, string, string) error {
panic("unused")
}
func (s stubStore) GetLibraryPlaybackPreference(context.Context, string, int) (*userstore.LibraryPlaybackPreference, error) {
panic("unused")
}
-3
View File
@@ -9,7 +9,6 @@ import (
)
func requestAccessFilter(r *http.Request) catalog.AccessFilter {
deviceID := deviceMetadataFromRequest(r).DeviceID
if scope, ok := access.GetScope(r.Context()); ok {
return catalog.AccessFilter{
AllowedLibraryIDs: scope.AllowedLibraryIDs,
@@ -18,12 +17,10 @@ func requestAccessFilter(r *http.Request) catalog.AccessFilter {
MaxPlaybackQuality: scope.MaxPlaybackQuality,
UserID: apimw.GetUserID(r.Context()),
ProfileID: apimw.GetProfileID(r.Context()),
DeviceID: deviceID,
}
}
return catalog.AccessFilter{
UserID: apimw.GetUserID(r.Context()),
ProfileID: apimw.GetProfileID(r.Context()),
DeviceID: deviceID,
}
}
@@ -1,18 +0,0 @@
package handlers
import (
"net/http/httptest"
"testing"
)
func TestCatalogAccessFiltersCarryExplicitDeviceIdentity(t *testing.T) {
req := httptest.NewRequest("GET", "/api/v1/items/movie-1", nil)
req.Header.Set(deviceIDHeader, " apple-tv ")
if got := requestAccessFilter(req).DeviceID; got != "apple-tv" {
t.Fatalf("requestAccessFilter().DeviceID = %q, want apple-tv", got)
}
if got := (&ItemsHandler{}).accessFilter(req).DeviceID; got != "apple-tv" {
t.Fatalf("ItemsHandler.accessFilter().DeviceID = %q, want apple-tv", got)
}
}
-16
View File
@@ -59,16 +59,6 @@ type catalogResponse struct {
Items []itemListResponse `json:"items"`
Snapshot string `json:"snapshot,omitempty"`
SearchDiagnostics *searchDiagnostics `json:"search_diagnostics,omitempty"`
// EffectiveSort reports the order a collection source actually resolved in
// once the viewer's saved override and the collection's configured default
// were applied. Omitted for non-collection sources and when the collection
// resolved in its own source order.
EffectiveSort *effectiveSortResponse `json:"effective_sort,omitempty"`
}
type effectiveSortResponse struct {
Field string `json:"field"`
Order string `json:"order"`
}
// searchDiagnostics is an additive, per-query observability object emitted on
@@ -254,11 +244,6 @@ func (h *CatalogHandler) writeCatalogResponse(w http.ResponseWriter, result *cat
}
}
var effectiveSort *effectiveSortResponse
if field := strings.TrimSpace(result.EffectiveSort.Field); field != "" {
effectiveSort = &effectiveSortResponse{Field: field, Order: result.EffectiveSort.Order}
}
writeJSON(w, http.StatusOK, catalogResponse{
Total: result.Total,
TotalExact: result.TotalExact && !groupedByWork,
@@ -266,7 +251,6 @@ func (h *CatalogHandler) writeCatalogResponse(w http.ResponseWriter, result *cat
Items: items,
Snapshot: snapshot,
SearchDiagnostics: diag,
EffectiveSort: effectiveSort,
})
}
@@ -1,195 +0,0 @@
package handlers
import (
"context"
"encoding/json"
"net/http"
"strings"
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/userstore"
)
type collectionPreferenceLibraryReader interface {
GetByID(ctx context.Context, id string) (*models.LibraryCollection, error)
}
// collectionSortPreferenceRequest saves the sort a viewer chose while browsing a
// collection. An empty Field is meaningful — it pins the viewer to the
// collection's own source order even if its creator later configures a default.
// To go back to following the creator's default, DELETE the preference instead.
type collectionSortPreferenceRequest struct {
CollectionKind string `json:"collection_kind"`
CollectionID string `json:"collection_id"`
Field string `json:"field"`
Order string `json:"order"`
}
type collectionSortPreferenceResponse struct {
CollectionKind string `json:"collection_kind"`
CollectionID string `json:"collection_id"`
Field string `json:"field"`
Order string `json:"order"`
}
// NormalizeCollectionSortConfig validates the default sort a collection's
// creator configured and returns its canonical sort_config JSON. Shared by the
// personal and library collection handlers so both id spaces accept exactly the
// same vocabulary.
//
// allowPersonalized must be false for library collection defaults because
// those defaults are shared across profiles. Viewer overrides are validated
// separately and may use profile-scoped fields for either collection kind.
func NormalizeCollectionSortConfig(raw json.RawMessage, allowPersonalized bool) (string, error) {
trimmed := strings.TrimSpace(string(raw))
if trimmed == "" || trimmed == jsonNullLiteral || trimmed == "{}" {
return "{}", nil
}
var cfg struct {
Field string `json:"field"`
Order string `json:"order"`
}
if err := json.Unmarshal(raw, &cfg); err != nil {
return "", errInvalidSortConfig
}
if strings.TrimSpace(cfg.Field) == "" {
// sort_config predates default sorting and may contain independent
// collection modes such as {"mode":"manual_pins"}. Preserve those
// configurations byte-for-byte instead of erasing them merely because
// they do not contain a sort field.
return string(raw), nil
}
qs, ok := catalog.NormalizeCollectionSort(cfg.Field, cfg.Order, allowPersonalized)
if !ok {
return "", errInvalidSortConfig
}
return catalog.EncodeCollectionDefaultSort(qs, true)
}
var errInvalidSortConfig = &sortConfigError{}
type sortConfigError struct{}
func (*sortConfigError) Error() string {
return "sort_config must be {} or {\"field\":\"<supported sort>\",\"order\":\"asc|desc\"}"
}
// HandleSetCollectionSortPreference handles PUT /collections/sort-preference.
func (h *CollectionHandler) HandleSetCollectionSortPreference(w http.ResponseWriter, r *http.Request) {
userID := apimw.GetUserID(r.Context())
profileID := apimw.GetProfileID(r.Context())
var req collectionSortPreferenceRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
return
}
kind, collectionID, ok := normalizeCollectionRef(req.CollectionKind, req.CollectionID)
if !ok {
writeError(w, http.StatusBadRequest, "bad_request", "collection_kind must be 'library' or 'user' and collection_id is required")
return
}
field := strings.ToLower(strings.TrimSpace(req.Field))
order := ""
if field != "" {
qs, valid := catalog.NormalizeCollectionSort(field, req.Order, true)
if !valid {
writeError(w, http.StatusBadRequest, "bad_request", "Unsupported sort field or order for this collection")
return
}
field, order = qs.Field, qs.Order
}
store, err := h.storeProvider.ForUser(r.Context(), userID)
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to access user store")
return
}
if !h.canSaveCollectionSortPreference(r, store, kind, collectionID) {
// Use one not-found response for absent and inaccessible collections so
// the preference endpoint cannot be used to enumerate hidden IDs.
writeError(w, http.StatusNotFound, "not_found", "Collection not found")
return
}
if err := store.SetCollectionSortPreference(r.Context(), userstore.CollectionSortPreference{
ProfileID: profileID,
CollectionKind: kind,
CollectionID: collectionID,
SortField: field,
SortOrder: order,
}); err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to save sort preference")
return
}
writeJSON(w, http.StatusOK, collectionSortPreferenceResponse{
CollectionKind: kind,
CollectionID: collectionID,
Field: field,
Order: order,
})
}
func (h *CollectionHandler) canSaveCollectionSortPreference(
r *http.Request,
store userstore.UserStore,
kind, collectionID string,
) bool {
switch kind {
case userstore.CollectionKindLibrary:
if h.LibraryCollections == nil {
return false
}
collection, err := h.LibraryCollections.GetByID(r.Context(), collectionID)
return err == nil && catalog.CanAccessLibraryCollection(collection, requestAccessFilter(r))
case userstore.CollectionKindUser:
collection, err := store.GetCollection(r.Context(), collectionID)
return err == nil && catalog.ProfileCanAccessCollection(collection, apimw.GetProfileID(r.Context()))
default:
return false
}
}
// HandleClearCollectionSortPreference handles DELETE /collections/sort-preference,
// returning the collection to whatever default its creator configured.
func (h *CollectionHandler) HandleClearCollectionSortPreference(w http.ResponseWriter, r *http.Request) {
userID := apimw.GetUserID(r.Context())
profileID := apimw.GetProfileID(r.Context())
values := r.URL.Query()
kind, collectionID, ok := normalizeCollectionRef(values.Get("collection_kind"), values.Get("collection_id"))
if !ok {
writeError(w, http.StatusBadRequest, "bad_request", "collection_kind must be 'library' or 'user' and collection_id is required")
return
}
store, err := h.storeProvider.ForUser(r.Context(), userID)
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to access user store")
return
}
if err := store.ClearCollectionSortPreference(r.Context(), profileID, kind, collectionID); err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to clear sort preference")
return
}
w.WriteHeader(http.StatusNoContent)
}
func normalizeCollectionRef(rawKind, rawID string) (string, string, bool) {
kind := strings.ToLower(strings.TrimSpace(rawKind))
if kind != userstore.CollectionKindLibrary && kind != userstore.CollectionKindUser {
return "", "", false
}
collectionID := strings.TrimSpace(rawID)
if collectionID == "" {
return "", "", false
}
return kind, collectionID, true
}
@@ -1,302 +0,0 @@
package handlers
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/Silo-Server/silo-server/internal/access"
"github.com/Silo-Server/silo-server/internal/api/middleware"
"github.com/Silo-Server/silo-server/internal/auth"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/userstore"
"github.com/Silo-Server/silo-server/internal/userstore/pgstore"
)
type sortPrefLibraryReader struct {
collection *models.LibraryCollection
}
func (r sortPrefLibraryReader) GetByID(_ context.Context, id string) (*models.LibraryCollection, error) {
if r.collection == nil || r.collection.ID != id {
return nil, catalog.ErrLibraryCollectionNotFound
}
return r.collection, nil
}
// These exercise the sort-preference endpoints through real HTTP handlers and a
// real store, using middleware.SetClaims / SetProfileID to supply the auth
// context the middleware would normally attach. Set SILO_TEST_DATABASE_URL to a
// migrated database to run them.
func sortPrefHandlerPool(t *testing.T) *pgxpool.Pool {
t.Helper()
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("SILO_TEST_DATABASE_URL is not set")
}
pool, err := pgxpool.New(context.Background(), dsn)
if err != nil {
t.Fatalf("connect test database: %v", err)
}
t.Cleanup(pool.Close)
var tableName *string
if err := pool.QueryRow(context.Background(),
`SELECT to_regclass('public.user_collection_sort_preferences')::text`).Scan(&tableName); err != nil {
t.Fatalf("check preference table: %v", err)
}
if tableName == nil || *tableName == "" {
t.Skip("test database has not applied the collection sort preference migration")
}
return pool
}
func sortPrefFixtureUser(t *testing.T, pool *pgxpool.Pool) int {
t.Helper()
var userID int
if err := pool.QueryRow(context.Background(), `
INSERT INTO users (username, email, password_hash, role)
VALUES ('sort-pref-handler-fixture', 'sort-pref-handler@invalid.test', '', 'user')
RETURNING id
`).Scan(&userID); err != nil {
t.Fatalf("seed fixture user: %v", err)
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), `DELETE FROM users WHERE id = $1`, userID)
})
return userID
}
// authed attaches the user + profile scope the middleware chain would normally
// have put on the request.
func authed(r *http.Request, userID int, profileID string) *http.Request {
ctx := middleware.SetClaims(r.Context(), &auth.Claims{UserID: userID, Role: "user"})
ctx = middleware.SetProfileID(ctx, profileID)
return r.WithContext(ctx)
}
func TestCollectionSortPreferenceEndpoints(t *testing.T) {
pool := sortPrefHandlerPool(t)
userID := sortPrefFixtureUser(t, pool)
const profileID = "profile-http"
const collectionID = "http-collection-1"
provider := pgstore.NewPostgresProvider(pool)
handler := NewCollectionHandler(provider)
handler.LibraryCollections = sortPrefLibraryReader{collection: &models.LibraryCollection{
ID: collectionID, Visibility: "visible", LibraryIDs: []int{7},
}}
store, err := provider.ForUser(context.Background(), userID)
if err != nil {
t.Fatalf("user store: %v", err)
}
if err := store.CreateProfile(context.Background(), userstore.Profile{ID: profileID, Name: "HTTP sort preference"}); err != nil {
t.Fatalf("seed profile: %v", err)
}
personalCollection, err := store.CreateCollection(context.Background(), userstore.CreateCollectionInput{
CreatorProfileID: profileID,
Name: "HTTP personal sort preference",
CollectionType: "manual",
})
if err != nil {
t.Fatalf("seed personal collection: %v", err)
}
t.Cleanup(func() {
_ = store.DeleteProfile(context.Background(), profileID)
})
put := func(body string) *httptest.ResponseRecorder {
req := authed(httptest.NewRequest(http.MethodPut, "/collections/sort-preference", strings.NewReader(body)), userID, profileID)
rec := httptest.NewRecorder()
handler.HandleSetCollectionSortPreference(rec, req)
return rec
}
t.Run("saves a sort the viewer picked", func(t *testing.T) {
rec := put(`{"collection_kind":"library","collection_id":"` + collectionID + `","field":"title","order":"asc"}`)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
}
var resp collectionSortPreferenceResponse
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode: %v", err)
}
if resp.Field != "title" || resp.Order != "asc" {
t.Fatalf("response = %+v", resp)
}
// Read it back through the store, not the response, so the assertion
// covers persistence rather than echo.
pref, err := store.GetCollectionSortPreference(context.Background(), profileID, userstore.CollectionKindLibrary, collectionID)
if err != nil || pref == nil {
t.Fatalf("stored preference = %v, err = %v", pref, err)
}
if pref.SortField != "title" || pref.SortOrder != "asc" {
t.Fatalf("stored preference = %+v", pref)
}
})
t.Run("fills in the field's default order", func(t *testing.T) {
rec := put(`{"collection_kind":"library","collection_id":"` + collectionID + `","field":"year"}`)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
}
pref, _ := store.GetCollectionSortPreference(context.Background(), profileID, userstore.CollectionKindLibrary, collectionID)
if pref == nil || pref.SortOrder != "desc" {
t.Fatalf("stored preference = %+v, want year/desc", pref)
}
})
t.Run("an empty field pins to source order", func(t *testing.T) {
rec := put(`{"collection_kind":"library","collection_id":"` + collectionID + `","field":""}`)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
}
pref, _ := store.GetCollectionSortPreference(context.Background(), profileID, userstore.CollectionKindLibrary, collectionID)
if pref == nil {
t.Fatal("pinning to source order removed the row; it must persist as an explicit choice")
}
if pref.SortField != "" {
t.Fatalf("stored field = %q, want empty", pref.SortField)
}
})
t.Run("accepts a personalized viewer sort on a library collection", func(t *testing.T) {
rec := put(`{"collection_kind":"library","collection_id":"` + collectionID + `","field":"progress"}`)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
}
pref, _ := store.GetCollectionSortPreference(context.Background(), profileID, userstore.CollectionKindLibrary, collectionID)
if pref == nil || pref.SortField != "progress" || pref.SortOrder != "desc" {
t.Fatalf("stored preference = %+v, want progress/desc", pref)
}
})
t.Run("rejects an invalid sort order", func(t *testing.T) {
rec := put(`{"collection_kind":"library","collection_id":"` + collectionID + `","field":"title","order":"sideways"}`)
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want 400; body = %s", rec.Code, rec.Body.String())
}
})
t.Run("accepts a personalized sort on a personal collection", func(t *testing.T) {
rec := put(`{"collection_kind":"user","collection_id":"` + personalCollection.ID + `","field":"progress"}`)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
}
t.Cleanup(func() {
_ = store.ClearCollectionSortPreference(context.Background(), profileID, userstore.CollectionKindUser, personalCollection.ID)
})
})
t.Run("rejects a nonexistent collection", func(t *testing.T) {
rec := put(`{"collection_kind":"library","collection_id":"missing","field":"title"}`)
if rec.Code != http.StatusNotFound {
t.Fatalf("status = %d, want 404; body = %s", rec.Code, rec.Body.String())
}
})
t.Run("rejects a library collection outside the viewer scope", func(t *testing.T) {
req := authed(httptest.NewRequest(http.MethodPut, "/collections/sort-preference",
strings.NewReader(`{"collection_kind":"library","collection_id":"`+collectionID+`","field":"title"}`)), userID, profileID)
req = req.WithContext(access.SetScope(req.Context(), access.Scope{AllowedLibraryIDs: []int{99}}))
rec := httptest.NewRecorder()
handler.HandleSetCollectionSortPreference(rec, req)
if rec.Code != http.StatusNotFound {
t.Fatalf("status = %d, want 404; body = %s", rec.Code, rec.Body.String())
}
})
t.Run("rejects an unknown collection kind", func(t *testing.T) {
rec := put(`{"collection_kind":"nonsense","collection_id":"x","field":"title"}`)
if rec.Code != http.StatusBadRequest {
t.Fatalf("status = %d, want 400", rec.Code)
}
})
t.Run("clearing removes the override", func(t *testing.T) {
req := authed(httptest.NewRequest(http.MethodDelete,
"/collections/sort-preference?collection_kind=library&collection_id="+collectionID, nil), userID, profileID)
rec := httptest.NewRecorder()
handler.HandleClearCollectionSortPreference(rec, req)
if rec.Code != http.StatusNoContent {
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
}
pref, err := store.GetCollectionSortPreference(context.Background(), profileID, userstore.CollectionKindLibrary, collectionID)
if err != nil {
t.Fatalf("reading cleared preference: %v", err)
}
if pref != nil {
t.Fatalf("preference still present after clear: %+v", pref)
}
})
}
// TestNormalizeCollectionSortConfigForCreators covers the validation the
// collection create/update handlers apply to a creator-configured default.
func TestNormalizeCollectionSortConfigForCreators(t *testing.T) {
cases := []struct {
name string
raw string
allowPersonalized bool
want string
wantErr bool
}{
{name: "absent", raw: "", want: "{}"},
{name: "explicit null", raw: `null`, want: "{}"},
{name: "legacy empty object", raw: `{}`, want: "{}"},
{name: "empty field remains source order", raw: `{"field":""}`, want: `{"field":""}`},
{name: "preserves manual pins mode", raw: `{"mode":"manual_pins"}`, want: `{"mode":"manual_pins"}`},
{name: "canonicalizes", raw: `{"field":" Title ","order":"ASC"}`, want: `{"field":"title","order":"asc"}`},
{name: "fills the default order", raw: `{"field":"year"}`, want: `{"field":"year","order":"desc"}`},
{name: "rejects unknown fields", raw: `{"field":"not_a_sort"}`, wantErr: true},
{name: "rejects invalid orders", raw: `{"field":"year","order":"sideways"}`, wantErr: true},
{name: "rejects personalized for library", raw: `{"field":"progress"}`, wantErr: true},
{
name: "allows personalized for personal",
raw: `{"field":"progress"}`,
allowPersonalized: true,
want: `{"field":"progress","order":"desc"}`,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got, err := NormalizeCollectionSortConfig(json.RawMessage(tc.raw), tc.allowPersonalized)
if tc.wantErr {
if err == nil {
t.Fatalf("expected an error, got %q", got)
}
return
}
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if got != tc.want {
t.Fatalf("got %s, want %s", got, tc.want)
}
})
}
}
func TestNormalizeCollectionSortConfigPreservesManualPinsAcrossUpdates(t *testing.T) {
const manualPins = `{"mode":"manual_pins"}`
created, err := NormalizeCollectionSortConfig(json.RawMessage(manualPins), false)
if err != nil {
t.Fatalf("normalize on create: %v", err)
}
updated, err := NormalizeCollectionSortConfig(json.RawMessage(created), false)
if err != nil {
t.Fatalf("normalize on update: %v", err)
}
if updated != manualPins {
t.Fatalf("manual pins config changed across create/read/update: got %s, want %s", updated, manualPins)
}
}
+9 -24
View File
@@ -22,12 +22,11 @@ import (
// CollectionHandler handles personal collection CRUD endpoints.
type CollectionHandler struct {
storeProvider userstore.UserStoreProvider
LibraryCollections collectionPreferenceLibraryReader
Executor *catalog.QueryExecutor
S3GP *s3client.Client
HTTPClient *http.Client
PresignTTL time.Duration
storeProvider userstore.UserStoreProvider
Executor *catalog.QueryExecutor
S3GP *s3client.Client
HTTPClient *http.Client
PresignTTL time.Duration
}
// NewCollectionHandler creates a new CollectionHandler.
@@ -107,11 +106,8 @@ type collectionCapabilitiesResponse struct {
// DisplayFilterFields are the catalog query fields a personal-collection
// display filter may use. Clients build a display_query_definition fragment
// from these rather than a bespoke enum.
DisplayFilterFields []string `json:"display_filter_fields"`
DisplayFilterPresets collectionDisplayFilterPresets `json:"display_filter_presets"`
CollectionDefaultSort bool `json:"collection_default_sort"`
CollectionSortPreferences bool `json:"collection_sort_preferences"`
EffectiveCollectionSort bool `json:"effective_collection_sort"`
DisplayFilterFields []string `json:"display_filter_fields"`
DisplayFilterPresets collectionDisplayFilterPresets `json:"display_filter_presets"`
}
type collectionDisplayFilterPresets struct {
@@ -237,9 +233,6 @@ func (h *CollectionHandler) HandleCapabilities(w http.ResponseWriter, r *http.Re
Watched: []string{"all", "watched", "unwatched"},
Media: []string{"all", "movie", "series"},
},
CollectionDefaultSort: true,
CollectionSortPreferences: true,
EffectiveCollectionSort: true,
})
}
@@ -281,11 +274,7 @@ func (h *CollectionHandler) HandleCreateCollection(w http.ResponseWriter, r *htt
}
}
queryDefinition := string(queryDefinitionJSON)
sortConfig, err := NormalizeCollectionSortConfig(req.SortConfig, true)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
}
sortConfig := string(defaultJSON(req.SortConfig))
displayQueryDefinition, err := catalog.NormalizeDisplayQueryFragment(req.DisplayQueryDefinition)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
@@ -369,11 +358,7 @@ func (h *CollectionHandler) HandleUpdateCollection(w http.ResponseWriter, r *htt
input.QueryDefinition = &value
}
if len(req.SortConfig) > 0 {
value, err := NormalizeCollectionSortConfig(req.SortConfig, true)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
}
value := string(req.SortConfig)
input.SortConfig = &value
}
input.IsShared = req.IsShared
@@ -1,25 +0,0 @@
package handlers
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
func TestCollectionCapabilitiesAdvertiseSortSupport(t *testing.T) {
rec := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/collections/capabilities", nil)
NewCollectionHandler(nil).HandleCapabilities(rec, req)
if rec.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
}
var got collectionCapabilitiesResponse
if err := json.Unmarshal(rec.Body.Bytes(), &got); err != nil {
t.Fatalf("decode response: %v", err)
}
if !got.CollectionDefaultSort || !got.CollectionSortPreferences || !got.EffectiveCollectionSort {
t.Fatalf("sort capabilities not fully advertised: %+v", got)
}
}
+17 -248
View File
@@ -4,15 +4,9 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net/http"
"net/url"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/go-chi/chi/v5"
@@ -20,9 +14,7 @@ import (
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/downloads"
"github.com/Silo-Server/silo-server/internal/httpstream"
"github.com/Silo-Server/silo-server/internal/nodepool"
"github.com/Silo-Server/silo-server/internal/playback"
"github.com/Silo-Server/silo-server/internal/streamtoken"
)
// DownloadService is the interface that the download handler depends on. A
@@ -52,51 +44,14 @@ type DownloadService interface {
// DownloadHandler handles download endpoints.
type DownloadHandler struct {
svc DownloadService
nodePlanner nodepool.DownloadPlanner
jwtSecret func() string
proxyHTTPClient *http.Client
preflightMu sync.Mutex
preflightCache map[string]proxyPreflightResult
}
type proxyPreflightResult struct {
reachable bool
expiresAt time.Time
svc DownloadService
}
// NewDownloadHandler creates a new DownloadHandler.
func NewDownloadHandler(svc DownloadService) *DownloadHandler {
return &DownloadHandler{
svc: svc,
preflightCache: make(map[string]proxyPreflightResult),
proxyHTTPClient: &http.Client{
Timeout: 3 * time.Second,
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
return http.ErrUseLastResponse
},
},
}
return &DownloadHandler{svc: svc}
}
// SetProxyDelivery enables stateless proxy-node egress after the API service
// has authorized and resolved a download target. It is optional; integrated
// deployments continue to serve bytes locally.
func (h *DownloadHandler) SetProxyDelivery(planner nodepool.DownloadPlanner, jwtSecret func() string) {
h.nodePlanner = planner
h.jwtSecret = jwtSecret
}
type downloadFileResolver interface {
ResolveDirectFile(ctx context.Context, userID, fileID int, format string, filter catalog.AccessFilter) (*downloads.FileTarget, error)
ResolveManagedFile(ctx context.Context, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter) (*downloads.FileTarget, error)
}
const (
proxyDownloadTokenTTL = 15 * time.Minute
proxyPreflightCacheTTL = 5 * time.Second
)
// downloadRequest represents the JSON body for POST /downloads.
type downloadRequest struct {
ContentID string `json:"content_id"`
@@ -169,7 +124,6 @@ type downloadCapabilityResponse struct {
SeasonDownload bool `json:"season_download"`
SeriesMonitoring bool `json:"series_monitoring"`
MonitoringModes []string `json:"monitoring_modes,omitempty"`
ProxyDelivery bool `json:"proxy_delivery"`
}
func toDownloadResponse(d *downloads.Download) downloadResponse {
@@ -225,7 +179,6 @@ func (h *DownloadHandler) HandleCapability(w http.ResponseWriter, r *http.Reques
return
}
_, _, proxyDelivery := h.proxyTarget()
writeJSON(w, http.StatusOK, downloadCapabilityResponse{
Enabled: capInfo.Enabled,
DownloadAllowed: capInfo.DownloadAllowed,
@@ -235,7 +188,6 @@ func (h *DownloadHandler) HandleCapability(w http.ResponseWriter, r *http.Reques
SeasonDownload: capInfo.SeasonDownload,
SeriesMonitoring: capInfo.SeriesMonitoring,
MonitoringModes: capInfo.MonitoringModes,
ProxyDelivery: proxyDelivery,
})
}
@@ -430,17 +382,6 @@ func (h *DownloadHandler) HandlePatchDownload(w http.ResponseWriter, r *http.Req
// HandleDownloadFile handles GET /downloads/{id}/file.
func (h *DownloadHandler) HandleDownloadFile(w http.ResponseWriter, r *http.Request) {
h.handleDownloadFile(w, r, false)
}
// HandleDownloadFileViaProxy serves the additive proxy-aware download route.
// Clients opt into its 307 response only after discovering proxy_delivery in
// the download capability; the established file route retains 200/206.
func (h *DownloadHandler) HandleDownloadFileViaProxy(w http.ResponseWriter, r *http.Request) {
h.handleDownloadFile(w, r, true)
}
func (h *DownloadHandler) handleDownloadFile(w http.ResponseWriter, r *http.Request, delegate bool) {
userID := apimw.GetUserID(r.Context())
if userID == 0 {
writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required")
@@ -458,56 +399,35 @@ func (h *DownloadHandler) handleDownloadFile(w http.ResponseWriter, r *http.Requ
profileID, deviceID, _, _ := managedIdentity(r)
filter := requestAccessFilter(r)
if delegate && deviceID != "" {
handled, err := h.redirectManagedDownload(r.Context(), w, r, userID, profileID, deviceID, id, filter)
if err != nil {
h.writeDownloadFileError(w, r, id, err)
return
}
if handled {
return
}
}
// Full media downloads outlive the server's absolute WriteTimeout; roll
// the write deadline with progress instead.
sw := httpstream.NewRollingDeadlineWriter(w)
if err := h.svc.ServeFile(r.Context(), sw, r, userID, profileID, deviceID, id, filter); err != nil {
if errors.Is(err, downloads.ErrResponseCommitted) {
if errors.Is(err, downloads.ErrNotFound) || errors.Is(err, catalog.ErrItemNotFound) {
writeError(w, http.StatusNotFound, "not_found", "Download not found")
return
}
if errors.Is(err, downloads.ErrDownloadNotActive) {
writeError(w, http.StatusConflict, "download_inactive", "This download is no longer active")
return
}
if errors.Is(err, downloads.ErrProfileRequired) {
writeError(w, http.StatusBadRequest, "profile_required", "A profile is required for managed downloads")
return
}
if errors.Is(err, downloads.ErrFeatureDisabled) || errors.Is(err, downloads.ErrDownloadNotAllowed) {
writeError(w, http.StatusForbidden, "forbidden", "You are not allowed to download")
return
}
h.writeDownloadFileError(w, r, id, err)
}
}
func (h *DownloadHandler) writeDownloadFileError(w http.ResponseWriter, r *http.Request, id string, err error) {
switch {
case errors.Is(err, downloads.ErrNotFound), errors.Is(err, catalog.ErrItemNotFound):
writeError(w, http.StatusNotFound, "not_found", "Download not found")
case errors.Is(err, downloads.ErrDownloadNotActive):
writeError(w, http.StatusConflict, "download_inactive", "This download is no longer active")
case errors.Is(err, downloads.ErrProfileRequired):
writeError(w, http.StatusBadRequest, "profile_required", "A profile is required for managed downloads")
case errors.Is(err, downloads.ErrFeatureDisabled), errors.Is(err, downloads.ErrDownloadNotAllowed):
writeError(w, http.StatusForbidden, "forbidden", "You are not allowed to download")
default:
slog.ErrorContext(r.Context(), "failed to serve download file", "component", "api", "download_id", id, "error", err)
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to serve download")
return
}
}
// HandleDirectDownload handles GET /direct-download?file_id=N. A legacy
// format=original query is accepted, but direct downloads remain original-only.
func (h *DownloadHandler) HandleDirectDownload(w http.ResponseWriter, r *http.Request) {
h.handleDirectDownload(w, r, false)
}
// HandleDirectDownloadViaProxy serves the additive proxy-aware direct-download
// route while the established endpoint retains its 200/206 response contract.
func (h *DownloadHandler) HandleDirectDownloadViaProxy(w http.ResponseWriter, r *http.Request) {
h.handleDirectDownload(w, r, true)
}
func (h *DownloadHandler) handleDirectDownload(w http.ResponseWriter, r *http.Request, delegate bool) {
userID := apimw.GetUserID(r.Context())
if userID == 0 {
writeError(w, http.StatusUnauthorized, "unauthorized", "Authentication required")
@@ -526,163 +446,12 @@ func (h *DownloadHandler) handleDirectDownload(w http.ResponseWriter, r *http.Re
}
filter := requestAccessFilter(r)
if delegate {
if handled, redirectErr := h.redirectDirectDownload(r.Context(), w, r, userID, fileID, r.URL.Query().Get("format"), filter); redirectErr != nil {
h.writeDownloadError(w, redirectErr)
return
} else if handled {
return
}
}
if err := h.svc.ServeDirect(r.Context(), w, r, userID, fileID, r.URL.Query().Get("format"), filter); err != nil {
h.writeDownloadError(w, err)
return
}
}
func (h *DownloadHandler) redirectDirectDownload(ctx context.Context, w http.ResponseWriter, r *http.Request, userID, fileID int, format string, filter catalog.AccessFilter) (bool, error) {
resolver, secret, ok := h.proxyTarget()
if !ok {
return false, nil
}
target, err := resolver.ResolveDirectFile(ctx, userID, fileID, format, filter)
if err != nil {
return false, err
}
return h.redirectToProxy(w, r, secret, target, userID, "")
}
func (h *DownloadHandler) redirectManagedDownload(ctx context.Context, w http.ResponseWriter, r *http.Request, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter) (bool, error) {
resolver, secret, ok := h.proxyTarget()
if !ok {
return false, nil
}
target, err := resolver.ResolveManagedFile(ctx, userID, profileID, deviceID, downloadID, filter)
if err != nil {
return false, err
}
return h.redirectToProxy(w, r, secret, target, userID, profileID)
}
func (h *DownloadHandler) proxyTarget() (downloadFileResolver, string, bool) {
resolver, ok := h.svc.(downloadFileResolver)
if !ok || h.nodePlanner == nil || h.jwtSecret == nil {
return nil, "", false
}
secret := strings.TrimSpace(h.jwtSecret())
if secret == "" {
return nil, "", false
}
return resolver, secret, true
}
func (h *DownloadHandler) redirectToProxy(w http.ResponseWriter, r *http.Request, secret string, target *downloads.FileTarget, userID int, profileID string) (bool, error) {
if target == nil || (strings.TrimSpace(target.Path) == "" && target.OriginArtifactID == "") {
return false, fmt.Errorf("download target is empty")
}
if target.OriginArtifactID != "" && strings.TrimSpace(target.OriginNodeURL) == "" {
return false, nil
}
if !target.ProxyEligible {
return false, nil
}
reservationKey := target.DownloadID
if reservationKey == "" {
reservationKey = fmt.Sprintf("direct-%d-%d", userID, target.MediaFileID)
}
sessionID := fmt.Sprintf("download-%s-%d", reservationKey, time.Now().UnixNano())
plan := h.nodePlanner.PlanDownload(sessionID, target.OriginNodeGroup)
if plan.ProxyNode == nil {
return false, nil
}
releaseReservation := func() { h.nodePlanner.ReleaseSession(sessionID) }
mediaPath := target.Path
downloadFilename := ""
if target.OriginArtifactID != "" {
mediaPath = ""
if strings.TrimSpace(target.Path) != "" {
downloadFilename = filepath.Base(target.Path)
}
}
token, err := streamtoken.Sign(streamtoken.Claims{
SessionID: sessionID,
MediaPath: mediaPath,
PlayMethod: streamtoken.PlayMethodDownload,
TranscodeNode: target.OriginNodeURL,
DownloadArtifactID: target.OriginArtifactID,
DownloadArtifactRowID: target.ArtifactID,
DownloadFilename: downloadFilename,
UserID: userID,
ProfileID: profileID,
MediaFileID: target.MediaFileID,
}, secret, proxyDownloadTokenTTL)
if err != nil {
releaseReservation()
return false, fmt.Errorf("sign proxy download token: %w", err)
}
location := strings.TrimRight(plan.ProxyNode.URL, "/") + "/downloads/file/" + url.PathEscape(token)
targetKey := target.Path
if target.OriginArtifactID != "" {
targetKey = target.OriginNodeURL + "\x00" + target.OriginArtifactID
}
cacheKey := strings.TrimRight(plan.ProxyNode.URL, "/") + "\x00" + targetKey
if !h.proxyCanServe(r.Context(), cacheKey, location) {
releaseReservation()
return false, nil
}
// A HEAD request only proves that the proxy can read the selected path; the
// proxy deliberately does not count it as an active transfer. Release the
// provisional slot before redirecting so a probe cannot hold max_jobs
// capacity until the planner's reservation bridge expires.
if r.Method == http.MethodHead {
releaseReservation()
}
http.Redirect(w, r, location, http.StatusTemporaryRedirect)
return true, nil
}
func (h *DownloadHandler) proxyCanServe(ctx context.Context, cacheKey, location string) bool {
now := time.Now()
h.preflightMu.Lock()
for key, cached := range h.preflightCache {
if !now.Before(cached.expiresAt) {
delete(h.preflightCache, key)
}
}
if cached, ok := h.preflightCache[cacheKey]; ok && now.Before(cached.expiresAt) {
h.preflightMu.Unlock()
return cached.reachable
}
h.preflightMu.Unlock()
req, err := http.NewRequestWithContext(ctx, http.MethodHead, location, nil)
if err != nil {
return false
}
client := h.proxyHTTPClient
if client == nil {
client = http.DefaultClient
}
resp, err := client.Do(req)
reachable := false
if err != nil {
slog.DebugContext(ctx, "download proxy preflight failed; serving locally", "component", "api", "error", err)
} else {
_ = resp.Body.Close()
reachable = resp.StatusCode >= http.StatusOK && resp.StatusCode < http.StatusMultipleChoices
}
if err == nil && !reachable {
slog.DebugContext(ctx, "download proxy preflight unavailable; serving locally", "component", "api", "status", resp.StatusCode)
}
h.preflightMu.Lock()
if h.preflightCache == nil {
h.preflightCache = make(map[string]proxyPreflightResult)
}
h.preflightCache[cacheKey] = proxyPreflightResult{reachable: reachable, expiresAt: now.Add(proxyPreflightCacheTTL)}
h.preflightMu.Unlock()
return reachable
}
// requireManaged validates a managed (device-scoped) request: authentication, a
// configured service, and the device + profile identity (device_id from the
// X-Silo-Device-Id header only, never the body). On failure it writes the error
-330
View File
@@ -4,10 +4,8 @@ import (
"bytes"
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/go-chi/chi/v5"
@@ -16,8 +14,6 @@ import (
"github.com/Silo-Server/silo-server/internal/auth"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/downloads"
"github.com/Silo-Server/silo-server/internal/nodepool"
"github.com/Silo-Server/silo-server/internal/streamtoken"
)
// fakeDownloadService is a programmable DownloadService for handler tests. It
@@ -35,7 +31,6 @@ type fakeDownloadService struct {
deleteErr error
patchErr error
serveErr error
serveBody string
directErr error
manifest *downloads.OfflineManifest
batchManifests []*downloads.OfflineManifest
@@ -86,21 +81,6 @@ type identityCall struct {
downloadID string
}
type proxyDownloadService struct {
*fakeDownloadService
directTarget *downloads.FileTarget
managedTarget *downloads.FileTarget
resolveErr error
}
func (s *proxyDownloadService) ResolveDirectFile(context.Context, int, int, string, catalog.AccessFilter) (*downloads.FileTarget, error) {
return s.directTarget, s.resolveErr
}
func (s *proxyDownloadService) ResolveManagedFile(context.Context, int, string, string, string, catalog.AccessFilter) (*downloads.FileTarget, error) {
return s.managedTarget, s.resolveErr
}
func (f *fakeDownloadService) Capability(context.Context, int) (downloads.Capability, error) {
return f.capability, nil
}
@@ -186,9 +166,6 @@ func (f *fakeDownloadService) ServeDirect(_ context.Context, w http.ResponseWrit
func (f *fakeDownloadService) ServeFile(_ context.Context, w http.ResponseWriter, _ *http.Request, userID int, profileID, deviceID, downloadID string, _ catalog.AccessFilter) error {
f.gotServe = identityCall{userID, profileID, deviceID, downloadID}
if f.serveBody != "" {
_, _ = w.Write([]byte(f.serveBody))
}
if f.serveErr != nil {
return f.serveErr
}
@@ -306,25 +283,6 @@ func TestHandleCapability(t *testing.T) {
if len(resp.QualityPresets) != 1 || resp.QualityPresets[0] != downloads.QualityOriginal {
t.Fatalf("quality presets = %v, want [original]", resp.QualityPresets)
}
if resp.ProxyDelivery {
t.Fatal("proxy delivery advertised without a configured planner")
}
}
func TestHandleCapabilityAdvertisesAdditiveProxyDelivery(t *testing.T) {
svc := &proxyDownloadService{fakeDownloadService: &fakeDownloadService{}}
h := NewDownloadHandler(svc)
h.SetProxyDelivery(nodepool.NewPlanner(nodepool.NewProxyPool(), nodepool.NewTranscodePool()), func() string { return "secret" })
rec := httptest.NewRecorder()
h.HandleCapability(rec, downloadTestRequest(http.MethodGet, "/downloads/capability", nil, 7, "", ""))
var resp downloadCapabilityResponse
if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if !resp.ProxyDelivery {
t.Fatalf("capability = %+v", resp)
}
}
func TestHandleCapabilityUnauthorized(t *testing.T) {
@@ -471,139 +429,6 @@ func TestManagedFileThreadsIdentity(t *testing.T) {
}
}
func TestManagedFileDoesNotAppendJSONAfterRelayResponseCommitted(t *testing.T) {
svc := &fakeDownloadService{
serveBody: "partial",
serveErr: fmt.Errorf("remote relay failed: %w", downloads.ErrResponseCommitted),
}
h := NewDownloadHandler(svc)
req := withChiID(downloadTestRequest(http.MethodGet, "/downloads/dl1/file", nil, 7, "pA", "devA"), "dl1")
rec := httptest.NewRecorder()
h.HandleDownloadFile(rec, req)
if rec.Code != http.StatusOK || rec.Body.String() != "partial" {
t.Fatalf("response status=%d body=%q", rec.Code, rec.Body.String())
}
}
func TestManagedFileRedirectsAuthorizedArtifactToProxy(t *testing.T) {
const secret = "download-proxy-secret"
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodHead {
t.Errorf("preflight method = %s", r.Method)
}
w.WriteHeader(http.StatusOK)
}))
defer proxy.Close()
svc := &proxyDownloadService{
fakeDownloadService: &fakeDownloadService{},
managedTarget: &downloads.FileTarget{Path: "/artifacts/movie.mp4", DownloadID: "dl1", MediaFileID: 42, ProxyEligible: true},
}
proxies := nodepool.NewProxyPool()
proxies.SetNodes([]*nodepool.Node{{URL: proxy.URL, Enabled: true, Healthy: true}})
h := NewDownloadHandler(svc)
h.SetProxyDelivery(nodepool.NewPlanner(proxies, nodepool.NewTranscodePool()), func() string { return secret })
req := withChiID(downloadTestRequest(http.MethodGet, "/downloads/dl1/file-proxy", nil, 7, "pA", "devA"), "dl1")
rec := httptest.NewRecorder()
h.HandleDownloadFileViaProxy(rec, req)
if rec.Code != http.StatusTemporaryRedirect {
t.Fatalf("status = %d, want 307 (body: %s)", rec.Code, rec.Body.String())
}
location := rec.Header().Get("Location")
prefix := proxy.URL + "/downloads/file/"
if !strings.HasPrefix(location, prefix) {
t.Fatalf("Location = %q", location)
}
claims, err := streamtoken.Verify(strings.TrimPrefix(location, prefix), secret)
if err != nil {
t.Fatal(err)
}
if claims.PlayMethod != streamtoken.PlayMethodDownload || claims.MediaPath != "/artifacts/movie.mp4" || claims.UserID != 7 || claims.ProfileID != "pA" {
t.Fatalf("claims = %+v", claims)
}
if svc.gotServe != (identityCall{}) {
t.Fatalf("local ServeFile was called: %+v", svc.gotServe)
}
}
func TestManagedFileRedirectsNodeLocalArtifactThroughSameGroupProxy(t *testing.T) {
const secret = "download-proxy-secret"
proxyA := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
t.Error("wrong-group proxy was selected")
w.WriteHeader(http.StatusOK)
}))
defer proxyA.Close()
proxyB := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodHead {
t.Errorf("preflight method = %s", r.Method)
}
w.WriteHeader(http.StatusOK)
}))
defer proxyB.Close()
svc := &proxyDownloadService{
fakeDownloadService: &fakeDownloadService{},
managedTarget: &downloads.FileTarget{
DownloadID: "dl-remote",
ArtifactID: "artifact-row",
Path: "/prepared/Movie Final.mp4",
MediaFileID: 42,
OriginNodeURL: "http://transcode-b.internal:8096",
OriginNodeGroup: "host-b",
OriginArtifactID: "artifact-remote",
ProxyEligible: true,
},
}
groupA, groupB := "host-a", "host-b"
proxies := nodepool.NewProxyPool()
proxies.SetNodes([]*nodepool.Node{
{URL: proxyA.URL, Group: &groupA, Enabled: true, Healthy: true},
{URL: proxyB.URL, Group: &groupB, Enabled: true, Healthy: true},
})
h := NewDownloadHandler(svc)
h.SetProxyDelivery(nodepool.NewPlanner(proxies, nodepool.NewTranscodePool()), func() string { return secret })
req := withChiID(downloadTestRequest(http.MethodGet, "/downloads/dl-remote/file-proxy", nil, 7, "pA", "devA"), "dl-remote")
rec := httptest.NewRecorder()
h.HandleDownloadFileViaProxy(rec, req)
if rec.Code != http.StatusTemporaryRedirect {
t.Fatalf("status = %d body=%s", rec.Code, rec.Body.String())
}
location := rec.Header().Get("Location")
prefix := proxyB.URL + "/downloads/file/"
if !strings.HasPrefix(location, prefix) {
t.Fatalf("Location = %q, want proxy-b", location)
}
claims, err := streamtoken.Verify(strings.TrimPrefix(location, prefix), secret)
if err != nil {
t.Fatal(err)
}
if claims.MediaPath != "" || claims.DownloadFilename != "Movie Final.mp4" || claims.TranscodeNode != "http://transcode-b.internal:8096" ||
claims.DownloadArtifactID != "artifact-remote" || claims.DownloadArtifactRowID != "artifact-row" {
t.Fatalf("claims = %+v", claims)
}
}
func TestManagedFileFallsBackWhenRemoteLocatorHasNoOriginURL(t *testing.T) {
svc := &proxyDownloadService{
fakeDownloadService: &fakeDownloadService{},
managedTarget: &downloads.FileTarget{
DownloadID: "dl-incomplete",
OriginArtifactID: "artifact-without-origin",
ProxyEligible: true,
},
}
proxies := nodepool.NewProxyPool()
proxies.SetNodes([]*nodepool.Node{{URL: "http://proxy.invalid", Enabled: true, Healthy: true}})
h := NewDownloadHandler(svc)
h.SetProxyDelivery(nodepool.NewPlanner(proxies, nodepool.NewTranscodePool()), func() string { return "secret" })
req := withChiID(downloadTestRequest(http.MethodGet, "/downloads/dl-incomplete/file-proxy", nil, 7, "pA", "devA"), "dl-incomplete")
rec := httptest.NewRecorder()
h.HandleDownloadFileViaProxy(rec, req)
if rec.Code != http.StatusOK || rec.Body.String() != "served" {
t.Fatalf("status=%d body=%q", rec.Code, rec.Body.String())
}
}
func TestManagedListThreadsIdentity(t *testing.T) {
svc := &fakeDownloadService{}
h := NewDownloadHandler(svc)
@@ -689,161 +514,6 @@ func TestHandleDirectDownloadThreadsOriginalFormat(t *testing.T) {
}
}
func TestHandleDirectDownloadViaProxyRedirectsToProxy(t *testing.T) {
const secret = "download-proxy-secret"
maxJobs := 1
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer proxy.Close()
svc := &proxyDownloadService{
fakeDownloadService: &fakeDownloadService{},
directTarget: &downloads.FileTarget{Path: "/media/movie.mkv", MediaFileID: 42, ProxyEligible: true},
}
proxies := nodepool.NewProxyPool()
proxies.SetNodes([]*nodepool.Node{{URL: proxy.URL + "/", Enabled: true, Healthy: true, MaxJobs: &maxJobs}})
planner := nodepool.NewPlanner(proxies, nodepool.NewTranscodePool())
h := NewDownloadHandler(svc)
h.SetProxyDelivery(planner, func() string { return secret })
rec := httptest.NewRecorder()
h.HandleDirectDownloadViaProxy(rec, downloadTestRequest(http.MethodHead, "/direct-download-proxy?file_id=42", nil, 7, "", ""))
if rec.Code != http.StatusTemporaryRedirect {
t.Fatalf("status = %d, want 307 (body: %s)", rec.Code, rec.Body.String())
}
location := rec.Header().Get("Location")
claims, err := streamtoken.Verify(strings.TrimPrefix(location, proxy.URL+"/downloads/file/"), secret)
if err != nil {
t.Fatal(err)
}
if claims.MediaPath != "/media/movie.mkv" || claims.MediaFileID != 42 {
t.Fatalf("claims = %+v", claims)
}
if svc.gotDirectFileID != 0 {
t.Fatalf("local ServeDirect called for file %d", svc.gotDirectFileID)
}
// HEAD never becomes an active proxy transfer, so it must return the only
// available slot immediately rather than waiting for reservation expiry.
if plan := planner.PlanDownload("after-head"); plan.ProxyNode == nil {
t.Fatal("HEAD request retained the proxy's only job slot")
}
planner.ReleaseSession("after-head")
}
func TestHandleDirectDownloadFallsBackWithoutHealthyProxy(t *testing.T) {
svc := &proxyDownloadService{
fakeDownloadService: &fakeDownloadService{},
directTarget: &downloads.FileTarget{Path: "/media/movie.mkv", MediaFileID: 42, ProxyEligible: true},
}
proxies := nodepool.NewProxyPool()
proxies.SetNodes([]*nodepool.Node{{URL: "https://proxy.example", Enabled: true, Healthy: false}})
h := NewDownloadHandler(svc)
h.SetProxyDelivery(nodepool.NewPlanner(proxies, nodepool.NewTranscodePool()), func() string { return "secret" })
rec := httptest.NewRecorder()
h.HandleDirectDownloadViaProxy(rec, downloadTestRequest(http.MethodGet, "/direct-download-proxy?file_id=42", nil, 7, "", ""))
if rec.Code != http.StatusOK || svc.gotDirectFileID != 42 {
t.Fatalf("status = %d file = %d body = %s", rec.Code, svc.gotDirectFileID, rec.Body.String())
}
}
func TestHandleDirectDownloadFallsBackWhenProxyCannotServePath(t *testing.T) {
proxy := httptest.NewServer(http.NotFoundHandler())
defer proxy.Close()
svc := &proxyDownloadService{
fakeDownloadService: &fakeDownloadService{},
directTarget: &downloads.FileTarget{Path: "/media/movie.mkv", MediaFileID: 42, ProxyEligible: true},
}
proxies := nodepool.NewProxyPool()
proxies.SetNodes([]*nodepool.Node{{URL: proxy.URL, Enabled: true, Healthy: true}})
h := NewDownloadHandler(svc)
h.SetProxyDelivery(nodepool.NewPlanner(proxies, nodepool.NewTranscodePool()), func() string { return "secret" })
rec := httptest.NewRecorder()
h.HandleDirectDownloadViaProxy(rec, downloadTestRequest(http.MethodGet, "/direct-download-proxy?file_id=42", nil, 7, "", ""))
if rec.Code != http.StatusOK || svc.gotDirectFileID != 42 {
t.Fatalf("status = %d file = %d body = %s", rec.Code, svc.gotDirectFileID, rec.Body.String())
}
}
func TestHandleDirectDownloadEstablishedRouteKeepsLocalSuccessStatus(t *testing.T) {
proxyRequests := 0
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
proxyRequests++
w.WriteHeader(http.StatusOK)
}))
defer proxy.Close()
svc := &proxyDownloadService{
fakeDownloadService: &fakeDownloadService{},
directTarget: &downloads.FileTarget{Path: "/media/movie.mkv", MediaFileID: 42, ProxyEligible: true},
}
proxies := nodepool.NewProxyPool()
proxies.SetNodes([]*nodepool.Node{{URL: proxy.URL, Enabled: true, Healthy: true}})
h := NewDownloadHandler(svc)
h.SetProxyDelivery(nodepool.NewPlanner(proxies, nodepool.NewTranscodePool()), func() string { return "secret" })
rec := httptest.NewRecorder()
h.HandleDirectDownload(rec, downloadTestRequest(http.MethodGet, "/direct-download?file_id=42", nil, 7, "", ""))
if rec.Code != http.StatusOK || svc.gotDirectFileID != 42 || proxyRequests != 0 {
t.Fatalf("status = %d file = %d proxy requests = %d", rec.Code, svc.gotDirectFileID, proxyRequests)
}
}
func TestHandleDirectDownloadViaProxyStaysLocalWhenTargetIneligible(t *testing.T) {
proxyRequests := 0
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
proxyRequests++
w.WriteHeader(http.StatusOK)
}))
defer proxy.Close()
svc := &proxyDownloadService{
fakeDownloadService: &fakeDownloadService{},
directTarget: &downloads.FileTarget{Path: "/media/movie.mkv", MediaFileID: 42, ProxyEligible: false},
}
proxies := nodepool.NewProxyPool()
proxies.SetNodes([]*nodepool.Node{{URL: proxy.URL, Enabled: true, Healthy: true}})
h := NewDownloadHandler(svc)
h.SetProxyDelivery(nodepool.NewPlanner(proxies, nodepool.NewTranscodePool()), func() string { return "secret" })
rec := httptest.NewRecorder()
h.HandleDirectDownloadViaProxy(rec, downloadTestRequest(http.MethodGet, "/direct-download-proxy?file_id=42", nil, 7, "", ""))
if rec.Code != http.StatusOK || svc.gotDirectFileID != 42 || proxyRequests != 0 {
t.Fatalf("status = %d file = %d proxy requests = %d", rec.Code, svc.gotDirectFileID, proxyRequests)
}
}
func TestHandleDirectDownloadViaProxyMapsResolverError(t *testing.T) {
svc := &proxyDownloadService{fakeDownloadService: &fakeDownloadService{}, resolveErr: catalog.ErrItemNotFound}
h := NewDownloadHandler(svc)
h.SetProxyDelivery(nodepool.NewPlanner(nodepool.NewProxyPool(), nodepool.NewTranscodePool()), func() string { return "secret" })
rec := httptest.NewRecorder()
h.HandleDirectDownloadViaProxy(rec, downloadTestRequest(http.MethodGet, "/direct-download-proxy?file_id=42", nil, 7, "", ""))
if rec.Code != http.StatusNotFound {
t.Fatalf("status = %d, body = %s", rec.Code, rec.Body.String())
}
}
func TestProxyPreflightCachesReachabilityByNodeAndPath(t *testing.T) {
requests := 0
proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
requests++
w.WriteHeader(http.StatusOK)
}))
defer proxy.Close()
h := NewDownloadHandler(&fakeDownloadService{})
location := proxy.URL + "/downloads/file/token"
if !h.proxyCanServe(context.Background(), proxy.URL+"\x00/media/movie.mkv", location) {
t.Fatal("first preflight failed")
}
if !h.proxyCanServe(context.Background(), proxy.URL+"\x00/media/movie.mkv", location) {
t.Fatal("cached preflight failed")
}
if requests != 1 {
t.Fatalf("preflight requests = %d, want 1", requests)
}
}
func TestHandleDirectDownloadMissingFileID(t *testing.T) {
h := NewDownloadHandler(&fakeDownloadService{})
rec := httptest.NewRecorder()
+12 -32
View File
@@ -2115,7 +2115,6 @@ func filterSortClause(sort, order string) string {
}
func (h *ItemsHandler) accessFilter(r *http.Request) catalog.AccessFilter {
deviceID := deviceMetadataFromRequest(r).DeviceID
selectedFileID := 0
if fileIDRaw := strings.TrimSpace(r.URL.Query().Get("fileId")); fileIDRaw != "" {
if fileID, err := strconv.Atoi(fileIDRaw); err == nil && fileID > 0 {
@@ -2142,7 +2141,6 @@ func (h *ItemsHandler) accessFilter(r *http.Request) catalog.AccessFilter {
SelectedFileID: selectedFileID,
UserID: apimw.GetUserID(r.Context()),
ProfileID: apimw.GetProfileID(r.Context()),
DeviceID: deviceID,
}
}
@@ -2170,7 +2168,6 @@ func (h *ItemsHandler) accessFilter(r *http.Request) catalog.AccessFilter {
SelectedFileID: selectedFileID,
UserID: apimw.GetUserID(r.Context()),
ProfileID: apimw.GetProfileID(r.Context()),
DeviceID: deviceID,
}
}
@@ -2264,42 +2261,17 @@ func (h *ItemsHandler) resolveWatchedTargets(ctx context.Context, contentID stri
}}, nil
}
// episodeTargets resolves the watched leaf targets for a season or series.
// Durations come from one batched file lookup rather than a query per episode:
// marking a long series used to issue one GetByEpisodeID per episode before it
// wrote anything.
func (h *ItemsHandler) episodeTargets(ctx context.Context, episodes []*models.Episode) []watchedLeafTarget {
episodeIDs := make([]string, 0, len(episodes))
for _, episode := range episodes {
episodeIDs = append(episodeIDs, episode.ContentID)
}
filesByEpisode := h.listEpisodeFiles(ctx, episodeIDs)
targets := make([]watchedLeafTarget, 0, len(episodes))
for _, episode := range episodes {
targets = append(targets, watchedLeafTarget{
ContentID: episode.ContentID,
DurationSeconds: mediaFileDurationSeconds(filesByEpisode[episode.ContentID], episode.Runtime),
DurationSeconds: h.contentDurationSeconds(ctx, "", episode.ContentID, episode.Runtime),
})
}
return targets
}
// mediaFileDurationSeconds picks a playable duration from an item's files,
// falling back to the catalog runtime. Mirrors contentDurationSeconds's
// preference order so the batched and single-item paths agree.
func mediaFileDurationSeconds(files []*models.MediaFile, fallbackRuntimeMinutes int) float64 {
for _, file := range files {
if file != nil && file.Duration > 0 {
return float64(file.Duration)
}
}
if fallbackRuntimeMinutes > 0 {
return float64(fallbackRuntimeMinutes * 60)
}
return 0
}
func (h *ItemsHandler) contentDurationSeconds(ctx context.Context, contentID, episodeID string, fallbackRuntimeMinutes int) float64 {
if h.fileRepo == nil {
if fallbackRuntimeMinutes > 0 {
@@ -2319,10 +2291,18 @@ func (h *ItemsHandler) contentDurationSeconds(ctx context.Context, contentID, ep
case contentID != "":
files, err = h.fileRepo.GetByContentID(ctx, contentID)
}
if err != nil {
files = nil
if err == nil {
for _, file := range files {
if file != nil && file.Duration > 0 {
return float64(file.Duration)
}
}
}
return mediaFileDurationSeconds(files, fallbackRuntimeMinutes)
if fallbackRuntimeMinutes > 0 {
return float64(fallbackRuntimeMinutes * 60)
}
return 0
}
// isNotFound checks if an error is a "not found" sentinel.
+60 -106
View File
@@ -33,28 +33,26 @@ import (
"github.com/Silo-Server/silo-server/internal/s3client"
"github.com/Silo-Server/silo-server/internal/sections"
"github.com/Silo-Server/silo-server/internal/usercollections"
"github.com/Silo-Server/silo-server/internal/userstore"
)
type LibraryCollectionHandler struct {
repo *catalog.LibraryCollectionRepository
service *catalog.LibraryCollectionService
itemRepo *catalog.ItemRepository
Executor *catalog.QueryExecutor
detailSvc *catalog.DetailService
presignTTL time.Duration
httpClient *http.Client
s3GP *s3client.Client
FrontendFS fs.FS
SectionRepo *sections.Repository
UserCollectionPool *pgxpool.Pool
GroupRepo *catalog.LibraryCollectionGroupRepository
FolderRepo *catalog.FolderRepository
TemplateRegistry *templates.Registry
SmartCountRefresher *catalog.SmartCountRefresher
JobRepo *adminjob.Repository
EventsHub *evt.Hub
SortPreferenceCleaner *userstore.CollectionSortPreferenceCleaner
repo *catalog.LibraryCollectionRepository
service *catalog.LibraryCollectionService
itemRepo *catalog.ItemRepository
Executor *catalog.QueryExecutor
detailSvc *catalog.DetailService
presignTTL time.Duration
httpClient *http.Client
s3GP *s3client.Client
FrontendFS fs.FS
SectionRepo *sections.Repository
UserCollectionPool *pgxpool.Pool
GroupRepo *catalog.LibraryCollectionGroupRepository
FolderRepo *catalog.FolderRepository
TemplateRegistry *templates.Registry
SmartCountRefresher *catalog.SmartCountRefresher
JobRepo *adminjob.Repository
EventsHub *evt.Hub
}
var errLibraryCollectionInUse = errors.New("collection is used by one or more sections")
@@ -379,22 +377,21 @@ type updateLibraryCollectionRequest struct {
}
type importMDBListRequest struct {
SortConfig json.RawMessage `json:"sort_config,omitempty"`
LibraryID int `json:"library_id"`
LibraryIDs []int `json:"library_ids"`
Title string `json:"title"`
Description string `json:"description"`
URL string `json:"url"`
Limit *int `json:"limit,omitempty"`
Featured bool `json:"featured"`
SortOrder int `json:"sort_order,omitempty"`
PosterURL string `json:"poster_url"`
PosterSourceURL string `json:"poster_source_url"`
BackdropSourceURL string `json:"backdrop_source_url"`
SyncSchedule string `json:"sync_schedule"`
ManagementMode string `json:"management_mode,omitempty"`
ManagementSource string `json:"management_source,omitempty"`
ManagementKey string `json:"management_key,omitempty"`
LibraryID int `json:"library_id"`
LibraryIDs []int `json:"library_ids"`
Title string `json:"title"`
Description string `json:"description"`
URL string `json:"url"`
Limit *int `json:"limit,omitempty"`
Featured bool `json:"featured"`
SortOrder int `json:"sort_order,omitempty"`
PosterURL string `json:"poster_url"`
PosterSourceURL string `json:"poster_source_url"`
BackdropSourceURL string `json:"backdrop_source_url"`
SyncSchedule string `json:"sync_schedule"`
ManagementMode string `json:"management_mode,omitempty"`
ManagementSource string `json:"management_source,omitempty"`
ManagementKey string `json:"management_key,omitempty"`
// PosterFromTemplate marks the poster as supplied by a template bundle so
// future template applies can refresh it. Set programmatically by
// createCollectionFromTemplate; not user-settable via JSON.
@@ -402,25 +399,24 @@ type importMDBListRequest struct {
}
type importTMDBRequest struct {
SortConfig json.RawMessage `json:"sort_config,omitempty"`
LibraryID int `json:"library_id"`
LibraryIDs []int `json:"library_ids"`
Title string `json:"title"`
Description string `json:"description"`
Preset string `json:"preset"`
TimeWindow string `json:"time_window"`
MediaType string `json:"media_type"`
Limit *int `json:"limit,omitempty"`
Featured bool `json:"featured"`
SortOrder int `json:"sort_order,omitempty"`
PosterURL string `json:"poster_url"`
PosterSourceURL string `json:"poster_source_url"`
BackdropSourceURL string `json:"backdrop_source_url"`
SyncSchedule string `json:"sync_schedule"`
ManagementMode string `json:"management_mode,omitempty"`
ManagementSource string `json:"management_source,omitempty"`
ManagementKey string `json:"management_key,omitempty"`
PosterFromTemplate bool `json:"-"`
LibraryID int `json:"library_id"`
LibraryIDs []int `json:"library_ids"`
Title string `json:"title"`
Description string `json:"description"`
Preset string `json:"preset"`
TimeWindow string `json:"time_window"`
MediaType string `json:"media_type"`
Limit *int `json:"limit,omitempty"`
Featured bool `json:"featured"`
SortOrder int `json:"sort_order,omitempty"`
PosterURL string `json:"poster_url"`
PosterSourceURL string `json:"poster_source_url"`
BackdropSourceURL string `json:"backdrop_source_url"`
SyncSchedule string `json:"sync_schedule"`
ManagementMode string `json:"management_mode,omitempty"`
ManagementSource string `json:"management_source,omitempty"`
ManagementKey string `json:"management_key,omitempty"`
PosterFromTemplate bool `json:"-"`
}
// importTMDBFranchiseRequest is the request body for creating a collection
@@ -491,14 +487,13 @@ type importTMDBDiscoverSpecBody struct {
}
type importTraktRequest struct {
SortConfig json.RawMessage `json:"sort_config,omitempty"`
LibraryID int `json:"library_id"`
LibraryIDs []int `json:"library_ids"`
Title string `json:"title"`
Description string `json:"description"`
Preset string `json:"preset"`
MediaType string `json:"media_type"`
ProfileID string `json:"profile_id,omitempty"`
LibraryID int `json:"library_id"`
LibraryIDs []int `json:"library_ids"`
Title string `json:"title"`
Description string `json:"description"`
Preset string `json:"preset"`
MediaType string `json:"media_type"`
ProfileID string `json:"profile_id,omitempty"`
// ListURL selects a user-authored Trakt list
// (https://trakt.tv/users/{user}/lists/{slug}) instead of a preset.
ListURL string `json:"list_url,omitempty"`
@@ -1142,14 +1137,6 @@ func (h *LibraryCollectionHandler) HandleCreateAdminCollection(w http.ResponseWr
}
}
// Library collections resolve with user scope stripped, so per-profile sort
// fields are rejected here rather than failing at browse time.
sortConfig, err := NormalizeCollectionSortConfig(req.SortConfig, false)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
}
var syncSchedule *string
if s := strings.TrimSpace(req.SyncSchedule); s != "" {
if err := catalog.ParseCronExpression(s); err != nil {
@@ -1179,7 +1166,7 @@ func (h *LibraryCollectionHandler) HandleCreateAdminCollection(w http.ResponseWr
BackdropURL: req.BackdropURL,
SourceURL: req.SourceURL,
QueryDefinition: queryDefinition,
SortConfig: json.RawMessage(sortConfig),
SortConfig: defaultJSON(req.SortConfig),
SourceConfig: defaultCollectionSourceConfig(req.SourceConfig),
ManagementMode: managementMode,
ManagementSource: managementSource,
@@ -1238,16 +1225,6 @@ func (h *LibraryCollectionHandler) HandleUpdateAdminCollection(w http.ResponseWr
}
}
sortConfig := req.SortConfig
if len(req.SortConfig) > 0 {
normalized, err := NormalizeCollectionSortConfig(req.SortConfig, false)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
}
sortConfig = json.RawMessage(normalized)
}
// Validate sync_schedule if provided.
if req.SyncSchedule != nil && *req.SyncSchedule != "" {
if err := catalog.ParseCronExpression(*req.SyncSchedule); err != nil {
@@ -1280,7 +1257,7 @@ func (h *LibraryCollectionHandler) HandleUpdateAdminCollection(w http.ResponseWr
BackdropURL: req.BackdropURL,
SourceURL: req.SourceURL,
QueryDefinition: queryDefinition,
SortConfig: sortConfig,
SortConfig: req.SortConfig,
SourceConfig: req.SourceConfig,
ManagementMode: normalizeOptionalCollectionManagementMode(req.ManagementMode),
ManagementSource: req.ManagementSource,
@@ -1454,13 +1431,7 @@ func (h *LibraryCollectionHandler) deleteServerCollection(ctx context.Context, c
}
}
if err := h.repo.Delete(ctx, collectionID); err != nil {
return err
}
if h.SortPreferenceCleaner != nil {
h.SortPreferenceCleaner.DeleteForCollection(ctx, userstore.CollectionKindLibrary, collectionID)
}
return nil
return h.repo.Delete(ctx, collectionID)
}
type adminReorderCollectionsRequest struct {
@@ -2665,10 +2636,6 @@ func (h *LibraryCollectionHandler) createMDBListCollection(
if err != nil {
return nil, requestValidationError{err: err}
}
sortConfig, err := NormalizeCollectionSortConfig(req.SortConfig, false)
if err != nil {
return nil, requestValidationError{err: err}
}
collection, err := h.repo.Create(ctx, catalog.CreateLibraryCollectionInput{
LibraryID: req.LibraryID,
LibraryIDs: req.LibraryIDs,
@@ -2676,7 +2643,6 @@ func (h *LibraryCollectionHandler) createMDBListCollection(
Title: req.Title,
Description: req.Description,
CollectionType: "mdblist",
SortConfig: json.RawMessage(sortConfig),
Visibility: "visible",
Featured: req.Featured,
SortOrder: req.SortOrder,
@@ -2730,10 +2696,6 @@ func (h *LibraryCollectionHandler) createTMDBCollection(
if err != nil {
return nil, requestValidationError{err: err}
}
tmdbSortConfig, err := NormalizeCollectionSortConfig(req.SortConfig, false)
if err != nil {
return nil, requestValidationError{err: err}
}
collection, err := h.repo.Create(ctx, catalog.CreateLibraryCollectionInput{
LibraryID: req.LibraryID,
LibraryIDs: req.LibraryIDs,
@@ -2741,7 +2703,6 @@ func (h *LibraryCollectionHandler) createTMDBCollection(
Title: req.Title,
Description: req.Description,
CollectionType: "tmdb",
SortConfig: json.RawMessage(tmdbSortConfig),
Visibility: "visible",
Featured: req.Featured,
SortOrder: req.SortOrder,
@@ -3087,12 +3048,6 @@ func (h *LibraryCollectionHandler) HandleImportTraktCollection(w http.ResponseWr
return
}
traktSortConfig, err := NormalizeCollectionSortConfig(req.SortConfig, false)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
}
collection, err := h.repo.Create(r.Context(), catalog.CreateLibraryCollectionInput{
LibraryID: req.LibraryID,
LibraryIDs: req.LibraryIDs,
@@ -3100,7 +3055,6 @@ func (h *LibraryCollectionHandler) HandleImportTraktCollection(w http.ResponseWr
Title: req.Title,
Description: req.Description,
CollectionType: "trakt",
SortConfig: json.RawMessage(traktSortConfig),
Visibility: "visible",
Featured: req.Featured,
PosterURL: req.PosterURL,
+74 -79
View File
@@ -33,6 +33,7 @@ import (
"github.com/Silo-Server/silo-server/internal/settingsresolve"
"github.com/Silo-Server/silo-server/internal/streamtoken"
"github.com/Silo-Server/silo-server/internal/subtitles"
"github.com/Silo-Server/silo-server/internal/transcodeproxy"
"github.com/Silo-Server/silo-server/internal/userstore"
"github.com/Silo-Server/silo-server/internal/watchstate"
"github.com/Silo-Server/silo-server/internal/watchsync"
@@ -43,18 +44,12 @@ import (
type SessionManagerInterface interface {
StartSession(userID int, profileID string, fileID int, method playback.PlayMethod, transcodeAudio bool) (*playback.Session, error)
StartSessionWithFiles(userID int, profileID string, effectiveFileID int, requestedFileID int, method playback.PlayMethod, transcodeAudio bool) (*playback.Session, error)
// StartSessionWithFilesContext is required rather than probed for at run
// time: the context is how the reporting client's identity reaches the new
// session, so an implementation without it would start sessions that
// silently carry no client name, version, build or channel.
StartSessionWithFilesContext(ctx context.Context, userID int, profileID string, effectiveFileID int, requestedFileID int, method playback.PlayMethod, transcodeAudio bool) (*playback.Session, error)
UpdateProgress(sessionID string, position float64, isPaused bool) error
UpdateAudioTrack(sessionID string, audioTrackIndex int, method playback.PlayMethod) error
UpdateStreamState(sessionID string, state playback.SessionStreamState) error
TouchActivity(sessionID string) error
BeginTransport(sessionID string) error
EndTransport(sessionID string) error
SetRemoteTransport(sessionID string, remote bool) error
SetEffectiveMediaFileID(sessionID string, fileID int) error
SetTranscodeNodeURL(sessionID, url string) error
SetTranscodeRoute(sessionID string, route playback.TranscodeRoute) error
@@ -68,6 +63,10 @@ type SessionManagerInterface interface {
GetSession(sessionID string) (*playback.Session, error)
}
type sessionStarterWithFilesContext interface {
StartSessionWithFilesContext(ctx context.Context, userID int, profileID string, effectiveFileID int, requestedFileID int, method playback.PlayMethod, transcodeAudio bool) (*playback.Session, error)
}
type transcodePermissionChecker interface {
CheckTranscodingAllowed(ctx context.Context, userID int, requiresVideoTranscode bool) error
}
@@ -135,14 +134,6 @@ type PlaybackOriginalLanguageLookup interface {
GetOriginalLanguage(ctx context.Context, contentID string) (string, error)
}
type copySeekAnchorResolver func(
ctx context.Context,
ffmpegPath string,
inputPath string,
requestedSeekSeconds float64,
segmentDuration int,
) (float64, int, error)
// PlaybackHandler handles playback session HTTP endpoints.
type PlaybackHandler struct {
sessionMgr SessionManagerInterface
@@ -185,7 +176,6 @@ type PlaybackHandler struct {
// playbackConfig(), which falls back to defaults when unset.
PlaybackConfig func() config.PlaybackConfig
FFmpegLogSink playback.FFmpegLogSink
copySeekAnchor copySeekAnchorResolver
realtimeCommandMu sync.Mutex
realtimeCommands map[string]playbackCommandRecord
// tm owns the transcode-session lifecycle (live map, recipe cards, and
@@ -240,10 +230,11 @@ func NewPlaybackHandler(sessionMgr SessionManagerInterface, opts ...FilePathReso
h.tm.Config = func() playback.TranscodeRuntimeConfig {
c := h.playbackConfig()
return playback.TranscodeRuntimeConfig{
TranscodeDir: c.TranscodeDir,
FFmpegPath: c.FFmpegPath,
HWAccel: c.HWAccel,
HWDevice: c.HWDevice,
TranscodeDir: c.TranscodeDir,
FFmpegPath: c.FFmpegPath,
HWAccel: c.HWAccel,
HWDevice: c.HWDevice,
SegmentRetentionSeconds: c.SegmentRetentionSeconds,
}
}
h.tm.StartThrottler = func(ctx context.Context, ts *playback.TranscodeSession) {
@@ -398,20 +389,12 @@ const streamTokenParam = "st"
// reconstruction recipe. Returns "" when no signing secret is configured
// (reconstruct effectively disabled, e.g. in tests).
func (h *PlaybackHandler) signSessionToken(card playback.RecipeCard) string {
return h.signStreamClaims(card.ToClaims())
}
// signStreamClaims mints a stream token from claims that are already assembled.
// Callers serving a session from another node use it to add the claims a
// RecipeCard does not model (the file's Dolby Vision profile, audio-only flag),
// which a remote executor cannot look up for itself.
func (h *PlaybackHandler) signStreamClaims(claims streamtoken.Claims) string {
if h.JWTSecret == "" {
return ""
}
token, err := streamtoken.Sign(claims, h.JWTSecret, playback.MaxTokenTTL)
token, err := streamtoken.Sign(card.ToClaims(), h.JWTSecret, playback.MaxTokenTTL)
if err != nil {
slog.Warn("sign stream token failed", "error", err, "session", claims.SessionID, "playback_session_id", claims.SessionID)
slog.Warn("sign stream token failed", "error", err, "session", card.SessionID, "playback_session_id", card.SessionID)
return ""
}
return token
@@ -634,33 +617,32 @@ func (h *PlaybackHandler) resolveOriginalLanguage(ctx context.Context, file *mod
return lang
}
// resolvedPlaybackAudioLanguage returns the effective playback.audio_language
// for one canonical settings context. It may return
// playback.OriginalLanguageSentinel, which the caller resolves to a concrete
// language. Returns "" when nothing is stored: the contract default is null,
// "no preference". Resolution and decoding failures are returned so playback
// does not silently substitute a different track.
func resolvedPlaybackAudioLanguage(ctx context.Context, store userstore.UserStore, rc settingsresolve.Context) (string, error) {
if store == nil || rc.ProfileID == "" {
return "", nil
// resolvedProfileAudioLanguage returns the effective playback.audio_language
// for the profile with no content context, resolved through the settings
// contract — the canonical replacement for reading the legacy
// user_profiles.language column, matching catalog's detail resolution. It may
// return playback.OriginalLanguageSentinel, which the caller resolves to a
// concrete language. Returns "" when nothing is stored: the contract default
// is null, "no preference".
func resolvedProfileAudioLanguage(ctx context.Context, store userstore.UserStore, profileID string) string {
if store == nil || profileID == "" {
return ""
}
contract, err := settingscontract.Load()
if err != nil {
return "", fmt.Errorf("loading settings contract: %w", err)
return ""
}
resolved, err := settingsresolve.New(contract).Resolve(ctx, store, rc,
resolved, err := settingsresolve.New(contract).Resolve(ctx, store,
settingsresolve.Context{ProfileID: profileID},
[]string{settingskeys.PlaybackAudioLanguage}, nil)
if err != nil {
return "", fmt.Errorf("resolving playback audio language: %w", err)
}
if len(resolved) == 0 {
return "", nil
if err != nil || len(resolved) == 0 {
return ""
}
var language string
if err := json.Unmarshal(resolved[0].Value, &language); err != nil {
return "", fmt.Errorf("decoding playback audio language: %w", err)
if json.Unmarshal(resolved[0].Value, &language) != nil {
return ""
}
return strings.TrimSpace(language), nil
return strings.TrimSpace(language)
}
// --- Persistence helpers ---
@@ -953,9 +935,7 @@ func (h *PlaybackHandler) handleExpiredSession(session *playback.Session) {
}
sessionCopy := *session
go func() {
slog.Info("expired inactive playback session", append([]any{
"session", sessionCopy.ID, "playback_session_id", sessionCopy.ID,
}, sessionCopy.ClientInfo().LogAttrs()...)...)
slog.Info("expired inactive playback session", "session", sessionCopy.ID, "playback_session_id", sessionCopy.ID)
// Expiry is a liveness reap, not a user stop — keep the recipe card so a
// resume reconstructs under the same id (the card's own TTL reaps it if
// the session is truly abandoned).
@@ -1079,18 +1059,11 @@ func playbackClientInfoFromRequest(r *http.Request) playback.ClientInfo {
if r == nil {
return playback.ClientInfo{}
}
// Clamped here, at the boundary, rather than only where the session stamps
// them: the decision logs and playback_route_events are written from this
// value directly, so a client sending a header-sized build would otherwise
// reach both despite the published bound. Values stay opaque — trimmed and
// length-clamped, never parsed or validated against an enum.
return playback.ClientInfo{
Name: r.Header.Get("X-Silo-Client"),
Version: r.Header.Get("X-Silo-Client-Version"),
Build: r.Header.Get("X-Silo-Client-Build"),
Channel: r.Header.Get("X-Silo-Client-Channel"),
Name: strings.TrimSpace(r.Header.Get("X-Silo-Client")),
Version: strings.TrimSpace(r.Header.Get("X-Silo-Client-Version")),
UserAgent: r.UserAgent(),
}.Normalized()
}
}
// HandleUpdateProgress handles POST /playback/{session_id}/progress.
@@ -1403,7 +1376,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
h.touchSessionActivity(sessionID)
segmentName := chi.URLParam(r, "name")
segmentPath, err := transcodeSession.GetSegment(segmentName)
segmentLease, err := transcodeSession.OpenSegment(segmentName)
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) {
segNum, parseErr := playback.ParseSegmentNumber(segmentName)
if parseErr == nil {
@@ -1440,7 +1413,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
"session", sessionID,
"playback_session_id", sessionID,
)
segmentPath, err = transcodeSession.WaitForSegment(segmentName, decision.WaitTimeout)
segmentLease, err = transcodeSession.WaitForOpenSegment(segmentName, decision.WaitTimeout)
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) {
slog.InfoContext(r.Context(), "transcode segment wait timeout", "component", "api",
"segment", segmentName,
@@ -1502,7 +1475,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
); restartErr == nil {
// Throttler + exit monitor re-arm via the session's
// restart hook.
segmentPath, err = transcodeSession.WaitForSegment(segmentName, 30*time.Second)
segmentLease, err = transcodeSession.WaitForOpenSegment(segmentName, 30*time.Second)
if err == nil && strings.EqualFold(transcodeSession.Opts().TargetCodecVideo, "copy") {
// Copy-mode seeks can resume as soon as the target segment
// exists, but that sometimes leaves the player one segment
@@ -1510,7 +1483,9 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
// for a single lookahead fragment when available so the
// first resumed playback window is less brittle.
nextSegmentName := fmt.Sprintf("seg_%05d.m4s", segNum+1)
_, _ = transcodeSession.WaitForSegment(nextSegmentName, 1200*time.Millisecond)
if nextSegment, nextErr := transcodeSession.WaitForOpenSegment(nextSegmentName, 1200*time.Millisecond); nextErr == nil {
_ = nextSegment.Close()
}
}
}
}
@@ -1518,7 +1493,7 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
} else if transcodeSession.IsRunning() {
// Non-numbered segment (e.g., init.mp4 for fMP4 HLS).
// Wait briefly — the init segment is written almost immediately.
segmentPath, err = transcodeSession.WaitForSegment(segmentName, 10*time.Second)
segmentLease, err = transcodeSession.WaitForOpenSegment(segmentName, 10*time.Second)
}
}
if err != nil {
@@ -1530,14 +1505,17 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
return
}
// Report segment download for throttle tracking.
if segNum, parseErr := playback.ParseSegmentNumber(segmentName); parseErr == nil {
transcodeSession.ReportSegmentDownloaded(segNum)
}
w.Header().Set("Cache-Control", "no-store, max-age=0")
w.Header().Set("Pragma", "no-cache")
http.ServeFile(w, r, segmentPath)
defer func() { _ = segmentLease.Close() }()
sw := httpstream.NewRollingDeadlineWriter(w)
http.ServeContent(sw, r, segmentLease.Info.Name(), segmentLease.Info.ModTime(), segmentLease.File)
if r.Method == http.MethodGet &&
sw.CompletedFullResponse(segmentLease.Info.Size()) {
if segNum, parseErr := playback.ParseSegmentNumber(segmentName); parseErr == nil {
transcodeSession.ReportSegmentDownloadedForGeneration(segNum, segmentLease.Generation)
}
}
}
// buildProxyManifestURL signs a stream token carrying the session's full
@@ -1560,6 +1538,9 @@ func (h *PlaybackHandler) buildProxyManifestURL(card playback.RecipeCard, proxyN
func (h *PlaybackHandler) proxyToTranscodeNode(w http.ResponseWriter, r *http.Request, transcodeNodeURL, path string) {
sessionID := chi.URLParam(r, "session_id")
targetURL := transcodeNodeURL + path
isSegmentRoute := strings.Contains(path, "/segment/")
_, segmentParseErr := playback.ParseSegmentNumber(filepath.Base(path))
isMediaSegment := segmentParseErr == nil
// Capture the signed stream token ("st") before stripping it from the URL.
// We forward it out-of-band as a header so the node can reconstruct after a
// self-restart, while keeping it out of the forwarded/logged URL.
@@ -1579,6 +1560,14 @@ func (h *PlaybackHandler) proxyToTranscodeNode(w http.ResponseWriter, r *http.Re
return
}
req.Header.Set("Authorization", "Bearer "+h.JWTSecret)
if isSegmentRoute {
// The node's immediate transport peer is this API process, so receiving a
// complete response there does not prove that the browser received it.
// Suppress node-local accounting and acknowledge only after the downstream
// writer completes. Forward range validators so both hops serve the same
// representation.
transcodeproxy.PrepareRequest(req, r)
}
// Best-effort forward of the stream token as a header so the node's
// reconstruct path (X-Silo-Stream-Token) can rebuild after a self-restart.
// Verify at the API boundary and confirm it belongs to this session; an
@@ -1602,7 +1591,7 @@ func (h *PlaybackHandler) proxyToTranscodeNode(w http.ResponseWriter, r *http.Re
http.Error(w, "transcode node unavailable", http.StatusBadGateway)
return
}
defer resp.Body.Close()
defer func() { _ = resp.Body.Close() }()
// The node strips "st" from the request query (kept out of node URLs/logs),
// so the segment/init URIs in the manifest it builds carry no token. Without
@@ -1632,16 +1621,22 @@ func (h *PlaybackHandler) proxyToTranscodeNode(w http.ResponseWriter, r *http.Re
return
}
for k, vv := range resp.Header {
for _, v := range vv {
w.Header().Add(k, v)
}
}
generation := resp.Header.Get(transcodeproxy.GenerationHeader)
transcodeproxy.CopyResponseHeaders(w.Header(), resp.Header)
// Proxied transcode output can stream past the server's absolute
// WriteTimeout; roll the write deadline with progress instead.
sw := httpstream.NewRollingDeadlineWriter(w)
sw.WriteHeader(resp.StatusCode)
io.Copy(sw, resp.Body)
if _, copyErr := io.Copy(sw, resp.Body); copyErr != nil {
return
}
fullSize := transcodeproxy.FullRepresentationSize(resp)
if isMediaSegment && generation != "" && r.Method == http.MethodGet &&
sw.CompletedFullResponse(fullSize) {
if ackErr := transcodeproxy.Acknowledge(r.Context(), http.DefaultClient, transcodeNodeURL+path, h.JWTSecret, generation); ackErr != nil {
slog.WarnContext(r.Context(), "acknowledge transcode segment completion", "component", "api", "error", ackErr, "playback_session_id", sessionID)
}
}
}
// maybeStartThrottler reads throttle settings and starts the throttler if enabled.
@@ -0,0 +1,19 @@
package handlers
import (
"testing"
"github.com/Silo-Server/silo-server/internal/config"
"github.com/Silo-Server/silo-server/internal/playback"
)
func TestPlaybackHandlerTranscodeRuntimeConfigIncludesSegmentRetention(t *testing.T) {
handler := NewPlaybackHandler(playback.NewSessionManager(0, 0))
handler.PlaybackConfig = func() config.PlaybackConfig {
return config.PlaybackConfig{SegmentRetentionSeconds: 300}
}
if got := handler.tm.Config().SegmentRetentionSeconds; got != 300 {
t.Fatalf("configured segment retention = %d, want 300", got)
}
}
@@ -0,0 +1,127 @@
package handlers
import (
"io"
"net/http"
"net/http/httptest"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/transcodeproxy"
)
func TestProxyToTranscodeNodeAcknowledgesOnlyFullDownstreamResponse(t *testing.T) {
const (
body = "complete segment"
generation = "17"
)
tests := []struct {
name string
rangeHeader string
wantStatus int
wantAck int32
}{
{name: "ordinary get", wantStatus: http.StatusOK, wantAck: 1},
{name: "whole-file range", rangeHeader: "bytes=0-", wantStatus: http.StatusPartialContent, wantAck: 1},
{name: "partial range", rangeHeader: "bytes=1-", wantStatus: http.StatusPartialContent},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
var acknowledgements atomic.Int32
seenRange := make(chan string, 1)
node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
if r.Header.Get(transcodeproxy.RequestHeader) != "1" {
t.Error("segment request omitted proxy marker")
}
seenRange <- r.Header.Get("Range")
w.Header().Set(transcodeproxy.GenerationHeader, generation)
http.ServeContent(w, r, "seg_00007.ts", time.Time{}, strings.NewReader(body))
case http.MethodPost:
if r.Header.Get(transcodeproxy.GenerationHeader) != generation {
t.Errorf("ack generation = %q, want %q", r.Header.Get(transcodeproxy.GenerationHeader), generation)
}
acknowledgements.Add(1)
w.WriteHeader(http.StatusNoContent)
default:
http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
}
}))
defer node.Close()
handler := &PlaybackHandler{JWTSecret: "proxy-test-secret"}
req := withPlaybackRouteParam(
httptest.NewRequest(http.MethodGet, "/playback/transcode/public/segment/seg_00007.ts", nil),
"session_id",
"public",
)
req.Header.Set("Range", tt.rangeHeader)
rr := httptest.NewRecorder()
handler.proxyToTranscodeNode(rr, req, node.URL, "/transcode/remote/segment/seg_00007.ts")
if rr.Code != tt.wantStatus {
t.Fatalf("status = %d, want %d; body = %q", rr.Code, tt.wantStatus, rr.Body.String())
}
if got := <-seenRange; got != tt.rangeHeader {
t.Fatalf("forwarded Range = %q, want %q", got, tt.rangeHeader)
}
if got := rr.Header().Get(transcodeproxy.GenerationHeader); got != "" {
t.Fatalf("internal generation leaked downstream: %q", got)
}
if got := acknowledgements.Load(); got != tt.wantAck {
t.Fatalf("acknowledgements = %d, want %d", got, tt.wantAck)
}
})
}
}
func TestProxyToTranscodeNodeDoesNotAcknowledgeFailedDownstreamWrite(t *testing.T) {
const body = "complete segment"
var acknowledgements atomic.Int32
node := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost {
acknowledgements.Add(1)
w.WriteHeader(http.StatusNoContent)
return
}
w.Header().Set(transcodeproxy.GenerationHeader, "21")
http.ServeContent(w, r, "seg_00007.ts", time.Time{}, strings.NewReader(body))
}))
defer node.Close()
handler := &PlaybackHandler{JWTSecret: "proxy-test-secret"}
req := withPlaybackRouteParam(
httptest.NewRequest(http.MethodGet, "/playback/transcode/public/segment/seg_00007.ts", nil),
"session_id",
"public",
)
w := &failingProxyResponseWriter{header: make(http.Header), remaining: 5}
handler.proxyToTranscodeNode(w, req, node.URL, "/transcode/remote/segment/seg_00007.ts")
if got := acknowledgements.Load(); got != 0 {
t.Fatalf("failed downstream transfer produced %d acknowledgement(s)", got)
}
}
type failingProxyResponseWriter struct {
header http.Header
status int
remaining int
}
func (w *failingProxyResponseWriter) Header() http.Header { return w.header }
func (w *failingProxyResponseWriter) WriteHeader(status int) { w.status = status }
func (w *failingProxyResponseWriter) Write(p []byte) (int, error) {
if w.remaining <= 0 {
return 0, io.ErrClosedPipe
}
n := min(len(p), w.remaining)
w.remaining -= n
return n, io.ErrClosedPipe
}
+2 -58
View File
@@ -48,10 +48,7 @@ type playbackSessionRow struct {
ClientIP string `json:"client_ip,omitempty"`
ClientName string `json:"client_name,omitempty"`
ClientVersion string `json:"client_version,omitempty"`
ClientBuild string `json:"client_build,omitempty"`
ClientChannel string `json:"client_channel,omitempty"`
ClientLabel string `json:"client_label,omitempty"`
ClientLabelFull string `json:"client_label_full,omitempty"`
ClientUserAgent string `json:"client_user_agent,omitempty"`
AudioTrackIndex int `json:"audio_track_index"`
TranscodeAudio bool `json:"transcode_audio"`
@@ -92,11 +89,6 @@ type playbackSessionsCapabilitiesResponse struct {
EffectivePlayMethodValues []string `json:"effective_play_method_values"`
// IsJellyfinClient reports that rows carry is_jellyfin_client.
IsJellyfinClient bool `json:"is_jellyfin_client"`
// ClientBuild reports that rows carry client_build (and the exact-version
// client_label_full derived from it).
ClientBuild bool `json:"client_build"`
// ClientChannel reports that rows carry client_channel.
ClientChannel bool `json:"client_channel"`
}
// HandleGetSessionsCapabilities exposes additive feature support for the live
@@ -106,8 +98,6 @@ func (h *AdminHandler) HandleGetSessionsCapabilities(w http.ResponseWriter, _ *h
EffectivePlayMethod: true,
EffectivePlayMethodValues: []string{"direct", "remux", "transcode", "audio"},
IsJellyfinClient: true,
ClientBuild: true,
ClientChannel: true,
})
}
@@ -192,8 +182,6 @@ func (l *PlaybackSessionsLoader) Load(
COALESCE(HOST(s.client_ip), ''),
COALESCE(s.client_name, ''),
COALESCE(s.client_version, ''),
COALESCE(s.client_build, ''),
COALESCE(s.client_channel, ''),
COALESCE(s.client_user_agent, ''),
COALESCE(s.audio_track_index, 0),
COALESCE(s.transcode_audio, FALSE),
@@ -251,7 +239,6 @@ func (l *PlaybackSessionsLoader) Load(
&posterPath,
&s.PlayMethod, &s.ReportingNode, &s.NodeDisplayName, &s.FileDuration, &s.StartedAt, &s.UpdatedAt,
&s.PositionSeconds, &s.IsPaused, &s.HasPlaybackControl, &s.ClientIP, &s.ClientName, &s.ClientVersion,
&s.ClientBuild, &s.ClientChannel,
&s.ClientUserAgent, &s.AudioTrackIndex, &s.TranscodeAudio, &streamBitrateKbps,
&s.TranscodeNodeURL, &s.TargetResolution, &s.TargetVideoCodec, &s.TargetAudioCodec, &targetBitrateKbps,
&s.TranscodeHWAccel, &s.SourceContainer, &sourceBitrateKbps, &s.SourceVideoCodec, &s.SourceVideoResolution,
@@ -266,13 +253,6 @@ func (l *PlaybackSessionsLoader) Load(
s.SourceBitrateKbps = sourceBitrateKbps
s.SourceAudioChannels = sourceAudioChannels
s.ClientLabel = playbackClientDisplayName(s.ClientName, s.ClientVersion, s.ClientUserAgent)
// Only worth the bytes when it says more than the compact label — which
// it does not for any client without a build or a non-release channel,
// i.e. most of a 200-row page. Clients read client_label when
// client_label_full is absent, so omitting the duplicate costs nothing.
if full := playbackClientFullDisplayName(s.ClientName, s.ClientVersion, s.ClientBuild, s.ClientChannel, s.ClientUserAgent); full != s.ClientLabel {
s.ClientLabelFull = full
}
enrichPlaybackSessionRow(&s, audioTracksJSON)
sessions = append(sessions, s)
}
@@ -467,47 +447,11 @@ func firstNonEmptyValue(values ...string) string {
return ""
}
// playbackClientFullDisplayName renders the client's exact identity — version,
// opaque build, and non-default channel — for surfaces that can afford the
// width (expanded session details, tooltips). The name-and-version half is
// whatever the compact label resolved, so a client identified only by its user
// agent still gets its build named: a client that bothered to report a build
// has earned having it displayed, whether or not it also named itself.
func playbackClientFullDisplayName(name, version, build, channel, userAgent string) string {
label := playbackClientDisplayName(name, version, userAgent)
if label == "" {
return ""
}
build = strings.TrimSpace(build)
channel = strings.TrimSpace(channel)
// "release" is the assumed channel, so naming it adds width without adding
// information; every other channel is worth calling out.
if strings.EqualFold(channel, "release") {
channel = ""
}
var qualifiers []string
if build != "" {
qualifiers = append(qualifiers, "build "+build)
}
if channel != "" {
qualifiers = append(qualifiers, channel)
}
if len(qualifiers) > 0 {
label += " (" + strings.Join(qualifiers, ", ") + ")"
}
return label
}
// playbackClientDisplayName renders the compact label the session lists show.
// A client that reports its own name keeps its version verbatim — truncating
// "1.0.0" to "1" hid the only field that identifies the running build. Labels
// derived from a user agent stay shortened, where a full browser version
// ("120.0.6099.109") is noise.
func playbackClientDisplayName(name, version, userAgent string) string {
name = strings.TrimSpace(name)
version = shortPlaybackClientVersion(version)
if name != "" {
if version = strings.TrimSpace(version); version != "" {
if version != "" {
return name + " " + version
}
return name
+2 -274
View File
@@ -4,12 +4,7 @@ import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"unicode"
"unicode/utf8"
"github.com/Silo-Server/silo-server/internal/playback"
)
func TestSessionComponentDecisionLabelsCopiedAudioDuringHLSAsRemux(t *testing.T) {
@@ -70,8 +65,8 @@ func TestSessionsCapabilitiesAdvertisesActivityFields(t *testing.T) {
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode capabilities: %v", err)
}
if !resp.EffectivePlayMethod || !resp.IsJellyfinClient || !resp.ClientBuild || !resp.ClientChannel {
t.Fatalf("capabilities must advertise every additive field: %+v", resp)
if !resp.EffectivePlayMethod || !resp.IsJellyfinClient {
t.Fatalf("capabilities must advertise both fields: %+v", resp)
}
want := []string{"direct", "remux", "transcode", "audio"}
if len(resp.EffectivePlayMethodValues) != len(want) {
@@ -218,273 +213,6 @@ func TestPlaybackClientDisplayNameAndroidDevices(t *testing.T) {
}
}
func TestPlaybackClientDisplayNameKeepsNamedClientVersionExact(t *testing.T) {
cases := []struct {
name string
clientName string
clientVersion string
want string
}{
{name: "patch version survives", clientName: "Silo Android TV", clientVersion: "1.0.0", want: "Silo Android TV 1.0.0"},
{name: "prerelease suffix survives", clientName: "Silo iOS", clientVersion: "2.1.0-rc.3", want: "Silo iOS 2.1.0-rc.3"},
{name: "no version", clientName: "Silo Android TV", clientVersion: "", want: "Silo Android TV"},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := playbackClientDisplayName(tc.clientName, tc.clientVersion, "")
if got != tc.want {
t.Fatalf("playbackClientDisplayName(%q, %q, \"\") = %q, want %q", tc.clientName, tc.clientVersion, got, tc.want)
}
})
}
}
func TestPlaybackClientFullDisplayName(t *testing.T) {
cases := []struct {
name string
client string
version string
build string
channel string
userAgent string
want string
}{
{
name: "version build and channel",
client: "Silo Android TV",
version: "1.0.0",
build: "5",
channel: "dev",
want: "Silo Android TV 1.0.0 (build 5, dev)",
},
{
name: "release channel omitted",
client: "Silo Android TV",
version: "1.0.0",
build: "5",
channel: "release",
want: "Silo Android TV 1.0.0 (build 5)",
},
{
name: "release channel omitted case insensitively",
client: "Silo Android TV",
version: "1.0.0",
build: "5",
channel: "Release",
want: "Silo Android TV 1.0.0 (build 5)",
},
{
name: "empty channel omitted",
client: "Silo Android TV",
version: "1.0.0",
build: "5",
want: "Silo Android TV 1.0.0 (build 5)",
},
{
name: "channel without build",
client: "Silo Android TV",
version: "1.0.0",
channel: "dev",
want: "Silo Android TV 1.0.0 (dev)",
},
{
name: "build and channel empty",
client: "Silo Android TV",
version: "1.0.0",
want: "Silo Android TV 1.0.0",
},
{
name: "version empty",
client: "Silo Android TV",
want: "Silo Android TV",
},
{
name: "opaque build is not parsed",
client: "Silo tvOS",
version: "1.0.0",
build: "2026.08.13-abcdef",
want: "Silo tvOS 1.0.0 (build 2026.08.13-abcdef)",
},
{
// A client that reports a build but not a name is still named by its
// build: the user-agent label supplies the product half, and dropping
// the qualifiers would hide the exact field this surface exists for.
name: "no client name keeps build and channel on the user agent label",
userAgent: "Mozilla/5.0 (X11; Linux x86_64) Chrome/120.0.0.0 Safari/537.36",
build: "5",
channel: "dev",
want: "Chrome 120 (build 5, dev)",
},
{
name: "no client name and no qualifiers keeps the plain user agent label",
userAgent: "Mozilla/5.0 (X11; Linux x86_64) Chrome/120.0.0.0 Safari/537.36",
want: "Chrome 120",
},
{
name: "no client name and no user agent",
want: "",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
got := playbackClientFullDisplayName(tc.client, tc.version, tc.build, tc.channel, tc.userAgent)
if got != tc.want {
t.Fatalf("playbackClientFullDisplayName() = %q, want %q", got, tc.want)
}
})
}
}
func TestPlaybackClientInfoForStartV3(t *testing.T) {
cases := []struct {
name string
headers map[string]string
context playback.ClientPlaybackContextV3
want playback.ClientInfo
}{
{
name: "headers win over the body",
headers: map[string]string{"X-Silo-Client": "Silo iOS", "X-Silo-Client-Version": "2.1.0", "X-Silo-Client-Build": "9", "X-Silo-Client-Channel": "beta"},
context: playback.ClientPlaybackContextV3{AppVersion: "1.0.0", AppBuild: "1", AppChannel: "release"},
want: playback.ClientInfo{Name: "Silo iOS", Version: "2.1.0", Build: "9", Channel: "beta"},
},
{
// The fallback is per field, not per struct: a client may set the
// name and version headers on every request and still report the
// build it only knows at start time in the body.
name: "body fills only the fields the headers omit",
headers: map[string]string{"X-Silo-Client": "Silo tvOS", "X-Silo-Client-Version": "2.1.0"},
context: playback.ClientPlaybackContextV3{AppVersion: "1.0.0", AppBuild: "77", AppChannel: "sideload"},
want: playback.ClientInfo{Name: "Silo tvOS", Version: "2.1.0", Build: "77", Channel: "sideload"},
},
{
name: "body supplies everything but the name",
headers: map[string]string{"X-Silo-Client": "Silo Android TV"},
context: playback.ClientPlaybackContextV3{AppVersion: "1.0.0", AppBuild: "5", AppChannel: "dev"},
want: playback.ClientInfo{Name: "Silo Android TV", Version: "1.0.0", Build: "5", Channel: "dev"},
},
{
// The regression this guards: the web player reports the literal
// "web" as its app_version and sends no X-Silo-Client. Taking the
// body anyway would stamp "web" onto client_version — the one field
// the contract promises is a marketing version — for every browser
// session, and onto every route event and decision log with it.
name: "nameless client keeps the body out of client_version",
context: playback.ClientPlaybackContextV3{AppVersion: "web"},
want: playback.ClientInfo{},
},
{
name: "nameless client takes no build or channel either",
context: playback.ClientPlaybackContextV3{AppVersion: "web", AppBuild: "5", AppChannel: "dev"},
want: playback.ClientInfo{},
},
{
name: "whitespace-only header falls through to the body",
headers: map[string]string{"X-Silo-Client": "Silo iOS", "X-Silo-Client-Build": " "},
context: playback.ClientPlaybackContextV3{AppBuild: " 42 "},
want: playback.ClientInfo{Name: "Silo iOS", Build: "42"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", nil)
for name, value := range tc.headers {
req.Header.Set(name, value)
}
got := playbackClientInfoForStartV3(req, tc.context)
got.UserAgent = ""
if got != tc.want {
t.Fatalf("playbackClientInfoForStartV3() = %+v, want %+v", got, tc.want)
}
})
}
}
// TestPlaybackClientInfoFromRequestClampsHeaders pins the bound at the request
// boundary rather than at session creation. The resolved identity is written
// straight to the plan-decision log and to playback_route_events, so a client
// sending a header-sized build would reach both if the clamp only happened
// where the session stamps its fields.
func TestPlaybackClientInfoFromRequestClampsHeaders(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", nil)
req.Header.Set("X-Silo-Client", " "+strings.Repeat("N", 200)+" ")
req.Header.Set("X-Silo-Client-Version", strings.Repeat("v", 100))
req.Header.Set("X-Silo-Client-Build", strings.Repeat("b", 100))
req.Header.Set("X-Silo-Client-Channel", strings.Repeat("c", 100))
got := playbackClientInfoFromRequest(req)
for _, tc := range []struct {
field string
value string
want int
}{
{"Name", got.Name, 128},
{"Version", got.Version, 64},
{"Build", got.Build, 64},
{"Channel", got.Channel, 32},
} {
if len(tc.value) != tc.want {
t.Errorf("%s length = %d, want %d", tc.field, len(tc.value), tc.want)
}
}
}
// TestNormalizeClientMetadataCountsRunes guards the two ways a clamp can go
// wrong on a client-supplied header: the published bound is JSON Schema
// maxLength, which counts characters, and a byte-wise cut through a multi-byte
// rune yields invalid UTF-8 that Postgres refuses — failing the whole per-node
// session-sync transaction, not just the offending row.
func TestNormalizeClientMetadataCountsRunes(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", nil)
// 40 three-byte runes: within the 64-character channel bound as the schema
// counts it, well past it as bytes.
req.Header.Set("X-Silo-Client-Channel", strings.Repeat("δ", 40))
got := playbackClientInfoFromRequest(req)
if runes := utf8.RuneCountInString(got.Channel); runes != 32 {
t.Errorf("Channel runes = %d, want the 32-character bound", runes)
}
if !utf8.ValidString(got.Channel) {
t.Errorf("Channel = %q is not valid UTF-8; a text column would reject it", got.Channel)
}
}
// TestPlaybackClientInfoStripsControlCharacters guards the one malformed value
// that reaches a text column looking perfectly well-formed. A JSON NUL escape in a
// v3 start body decodes to a real NUL, which is valid UTF-8 — so the UTF-8
// repair above leaves it and TrimSpace does not consider it whitespace — but
// Postgres refuses NUL in text. The per-node session upserts share one
// transaction, so one such start would stop every live session on that node
// from reconciling. Headers cannot carry it (net/http rejects bytes below
// 0x20), which is exactly why the body path needs its own guard.
func TestPlaybackClientInfoStripsControlCharacters(t *testing.T) {
req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start", nil)
req.Header.Set("X-Silo-Client", "Silo Android TV")
got := playbackClientInfoForStartV3(req, playback.ClientPlaybackContextV3{
AppVersion: "1.0\x00.0",
AppBuild: "5\x00",
AppChannel: "de\nv",
})
if want := "1.0.0"; got.Version != want {
t.Errorf("Version = %q, want %q", got.Version, want)
}
if want := "5"; got.Build != want {
t.Errorf("Build = %q, want %q", got.Build, want)
}
if want := "dev"; got.Channel != want {
t.Errorf("Channel = %q, want %q", got.Channel, want)
}
for _, field := range []string{got.Name, got.Version, got.Build, got.Channel} {
if strings.ContainsFunc(field, unicode.IsControl) {
t.Errorf("%q still carries a control character; a text column would reject it", field)
}
}
}
func TestEnrichPlaybackSessionRowUsesCompatOrigin(t *testing.T) {
row := playbackSessionRow{
ClientName: "Unrecognized Client",
+10 -110
View File
@@ -167,10 +167,6 @@ func (failingSessionManager) StartSessionWithFiles(int, string, int, int, playba
return nil, errors.New("boom")
}
func (failingSessionManager) StartSessionWithFilesContext(context.Context, int, string, int, int, playback.PlayMethod, bool) (*playback.Session, error) {
return nil, errors.New("boom")
}
func (failingSessionManager) UpdateProgress(string, float64, bool) error { return nil }
func (failingSessionManager) UpdateAudioTrack(string, int, playback.PlayMethod) error { return nil }
@@ -185,8 +181,6 @@ func (failingSessionManager) BeginTransport(string) error { return nil }
func (failingSessionManager) EndTransport(string) error { return nil }
func (failingSessionManager) SetRemoteTransport(string, bool) error { return nil }
func (failingSessionManager) SetEffectiveMediaFileID(string, int) error { return nil }
func (failingSessionManager) SetTranscodeNodeURL(string, string) error { return nil }
@@ -557,8 +551,6 @@ func TestHandleStartPlaybackV3_AudioLanguageResolvesCanonically(t *testing.T) {
file.AudioTracks = []models.AudioTrack{
{Language: "eng", Codec: "aac", Channels: 2, Layout: "stereo", Default: true},
{Language: "jpn", Codec: "aac", Channels: 2, Layout: "stereo"},
{Language: "spa", Codec: "aac", Channels: 2, Layout: "stereo"},
{Language: "fra", Codec: "aac", Channels: 2, Layout: "stereo"},
}
return file
}
@@ -572,49 +564,18 @@ func TestHandleStartPlaybackV3_AudioLanguageResolvesCanonically(t *testing.T) {
}
}
setCanonicalLanguage := func(t *testing.T, store userstore.UserStore, identity userstore.SettingIdentity, language string) {
t.Helper()
identity.Key = settingskeys.PlaybackAudioLanguage
identity.ProfileID = "profile-1"
encoded, err := json.Marshal(language)
if err != nil {
t.Fatalf("encode canonical audio language: %v", err)
}
if _, err := store.UpsertSettingValue(context.Background(), identity, encoded); err != nil {
t.Fatalf("seed canonical audio language: %v", err)
}
}
startPlayback := func(
t *testing.T,
store userstore.UserStore,
file *models.MediaFile,
deviceID string,
mutate func(*playback.StartRequestV3),
) int {
startPlayback := func(t *testing.T, store userstore.UserStore, file *models.MediaFile) int {
t.Helper()
manager := playback.NewSessionManager(0, 0)
handler := NewPlaybackHandler(manager, testPlaybackFileResolver{file: file})
handler.SettingsRepo = &mutablePlaybackSettingsV3{values: map[string]string{}}
handler.StoreProvider = testUserStoreProvider{store: store}
handler.ItemAccess = allowAllPlaybackItemAccess{}
if file.EpisodeID != "" {
handler.EpisodeLookup = testEpisodeLookup{episode: &models.Episode{ContentID: file.EpisodeID, SeriesID: "series-1"}}
}
startRequest := v3HandlerStartRequest()
startRequest.ClientPlaybackContext.Deliveries[playback.DeliveryClassProgressiveV3] = playback.DeliveryCapabilityV3{Enabled: true, SupportedOnDevice: true}
if mutate != nil {
mutate(&startRequest)
}
req := httptest.NewRequest(http.MethodPost, "/api/v1/playback/start",
strings.NewReader(marshalV3StartRequest(t, startRequest))).WithContext(newAuthorizedPlaybackContext())
// A client name or user agent is not a stable settings identity. Only
// the explicit device header may select a profile_device row.
req.Header.Set("User-Agent", "SiloTV/apple-tv")
if deviceID != "" {
req.Header.Set(deviceIDHeader, deviceID)
}
rr := httptest.NewRecorder()
handler.HandleStartPlayback(rr, req)
if rr.Code != http.StatusCreated {
@@ -637,11 +598,15 @@ func TestHandleStartPlaybackV3_AudioLanguageResolvesCanonically(t *testing.T) {
t.Run("canonical value wins over legacy column", func(t *testing.T) {
store := newPlaybackTestStore(t)
setLegacyLanguage(t, store, "eng")
setCanonicalLanguage(t, store, userstore.SettingIdentity{
Scope: settingscontract.ScopeProfile,
}, "ja")
if _, err := store.UpsertSettingValue(context.Background(), userstore.SettingIdentity{
Key: settingskeys.PlaybackAudioLanguage,
Scope: settingscontract.ScopeProfile,
ProfileID: "profile-1",
}, json.RawMessage(`"ja"`)); err != nil {
t.Fatalf("seed canonical audio language: %v", err)
}
if index := startPlayback(t, store, newFile(t), "", nil); index != 1 {
if index := startPlayback(t, store, newFile(t)); index != 1 {
t.Fatalf("AudioTrackIndex = %d, want 1 (canonical \"ja\" track)", index)
}
})
@@ -652,75 +617,10 @@ func TestHandleStartPlaybackV3_AudioLanguageResolvesCanonically(t *testing.T) {
// No canonical value stored: the contract default is "no preference",
// so selection falls to the file's default track, not the column's.
if index := startPlayback(t, store, newFile(t), "", nil); index != 0 {
if index := startPlayback(t, store, newFile(t)); index != 0 {
t.Fatalf("AudioTrackIndex = %d, want 0 (file default track)", index)
}
})
t.Run("device value applies only to the identified device", func(t *testing.T) {
store := newPlaybackTestStore(t)
setCanonicalLanguage(t, store, userstore.SettingIdentity{Scope: settingscontract.ScopeProfile}, "en")
setCanonicalLanguage(t, store, userstore.SettingIdentity{
Scope: settingscontract.ScopeProfileDevice,
DeviceID: "apple-tv",
}, "ja")
if index := startPlayback(t, store, newFile(t), "apple-tv", nil); index != 1 {
t.Fatalf("AudioTrackIndex = %d, want 1 (device \"ja\" track)", index)
}
if index := startPlayback(t, store, newFile(t), "", nil); index != 0 {
t.Fatalf("AudioTrackIndex = %d, want 0 (profile \"en\" track without device identity)", index)
}
if index := startPlayback(t, store, newFile(t), "iphone", nil); index != 0 {
t.Fatalf("AudioTrackIndex = %d, want 0 (profile \"en\" track on another device)", index)
}
})
t.Run("series and library values outrank device value", func(t *testing.T) {
store := newPlaybackTestStore(t)
setCanonicalLanguage(t, store, userstore.SettingIdentity{Scope: settingscontract.ScopeProfile}, "en")
setCanonicalLanguage(t, store, userstore.SettingIdentity{
Scope: settingscontract.ScopeProfileDevice,
DeviceID: "apple-tv",
}, "ja")
setCanonicalLanguage(t, store, userstore.SettingIdentity{
Scope: settingscontract.ScopeProfileLibrary,
LibraryID: 12,
}, "es")
setCanonicalLanguage(t, store, userstore.SettingIdentity{
Scope: settingscontract.ScopeProfileSeries,
SeriesID: "series-1",
}, "fr")
movie := newFile(t)
movie.MediaFolderID = 12
if index := startPlayback(t, store, movie, "apple-tv", nil); index != 2 {
t.Fatalf("AudioTrackIndex = %d, want 2 (library \"es\" track)", index)
}
episode := newFile(t)
episode.MediaFolderID = 12
episode.EpisodeID = "episode-1"
if index := startPlayback(t, store, episode, "apple-tv", nil); index != 3 {
t.Fatalf("AudioTrackIndex = %d, want 3 (series \"fr\" track)", index)
}
})
t.Run("explicit track selection outranks saved settings", func(t *testing.T) {
store := newPlaybackTestStore(t)
setCanonicalLanguage(t, store, userstore.SettingIdentity{
Scope: settingscontract.ScopeProfileDevice,
DeviceID: "apple-tv",
}, "ja")
file := newFile(t)
if index := startPlayback(t, store, file, "apple-tv", func(request *playback.StartRequestV3) {
explicit := 0
request.AudioTrackIndex = &explicit
request.AudioTrackID = playback.TrackIDV3(file.ID, "audio", explicit)
}); index != 0 {
t.Fatalf("AudioTrackIndex = %d, want 0 (explicit \"en\" track)", index)
}
})
}
// An omitted start_position asks the server for its resume policy; an explicit
+124 -510
View File
@@ -26,23 +26,21 @@ import (
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/nodepool"
"github.com/Silo-Server/silo-server/internal/playback"
"github.com/Silo-Server/silo-server/internal/settingsresolve"
"github.com/Silo-Server/silo-server/internal/subtitles"
"github.com/Silo-Server/silo-server/internal/transcodenode"
)
const (
maxPlaybackV3BodyBytes = 256 << 10
maxPlaybackV3EventBodyBytes = 32 << 10
replanLeaseDurationV3 = 15 * time.Second
replanReleaseTimeoutV3 = 3 * time.Second
v3NodeCapabilityTTL = time.Minute
playbackNodeIntegratedV3 = "integrated"
subtitleFormatVTTV3 = "vtt"
subtitleMIMEVTTV3 = "text/vtt"
subtitleUnavailableReasonV3 = "subtitle_artifact_unavailable"
transcodeStartFailedReasonV3 = "transcode_start_failed"
seekRestorationPlayerV3 = "player_position"
maxPlaybackV3BodyBytes = 256 << 10
maxPlaybackV3EventBodyBytes = 32 << 10
replanLeaseDurationV3 = 15 * time.Second
replanReleaseTimeoutV3 = 3 * time.Second
v3NodeCapabilityTTL = time.Minute
playbackNodeIntegratedV3 = "integrated"
subtitleFormatVTTV3 = "vtt"
subtitleMIMEVTTV3 = "text/vtt"
subtitleUnavailableReasonV3 = "subtitle_artifact_unavailable"
seekRestorationPlayerV3 = "player_position"
// Failed capability fetches are memoized briefly so an unreachable node
// costs one timeout per window instead of one per planning request.
v3NodeCapabilityErrorTTL = 15 * time.Second
@@ -70,13 +68,6 @@ type preparedTransportV3 struct {
afterDurableCommit func()
}
type preparedTimelineV3 struct {
seekSeconds float64
streamOriginSeconds float64
startSegmentNumber int
copySeekAnchorResolved bool
}
type transportErrorV3 struct {
reason string
message string
@@ -198,12 +189,6 @@ type transcodeNodeEnumeratorV3 interface {
TranscodeNodeURLs() []string
}
// proxyNodeEnumeratorV3 is the proxy-pool counterpart, letting identity
// planning narrow selection to proxies that can execute the plan's recipe.
type proxyNodeEnumeratorV3 interface {
ProxyNodeURLs() []string
}
// hlsPlanningRegistryV3 returns the registry HLS deliveries plan against: the
// local probe plus every pooled transcode node's advertised transformations.
// Only availability of locally-defined specs widens (name and recipe version
@@ -369,11 +354,11 @@ func (h *PlaybackHandler) handleStartPlaybackV3(w http.ResponseWriter, r *http.R
return
}
userID := apimw.GetUserID(r.Context())
deviceID := deviceMetadataFromRequest(r).DeviceID
requestDigests := newPlaybackStartRequestDigestsV3(body, deviceID)
digestBytes := sha256.Sum256(body)
requestDigest := hex.EncodeToString(digestBytes[:])
if existing, lookupErr := h.PlanStoreV3.GetAttemptByPlaybackAttemptID(r.Context(), req.PlaybackAttemptID); lookupErr == nil {
if existing.UserID != userID || existing.ProfileID != profileID || existing.RequestedMediaFileID != req.FileID ||
!requestDigests.matches(existing.RequestDigest) {
(existing.RequestDigest != "" && existing.RequestDigest != requestDigest) {
writeError(w, http.StatusConflict, "playback_attempt_reused", "The playback attempt ID belongs to a different request")
return
}
@@ -411,7 +396,7 @@ func (h *PlaybackHandler) handleStartPlaybackV3(w http.ResponseWriter, r *http.R
return
}
if req.AudioTrackID == "" && req.AudioTrackIndex == nil {
audioIndex, err = h.preferredAudioTrackIndexV3(r.Context(), userID, profileID, deviceID, requestedFile)
audioIndex, err = h.preferredAudioTrackIndexV3(r.Context(), userID, profileID, requestedFile)
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load the saved audio preference")
return
@@ -436,12 +421,12 @@ func (h *PlaybackHandler) handleStartPlaybackV3(w http.ResponseWriter, r *http.R
Registry: h.transformationRegistryV3(r.Context()), HLSRegistry: h.lazyHLSPlanningRegistryV3(r.Context()), DVRPUStrippable: h.lazyDVRPUStrippableV3(r.Context(), effectiveFile), Now: time.Now(),
AdditionalSubtitles: h.downloadedSubtitleInventoryV3(r.Context(), effectiveFile),
})
if terminalAllowsAlternateFileV3(result.Terminal) && shouldTryAlternateFileV3(req.QualityPreference) {
if result.Terminal != nil && result.Terminal.Reason == "no_alternate_version" && shouldTryAlternateFileV3(req.QualityPreference) {
if alternate, alternateErr := h.findAlternateFile(r.Context(), requestedFile); alternateErr == nil && alternate != nil {
effectiveFile = h.ensurePlaybackProbe(r.Context(), alternate)
audioIndex = remapAudioIndexV3(requestedFile, effectiveFile, audioIndex)
if err := h.remapSubtitleSelectionV3(r.Context(), requestedFile, effectiveFile, &req); err != nil {
response, persistErr := h.persistTerminalStartDecisionV3(r.Context(), userID, profileID, req, requestDigests, requestedFile.ID, effectiveFile.ID, playback.NewTerminalResponseV3("subtitle_unavailable_in_version", err.Error(), false))
response, persistErr := h.persistTerminalStartDecisionV3(r.Context(), userID, profileID, req, requestDigest, requestedFile.ID, effectiveFile.ID, playback.NewTerminalResponseV3("subtitle_unavailable_in_version", err.Error(), false))
if persistErr != nil {
writeStartAttemptPersistenceErrorV3(w, persistErr)
return
@@ -457,25 +442,20 @@ func (h *PlaybackHandler) handleStartPlaybackV3(w http.ResponseWriter, r *http.R
}
}
h.clarifyOriginalQuality4KTerminalV3(r.Context(), result.Terminal, requestedFile, !shouldTryAlternateFileV3(req.QualityPreference))
// The exact app identity is logged with every decision so a route or
// terminal reported against one build is attributable without asking the
// user which version they are running.
clientInfo := playbackClientInfoForStartV3(r, req.ClientPlaybackContext)
if result.Terminal != nil {
slog.InfoContext(r.Context(), "playback plan decided", append([]any{
logComponentKey, "playback",
slog.InfoContext(r.Context(), "playback plan decided", "component", "playback",
"outcome", "terminal",
"reason", result.Terminal.Reason,
"file_id", effectiveFile.ID,
"quality_preference", req.QualityPreference,
}, clientInfo.LogAttrs()...)...)
response, persistErr := h.persistTerminalStartDecisionV3(r.Context(), userID, profileID, req, requestDigests, requestedFile.ID, effectiveFile.ID, playback.NewTerminalResponseV3(result.Terminal.Reason, result.Terminal.Message, result.Terminal.Retryable))
)
response, persistErr := h.persistTerminalStartDecisionV3(r.Context(), userID, profileID, req, requestDigest, requestedFile.ID, effectiveFile.ID, playback.NewTerminalResponseV3(result.Terminal.Reason, result.Terminal.Message, result.Terminal.Retryable))
if persistErr != nil {
writeStartAttemptPersistenceErrorV3(w, persistErr)
return
}
if response.Terminal != nil {
h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: playback.RouteEventV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: req.PlaybackAttemptID, Event: playback.RouteEventTerminalV3, FallbackReason: response.Terminal.Reason, OutputContextID: req.ClientPlaybackContext.Output.OutputContextID}, UserID: userID, ProfileID: profileID, ClientName: clientInfo.Name, ClientVersion: clientInfo.Version, ClientBuild: clientInfo.Build, ClientChannel: clientInfo.Channel, ClientModel: req.ClientPlaybackContext.Device.Model})
h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: playback.RouteEventV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: req.PlaybackAttemptID, Event: playback.RouteEventTerminalV3, FallbackReason: response.Terminal.Reason, OutputContextID: req.ClientPlaybackContext.Output.OutputContextID}, UserID: userID, ProfileID: profileID, ClientName: playbackClientInfoFromRequest(r).Name, ClientVersion: playbackClientInfoFromRequest(r).Version, ClientModel: req.ClientPlaybackContext.Device.Model})
}
writeJSON(w, http.StatusCreated, response)
return
@@ -483,8 +463,7 @@ func (h *PlaybackHandler) handleStartPlaybackV3(w http.ResponseWriter, r *http.R
// One line per plan decision so route selection is reconstructible from
// server logs alone (finding a mis-planned route previously required
// correlating client logcat, ffmpeg commands, and session rows).
slog.InfoContext(r.Context(), "playback plan decided", append([]any{
logComponentKey, "playback",
slog.InfoContext(r.Context(), "playback plan decided", "component", "playback",
"outcome", "plan",
"decision_reason", result.Plan.DecisionReason,
"delivery", result.Plan.Delivery,
@@ -497,9 +476,9 @@ func (h *PlaybackHandler) handleStartPlaybackV3(w http.ResponseWriter, r *http.R
"target_bitrate_kbps", result.TargetBitrateKbps,
"quality_preference", req.QualityPreference,
"bandwidth_estimate_kbps", intOrZeroHandlerV3(req.BandwidthEstimateKbps),
}, clientInfo.LogAttrs()...)...)
)
result.Plan.DegradationWarnings = append(result.Plan.DegradationWarnings, warnings...)
response, statusErr := h.startPlannedPlaybackV3(r, userID, profileID, req, requestDigests, requestedFile, effectiveFile, audioIndex, result, clientInfo)
response, statusErr := h.startPlannedPlaybackV3(r, userID, profileID, req, requestDigest, requestedFile, effectiveFile, audioIndex, result)
if statusErr != nil {
if statusErr.reason == "playback_attempt_reused" {
writeError(w, http.StatusConflict, "playback_attempt_reused", statusErr.message)
@@ -508,7 +487,7 @@ func (h *PlaybackHandler) handleStartPlaybackV3(w http.ResponseWriter, r *http.R
if statusErr.reason == "internal_error" {
slog.ErrorContext(r.Context(), "protocol v3 start failed", "component", "api", "reason", statusErr.reason, "error", statusErr.cause)
}
persistedResponse, persistErr := h.startFailureDecisionV3(r.Context(), userID, profileID, req, requestDigests, requestedFile.ID, effectiveFile.ID, statusErr)
persistedResponse, persistErr := h.startFailureDecisionV3(r.Context(), userID, profileID, req, requestDigest, requestedFile.ID, effectiveFile.ID, statusErr)
if persistErr != nil {
writeStartAttemptPersistenceErrorV3(w, persistErr)
return
@@ -519,96 +498,7 @@ func (h *PlaybackHandler) handleStartPlaybackV3(w http.ResponseWriter, r *http.R
writeJSON(w, http.StatusCreated, response)
}
type playbackStartRequestDigestsV3 struct {
current string
legacy string
}
// playbackClientInfoForStartV3 resolves the client's app identity for a v3
// start request. The X-Silo-Client-* headers win because they are present on
// every request; the start body's client_playback_context is the fallback for
// clients that report their app identity only there. All values stay opaque —
// they are trimmed and length-clamped when the session stamps them.
//
// The fallback applies only to a client that named itself. client_playback_context
// carries no app name, so nothing in the body can identify a nameless client
// anyway — it is labeled from its user agent, and its app_version is a
// free-form platform string rather than the marketing version client_version
// promises. The web player, for one, reports the literal "web" there; taking it
// unconditionally would write "web" into the one field that is contractually
// semver, on every browser session.
func playbackClientInfoForStartV3(r *http.Request, clientContext playback.ClientPlaybackContextV3) playback.ClientInfo {
info := playbackClientInfoFromRequest(r)
if info.Name == "" {
return info
}
if info.Version == "" {
info.Version = strings.TrimSpace(clientContext.AppVersion)
}
if info.Build == "" {
info.Build = strings.TrimSpace(clientContext.AppBuild)
}
if info.Channel == "" {
info.Channel = strings.TrimSpace(clientContext.AppChannel)
}
// The header half is already normalized; body-sourced values have to be
// clamped too before they reach the decision log and the route event.
return info.Normalized()
}
// playbackClientInfoWithSessionFallbackV3 completes a header-derived identity
// from the session the event belongs to. Route events posted out of band carry
// no client_playback_context, so without this a client that reports its build
// only in the start body would attribute its plan_selected event to a build and
// every later event of the same attempt to none.
func (h *PlaybackHandler) playbackClientInfoWithSessionFallbackV3(sessionID string, info playback.ClientInfo) playback.ClientInfo {
if sessionID == "" || h.sessionMgr == nil {
return info
}
if info.Name != "" && info.Version != "" && info.Build != "" && info.Channel != "" {
return info
}
session, err := h.sessionMgr.GetSession(sessionID)
if err != nil || session == nil {
return info
}
stamped := session.ClientInfo()
if info.Name == "" {
info.Name = stamped.Name
}
if info.Version == "" {
info.Version = stamped.Version
}
if info.Build == "" {
info.Build = stamped.Build
}
if info.Channel == "" {
info.Channel = stamped.Channel
}
return info
}
// newPlaybackStartRequestDigestsV3 fingerprints both the body and normalized
// device identity because either can change the selected playback plan. It
// also retains the pre-device digest while attempts written by an older
// server can still be replayed during a rolling deployment.
func newPlaybackStartRequestDigestsV3(body []byte, deviceID string) playbackStartRequestDigestsV3 {
hasher := sha256.New()
_, _ = fmt.Fprintf(hasher, "%d:", len(body))
_, _ = hasher.Write(body)
_, _ = hasher.Write([]byte(deviceID))
legacy := sha256.Sum256(body)
return playbackStartRequestDigestsV3{
current: hex.EncodeToString(hasher.Sum(nil)),
legacy: hex.EncodeToString(legacy[:]),
}
}
func (d playbackStartRequestDigestsV3) matches(stored string) bool {
return stored == "" || stored == d.current || stored == d.legacy
}
func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, profileID string, req playback.StartRequestV3, requestDigests playbackStartRequestDigestsV3, requestedFile, effectiveFile *models.MediaFile, audioIndex int, result playback.PlannerResultV3, clientInfo playback.ClientInfo) (playback.DecisionResponseV3, *transportErrorV3) {
func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, profileID string, req playback.StartRequestV3, requestDigest string, requestedFile, effectiveFile *models.MediaFile, audioIndex int, result playback.PlannerResultV3) (playback.DecisionResponseV3, *transportErrorV3) {
if result.Plan == nil {
return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "The server produced no playback plan."}
}
@@ -621,8 +511,15 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr
return playback.DecisionResponseV3{}, &transportErrorV3{reason: reason, message: "The selected server adaptation is disabled for this user."}
}
}
clientInfo := playbackClientInfoFromRequest(r)
ctx := playback.WithClientInfo(r.Context(), clientInfo)
session, err := h.sessionMgr.StartSessionWithFilesContext(ctx, userID, profileID, effectiveFile.ID, requestedFile.ID, result.PlayMethod, result.TranscodeAudio)
var session *playback.Session
var err error
if starter, ok := h.sessionMgr.(sessionStarterWithFilesContext); ok {
session, err = starter.StartSessionWithFilesContext(ctx, userID, profileID, effectiveFile.ID, requestedFile.ID, result.PlayMethod, result.TranscodeAudio)
} else {
session, err = h.sessionMgr.StartSessionWithFiles(userID, profileID, effectiveFile.ID, requestedFile.ID, result.PlayMethod, result.TranscodeAudio)
}
if err != nil {
return playback.DecisionResponseV3{}, sessionStartErrorV3(err)
}
@@ -669,7 +566,7 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr
return playback.DecisionResponseV3{}, subtitleArtifactErrorV3("Failed to prepare the selected subtitle artifact.", err)
}
response := playback.DecisionResponseV3{ProtocolVersion: playback.ProtocolV3, ServerFeatures: playback.ServerFeaturesV3(), Outcome: playback.OutcomePlayableV3, SessionID: session.ID, PlaybackPlan: result.Plan}
record := playback.AttemptRecordV3{PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, UserID: userID, ProfileID: profileID, RequestedMediaFileID: requestedFile.ID, EffectiveMediaFileID: effectiveFile.ID, CurrentPlanID: result.Plan.PlanID, CurrentPlan: *result.Plan, FrozenRecipe: frozenRecipe, NormalizedRequest: req, StartResponse: response, RequestDigest: requestDigests.current, ExpiresAt: time.Now().Add(playback.MaxTokenTTL)}
record := playback.AttemptRecordV3{PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, UserID: userID, ProfileID: profileID, RequestedMediaFileID: requestedFile.ID, EffectiveMediaFileID: effectiveFile.ID, CurrentPlanID: result.Plan.PlanID, CurrentPlan: *result.Plan, FrozenRecipe: frozenRecipe, NormalizedRequest: req, StartResponse: response, RequestDigest: requestDigest, ExpiresAt: time.Now().Add(playback.MaxTokenTTL)}
if err := h.updateV3SessionState(r.Context(), session, effectiveFile, result, transport); err != nil {
transport.rollback()
abort()
@@ -678,9 +575,12 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr
if err := h.PlanStoreV3.SaveAttempt(r.Context(), record); err != nil {
transport.rollback()
abort()
if errors.Is(err, playback.ErrPlaybackAttemptExistsV3) || errors.Is(err, playback.ErrIdempotencyKeyReusedV3) {
if errors.Is(err, playback.ErrIdempotencyKeyReusedV3) {
return playback.DecisionResponseV3{}, &transportErrorV3{reason: "playback_attempt_reused", message: "The playback attempt ID was reused with different input."}
}
if errors.Is(err, playback.ErrPlaybackAttemptExistsV3) {
existing, lookupErr := h.PlanStoreV3.GetAttemptByPlaybackAttemptID(r.Context(), req.PlaybackAttemptID)
if lookupErr == nil && existing.UserID == userID && existing.ProfileID == profileID && existing.RequestedMediaFileID == req.FileID && requestDigests.matches(existing.RequestDigest) {
if lookupErr == nil && existing.UserID == userID && existing.ProfileID == profileID && existing.RequestedMediaFileID == req.FileID {
// Replaying a concurrent duplicate is only valid while its
// session is alive; otherwise tell the client to mint a new
// attempt rather than hand it a plan it can never stream.
@@ -689,9 +589,6 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr
}
return decisionResponseFromAttemptV3(existing), nil
}
if errors.Is(err, playback.ErrIdempotencyKeyReusedV3) {
return playback.DecisionResponseV3{}, &transportErrorV3{reason: "playback_attempt_reused", message: "The playback attempt ID was reused with different input."}
}
}
return playback.DecisionResponseV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to persist the playback plan.", cause: err}
}
@@ -721,7 +618,7 @@ func (h *PlaybackHandler) startPlannedPlaybackV3(r *http.Request, userID int, pr
h.maybeQueueLazyPlaybackMarkers(r.Context(), session, effectiveFile)
h.persistSeriesSelectionsV3(r.Context(), userID, profileID, effectiveFile, plannedAudioTrackIndexV3(result, audioIndex))
h.syncSessionsNow(r.Context(), "v3_start")
h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: playback.RouteEventV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, PlanID: result.Plan.PlanID, Event: playback.RouteEventPlanSelectedV3, AppliedQuirkIDs: appliedQuirkIDsV3(result.Plan), QuirkRegistryRevision: appliedQuirkRevisionV3(result.Plan), OutputContextID: req.ClientPlaybackContext.Output.OutputContextID}, UserID: userID, ProfileID: profileID, ClientName: clientInfo.Name, ClientVersion: clientInfo.Version, ClientBuild: clientInfo.Build, ClientChannel: clientInfo.Channel, ClientModel: req.ClientPlaybackContext.Device.Model})
h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: playback.RouteEventV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, PlanID: result.Plan.PlanID, Event: playback.RouteEventPlanSelectedV3, AppliedQuirkIDs: appliedQuirkIDsV3(result.Plan), QuirkRegistryRevision: appliedQuirkRevisionV3(result.Plan), OutputContextID: req.ClientPlaybackContext.Output.OutputContextID}, UserID: userID, ProfileID: profileID, ClientName: clientInfo.Name, ClientVersion: clientInfo.Version, ClientModel: req.ClientPlaybackContext.Device.Model})
return response, nil
}
@@ -736,12 +633,8 @@ func (h *PlaybackHandler) persistSeriesSelectionsV3(ctx context.Context, userID
}
func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3) (preparedTransportV3, *transportErrorV3) {
timeline, timelineErr := h.prepareTransportTimelineV3(r.Context(), session, file, result)
if timelineErr != nil {
return preparedTransportV3{}, timelineErr
}
if result.Plan.Delivery != playback.DeliveryTranscodeHLSV3 && result.Plan.Delivery != playback.DeliveryRemuxHLSV3 {
return h.prepareIdentityTransportV3(r, session, file, result, timeline)
return h.prepareIdentityTransportV3(session, result), nil
}
if h.NodePlanner != nil {
plan := h.planNodeSessionV3(r.Context(), session, result)
@@ -751,7 +644,7 @@ func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback.
err = validateAdvertisedTransformationsV3(result.Plan, transformations)
}
if err == nil {
transport, transportErr := h.prepareRemoteTransportV3(r, session, file, result, plan, timeline)
transport, transportErr := h.prepareRemoteTransportV3(r, session, file, result, plan)
if transportErr != nil {
if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok {
releaser.ReleaseSession(session.ID)
@@ -781,54 +674,7 @@ func (h *PlaybackHandler) prepareTransportV3(r *http.Request, session *playback.
return preparedTransportV3{}, &transportErrorV3{reason: "transcode_node_capability_unavailable", message: "No available transcode executor can run the selected playback recipe.", retryable: true, cause: err}
}
}
return h.prepareLocalTransportV3(r, session, file, result, timeline)
}
func (h *PlaybackHandler) prepareTransportTimelineV3(ctx context.Context, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3) (preparedTimelineV3, *transportErrorV3) {
if result.Plan == nil {
return preparedTimelineV3{}, nil
}
requested := result.Plan.Timeline.SourceStartSeconds
switch result.Plan.Delivery {
case playback.DeliveryRemuxProgressiveV3, playback.DeliveryRemuxHLSV3:
// Audio-only remuxes have no copied video keyframe to resolve. Keep the
// requested position as their exact stream origin: the chunked output
// clock restarts at zero and later seeks require another server reanchor.
if file != nil && file.IsAudioOnly() {
configureCopyRemuxTimelineV3(result.Plan, requested)
return preparedTimelineV3{seekSeconds: requested, streamOriginSeconds: requested}, nil
}
origin, startSegment := 0.0, 0
if requested > 0 {
if file == nil || strings.TrimSpace(file.FilePath) == "" {
return preparedTimelineV3{}, &transportErrorV3{reason: transcodeStartFailedReasonV3, message: "Failed to resolve remux seek position.", retryable: true, cause: errors.New("copy seek anchor requires a media file path")}
}
resolver := h.copySeekAnchor
if resolver == nil {
resolver = playback.ResolveCopySeekAnchor
}
var err error
origin, startSegment, err = resolver(ctx, h.playbackConfig().FFmpegPath, file.FilePath, requested, 2)
if err != nil {
slog.ErrorContext(ctx, "failed to resolve protocol v3 copy-video seek anchor",
"component", "api",
"playback_session_id", session.ID,
"requested_seek_seconds", requested,
"error", err,
)
return preparedTimelineV3{}, &transportErrorV3{reason: transcodeStartFailedReasonV3, message: "Failed to resolve remux seek position.", retryable: true, cause: err}
}
}
configureCopyRemuxTimelineV3(result.Plan, origin)
return preparedTimelineV3{seekSeconds: requested, streamOriginSeconds: origin, startSegmentNumber: startSegment, copySeekAnchorResolved: true}, nil
case playback.DeliveryTranscodeHLSV3:
sourceMetadata := sourceExecutionMetadataV3(file, result)
seekSeconds, startSegment := configureHLSTimelineV3(result.Plan, result.TargetVideoCodec, 2, sourceMetadata.DurationSeconds)
return preparedTimelineV3{seekSeconds: seekSeconds, streamOriginSeconds: result.Plan.Timeline.StreamOriginSeconds, startSegmentNumber: startSegment}, nil
default:
return preparedTimelineV3{}, nil
}
return h.prepareLocalTransportV3(r, session, file, result)
}
// planRequiresServerTransformationsV3 reports whether the plan carries any
@@ -846,7 +692,7 @@ func planRequiresServerTransformationsV3(plan *playback.PlanV3) bool {
return false
}
func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, timeline preparedTimelineV3) (preparedTransportV3, *transportErrorV3) {
func (h *PlaybackHandler) prepareIdentityTransportV3(session *playback.Session, result playback.PlannerResultV3) preparedTransportV3 {
routeSession := *session
routeSession.PlayMethod = result.PlayMethod
routeSession.BasePlayMethod = result.PlayMethod
@@ -857,33 +703,14 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p
routeSession.TargetAudioChannels = result.TargetAudioChannels
routeSession.TargetAudioBitrateKbps = result.TargetAudioBitrateKbps
routeSession.RemuxDVMode = remuxDVModeForPlanV3(result.Plan)
proxyNode, proxyErr := h.planIdentityProxyV3(r, session.ID, result)
if proxyErr != nil {
return preparedTransportV3{}, proxyErr
}
streamURL, servedByProxy := h.identityStreamURLV3(&routeSession, file, proxyNode)
releaseProxyReservation := func() {
if releaser, ok := h.NodePlanner.(sessionReservationReleaserV3); ok {
releaser.ReleaseSession(session.ID)
}
}
if proxyNode != nil && !servedByProxy {
// A planned proxy that could not be addressed (no signable token, no
// file record) falls back to the local path, so its reservation must be
// dropped now rather than pinning that node's budget until it ages out.
releaseProxyReservation()
if err := h.refuseLocalIdentityWorkV3(r, result); err != nil {
return preparedTransportV3{}, err
}
}
previousNodeURL := session.TranscodeNodeURL
previousTransportID := remoteTransportID(session)
unlock := h.tm.LockSessionLifecycle(session.ID)
committed := false
streamURL := h.playbackStreamURL(&routeSession)
if result.Plan != nil && result.Plan.Delivery == playback.DeliveryRemuxProgressiveV3 {
if seek := timeline.seekSeconds; seek > 0 {
configureProgressiveRemuxTimelineV3(result.Plan)
if seek := result.Plan.Timeline.StreamOriginSeconds; seek > 0 {
streamURL = appendPlaybackQueryV3(streamURL, "seek", strconv.FormatFloat(seek, 'f', -1, 64))
}
}
@@ -898,7 +725,6 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p
if previousNodeURL != "" {
h.tm.StopRemoteTranscode(previousTransportID, previousNodeURL)
}
h.applyRemoteTransportMarkV3(r.Context(), session.ID, servedByProxy)
unlock()
},
rollback: func() {
@@ -906,158 +732,9 @@ func (h *PlaybackHandler) prepareIdentityTransportV3(r *http.Request, session *p
return
}
committed = true
// The session never reached the client, so a proxy admitted for it
// must not keep consuming that node's job/bandwidth budget until the
// reservation ages out.
if servedByProxy {
releaseProxyReservation()
}
unlock()
},
}, nil
}
// planIdentityProxyV3 selects the proxy node that will serve a direct-play or
// progressive-remux session. These deliveries need no transcode node — the
// bytes are either the source file or a single remux pipe — so the planner is
// asked for a proxy alone, exactly as the Jellyfin-compat transport does.
//
// A nil node (no planner, no signing secret, or no eligible proxy) means the
// API server serves the stream itself, which is both the single-node case and
// the correct degradation when every proxy is unhealthy or at capacity. The one
// exception is a remux that must run ffmpeg: that is transcode work, so it
// honors the same local-fallback gate as the HLS routes rather than quietly
// spawning an encoder on an API-only node.
func (h *PlaybackHandler) planIdentityProxyV3(r *http.Request, sessionID string, result playback.PlannerResultV3) (*nodepool.Node, *transportErrorV3) {
if h.NodePlanner == nil || h.JWTSecret == "" {
return nil, h.refuseLocalIdentityWorkV3(r, result)
}
// Reserve against the session id the rest of the transport uses, so a
// re-plan replaces its own reservation instead of double-counting, and the
// rollback path can release it.
plan := h.planIdentityProxySessionV3(r.Context(), sessionID, result)
if plan.ProxyNode == nil {
return nil, h.refuseLocalIdentityWorkV3(r, result)
}
return plan.ProxyNode, nil
}
// applyRemoteTransportMarkV3 records whether the committed route's media bytes
// leave this server or another node.
//
// Every committed transition calls it, including the ones that serve locally:
// a session replanned from a proxy onto the integrated transcoder would
// otherwise keep a stale mark, and the widened idle grace it grants would hold
// that session's stream and transcode slots for five minutes after the local
// stream disconnected without an explicit stop.
func (h *PlaybackHandler) applyRemoteTransportMarkV3(ctx context.Context, sessionID string, remote bool) {
if err := h.sessionMgr.SetRemoteTransport(sessionID, remote); err != nil &&
!errors.Is(err, playback.ErrSessionNotFound) {
slog.WarnContext(ctx, "failed to record transport locality",
"component", "api", "playback_session_id", sessionID, "remote", remote, "error", err)
}
}
// planIdentityProxySessionV3 asks the planner for a proxy, narrowing selection
// to proxies that can execute the plan's frozen recipe when one is required.
//
// The narrowing happens before selection rather than after, mirroring how HLS
// filters transcode nodes: rejecting a single round-robin pick would abandon
// the whole pool, so a capable sibling with free capacity would sit unused
// while playback fell back to the API — or was refused outright when local
// fallback is disabled.
//
// Direct play copies bytes and needs no recipe, so it skips the probe entirely
// and any healthy proxy serves it.
func (h *PlaybackHandler) planIdentityProxySessionV3(ctx context.Context, sessionID string, result playback.PlannerResultV3) nodepool.Plan {
estBitrate := identityStreamBitrateKbpsV3(result)
selector, selectable := h.NodePlanner.(capabilitySessionPlannerV3)
enumerator, enumerable := h.NodePlanner.(proxyNodeEnumeratorV3)
if !selectable || !enumerable || !planRequiresServerTransformationsV3(result.Plan) {
return h.NodePlanner.PlanSession(sessionID, "", false, estBitrate)
}
capable := make(map[string]struct{})
for nodeURL, advertised := range h.pooledNodeTransformationsV3(ctx, enumerator.ProxyNodeURLs()) {
if validateAdvertisedTransformationsV3(result.Plan, advertised) == nil {
capable[nodeURL] = struct{}{}
}
}
if len(capable) == 0 {
slog.WarnContext(ctx, "protocol v3 no proxy advertises the planned recipe",
"component", "api", "delivery", result.Plan.Delivery)
}
// The predicate runs under the planner lock: a set lookup only.
return selector.PlanSessionWith(sessionID, "", false, estBitrate, func(node *nodepool.Node) bool {
if node == nil {
return false
}
_, ok := capable[node.URL]
return ok
})
}
// refuseLocalIdentityWorkV3 enforces playback.local_transcode_fallback for the
// identity deliveries. Direct play moves bytes and always falls back locally;
// a progressive remux that must convert audio is ffmpeg work, so on an API-only
// node it is refused for the same reason the HLS routes refuse it, instead of
// silently spawning an encoder the operator disabled.
func (h *PlaybackHandler) refuseLocalIdentityWorkV3(r *http.Request, result playback.PlannerResultV3) *transportErrorV3 {
if result.Plan == nil || result.Plan.Delivery != playback.DeliveryRemuxProgressiveV3 ||
!planRequiresServerTransformationsV3(result.Plan) ||
nodepool.LocalTranscodeFallbackAllowed(r.Context(), h.SettingsRepo) {
return nil
}
return &transportErrorV3{reason: "capacity_unavailable", message: "No proxy node is available and local fallback is disabled.", retryable: true}
}
// identityStreamBitrateKbpsV3 estimates the bitrate a proxy will egress for an
// identity delivery, so bandwidth-capped proxies admit it accurately. The plan's
// effective recipe is authoritative (a remux that downmixes audio egresses less
// than the source); the source descriptor is the fallback.
func identityStreamBitrateKbpsV3(result playback.PlannerResultV3) int {
if result.Plan == nil {
return 0
}
if effective := result.Plan.EffectiveRecipe.BitrateKbps; effective != nil && *effective > 0 {
return *effective
}
if source := result.Plan.Source.BitrateKbps; source > 0 {
return source
}
return 0
}
// identityStreamURLV3 builds the stream URL for a direct-play or
// progressive-remux session: an absolute proxy URL when one was planned,
// otherwise the API-local path.
//
// The proxy serves from the token alone, so the token has to carry everything
// the API-local path would have read from the session and the file record: the
// media path it opens, and the Dolby Vision profile its remux needs to strip a
// dangling Profile 7 RPU. Omitting either would not fail loudly — the proxy
// would serve a subtly different stream than the plan promised.
//
// The bool reports whether the returned URL is actually a proxy URL, so the
// caller can release the planner reservation when it is not.
func (h *PlaybackHandler) identityStreamURLV3(s *playback.Session, file *models.MediaFile, proxyNode *nodepool.Node) (string, bool) {
if proxyNode == nil || file == nil {
return h.playbackStreamURL(s), false
}
card := identityRecipeCard(s)
card.InputPath = file.FilePath
claims := card.ToClaims()
claims.DVProfile = file.PrimaryDVProfile()
claims.AudioOnly = file.IsAudioOnly()
token := h.signStreamClaims(claims)
if token == "" {
return h.playbackStreamURL(s), false
}
base := strings.TrimRight(proxyNode.URL, "/")
if s.PlayMethod == playback.PlayRemux {
return base + "/stream/remux/" + token, true
}
return base + "/stream/direct/" + token, true
}
// sessionOwnsResumeTimelineV3 reports whether the session's own position is a
@@ -1079,14 +756,14 @@ func sessionOwnsResumeTimelineV3(file *models.MediaFile) bool {
}
// preferredAudioTrackIndexV3 answers what an omitted audio track means: the
// language this profile has settled on for this series, this library, this
// device, or generally — the same resolution the catalog performs when it
// publishes `effective_audio_track_index`, so the track a client sees on the
// detail page is the track that plays when it does not ask for one.
// language this profile has settled on for this series, this library, or
// generally — the same resolution the catalog performs when it publishes
// `effective_audio_track_index`, so the track a client sees on the detail page
// is the track that plays when it does not ask for one.
//
// The client sends a track identity only when the viewer picked one. Defaulting
// to ordinal zero instead would silently play the first track on the reel.
func (h *PlaybackHandler) preferredAudioTrackIndexV3(ctx context.Context, userID int, profileID, deviceID string, file *models.MediaFile) (int, error) {
func (h *PlaybackHandler) preferredAudioTrackIndexV3(ctx context.Context, userID int, profileID string, file *models.MediaFile) (int, error) {
if file == nil || len(file.AudioTracks) == 0 || h.StoreProvider == nil {
return 0, nil
}
@@ -1095,9 +772,8 @@ func (h *PlaybackHandler) preferredAudioTrackIndexV3(ctx context.Context, userID
slog.ErrorContext(ctx, "protocol v3 start: audio preference store lookup failed", "component", "api", "user_id", userID, "error", err)
return 0, err
}
seriesID := h.resolveSeriesID(ctx, file)
var seriesPref *playback.AudioTrackPreference
if seriesID != "" {
if seriesID := h.resolveSeriesID(ctx, file); seriesID != "" {
stored, prefErr := store.GetAudioPreference(ctx, profileID, seriesID)
if prefErr != nil {
slog.ErrorContext(ctx, "protocol v3 start: series audio preference lookup failed", "component", "api", "profile_id", profileID, "series_id", seriesID, "error", prefErr)
@@ -1105,38 +781,36 @@ func (h *PlaybackHandler) preferredAudioTrackIndexV3(ctx context.Context, userID
}
if stored != nil {
seriesPref = &playback.AudioTrackPreference{AudioTrackIndex: stored.AudioTrackIndex, AudioLanguage: stored.AudioLanguage, TrackSignature: stored.TrackSignature}
}
}
rc := settingsresolve.Context{
ProfileID: profileID,
DeviceID: deviceID,
LibraryIDs: []int{file.MediaFolderID},
}
if seriesID != "" {
rc.SeriesIDs = []string{seriesID}
}
preferredLang, err := resolvedPlaybackAudioLanguage(ctx, store, rc)
if err != nil {
slog.ErrorContext(ctx, "protocol v3 start: canonical audio preference lookup failed", "component", "api", "profile_id", profileID, "device_id", deviceID, "error", err)
return 0, err
}
if preferredLang == playback.OriginalLanguageSentinel {
preferredLang = h.resolveOriginalLanguage(ctx, file)
if preferredLang == "" {
preferredLang, err = resolvedPlaybackAudioLanguage(ctx, store, settingsresolve.Context{ProfileID: profileID})
if err != nil {
slog.ErrorContext(ctx, "protocol v3 start: profile audio preference fallback failed", "component", "api", "profile_id", profileID, "error", err)
return 0, err
}
if preferredLang == playback.OriginalLanguageSentinel {
preferredLang = h.resolveOriginalLanguage(ctx, file)
if seriesPref.AudioLanguage == playback.OriginalLanguageSentinel {
seriesPref.AudioLanguage = h.resolveOriginalLanguage(ctx, file)
}
}
}
if seriesPref != nil {
// The specialized row supplies concrete track identity; canonical
// settings own the language and its scope precedence.
seriesPref.AudioLanguage = preferredLang
preferredLang := resolvedProfileAudioLanguage(ctx, store, profileID)
// A library override applies only where no series-sticky choice exists;
// having watched this series in a language outranks the library default.
libraryLang := ""
if seriesPref == nil {
stored, prefErr := store.GetLibraryPlaybackPreference(ctx, profileID, file.MediaFolderID)
if prefErr != nil {
slog.ErrorContext(ctx, "protocol v3 start: library audio preference lookup failed", "component", "api", "profile_id", profileID, "library_id", file.MediaFolderID, "error", prefErr)
return 0, prefErr
}
if stored != nil {
libraryLang = stored.AudioLanguage
}
}
if preferredLang == playback.OriginalLanguageSentinel || libraryLang == playback.OriginalLanguageSentinel {
originalLang := h.resolveOriginalLanguage(ctx, file)
if preferredLang == playback.OriginalLanguageSentinel {
preferredLang = originalLang
}
if libraryLang == playback.OriginalLanguageSentinel {
libraryLang = originalLang
}
}
if libraryLang != "" {
preferredLang = libraryLang
}
return normalizeAudioTrackIndex(file, playback.SelectAudioTrack(file.AudioTracks, preferredLang, seriesPref)), nil
}
@@ -1175,16 +849,16 @@ func (h *PlaybackHandler) resumePositionV3(ctx context.Context, userID int, prof
return &position, nil
}
// A copy remux starts at the preceding keyframe selected by the demuxer, not
// necessarily at the requested source position. Its player clock therefore
// begins at the resolved stream origin and advances through the copied pre-roll
// before reaching the requested position. Neither progressive nor growing HLS
// copy transports can seek arbitrarily inside their current response.
func configureCopyRemuxTimelineV3(plan *playback.PlanV3, origin float64) {
// A progressive remux is a freshly generated, chunked MP4 response and does
// not implement byte ranges. Its player clock therefore begins at zero at the
// requested source origin, and arbitrary seeks must request another server
// reanchor rather than issuing a Range request against the remux pipe.
func configureProgressiveRemuxTimelineV3(plan *playback.PlanV3) {
if plan == nil {
return
}
plan.Timeline.PlayerStartSeconds = max(0, plan.Timeline.SourceStartSeconds-origin)
origin := plan.Timeline.SourceStartSeconds
plan.Timeline.PlayerStartSeconds = 0
plan.Timeline.StreamOriginSeconds = origin
plan.Timeline.TimelineOffsetSeconds = origin
plan.Timeline.SeekWindowStartSeconds = &origin
@@ -1201,7 +875,7 @@ func appendPlaybackQueryV3(rawURL, key, value string) string {
return rawURL + separator + key + "=" + value
}
func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, timeline preparedTimelineV3) (preparedTransportV3, *transportErrorV3) {
func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3) (preparedTransportV3, *transportErrorV3) {
cfg := h.playbackConfig()
if err := os.MkdirAll(cfg.TranscodeDir, 0o755); err != nil {
return preparedTransportV3{}, &transportErrorV3{reason: "internal_error", message: "Failed to prepare the transcode directory.", cause: err}
@@ -1214,12 +888,13 @@ func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *play
}
sourceMetadata := sourceExecutionMetadataV3(file, result)
sourceProfile, sourceBitDepth := sourceVideoTranscodeFactsV3(file, result)
seekSeconds, startSegment := configureHLSTimelineV3(result.Plan, videoCodec, 2, sourceMetadata.DurationSeconds)
unlock := h.tm.LockSessionLifecycle(session.ID)
opts := playback.TranscodeOpts{InputPath: file.FilePath, OutputDir: outputDir, OutputSubdir: outputSubdir, SessionID: session.ID, SourceVideoCodec: sourceMetadata.VideoCodec, SourceVideoProfile: sourceProfile, SourceVideoBitDepth: sourceBitDepth, SoftwareVideoDecode: sourceMetadata.SoftwareVideoDecode, VideoBitstreamFilter: videoBitstreamFilterForPlanV3(result.Plan), SeekSeconds: timeline.seekSeconds, StreamOriginSeconds: timeline.streamOriginSeconds, CopySeekAnchorResolved: timeline.copySeekAnchorResolved, StartSegmentNumber: timeline.startSegmentNumber, TargetResolution: result.TargetResolution, TargetCodecVideo: videoCodec, TargetCodecAudio: result.TargetAudioCodec, TargetAudioChannels: result.TargetAudioChannels, TargetAudioBitrateKbps: result.TargetAudioBitrateKbps, TargetBitrateKbps: result.TargetBitrateKbps, SegmentDuration: 2, FFmpegPath: cfg.FFmpegPath, HWAccel: cfg.HWAccel, HWDevice: cfg.HWDevice, AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex), SubtitleTrackIndex: result.SubtitleTransportTrackIndex, SubtitleBurnIn: result.SubtitleBurnIn, SubtitleCodec: result.SubtitleCodec, TotalDuration: sourceMetadata.DurationSeconds, FastStart: true, NodeType: playbackNodeIntegratedV3, ExecutionMode: playbackNodeIntegratedV3, FFmpegLogSink: h.FFmpegLogSink}
opts := playback.TranscodeOpts{InputPath: file.FilePath, OutputDir: outputDir, OutputSubdir: outputSubdir, SessionID: session.ID, SourceVideoCodec: sourceMetadata.VideoCodec, SourceVideoProfile: sourceProfile, SourceVideoBitDepth: sourceBitDepth, SoftwareVideoDecode: sourceMetadata.SoftwareVideoDecode, VideoBitstreamFilter: videoBitstreamFilterForPlanV3(result.Plan), SeekSeconds: seekSeconds, StartSegmentNumber: startSegment, TargetResolution: result.TargetResolution, TargetCodecVideo: videoCodec, TargetCodecAudio: result.TargetAudioCodec, TargetAudioChannels: result.TargetAudioChannels, TargetAudioBitrateKbps: result.TargetAudioBitrateKbps, TargetBitrateKbps: result.TargetBitrateKbps, SegmentDuration: 2, SegmentRetentionSeconds: cfg.SegmentRetentionSeconds, FFmpegPath: cfg.FFmpegPath, HWAccel: cfg.HWAccel, HWDevice: cfg.HWDevice, AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex), SubtitleTrackIndex: result.SubtitleTransportTrackIndex, SubtitleBurnIn: result.SubtitleBurnIn, SubtitleCodec: result.SubtitleCodec, TotalDuration: sourceMetadata.DurationSeconds, FastStart: true, NodeType: playbackNodeIntegratedV3, ExecutionMode: playbackNodeIntegratedV3, FFmpegLogSink: h.FFmpegLogSink}
ts, err := h.startLocalPlaybackTransport(r.Context(), opts)
if err != nil {
unlock()
return preparedTransportV3{}, &transportErrorV3{reason: transcodeStartFailedReasonV3, message: "Failed to start the playback transport.", retryable: true, cause: err}
return preparedTransportV3{}, &transportErrorV3{reason: "transcode_start_failed", message: "Failed to start the playback transport.", retryable: true, cause: err}
}
if _, readyErr := ts.WaitForManifest(playback.ManifestStartupTimeout); readyErr != nil {
wasRunning := ts.IsRunning()
@@ -1246,7 +921,7 @@ func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *play
ts, err = h.startLocalPlaybackTransport(r.Context(), retryOpts)
if err != nil {
unlock()
return preparedTransportV3{}, &transportErrorV3{reason: transcodeStartFailedReasonV3, message: "Failed to start the playback transport.", retryable: true, cause: err}
return preparedTransportV3{}, &transportErrorV3{reason: "transcode_start_failed", message: "Failed to start the playback transport.", retryable: true, cause: err}
}
if _, retryReadyErr := ts.WaitForManifest(playback.ManifestStartupTimeout); retryReadyErr != nil {
transportErr = manifestStartupTransportErrorV3(ts.IsRunning(), retryReadyErr)
@@ -1268,7 +943,6 @@ func (h *PlaybackHandler) prepareLocalTransportV3(r *http.Request, session *play
}
committed = true
previous := h.tm.SwapTranscodeSession(session.ID, ts)
h.applyRemoteTransportMarkV3(r.Context(), session.ID, false)
unlock()
if previous != nil && previous != ts {
_ = previous.Close()
@@ -1299,10 +973,10 @@ func manifestStartupTransportErrorV3(running bool, cause error) *transportErrorV
if running {
message = "The playback transport did not become ready in time."
}
return &transportErrorV3{reason: transcodeStartFailedReasonV3, message: message, retryable: running, cause: cause}
return &transportErrorV3{reason: "transcode_start_failed", message: message, retryable: running, cause: cause}
}
func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, nodePlan nodepool.Plan, timeline preparedTimelineV3) (preparedTransportV3, *transportErrorV3) {
func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *playback.Session, file *models.MediaFile, result playback.PlannerResultV3, nodePlan nodepool.Plan) (preparedTransportV3, *transportErrorV3) {
node := nodePlan.TranscodeNode
transportID := transportGenerationV3(session.ID, result.Plan.PlanID)
videoCodec := result.TargetVideoCodec
@@ -1311,7 +985,8 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla
}
sourceMetadata := sourceExecutionMetadataV3(file, result)
sourceProfile, sourceBitDepth := sourceVideoTranscodeFactsV3(file, result)
req := transcodenode.TranscodeStartRequest{SessionID: transportID, InputPath: file.FilePath, SourceVideoCodec: sourceMetadata.VideoCodec, SourceVideoProfile: sourceProfile, SourceVideoBitDepth: sourceBitDepth, SoftwareVideoDecode: sourceMetadata.SoftwareVideoDecode, VideoBitstreamFilter: videoBitstreamFilterForPlanV3(result.Plan), SeekSeconds: timeline.seekSeconds, StreamOriginSeconds: timeline.streamOriginSeconds, CopySeekAnchorResolved: timeline.copySeekAnchorResolved, StartSegmentNumber: timeline.startSegmentNumber, TargetResolution: result.TargetResolution, TargetCodecVideo: videoCodec, TargetCodecAudio: result.TargetAudioCodec, TargetAudioChannels: result.TargetAudioChannels, TargetAudioBitrateKbps: result.TargetAudioBitrateKbps, TargetBitrateKbps: result.TargetBitrateKbps, SegmentDuration: 2, HWAccel: h.playbackConfig().HWAccel, AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex), SubtitleTrackIndex: result.SubtitleTransportTrackIndex, SubtitleBurnIn: result.SubtitleBurnIn, SubtitleCodec: result.SubtitleCodec, TotalDuration: sourceMetadata.DurationSeconds, RequireReady: true}
seekSeconds, startSegment := configureHLSTimelineV3(result.Plan, videoCodec, 2, sourceMetadata.DurationSeconds)
req := transcodenode.TranscodeStartRequest{SessionID: transportID, InputPath: file.FilePath, SourceVideoCodec: sourceMetadata.VideoCodec, SourceVideoProfile: sourceProfile, SourceVideoBitDepth: sourceBitDepth, SoftwareVideoDecode: sourceMetadata.SoftwareVideoDecode, VideoBitstreamFilter: videoBitstreamFilterForPlanV3(result.Plan), SeekSeconds: seekSeconds, StartSegmentNumber: startSegment, TargetResolution: result.TargetResolution, TargetCodecVideo: videoCodec, TargetCodecAudio: result.TargetAudioCodec, TargetAudioChannels: result.TargetAudioChannels, TargetAudioBitrateKbps: result.TargetAudioBitrateKbps, TargetBitrateKbps: result.TargetBitrateKbps, SegmentDuration: 2, HWAccel: h.playbackConfig().HWAccel, AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex), SubtitleTrackIndex: result.SubtitleTransportTrackIndex, SubtitleBurnIn: result.SubtitleBurnIn, SubtitleCodec: result.SubtitleCodec, TotalDuration: sourceMetadata.DurationSeconds, RequireReady: true}
nodeResp, status, err := h.startRemotePlaybackTransport(r.Context(), node.URL, req)
if err != nil {
// A timeout can fire after the node actually started the job; the
@@ -1322,15 +997,11 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla
}
if status != http.StatusAccepted {
h.tm.StopRemoteTranscode(transportID, node.URL)
return preparedTransportV3{}, &transportErrorV3{reason: transcodeStartFailedReasonV3, message: "The selected transcode node rejected the playback transport.", retryable: true}
return preparedTransportV3{}, &transportErrorV3{reason: "transcode_start_failed", message: "The selected transcode node rejected the playback transport.", retryable: true}
}
hw := firstNonEmptyHandlerV3(strings.TrimSpace(nodeResp.HWAccel), strings.TrimSpace(req.HWAccel))
card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, node.URL, playback.TranscodeOpts{InputPath: req.InputPath, SessionID: session.ID, TranscodeTransportID: transportID, SourceVideoCodec: req.SourceVideoCodec, SourceVideoProfile: req.SourceVideoProfile, SourceVideoBitDepth: req.SourceVideoBitDepth, SoftwareVideoDecode: req.SoftwareVideoDecode, VideoBitstreamFilter: req.VideoBitstreamFilter, SeekSeconds: req.SeekSeconds, StreamOriginSeconds: req.StreamOriginSeconds, CopySeekAnchorResolved: req.CopySeekAnchorResolved, StartSegmentNumber: req.StartSegmentNumber, TargetResolution: req.TargetResolution, TargetCodecVideo: req.TargetCodecVideo, TargetCodecAudio: req.TargetCodecAudio, TargetAudioChannels: req.TargetAudioChannels, TargetAudioBitrateKbps: req.TargetAudioBitrateKbps, TargetBitrateKbps: req.TargetBitrateKbps, SegmentDuration: req.SegmentDuration, HWAccel: hw, AudioTrackIndex: req.AudioTrackIndex, SubtitleTrackIndex: req.SubtitleTrackIndex, SubtitleBurnIn: req.SubtitleBurnIn, SubtitleCodec: req.SubtitleCodec, TotalDuration: req.TotalDuration})
card := playback.NewRecipeCard(session.UserID, session.ProfileID, file.ID, node.URL, playback.TranscodeOpts{InputPath: req.InputPath, SessionID: session.ID, TranscodeTransportID: transportID, SourceVideoCodec: req.SourceVideoCodec, SourceVideoProfile: req.SourceVideoProfile, SourceVideoBitDepth: req.SourceVideoBitDepth, SoftwareVideoDecode: req.SoftwareVideoDecode, VideoBitstreamFilter: req.VideoBitstreamFilter, SeekSeconds: req.SeekSeconds, StartSegmentNumber: req.StartSegmentNumber, TargetResolution: req.TargetResolution, TargetCodecVideo: req.TargetCodecVideo, TargetCodecAudio: req.TargetCodecAudio, TargetAudioChannels: req.TargetAudioChannels, TargetAudioBitrateKbps: req.TargetAudioBitrateKbps, TargetBitrateKbps: req.TargetBitrateKbps, SegmentDuration: req.SegmentDuration, HWAccel: hw, AudioTrackIndex: req.AudioTrackIndex, SubtitleTrackIndex: req.SubtitleTrackIndex, SubtitleBurnIn: req.SubtitleBurnIn, SubtitleCodec: req.SubtitleCodec, TotalDuration: req.TotalDuration})
url := h.buildProxyManifestURL(card, nodePlan.ProxyNode)
// buildProxyManifestURL only returns an absolute proxy URL when a proxy was
// planned and the token could be signed; otherwise the client fetches the
// manifest from this server and the local liveness path applies.
servedByProxy := nodePlan.ProxyNode != nil && strings.HasPrefix(url, "http")
committed := false
previousNodeURL := session.TranscodeNodeURL
previousTransportID := remoteTransportID(session)
@@ -1344,7 +1015,6 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla
if previousNodeURL != "" {
h.tm.StopRemoteTranscode(previousTransportID, previousNodeURL)
}
h.applyRemoteTransportMarkV3(r.Context(), session.ID, servedByProxy)
unlock()
}, rollback: func() {
if committed {
@@ -1766,11 +1436,10 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte
seekScopedRecovery := seekReanchor || seekFailureRecovery
trackChange := operation == playback.ReplanOperationTrackChangeV3
qualityChange := operation == playback.ReplanOperationQualityChangeV3
outputChange := operation == playback.ReplanOperationOutputChangeV3
// User-intent operations replace the legacy audio PATCH and client-recipe
// transcode start. Nothing failed, so their previous route stays eligible:
// neither attempted-key history nor the failed-plan exclusion applies.
userIntentOperation := trackChange || qualityChange || outputChange
userIntentOperation := trackChange || qualityChange
intentChange := false
if seekScopedRecovery {
if err := validateSeekRecoveryRequestV3(record, req); err != nil {
@@ -1805,8 +1474,6 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte
case qualityChange:
nextQuality, _ := playback.NormalizeQualityV3(req.QualityPreference)
intentChange = nextQuality != start.QualityPreference
case outputChange:
intentChange = true
default:
switch req.Failure.Classification {
case "quality_changed":
@@ -1886,7 +1553,6 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte
return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "source_unavailable", message: "The effective media source is unavailable."}
}
effectiveFile := currentEffectiveFile
currentEffectiveStart := start
if intentChange && !trackChange {
// Prefer returning to the requested edition, but a quality/output/track
// change must not abandon a healthy active alternate merely because the
@@ -1900,20 +1566,13 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte
// from an eng/ac3 commentary track to the identically-shaped main
// track on a quality change.
if currentEffectiveFile.ID != effectiveFile.ID {
candidateStart := start
remapErr := remapAudioSelectionV3(currentEffectiveFile, effectiveFile, &candidateStart)
if remapErr == nil && (candidateStart.SubtitleTrackIndex != nil || candidateStart.SubtitleTrackID != "") {
remapErr = h.remapSubtitleSelectionV3(r.Context(), currentEffectiveFile, effectiveFile, &candidateStart)
if err := remapAudioSelectionV3(currentEffectiveFile, effectiveFile, &start); err != nil {
return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "track_unavailable", message: err.Error()}
}
if remapErr != nil && outputChange {
// An output refresh may make the requested edition viable again,
// but it must not retire a healthy active alternate merely because
// the viewer selected a track unique to that alternate.
effectiveFile = currentEffectiveFile
} else if remapErr != nil {
return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "track_unavailable", message: remapErr.Error()}
} else {
start = candidateStart
if start.SubtitleTrackIndex != nil || start.SubtitleTrackID != "" {
if err := h.remapSubtitleSelectionV3(r.Context(), currentEffectiveFile, effectiveFile, &start); err != nil {
return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "track_unavailable", message: err.Error()}
}
}
}
}
@@ -1988,21 +1647,7 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte
} else {
result = playback.PlanPlaybackV3(playback.PlannerInputV3{Request: start, RequestedFile: plannerRequestedFile, EffectiveFile: effectiveFile, AudioTrackIndex: audioIndex, Settings: h.plannerSettingsV3(r.Context()), Registry: h.transformationRegistryV3(r.Context()), HLSRegistry: h.lazyHLSPlanningRegistryV3(r.Context()), DVRPUStrippable: h.lazyDVRPUStrippableV3(r.Context(), effectiveFile), Now: time.Now(), AttemptedKeys: attemptedKeys, AdditionalSubtitles: h.downloadedSubtitleInventoryV3(r.Context(), effectiveFile)})
}
if outputChange && result.Terminal != nil && effectiveFile.ID != currentEffectiveFile.ID {
// Returning to the requested edition is speculative during an output
// refresh. Any terminal from that probe must fall back to the edition
// already playing, not only HDR/alternate-selection terminals: its audio,
// subtitle, or delivery constraints may still differ from the active file.
start = currentEffectiveStart
start.FileID = currentEffectiveFile.ID
effectiveFile = currentEffectiveFile
audioIndex, err = resolveV3AudioIndex(effectiveFile, start.AudioTrackID, start.AudioTrackIndex)
if err != nil {
return playback.DecisionResponseV3{}, *record, nil, &transportErrorV3{reason: "track_unavailable", message: err.Error()}
}
result = playback.PlanPlaybackV3(playback.PlannerInputV3{Request: start, RequestedFile: plannerRequestedFile, EffectiveFile: effectiveFile, AudioTrackIndex: audioIndex, Settings: h.plannerSettingsV3(r.Context()), Registry: h.transformationRegistryV3(r.Context()), HLSRegistry: h.lazyHLSPlanningRegistryV3(r.Context()), DVRPUStrippable: h.lazyDVRPUStrippableV3(r.Context(), effectiveFile), Now: time.Now(), AttemptedKeys: attemptedKeys, AdditionalSubtitles: h.downloadedSubtitleInventoryV3(r.Context(), effectiveFile)})
}
if terminalAllowsAlternateFileV3(result.Terminal) && replanAllowsAlternateFileV3(operation, start.QualityPreference) {
if result.Terminal != nil && result.Terminal.Reason == "no_alternate_version" && replanAllowsAlternateFileV3(operation, start.QualityPreference) {
if alternate, alternateErr := h.findAlternateFile(r.Context(), requestedFile); alternateErr == nil && alternate != nil {
alternate = h.ensurePlaybackProbe(r.Context(), alternate)
remappedAudio := remapAudioIndexV3(effectiveFile, alternate, audioIndex)
@@ -2159,7 +1804,7 @@ func (h *PlaybackHandler) executeReplanV3(r *http.Request, record *playback.Atte
event = playback.RouteEventRuntimeCorrectionSucceededV3
clientModel = start.ClientPlaybackContext.Device.Model
}
h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: playback.RouteEventV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, PlanID: result.Plan.PlanID, PlanAttemptID: req.PlanAttemptID, PlanAttemptKey: playback.PlanAttemptKeyV3(*result.Plan, start.ClientPlaybackContext.Output.OutputContextID, nil), Event: event, FallbackReason: req.Failure.Classification, AppliedQuirkIDs: appliedQuirkIDsV3(result.Plan), QuirkRegistryRevision: appliedQuirkRevisionV3(result.Plan), OutputContextID: start.ClientPlaybackContext.Output.OutputContextID}, UserID: session.UserID, ProfileID: session.ProfileID, ClientName: session.ClientName, ClientVersion: session.ClientVersion, ClientBuild: session.ClientBuild, ClientChannel: session.ClientChannel, ClientModel: clientModel})
h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: playback.RouteEventV3{ProtocolVersion: playback.ProtocolV3, PlaybackAttemptID: req.PlaybackAttemptID, SessionID: session.ID, PlanID: result.Plan.PlanID, PlanAttemptID: req.PlanAttemptID, PlanAttemptKey: playback.PlanAttemptKeyV3(*result.Plan, start.ClientPlaybackContext.Output.OutputContextID, nil), Event: event, FallbackReason: req.Failure.Classification, AppliedQuirkIDs: appliedQuirkIDsV3(result.Plan), QuirkRegistryRevision: appliedQuirkRevisionV3(result.Plan), OutputContextID: start.ClientPlaybackContext.Output.OutputContextID}, UserID: session.UserID, ProfileID: session.ProfileID, ClientName: session.ClientName, ClientVersion: session.ClientVersion, ClientModel: clientModel})
}
transport.rollback = func() {
originalRollback()
@@ -2604,43 +2249,12 @@ func shouldTryAlternateFileV3(qualityPreference string) bool {
return !strings.EqualFold(strings.TrimSpace(qualityPreference), "original")
}
const (
terminalNoAlternateVersionV3 = "no_alternate_version"
terminalHDRTranscodeUnsupportedV3 = "hdr_transcode_unsupported"
terminalSubtitleConversionUnsupportedV3 = "subtitle_conversion_unsupported"
)
// terminalAllowsAlternateFileV3 reports whether a refusal is the kind another
// version of the same item could satisfy.
//
// subtitle_conversion_unsupported belongs here because it is not only a
// subtitle-format refusal: when a burn-in requirement is the sole trigger of an
// adaptation the source cannot take, the planner reports the blocker in terms
// of the subtitle rather than the HDR pipeline or the 4K policy. Those cases
// used to surface as hdr_transcode_unsupported / no_alternate_version and were
// the exact reason this gate exists — a bitmap subtitle needing burn-in that an
// HDR source cannot support while an SDR alternate can. Leaving the new reason
// out silently retired that fallback and refused playback outright.
func terminalAllowsAlternateFileV3(terminal *playback.TerminalV3) bool {
if terminal == nil {
return false
}
switch terminal.Reason {
case terminalNoAlternateVersionV3, terminalHDRTranscodeUnsupportedV3, terminalSubtitleConversionUnsupportedV3:
return true
default:
return false
}
}
func replanAllowsAlternateFileV3(operation playback.ReplanOperationV3, qualityPreference string) bool {
switch operation {
case playback.ReplanOperationFailureRecoveryV3, playback.ReplanOperationQualityChangeV3, playback.ReplanOperationOutputChangeV3, playback.ReplanOperationTrackChangeV3:
// Quality, output, and track changes can make another version the only
// viable route. In particular, a bitmap subtitle can require video burn-in
// that an HDR source cannot support while an SDR alternate can. The
// subtitle identity is remapped before the alternate is adopted; seek-only
// operations remain pinned to the mounted source.
case playback.ReplanOperationFailureRecoveryV3, playback.ReplanOperationQualityChangeV3:
// A quality change carries the same "another version may fit better"
// semantics its legacy quality_changed failure classification had; seek
// operations and track changes stay pinned to the mounted source.
return shouldTryAlternateFileV3(qualityPreference)
default:
return false
@@ -2651,11 +2265,11 @@ func replanAlternateFilePinnedByOriginalQualityV3(operation playback.ReplanOpera
if shouldTryAlternateFileV3(qualityPreference) {
return false
}
return operation == playback.ReplanOperationFailureRecoveryV3 || operation == playback.ReplanOperationQualityChangeV3 || operation == playback.ReplanOperationOutputChangeV3
return operation == playback.ReplanOperationFailureRecoveryV3 || operation == playback.ReplanOperationQualityChangeV3
}
func (h *PlaybackHandler) clarifyOriginalQuality4KTerminalV3(ctx context.Context, terminal *playback.TerminalV3, requestedFile *models.MediaFile, alternateFilePinned bool) {
if !alternateFilePinned || terminal == nil || terminal.Reason != terminalNoAlternateVersionV3 || terminal.Message != playback.TerminalMessage4KTranscodeDisabledV3 {
if !alternateFilePinned || terminal == nil || terminal.Reason != "no_alternate_version" || terminal.Message != playback.TerminalMessage4KTranscodeDisabledV3 {
return
}
if alternate, err := h.findAlternateFile(ctx, requestedFile); err == nil && alternate != nil {
@@ -2770,8 +2384,8 @@ func (h *PlaybackHandler) HandlePlaybackRouteEventV3(w http.ResponseWriter, r *h
return
}
event.Diagnostics = sanitizeDiagnosticsV3(event.Diagnostics)
client := h.playbackClientInfoWithSessionFallbackV3(firstNonEmptyValue(event.SessionID, identity.SessionID), playbackClientInfoFromRequest(r))
h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: event, UserID: userID, ProfileID: profileID, ClientName: client.Name, ClientVersion: client.Version, ClientBuild: client.Build, ClientChannel: client.Channel, ClientModel: event.Diagnostics["device_model"]})
client := playbackClientInfoFromRequest(r)
h.enqueueRouteEventV3(playback.RouteEventRecordV3{RouteEventV3: event, UserID: userID, ProfileID: profileID, ClientName: client.Name, ClientVersion: client.Version, ClientModel: event.Diagnostics["device_model"]})
w.WriteHeader(http.StatusAccepted)
}
@@ -3007,7 +2621,7 @@ func sessionStartErrorV3(err error) *transportErrorV3 {
}
}
func (h *PlaybackHandler) persistTerminalStartDecisionV3(ctx context.Context, userID int, profileID string, req playback.StartRequestV3, requestDigests playbackStartRequestDigestsV3, requestedFileID, effectiveFileID int, response playback.DecisionResponseV3) (playback.DecisionResponseV3, error) {
func (h *PlaybackHandler) persistTerminalStartDecisionV3(ctx context.Context, userID int, profileID string, req playback.StartRequestV3, requestDigest string, requestedFileID, effectiveFileID int, response playback.DecisionResponseV3) (playback.DecisionResponseV3, error) {
record := playback.AttemptRecordV3{
PlaybackAttemptID: req.PlaybackAttemptID,
UserID: userID,
@@ -3016,12 +2630,12 @@ func (h *PlaybackHandler) persistTerminalStartDecisionV3(ctx context.Context, us
EffectiveMediaFileID: effectiveFileID,
NormalizedRequest: req,
StartResponse: response,
RequestDigest: requestDigests.current,
RequestDigest: requestDigest,
ExpiresAt: time.Now().Add(playback.MaxTokenTTL),
}
if err := h.PlanStoreV3.SaveAttempt(ctx, record); err == nil {
return response, nil
} else if !errors.Is(err, playback.ErrPlaybackAttemptExistsV3) && !errors.Is(err, playback.ErrIdempotencyKeyReusedV3) {
} else if !errors.Is(err, playback.ErrPlaybackAttemptExistsV3) {
return playback.DecisionResponseV3{}, err
}
@@ -3030,15 +2644,15 @@ func (h *PlaybackHandler) persistTerminalStartDecisionV3(ctx context.Context, us
return playback.DecisionResponseV3{}, err
}
if existing.UserID != userID || existing.ProfileID != profileID ||
existing.RequestedMediaFileID != requestedFileID || !requestDigests.matches(existing.RequestDigest) {
existing.RequestedMediaFileID != requestedFileID || existing.RequestDigest != requestDigest {
return playback.DecisionResponseV3{}, playback.ErrIdempotencyKeyReusedV3
}
return decisionResponseFromAttemptV3(existing), nil
}
func (h *PlaybackHandler) startFailureDecisionV3(ctx context.Context, userID int, profileID string, req playback.StartRequestV3, requestDigests playbackStartRequestDigestsV3, requestedFileID, effectiveFileID int, failure *transportErrorV3) (playback.DecisionResponseV3, error) {
func (h *PlaybackHandler) startFailureDecisionV3(ctx context.Context, userID int, profileID string, req playback.StartRequestV3, requestDigest string, requestedFileID, effectiveFileID int, failure *transportErrorV3) (playback.DecisionResponseV3, error) {
response := playback.NewTerminalResponseV3(failure.reason, failure.message, failure.retryable)
return h.persistTerminalStartDecisionV3(ctx, userID, profileID, req, requestDigests, requestedFileID, effectiveFileID, response)
return h.persistTerminalStartDecisionV3(ctx, userID, profileID, req, requestDigest, requestedFileID, effectiveFileID, response)
}
func writeStartAttemptPersistenceErrorV3(w http.ResponseWriter, err error) {
@@ -3270,12 +2884,12 @@ func configureHLSTimelineV3(plan *playback.PlanV3, videoCodec string, segmentDur
seek := alignedSeekSeconds(requested, segmentDuration, videoCodec)
startSegment := computeStartSegment(seek, segmentDuration)
plan.Timeline.SourceStartSeconds = requested
usesGrowingManifest := !playback.CanGenerateSyntheticManifest(durationSeconds, segmentDuration)
usesGrowingManifest := strings.EqualFold(videoCodec, "copy") ||
!playback.CanGenerateSyntheticManifest(durationSeconds, segmentDuration)
if usesGrowingManifest {
// Encoded streams seek to the preceding segment boundary. Preserve the
// requested sub-segment offset so playback still begins at the exact
// requested source position. Copy remuxes are configured separately with
// their probed keyframe origin.
// requested source position. Copy seeks are already exact, making this 0.
plan.Timeline.PlayerStartSeconds = max(0, requested-seek)
plan.Timeline.StreamOriginSeconds = seek
plan.Timeline.TimelineOffsetSeconds = seek
File diff suppressed because it is too large Load Diff
+12 -15
View File
@@ -23,19 +23,18 @@ import (
// SectionHandler handles section management and batch section endpoints.
type SectionHandler struct {
repo *sections.Repository
fetcher *sections.Fetcher
previewFetcher sectionPreviewFetcher // set to fetcher at construction; separate for test injection
episodeFetcher sectionEpisodeFetcher
FolderRepo *catalog.FolderRepository
EpisodeRepo *catalog.EpisodeRepository
StoreProvider userstore.UserStoreProvider
UserRepo *auth.UserRepository
DetailSvc *catalog.DetailService
Settings catalog.SettingsStore
CollectionRepo *catalog.LibraryCollectionRepository
SortPreferenceCleaner *userstore.CollectionSortPreferenceCleaner
EbookProgress EbookReaderProgressLister
repo *sections.Repository
fetcher *sections.Fetcher
previewFetcher sectionPreviewFetcher // set to fetcher at construction; separate for test injection
episodeFetcher sectionEpisodeFetcher
FolderRepo *catalog.FolderRepository
EpisodeRepo *catalog.EpisodeRepository
StoreProvider userstore.UserStoreProvider
UserRepo *auth.UserRepository
DetailSvc *catalog.DetailService
Settings catalog.SettingsStore
CollectionRepo *catalog.LibraryCollectionRepository
EbookProgress EbookReaderProgressLister
}
// NewSectionHandler creates a new SectionHandler.
@@ -406,8 +405,6 @@ func (h *SectionHandler) deleteUnreferencedSectionManagedCollection(ctx context.
}
if err := h.CollectionRepo.Delete(ctx, collectionID); err != nil && !errors.Is(err, catalog.ErrLibraryCollectionNotFound) {
slog.WarnContext(ctx, "failed to delete unreferenced section-managed collection", "component", "api", "collection_id", collectionID, "error", err)
} else if err == nil && h.SortPreferenceCleaner != nil {
h.SortPreferenceCleaner.DeleteForCollection(ctx, userstore.CollectionKindLibrary, collectionID)
}
}
-56
View File
@@ -98,62 +98,6 @@ func (h *SubtitleSearchHandler) authorizeMediaFile(w http.ResponseWriter, r *htt
return authorizeMediaFileAccess(w, r, h.FileAuthorizer, fileID)
}
// subtitleProviderStatusResponse tells a client whether this deployment can
// search external subtitle providers at all, following the per-subsystem
// capability convention (/subtitles/ai/status, /items/trailers/capability).
//
// Without it a search on a server with no providers configured answers exactly
// like a search that ran and matched nothing, so a player has no way to tell
// "this server cannot do that" from "nothing found for this file" and ends up
// offering an entry point that can never succeed. A client that finds enabled
// false should disable the search action and say why rather than let the user
// run a query that is guaranteed to come back empty.
//
// Provider names are safe to return to any authenticated viewer: they already
// travel in every SubtitleResult.provider and DownloadedSubtitle.provider. The
// credentials behind them stay in the admin-only provider config.
type subtitleProviderStatusResponse struct {
SchemaVersion int `json:"schema_version"`
// Enabled reports that at least one provider is registered, so
// POST /subtitles/search can actually reach an upstream.
Enabled bool `json:"enabled"`
// Providers is the registered provider identifiers, sorted. Always a
// list — empty rather than null — so clients can iterate it unguarded.
Providers []string `json:"providers"`
}
// HandleProviderStatus reports whether external subtitle search is available
// here, so the player can show or hide the entry point.
// GET /api/v1/subtitles/providers/status
//
// It answers even when the subtitle subsystem is unwired, because enabled:false
// is the answer in that case; the router registers a fallback so a client never
// has to interpret a 404 on the probe itself.
func (h *SubtitleSearchHandler) HandleProviderStatus(w http.ResponseWriter, _ *http.Request) {
providers := []string{}
if h != nil && h.manager != nil {
providers = h.manager.ProviderNames()
}
writeJSON(w, http.StatusOK, subtitleProviderStatusResponse{
SchemaVersion: 1,
Enabled: len(providers) > 0,
Providers: providers,
})
}
// WriteSubtitleProvidersDisabledStatus answers the subtitle provider capability
// probe with a 200 {"enabled": false, "providers": []} when no subtitle handler
// is wired, so the client gets a clean negative instead of a 404 (the 2-segment
// /providers/status path is not shadowed by the 1-segment /{media_file_id}
// route — they never compete in chi's router).
func WriteSubtitleProvidersDisabledStatus(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, subtitleProviderStatusResponse{
SchemaVersion: 1,
Enabled: false,
Providers: []string{},
})
}
// HandleSearch handles POST /api/v1/subtitles/search
func (h *SubtitleSearchHandler) HandleSearch(w http.ResponseWriter, r *http.Request) {
var req searchSubtitlesRequest
@@ -235,109 +235,6 @@ func TestHandleDeleteRequiresAccessToMediaFile(t *testing.T) {
}
}
// stubSubtitleProvider is a registerable no-op provider; only its name matters
// to the capability probe.
type stubSubtitleProvider struct {
name string
}
func (s stubSubtitleProvider) Name() string { return s.name }
func (s stubSubtitleProvider) Search(context.Context, subtitles.SearchRequest) ([]subtitles.SubtitleResult, error) {
return nil, nil
}
func (s stubSubtitleProvider) Download(context.Context, string) ([]byte, subtitles.SubtitleFormat, error) {
return nil, subtitles.FormatSRT, nil
}
func decodeProviderStatus(t *testing.T, rr *httptest.ResponseRecorder) struct {
SchemaVersion int `json:"schema_version"`
Enabled bool `json:"enabled"`
Providers []string `json:"providers"`
} {
t.Helper()
var resp struct {
SchemaVersion int `json:"schema_version"`
Enabled bool `json:"enabled"`
Providers []string `json:"providers"`
}
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body = %s", rr.Code, rr.Body.String())
}
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
t.Fatalf("decode response: %v", err)
}
if resp.SchemaVersion != 1 {
t.Fatalf("schema_version = %d, want 1", resp.SchemaVersion)
}
return resp
}
func TestHandleProviderStatusWithoutProviders(t *testing.T) {
repo := newMockSubtitleRepoForHandler()
manager := subtitles.NewManager(repo, newMockS3ClientForHandler(), "test-bucket")
handler := NewSubtitleSearchHandler(manager, repo, stubSubtitleMediaResolver{})
rr := httptest.NewRecorder()
handler.HandleProviderStatus(rr, newSubtitleAuthRequest(http.MethodGet, "/subtitles/providers/status", nil))
resp := decodeProviderStatus(t, rr)
if resp.Enabled {
t.Fatal("enabled = true, want false with no providers registered")
}
if len(resp.Providers) != 0 {
t.Fatalf("providers = %v, want empty", resp.Providers)
}
// The empty list must reach the wire as [] — a null would read to a
// client as a missing field rather than "no providers here".
if !bytes.Contains(rr.Body.Bytes(), []byte(`"providers":[]`)) {
t.Fatalf("body = %s, want providers serialized as []", rr.Body.String())
}
}
func TestHandleProviderStatusWithProviders(t *testing.T) {
repo := newMockSubtitleRepoForHandler()
manager := subtitles.NewManager(repo, newMockS3ClientForHandler(), "test-bucket")
manager.RegisterProvider(stubSubtitleProvider{name: "subdl"})
manager.RegisterProvider(stubSubtitleProvider{name: "opensubtitles"})
handler := NewSubtitleSearchHandler(manager, repo, stubSubtitleMediaResolver{})
rr := httptest.NewRecorder()
handler.HandleProviderStatus(rr, newSubtitleAuthRequest(http.MethodGet, "/subtitles/providers/status", nil))
resp := decodeProviderStatus(t, rr)
if !resp.Enabled {
t.Fatal("enabled = false, want true with providers registered")
}
want := []string{"opensubtitles", "subdl"}
if len(resp.Providers) != len(want) {
t.Fatalf("providers = %v, want %v", resp.Providers, want)
}
for i, name := range want {
if resp.Providers[i] != name {
t.Fatalf("providers = %v, want %v (sorted)", resp.Providers, want)
}
}
}
func TestWriteSubtitleProvidersDisabledStatus(t *testing.T) {
rr := httptest.NewRecorder()
WriteSubtitleProvidersDisabledStatus(rr, newSubtitleAuthRequest(http.MethodGet, "/subtitles/providers/status", nil))
resp := decodeProviderStatus(t, rr)
if resp.Enabled {
t.Fatal("enabled = true, want false from the disabled fallback")
}
if len(resp.Providers) != 0 {
t.Fatalf("providers = %v, want empty", resp.Providers)
}
if !bytes.Contains(rr.Body.Bytes(), []byte(`"providers":[]`)) {
t.Fatalf("body = %s, want providers serialized as []", rr.Body.String())
}
}
type handlerMockSubtitleRepo struct {
subtitles map[int]*subtitles.DownloadedSubtitle
nextID int
@@ -75,7 +75,6 @@ type userImportSharedFields struct {
PosterURL string `json:"poster_url"`
LibraryIDs []int `json:"library_ids,omitempty"`
DisplayQueryDefinition json.RawMessage `json:"display_query_definition,omitempty"`
SortConfig json.RawMessage `json:"sort_config,omitempty"`
}
type userImportMDBListRequest struct {
@@ -218,11 +217,6 @@ func (h *UserCollectionImportHandler) createImportedCollection(
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
}
sortConfig, err := NormalizeCollectionSortConfig(shared.SortConfig, true)
if err != nil {
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
return
}
collection, err := store.CreateCollection(r.Context(), userstore.CreateCollectionInput{
CreatorProfileID: profileID,
@@ -231,7 +225,7 @@ func (h *UserCollectionImportHandler) createImportedCollection(
CollectionType: collectionType,
IsShared: shared.IsShared,
QueryDefinition: "{}",
SortConfig: sortConfig,
SortConfig: "{}",
SourceURL: cfg.DisplayURL(),
SourceConfig: sourceConfigJSON,
SyncSchedule: schedule,
-40
View File
@@ -1438,14 +1438,6 @@ func NewRouter(deps Dependencies) chi.Router {
}
libraryCollectionRepo := catalog.NewLibraryCollectionRepository(deps.DB)
var collectionSortCleaner *userstore.CollectionSortPreferenceCleaner
if userRepo != nil && deps.UserStoreProvider != nil {
collectionSortCleaner = userstore.NewCollectionSortPreferenceCleaner(userRepo, deps.UserStoreProvider)
}
if collectionHandler != nil {
collectionHandler.LibraryCollections = libraryCollectionRepo
}
sectionHandler.SortPreferenceCleaner = collectionSortCleaner
if libraryCollectionService == nil {
libraryCollectionService = catalog.NewLibraryCollectionService(
libraryCollectionRepo,
@@ -1545,7 +1537,6 @@ func NewRouter(deps Dependencies) chi.Router {
libraryCollectionHandler.SectionRepo = sectionRepo
libraryCollectionHandler.UserCollectionPool = deps.DB
libraryCollectionHandler.EventsHub = deps.EventsHub
libraryCollectionHandler.SortPreferenceCleaner = collectionSortCleaner
if deps.FolderRepo != nil {
libraryCollectionHandler.FolderRepo = deps.FolderRepo
} else {
@@ -1680,15 +1671,6 @@ func NewRouter(deps Dependencies) chi.Router {
// background worker.
downloadSvc.SetSubscriptions(downloads.NewSubscriptionRepository(deps.DB))
downloadHandler = handlers.NewDownloadHandler(downloadSvc)
if deps.NodePlanner != nil {
downloadHandler.SetProxyDelivery(deps.NodePlanner, func() string {
cfg := deps.CurrentConfig()
if cfg == nil {
return ""
}
return cfg.Auth.JWTSecret
})
}
if profileHandler != nil {
// Profiles may live outside Postgres (sqlite userdb backend), so
// deleting one cannot FK-cascade the shared user_devices table;
@@ -2300,8 +2282,6 @@ func NewRouter(deps Dependencies) chi.Router {
r.Use(apimw.RequireProfile)
r.Get("/", collectionHandler.HandleListCollections)
r.Get("/capabilities", collectionHandler.HandleCapabilities)
r.Put("/sort-preference", collectionHandler.HandleSetCollectionSortPreference)
r.Delete("/sort-preference", collectionHandler.HandleClearCollectionSortPreference)
if libraryCollectionHandler != nil {
// Aggregated server (admin-curated) collections across
// every accessible library. Separate from "/" (personal,
@@ -2586,11 +2566,6 @@ func NewRouter(deps Dependencies) chi.Router {
}
}
r.Route("/subtitles", func(r chi.Router) {
// Capability probe for external subtitle search. Two
// segments on purpose: a bare /providers would shadow
// the /{media_file_id} route below, while
// /providers/status never competes with it in chi.
r.Get("/providers/status", subtitleSearchHandler.HandleProviderStatus)
r.Post("/search", subtitleSearchHandler.HandleSearch)
r.Post("/download", subtitleSearchHandler.HandleDownload)
r.Post("/upload", subtitleSearchHandler.HandleUpload)
@@ -2611,17 +2586,6 @@ func NewRouter(deps Dependencies) chi.Router {
r.Get("/{media_file_id}", subtitleSearchHandler.HandleList)
r.Delete("/{id}", subtitleSearchHandler.HandleDelete)
})
} else {
// The whole group above is conditional (it needs the DB,
// S3 and the subtitle repo), so on a storage-less
// deployment the capability probe would 404 — leaving a
// client to interpret the same ambiguous status the probe
// exists to replace. Mount the probe alone, answering
// enabled:false, so feature detection always gets a real
// answer.
r.Route("/subtitles", func(r chi.Router) {
r.Get("/providers/status", handlers.WriteSubtitleProvidersDisabledStatus)
})
}
// Playback routes.
@@ -2708,16 +2672,12 @@ func NewRouter(deps Dependencies) chi.Router {
// HEAD natively.
r.Get("/{id}/file", downloadHandler.HandleDownloadFile)
r.Head("/{id}/file", downloadHandler.HandleDownloadFile)
r.Get("/{id}/file-proxy", downloadHandler.HandleDownloadFileViaProxy)
r.Head("/{id}/file-proxy", downloadHandler.HandleDownloadFileViaProxy)
r.Get("/{id}/manifest", downloadHandler.HandleManifest)
r.Get("/{id}/artwork/{kind}", downloadHandler.HandleArtwork)
r.Get("/{id}/subtitles/{ref}", downloadHandler.HandleSubtitle)
})
r.Get("/direct-download", downloadHandler.HandleDirectDownload)
r.Head("/direct-download", downloadHandler.HandleDirectDownload)
r.Get("/direct-download-proxy", downloadHandler.HandleDirectDownloadViaProxy)
r.Head("/direct-download-proxy", downloadHandler.HandleDirectDownloadViaProxy)
// Recipe gallery catalog (no profile required — purely static metadata).
recipeHandler := &handlers.RecipeHandler{}
-28
View File
@@ -8,8 +8,6 @@ import (
"github.com/Silo-Server/silo-server/internal/models"
)
const LibraryCollectionVisibilityVisible = "visible"
// AccessFilter captures effective viewer access constraints for catalog reads.
type AccessFilter struct {
AllowedLibraryIDs []int
@@ -32,9 +30,6 @@ type AccessFilter struct {
SelectedFileID int
UserID int
ProfileID string
// DeviceID identifies the requesting client for device-scoped setting
// resolution. It does not participate in catalog access control.
DeviceID string
// NamePrefix, when non-empty, restricts results to items whose
// LOWER(COALESCE(NULLIF(BTRIM(sort_title),''), title)) starts with the
// given (case-insensitive) prefix. Pushed into the SQL WHERE clause so
@@ -47,29 +42,6 @@ type AccessFilter struct {
ExcludedMediaTypes []string
}
// CanAccessLibraryCollection reports whether a visible server collection is
// reachable through at least one library in the viewer's effective scope.
// Collections without explicit library scope retain their legacy unrestricted
// visibility, but restricted viewers cannot address them by ID.
func CanAccessLibraryCollection(collection *models.LibraryCollection, filter AccessFilter) bool {
if collection == nil || collection.Visibility != LibraryCollectionVisibilityVisible {
return false
}
if len(collection.LibraryIDs) == 0 {
return filter.AllowedLibraryIDs == nil && len(filter.DisabledLibraryIDs) == 0
}
for _, libraryID := range collection.LibraryIDs {
if filter.AllowedLibraryIDs != nil && !intInSlice(libraryID, filter.AllowedLibraryIDs) {
continue
}
if intInSlice(libraryID, filter.DisabledLibraryIDs) {
continue
}
return true
}
return false
}
func applyAccessFilter(alias string, filter AccessFilter, conditions *[]string, args *[]any, argIdx *int) {
if filter.MaxContentRating != "" {
allowedRatings := access.AllowedRatingsUpTo(filter.MaxContentRating)
+5 -74
View File
@@ -37,12 +37,6 @@ type CatalogResult struct {
SemanticUsed bool
FallbackReason string
IndexPendingEvents int
// EffectiveSort is the order collection sources actually resolved in, after
// applying the request's own sort, the viewer's saved override, and the
// collection's configured default in that order. An empty Field means the
// collection's source order. Only collection sources populate it; clients
// use it to show which sort is active when the request carried none.
EffectiveSort QuerySort
}
type CatalogFiltersResult struct {
@@ -493,25 +487,10 @@ func (r *CatalogResolver) resolveSectionBrowseSource(ctx context.Context, req Ca
func (r *CatalogResolver) resolveLibraryCollectionSource(ctx context.Context, req CatalogRequest, access AccessFilter) (*CatalogResult, error) {
collectionRepo := NewLibraryCollectionRepository(r.itemRepo.pool)
collection, err := collectionRepo.GetByID(ctx, req.CollectionID)
if err != nil || !CanAccessLibraryCollection(collection, access) {
if err != nil || collection.Visibility != "visible" {
return nil, ErrCatalogSourceNotFound
}
return r.resolveCollectionWithEffectiveSort(
ctx, req, access, userstore.CollectionKindLibrary, collection.ID, collection.SortConfig,
func(req CatalogRequest) (*CatalogResult, error) {
return r.resolveLibraryCollectionItems(ctx, req, access, collection, collectionRepo)
},
)
}
func (r *CatalogResolver) resolveLibraryCollectionItems(
ctx context.Context,
req CatalogRequest,
access AccessFilter,
collection *models.LibraryCollection,
collectionRepo *LibraryCollectionRepository,
) (*CatalogResult, error) {
if IsLiveQueryType(collection.CollectionType) {
return r.resolveLiveLibraryCollectionSource(ctx, req, access, collection)
}
@@ -565,55 +544,10 @@ func (r *CatalogResolver) resolveUserCollectionSource(ctx context.Context, req C
}
collection, err := store.GetCollection(ctx, req.CollectionID)
if err != nil || !ProfileCanAccessCollection(collection, access.ProfileID) {
if err != nil || !catalogProfileCanAccessCollection(collection, access.ProfileID) {
return nil, ErrCatalogSourceNotFound
}
return r.resolveCollectionWithEffectiveSort(
ctx, req, access, userstore.CollectionKindUser, collection.ID, []byte(collection.SortConfig),
func(req CatalogRequest) (*CatalogResult, error) {
return r.resolveUserCollectionItems(ctx, req, access, store, collection)
},
)
}
// resolveCollectionWithEffectiveSort applies the shared collection-sort
// precedence, delegates item resolution, and stamps the resolved order on the
// response. Keeping this sequence in one place prevents the library and
// personal collection paths from drifting.
func (r *CatalogResolver) resolveCollectionWithEffectiveSort(
ctx context.Context,
req CatalogRequest,
access AccessFilter,
kind, collectionID string,
sortConfig []byte,
resolve func(CatalogRequest) (*CatalogResult, error),
) (*CatalogResult, error) {
// A sort in the request is the viewer's live choice and always wins; only
// when there is none do the saved override and the collection's configured
// default come into play.
if req.UseSourceOrder {
if qs, ok := r.EffectiveCollectionSort(ctx, access, kind, collectionID, sortConfig); ok {
req.Query.Sort = qs
req.UseSourceOrder = false
}
}
result, err := resolve(req)
if err != nil {
return nil, err
}
result.EffectiveSort = req.Query.Sort
return result, nil
}
func (r *CatalogResolver) resolveUserCollectionItems(
ctx context.Context,
req CatalogRequest,
access AccessFilter,
store userstore.UserStore,
collection *userstore.Collection,
) (*CatalogResult, error) {
if IsLiveQueryType(collection.CollectionType) {
def, err := parseCatalogCollectionQueryDefinition([]byte(collection.QueryDefinition))
if err != nil {
@@ -1752,10 +1686,7 @@ func (r *CatalogResolver) catalogStoreForAccess(ctx context.Context, access Acce
return store, nil
}
// ProfileCanAccessCollection reports whether profileID may view a personal
// collection. Creator access is represented in AllowedProfileIDs at write time,
// so this is the canonical check for both browse and preference endpoints.
func ProfileCanAccessCollection(collection *userstore.Collection, profileID string) bool {
func catalogProfileCanAccessCollection(collection *userstore.Collection, profileID string) bool {
if collection == nil || strings.TrimSpace(profileID) == "" {
return false
}
@@ -1875,7 +1806,7 @@ func (r *CatalogResolver) loadCollectionSourceBaseItems(ctx context.Context, req
case CatalogSourceLibraryCollection:
collectionRepo := NewLibraryCollectionRepository(r.itemRepo.pool)
collection, err := collectionRepo.GetByID(ctx, req.CollectionID)
if err != nil || collection.Visibility != LibraryCollectionVisibilityVisible {
if err != nil || collection.Visibility != "visible" {
return nil, ErrCatalogSourceNotFound
}
if IsLiveQueryType(collection.CollectionType) || catalogCollectionUsesLiveQuery(collection.QueryDefinition) {
@@ -1906,7 +1837,7 @@ func (r *CatalogResolver) loadCollectionSourceBaseItems(ctx context.Context, req
return nil, err
}
collection, err := store.GetCollection(ctx, req.CollectionID)
if err != nil || !ProfileCanAccessCollection(collection, access.ProfileID) {
if err != nil || !catalogProfileCanAccessCollection(collection, access.ProfileID) {
return nil, ErrCatalogSourceNotFound
}
var items []*models.MediaItem
-185
View File
@@ -1,185 +0,0 @@
package catalog
import (
"context"
"encoding/json"
"fmt"
"log/slog"
"strings"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/userstore"
)
// QueryCollectionItemsBySort loads collection members through the catalog
// query executor, applying an optional personal-collection display filter and
// the resolved sort before the database limit. This keeps rail queries bounded
// without changing the full-match total returned to callers.
func QueryCollectionItemsBySort(
ctx context.Context,
pool *pgxpool.Pool,
contentIDs []string,
qs QuerySort,
access AccessFilter,
limit int,
displayQueryDefinition string,
) ([]*models.MediaItem, int, error) {
if len(contentIDs) == 0 {
return []*models.MediaItem{}, 0, nil
}
def := QueryDefinition{Sort: qs}
if trimmed := strings.TrimSpace(displayQueryDefinition); trimmed != "" {
if err := json.Unmarshal([]byte(trimmed), &def); err != nil {
return nil, 0, fmt.Errorf("parsing collection display_query_definition: %w", err)
}
// Display fragments are filter-only. Never let stale or malformed stored
// execution fields expand scope, replace the selected sort, or set a
// different limit.
def.MediaScope = ""
def.LibraryIDs = nil
def.Sort = qs
def.Limit = nil
}
queryAccess := access
if access.AllowedContentIDs != nil {
queryAccess.AllowedContentIDs = intersectContentIDs(contentIDs, access.AllowedContentIDs)
} else {
queryAccess.AllowedContentIDs = contentIDs
}
if limit <= 0 || limit > len(contentIDs) {
limit = len(contentIDs)
}
executor := &QueryExecutor{Pool: pool}
items, total, err := executor.Preview(ctx, def.Normalize(), queryAccess, limit)
if err != nil {
return nil, 0, fmt.Errorf("querying collection items by %q: %w", qs.Field, err)
}
return items, total, nil
}
// EffectiveCollectionSort resolves the order a collection's items are shown in
// when the request itself carries no sort. Precedence, highest first:
//
// 1. the viewing profile's saved override — set by changing the sort while
// browsing the collection, and sticky from then on;
// 2. the default sort the collection's creator configured (sort_config);
// 3. the collection's own source order, signaled by ok == false.
//
// An override row with an empty sort field is a real choice ("keep source
// order") and short-circuits the creator's default, which is why it returns
// false rather than falling through.
//
// Creator defaults for library collections stay user-agnostic because they are
// shared. A viewer override is profile-scoped, however, so personalized fields
// (progress, date viewed, plays) are valid for either collection kind.
func (r *CatalogResolver) EffectiveCollectionSort(
ctx context.Context,
access AccessFilter,
kind string,
collectionID string,
sortConfig []byte,
) (QuerySort, bool) {
if pref := r.collectionSortOverride(ctx, access, kind, collectionID); pref != nil {
if strings.TrimSpace(pref.SortField) == "" {
return QuerySort{}, false
}
if qs, ok := NormalizeCollectionSort(pref.SortField, pref.SortOrder, true); ok {
return qs, true
}
// A stored override that no longer validates (a sort field retired from
// the vocabulary) falls back to the creator's default rather than
// failing the browse.
}
return ParseCollectionDefaultSort(sortConfig, kind == userstore.CollectionKindUser)
}
// OrderCollectionItemsBySort reorders already-loaded collection members by a
// resolved sort, returning them in that order.
//
// Ordering runs through the catalog query executor constrained to the member
// IDs, so a rail orders items exactly as the collection's browse page does —
// same NULLS LAST handling, same sort_title collation — rather than through a
// second, drifting in-memory comparator. An empty sort field returns items
// untouched (source order). Items the executor does not return (no longer
// visible to this access scope) are dropped, matching the browse path.
func OrderCollectionItemsBySort(
ctx context.Context,
pool *pgxpool.Pool,
items []*models.MediaItem,
qs QuerySort,
access AccessFilter,
) ([]*models.MediaItem, error) {
if strings.TrimSpace(qs.Field) == "" || len(items) == 0 {
return items, nil
}
ids := make([]string, 0, len(items))
for _, item := range items {
if item != nil && strings.TrimSpace(item.ContentID) != "" {
ids = append(ids, item.ContentID)
}
}
if len(ids) == 0 {
return items, nil
}
ordered, _, err := QueryCollectionItemsBySort(ctx, pool, ids, qs, access, len(ids), "")
if err != nil {
return nil, fmt.Errorf("ordering collection items by %q: %w", qs.Field, err)
}
// Preview returns freshly loaded rows; re-project onto the caller's items so
// any enrichment already applied to them survives the reorder.
byID := make(map[string]*models.MediaItem, len(items))
for _, item := range items {
if item != nil {
byID[item.ContentID] = item
}
}
result := make([]*models.MediaItem, 0, len(ordered))
for _, item := range ordered {
if item == nil {
continue
}
if original, ok := byID[item.ContentID]; ok {
result = append(result, original)
}
}
return result, nil
}
// collectionSortOverride loads the viewing profile's saved sort for a
// collection, or nil when there is none. A lookup failure is logged and treated
// as "no override": a browse must not fail because a preference row could not
// be read.
func (r *CatalogResolver) collectionSortOverride(
ctx context.Context,
access AccessFilter,
kind string,
collectionID string,
) *userstore.CollectionSortPreference {
if r.storeProvider == nil || access.UserID <= 0 || strings.TrimSpace(access.ProfileID) == "" {
return nil
}
store, err := r.storeProvider.ForUser(ctx, access.UserID)
if err != nil || store == nil {
if err != nil {
slog.WarnContext(ctx, "loading user store for collection sort preference",
"collection_kind", kind, "collection_id", collectionID, "error", err)
}
return nil
}
pref, err := store.GetCollectionSortPreference(ctx, access.ProfileID, kind, collectionID)
if err != nil {
slog.WarnContext(ctx, "loading collection sort preference",
"collection_kind", kind, "collection_id", collectionID, "error", err)
return nil
}
return pref
}
-289
View File
@@ -1,289 +0,0 @@
package catalog
import (
"context"
"os"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/userstore"
"github.com/Silo-Server/silo-server/internal/userstore/pgstore"
)
// These tests run the default-sort / saved-override flow against a real
// Postgres: real SQL ordering through the query executor, and a real
// PostgresUserStore writing the override row. They cover the reported
// behavior end to end — a collection's creator picks a default order, a viewer
// re-sorts, and the viewer's choice survives a fresh read.
//
// Set SILO_TEST_DATABASE_URL to a migrated database to run them.
func collectionSortTestPool(t *testing.T) *pgxpool.Pool {
t.Helper()
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("SILO_TEST_DATABASE_URL is not set")
}
ctx := context.Background()
pool, err := pgxpool.New(ctx, dsn)
if err != nil {
t.Fatalf("connect test database: %v", err)
}
t.Cleanup(pool.Close)
var tableName *string
if err := pool.QueryRow(ctx, `SELECT to_regclass('public.user_collection_sort_preferences')::text`).Scan(&tableName); err != nil {
t.Fatalf("check user_collection_sort_preferences table: %v", err)
}
if tableName == nil || *tableName == "" {
t.Skip("test database has not applied the collection sort preference migration")
}
return pool
}
// seedSortableItem inserts one sortable item. Callers choose title/year pairs
// that make a title sort and a year sort disagree, so an order assertion cannot
// pass by coincidence.
func seedSortableItem(t *testing.T, pool *pgxpool.Pool, contentID, title string, year int) {
t.Helper()
ctx := context.Background()
if _, err := pool.Exec(ctx, `
INSERT INTO media_items (content_id, type, title, sort_title, year, genres)
VALUES ($1, 'movie', $2, $2, $3, '{}'::text[])
`, contentID, title, year); err != nil {
t.Fatalf("seed media item %s: %v", contentID, err)
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), `DELETE FROM media_items WHERE content_id = $1`, contentID)
})
}
func seedSortTestUser(t *testing.T, pool *pgxpool.Pool) int {
t.Helper()
ctx := context.Background()
var userID int
// A bare fixture row: the override table has an FK to users(id) and the
// test needs something to point at. No credentials are set.
if err := pool.QueryRow(ctx, `
INSERT INTO users (username, email, password_hash, role)
VALUES ('collection-sort-fixture', 'collection-sort-fixture@invalid.test', '', 'user')
RETURNING id
`).Scan(&userID); err != nil {
t.Fatalf("seed fixture user: %v", err)
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), `DELETE FROM users WHERE id = $1`, userID)
})
return userID
}
func itemsForSort(contentIDs ...string) []*models.MediaItem {
items := make([]*models.MediaItem, 0, len(contentIDs))
for _, id := range contentIDs {
items = append(items, &models.MediaItem{ContentID: id})
}
return items
}
func orderedIDs(items []*models.MediaItem) []string {
ids := make([]string, 0, len(items))
for _, item := range items {
ids = append(ids, item.ContentID)
}
return ids
}
func assertOrder(t *testing.T, got []string, want ...string) {
t.Helper()
if len(got) != len(want) {
t.Fatalf("order = %v, want %v", got, want)
}
for i := range want {
if got[i] != want[i] {
t.Fatalf("order = %v, want %v", got, want)
}
}
}
// TestCollectionDefaultSortAndOverrideEndToEnd walks the whole reported
// scenario against real Postgres, for both collection kinds.
func TestCollectionDefaultSortAndOverrideEndToEnd(t *testing.T) {
pool := collectionSortTestPool(t)
ctx := context.Background()
// Years are shuffled against titles on purpose: title-asc gives A,B,C but
// year-desc gives B,C,A. If the two agreed, an assertion on the override's
// order would also pass when the override was ignored entirely.
seedSortableItem(t, pool, "cs-a", "Alpha", 2001)
seedSortableItem(t, pool, "cs-b", "Bravo", 2003)
seedSortableItem(t, pool, "cs-c", "Charlie", 2002)
// Source order is deliberately neither sorted order.
sourceOrder := itemsForSort("cs-c", "cs-a", "cs-b")
limited, total, err := QueryCollectionItemsBySort(
ctx,
pool,
[]string{"cs-c", "cs-a", "cs-b"},
QuerySort{Field: "title", Order: "asc"},
AccessFilter{},
2,
"",
)
if err != nil {
t.Fatalf("querying limited sorted collection: %v", err)
}
assertOrder(t, orderedIDs(limited), "cs-a", "cs-b")
if total != 3 {
t.Fatalf("limited sorted collection total = %d, want 3", total)
}
userID := seedSortTestUser(t, pool)
provider := pgstore.NewPostgresProvider(pool)
store, err := provider.ForUser(ctx, userID)
if err != nil {
t.Fatalf("user store: %v", err)
}
if err := store.CreateProfile(ctx, userstore.Profile{ID: "profile-e2e", Name: "E2E sort profile"}); err != nil {
t.Fatalf("seed profile: %v", err)
}
t.Cleanup(func() { _ = store.DeleteProfile(context.Background(), "profile-e2e") })
resolver := &CatalogResolver{storeProvider: provider}
access := AccessFilter{UserID: userID, ProfileID: "profile-e2e"}
for _, kind := range []string{userstore.CollectionKindLibrary, userstore.CollectionKindUser} {
t.Run(kind, func(t *testing.T) {
collectionID := "cs-collection-" + kind
t.Cleanup(func() {
_ = store.ClearCollectionSortPreference(ctx, access.ProfileID, kind, collectionID)
})
// 1. No default, no override — the collection keeps source order.
if _, ok := resolver.EffectiveCollectionSort(ctx, access, kind, collectionID, []byte(`{}`)); ok {
t.Fatal("a collection with no default reported a sort")
}
// 2. The creator sets a default of Title (A→Z).
defaultSort := []byte(`{"field":"title","order":"asc"}`)
qs, ok := resolver.EffectiveCollectionSort(ctx, access, kind, collectionID, defaultSort)
if !ok {
t.Fatal("configured default sort was not applied")
}
ordered, err := OrderCollectionItemsBySort(ctx, pool, sourceOrder, qs, access)
if err != nil {
t.Fatalf("ordering by default sort: %v", err)
}
assertOrder(t, orderedIDs(ordered), "cs-a", "cs-b", "cs-c")
// 3. The viewer re-sorts to Year (newest first) and it is saved.
if err := store.SetCollectionSortPreference(ctx, userstore.CollectionSortPreference{
ProfileID: access.ProfileID,
CollectionKind: kind,
CollectionID: collectionID,
SortField: "year",
SortOrder: "desc",
}); err != nil {
t.Fatalf("saving override: %v", err)
}
// 4. A fresh resolution — the "leave and come back" case — reads the
// override back out of the database and beats the creator's default.
freshResolver := &CatalogResolver{storeProvider: pgstore.NewPostgresProvider(pool)}
qs, ok = freshResolver.EffectiveCollectionSort(ctx, access, kind, collectionID, defaultSort)
if !ok {
t.Fatal("saved override was not applied on a fresh read")
}
if qs.Field != "year" || qs.Order != "desc" {
t.Fatalf("effective sort = %q/%q, want year/desc", qs.Field, qs.Order)
}
ordered, err = OrderCollectionItemsBySort(ctx, pool, sourceOrder, qs, access)
if err != nil {
t.Fatalf("ordering by override: %v", err)
}
// Distinct from both the default's A,B,C and source order's C,A,B.
assertOrder(t, orderedIDs(ordered), "cs-b", "cs-c", "cs-a")
assertOrder(t, orderedIDs(sourceOrder), "cs-c", "cs-a", "cs-b")
// 5. The viewer pins back to the collection's own order. That is a
// real choice and must suppress the creator's default, not fall
// through to it.
if err := store.SetCollectionSortPreference(ctx, userstore.CollectionSortPreference{
ProfileID: access.ProfileID,
CollectionKind: kind,
CollectionID: collectionID,
SortField: "",
SortOrder: "",
}); err != nil {
t.Fatalf("pinning to source order: %v", err)
}
if _, ok := resolver.EffectiveCollectionSort(ctx, access, kind, collectionID, defaultSort); ok {
t.Fatal("source-order override fell through to the creator's default")
}
// 6. Clearing the override returns the viewer to the default.
if err := store.ClearCollectionSortPreference(ctx, access.ProfileID, kind, collectionID); err != nil {
t.Fatalf("clearing override: %v", err)
}
qs, ok = resolver.EffectiveCollectionSort(ctx, access, kind, collectionID, defaultSort)
if !ok || qs.Field != "title" {
t.Fatalf("after clearing, effective sort = %q (ok=%v), want title", qs.Field, ok)
}
})
}
}
// TestCollectionSortOverrideIsPerProfile guards the isolation that makes a
// server-wide collection safe to re-sort: one profile's choice must not change
// what another profile sees.
func TestCollectionSortOverrideIsPerProfile(t *testing.T) {
pool := collectionSortTestPool(t)
ctx := context.Background()
userID := seedSortTestUser(t, pool)
provider := pgstore.NewPostgresProvider(pool)
store, err := provider.ForUser(ctx, userID)
if err != nil {
t.Fatalf("user store: %v", err)
}
for _, profile := range []userstore.Profile{
{ID: "profile-one", Name: "Profile One"},
{ID: "profile-two", Name: "Profile Two"},
} {
if err := store.CreateProfile(ctx, profile); err != nil {
t.Fatalf("seed profile %s: %v", profile.ID, err)
}
}
t.Cleanup(func() {
_ = store.DeleteProfile(context.Background(), "profile-two")
_ = store.DeleteProfile(context.Background(), "profile-one")
})
resolver := &CatalogResolver{storeProvider: provider}
const collectionID = "cs-shared-collection"
defaultSort := []byte(`{"field":"title","order":"asc"}`)
kind := userstore.CollectionKindLibrary
if err := store.SetCollectionSortPreference(ctx, userstore.CollectionSortPreference{
ProfileID: "profile-one",
CollectionKind: kind,
CollectionID: collectionID,
SortField: "year",
SortOrder: "desc",
}); err != nil {
t.Fatalf("saving override: %v", err)
}
t.Cleanup(func() {
_ = store.ClearCollectionSortPreference(context.Background(), "profile-one", kind, collectionID)
})
one, ok := resolver.EffectiveCollectionSort(ctx, AccessFilter{UserID: userID, ProfileID: "profile-one"}, kind, collectionID, defaultSort)
if !ok || one.Field != "year" {
t.Fatalf("profile-one sort = %q (ok=%v), want year", one.Field, ok)
}
two, ok := resolver.EffectiveCollectionSort(ctx, AccessFilter{UserID: userID, ProfileID: "profile-two"}, kind, collectionID, defaultSort)
if !ok || two.Field != "title" {
t.Fatalf("profile-two sort = %q (ok=%v), want the collection default title", two.Field, ok)
}
}
@@ -1,189 +0,0 @@
package catalog
import (
"context"
"errors"
"testing"
"github.com/Silo-Server/silo-server/internal/userstore"
)
// sortPrefStore is a UserStore that only answers collection-sort-preference
// lookups; every other method panics so a test that strays off this path fails
// loudly rather than silently exercising unrelated behavior.
type sortPrefStore struct {
userstore.UserStore
pref *userstore.CollectionSortPreference
err error
// calls records the lookups made, so tests can assert the store is not
// consulted when there is no user scope to consult it with.
calls int
}
func (s *sortPrefStore) GetCollectionSortPreference(_ context.Context, _, _, _ string) (*userstore.CollectionSortPreference, error) {
s.calls++
if s.err != nil {
return nil, s.err
}
return s.pref, nil
}
type sortPrefProvider struct {
store *sortPrefStore
err error
}
func (p *sortPrefProvider) ForUser(context.Context, int) (userstore.UserStore, error) {
if p.err != nil {
return nil, p.err
}
return p.store, nil
}
func (p *sortPrefProvider) Close() error { return nil }
func TestEffectiveCollectionSortPrecedence(t *testing.T) {
const (
libraryDefault = `{"field":"added_at","order":"desc"}`
noDefault = `{}`
)
scopedAccess := AccessFilter{UserID: 7, ProfileID: "profile-1"}
cases := []struct {
name string
access AccessFilter
sortConfig string
pref *userstore.CollectionSortPreference
storeErr error
wantOK bool
wantField string
wantOrder string
wantCalls int
}{
{
name: "no override falls back to the collection default",
access: scopedAccess,
sortConfig: libraryDefault,
wantOK: true,
wantField: "added_at",
wantOrder: "desc",
wantCalls: 1,
},
{
// The reported behavior: admin defaults to Date Added, the viewer
// switches to Title, and Title is what they get from then on.
name: "viewer override beats the collection default",
access: scopedAccess,
sortConfig: libraryDefault,
pref: &userstore.CollectionSortPreference{SortField: "title", SortOrder: "asc"},
wantOK: true,
wantField: "title",
wantOrder: "asc",
wantCalls: 1,
},
{
// An empty stored field is a deliberate "put it back to the list's
// own order" and must not fall through to the creator's default.
name: "override pinned to source order suppresses the default",
access: scopedAccess,
sortConfig: libraryDefault,
pref: &userstore.CollectionSortPreference{SortField: "", SortOrder: ""},
wantCalls: 1,
},
{
name: "no override and no default means source order",
access: scopedAccess,
sortConfig: noDefault,
wantCalls: 1,
},
{
name: "personalized viewer override is valid on a library collection",
access: scopedAccess,
sortConfig: libraryDefault,
pref: &userstore.CollectionSortPreference{SortField: "progress", SortOrder: "desc"},
wantOK: true,
wantField: "progress",
wantOrder: "desc",
wantCalls: 1,
},
{
name: "unusable override falls back to the default",
access: scopedAccess,
sortConfig: libraryDefault,
pref: &userstore.CollectionSortPreference{SortField: "retired_sort", SortOrder: "desc"},
wantOK: true,
wantField: "added_at",
wantOrder: "desc",
wantCalls: 1,
},
{
name: "store failure degrades to the default",
access: scopedAccess,
sortConfig: libraryDefault,
storeErr: errors.New("boom"),
wantOK: true,
wantField: "added_at",
wantOrder: "desc",
wantCalls: 1,
},
{
// Unauthenticated / profile-less reads (jellycompat, public paths)
// have no override to look up and must not query for one.
name: "no user scope skips the override lookup",
access: AccessFilter{},
sortConfig: libraryDefault,
pref: &userstore.CollectionSortPreference{SortField: "title", SortOrder: "asc"},
wantOK: true,
wantField: "added_at",
wantOrder: "desc",
wantCalls: 0,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
store := &sortPrefStore{pref: tc.pref, err: tc.storeErr}
resolver := &CatalogResolver{storeProvider: &sortPrefProvider{store: store}}
qs, ok := resolver.EffectiveCollectionSort(
context.Background(),
tc.access,
userstore.CollectionKindLibrary,
"collection-1",
[]byte(tc.sortConfig),
)
if ok != tc.wantOK {
t.Fatalf("ok = %v, want %v", ok, tc.wantOK)
}
if store.calls != tc.wantCalls {
t.Fatalf("override lookups = %d, want %d", store.calls, tc.wantCalls)
}
if !tc.wantOK {
return
}
if qs.Field != tc.wantField || qs.Order != tc.wantOrder {
t.Fatalf("got %q/%q, want %q/%q", qs.Field, qs.Order, tc.wantField, tc.wantOrder)
}
})
}
}
func TestEffectiveCollectionSortAllowsPersonalizedOnUserCollections(t *testing.T) {
store := &sortPrefStore{pref: &userstore.CollectionSortPreference{SortField: "progress", SortOrder: "desc"}}
resolver := &CatalogResolver{storeProvider: &sortPrefProvider{store: store}}
qs, ok := resolver.EffectiveCollectionSort(
context.Background(),
AccessFilter{UserID: 7, ProfileID: "profile-1"},
userstore.CollectionKindUser,
"collection-1",
[]byte(`{}`),
)
if !ok {
t.Fatal("personalized override rejected on a personal collection")
}
if qs.Field != "progress" || qs.Order != "desc" {
t.Fatalf("got %q/%q, want progress/desc", qs.Field, qs.Order)
}
}
-105
View File
@@ -1,105 +0,0 @@
package catalog
import "testing"
func TestNormalizeCollectionSort(t *testing.T) {
cases := []struct {
name string
field string
order string
allowPersonalized bool
wantOK bool
wantField string
wantOrder string
}{
{name: "applies the field's default order", field: "release_date", wantOK: true, wantField: "release_date", wantOrder: "desc"},
{name: "title defaults ascending", field: "title", wantOK: true, wantField: "title", wantOrder: "asc"},
{name: "explicit order wins", field: "title", order: "desc", wantOK: true, wantField: "title", wantOrder: "desc"},
{name: "trims and lowercases", field: " Title ", order: "ASC", wantOK: true, wantField: "title", wantOrder: "asc"},
{name: "garbage order is rejected", field: "year", order: "sideways"},
{name: "empty field means source order", field: ""},
{name: "unknown field is rejected", field: "not_a_sort"},
{name: "personalized rejected for library collections", field: "progress"},
{
name: "personalized allowed for personal collections",
field: "progress",
allowPersonalized: true,
wantOK: true,
wantField: "progress",
wantOrder: "desc",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
qs, ok := NormalizeCollectionSort(tc.field, tc.order, tc.allowPersonalized)
if ok != tc.wantOK {
t.Fatalf("ok = %v, want %v", ok, tc.wantOK)
}
if !tc.wantOK {
return
}
if qs.Field != tc.wantField || qs.Order != tc.wantOrder {
t.Fatalf("got %q/%q, want %q/%q", qs.Field, qs.Order, tc.wantField, tc.wantOrder)
}
})
}
}
func TestParseCollectionDefaultSort(t *testing.T) {
cases := []struct {
name string
raw string
wantOK bool
wantField string
wantOrder string
}{
{name: "empty column", raw: ""},
{name: "legacy empty object", raw: `{}`},
{name: "json null", raw: `null`},
{name: "unparseable json", raw: `{`},
// The unimplemented manual-pin mode carries no field, so it must not be
// mistaken for a configured sort.
{name: "manual pins mode", raw: `{"mode":"manual_pins"}`},
{name: "configured sort", raw: `{"field":"title","order":"asc"}`, wantOK: true, wantField: "title", wantOrder: "asc"},
{name: "order filled in", raw: `{"field":"added_at"}`, wantOK: true, wantField: "added_at", wantOrder: "desc"},
{name: "invalid order is rejected", raw: `{"field":"title","order":"sideways"}`},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
qs, ok := ParseCollectionDefaultSort([]byte(tc.raw), false)
if ok != tc.wantOK {
t.Fatalf("ok = %v, want %v", ok, tc.wantOK)
}
if !tc.wantOK {
return
}
if qs.Field != tc.wantField || qs.Order != tc.wantOrder {
t.Fatalf("got %q/%q, want %q/%q", qs.Field, qs.Order, tc.wantField, tc.wantOrder)
}
})
}
}
func TestEncodeCollectionDefaultSort(t *testing.T) {
encoded, err := EncodeCollectionDefaultSort(QuerySort{Field: "title", Order: "asc"}, true)
if err != nil {
t.Fatalf("encoding: %v", err)
}
if encoded != `{"field":"title","order":"asc"}` {
t.Fatalf("encoded = %s", encoded)
}
// Source order round-trips through the empty object the column defaults to.
encoded, err = EncodeCollectionDefaultSort(QuerySort{}, false)
if err != nil {
t.Fatalf("encoding source order: %v", err)
}
if encoded != "{}" {
t.Fatalf("source order encoded = %s, want {}", encoded)
}
if _, ok := ParseCollectionDefaultSort([]byte(encoded), false); ok {
t.Fatal("re-parsing the source-order encoding reported a configured sort")
}
}
+3 -6
View File
@@ -3105,7 +3105,6 @@ type audioPrefResolver struct {
store userstore.UserStore
valid bool
profileID string
deviceID string
contentID string
profileDone bool
@@ -3137,7 +3136,6 @@ func (s *DetailService) newAudioPrefResolver(ctx context.Context, filter AccessF
r := &audioPrefResolver{
svc: s,
profileID: filter.ProfileID,
deviceID: filter.DeviceID,
contentID: audioPreferenceContentID,
resolvedLang: map[int]string{},
}
@@ -3178,16 +3176,15 @@ func (r *audioPrefResolver) audioPreference(ctx context.Context) *playback.Audio
// audioLanguage resolves with every content identity in context. The language
// preference lives in the canonical table at profile_series/profile_library/
// profile_device/profile scopes; the specialized audio row supplies only
// concrete track identity. Caching by library keeps a multi-file item at one
// canonical read per distinct folder rather than one read per file.
// profile scopes; the specialized audio row supplies only concrete track
// identity. Caching by library keeps a multi-file item at one canonical read
// per distinct folder rather than one read per file.
func (r *audioPrefResolver) audioLanguage(ctx context.Context, libraryID int) string {
if lang, ok := r.resolvedLang[libraryID]; ok {
return lang
}
rc := settingsresolve.Context{
ProfileID: r.profileID,
DeviceID: r.deviceID,
LibraryIDs: []int{libraryID},
}
if strings.TrimSpace(r.contentID) != "" {
@@ -204,42 +204,6 @@ func TestEffectiveAudioTrackIndex_PrefersSeriesAudioPreferenceOverLibraryAndProf
}
}
func TestEffectiveAudioTrackIndex_ResolvesDeviceLanguageWithCanonicalPrecedence(t *testing.T) {
store := newDetailTestStore(t)
setProfileAudioLanguage(t, store, "en")
setScopedAudioLanguageForDevice(t, store, settingscontract.ScopeProfileDevice, "", 0, "apple-tv", "ja")
setScopedAudioLanguage(t, store, settingscontract.ScopeProfileLibrary, "", 12, "es")
setScopedAudioLanguage(t, store, settingscontract.ScopeProfileSeries, "series-1", 0, "fr")
service := &DetailService{}
service.SetUserStoreProvider(testDetailUserStoreProvider{store: store})
file := &models.MediaFile{
AudioTracks: []models.AudioTrack{
{Language: "en", Default: true},
{Language: "ja"},
{Language: "es"},
{Language: "fr"},
},
}
assertIndex := func(name, deviceID, contentID string, libraryID, want int) {
t.Helper()
file.MediaFolderID = libraryID
if got := service.effectiveAudioTrackIndex(context.Background(), AccessFilter{
UserID: 1,
ProfileID: "profile-1",
DeviceID: deviceID,
}, contentID, file); got != want {
t.Fatalf("%s: effectiveAudioTrackIndex() = %d, want %d", name, got, want)
}
}
assertIndex("device override", "apple-tv", "movie-1", 99, 1)
assertIndex("other device isolation", "iphone", "movie-1", 99, 0)
assertIndex("library over device", "apple-tv", "movie-1", 12, 2)
assertIndex("series over library and device", "apple-tv", "series-1", 12, 3)
}
func TestBuildPlaybackInfo_SetsEffectiveAudioLanguageFromOriginalWhenTrackLanguageMissing(t *testing.T) {
store := newDetailTestStore(t)
language := playback.OriginalLanguageSentinel
@@ -547,19 +511,6 @@ func setScopedAudioLanguage(
seriesID string,
libraryID int,
language string,
) {
t.Helper()
setScopedAudioLanguageForDevice(t, store, scope, seriesID, libraryID, "", language)
}
func setScopedAudioLanguageForDevice(
t *testing.T,
store userstore.UserStore,
scope settingscontract.Scope,
seriesID string,
libraryID int,
deviceID string,
language string,
) {
t.Helper()
encoded, err := json.Marshal(language)
@@ -572,7 +523,6 @@ func setScopedAudioLanguageForDevice(
ProfileID: "profile-1",
SeriesID: seriesID,
LibraryID: libraryID,
DeviceID: deviceID,
}, encoded); err != nil {
t.Fatalf("seeding %s: %v", settingskeys.PlaybackAudioLanguage, err)
}
+1 -1
View File
@@ -200,7 +200,7 @@ func (r *LibraryCollectionRepository) Create(ctx context.Context, input CreateLi
input.SortConfig = json.RawMessage(`{}`)
}
if input.Visibility == "" {
input.Visibility = LibraryCollectionVisibilityVisible
input.Visibility = "visible"
}
if input.CollectionType == "" {
input.CollectionType = "manual"
-65
View File
@@ -1,7 +1,6 @@
package catalog
import (
"bytes"
"encoding/json"
"fmt"
"sort"
@@ -214,70 +213,6 @@ func NormalizePersonalListSort(field, order string) (QuerySort, bool) {
return QuerySort{Field: field, Order: order}, true
}
// NormalizeCollectionSort validates a collection's default sort (its stored
// sort_config) or a viewer's saved override, applying the field's default order
// when none is given. Returns false when the field or a supplied order is
// unsupported. An empty field means the collection's own source order should
// be kept.
//
// allowPersonalized gates the per-profile sort fields (progress, date viewed,
// plays). They are legitimate for personal collections, which always resolve
// with the owning profile in scope, but not for library collections: those
// resolve with user scope stripped (see stripCatalogUserScope), so the query
// builder's ensureUserScope would reject the sort at browse time.
func NormalizeCollectionSort(field, order string, allowPersonalized bool) (QuerySort, bool) {
field = strings.ToLower(strings.TrimSpace(field))
def, ok := querySortDefs[field]
if !ok {
return QuerySort{}, false
}
if def.personalized && !allowPersonalized {
return QuerySort{}, false
}
order = strings.ToLower(strings.TrimSpace(order))
if order == "" {
order = def.defaultOrder
} else if order != "asc" && order != "desc" {
return QuerySort{}, false
}
return QuerySort{Field: field, Order: order}, true
}
// collectionSortConfig is the shape a collection's sort_config carries when it
// pins a default order. Everything else the column may hold (legacy `{}`, the
// unimplemented manual-pin mode) leaves the collection on source order.
type collectionSortConfig struct {
Field string `json:"field"`
Order string `json:"order"`
}
// ParseCollectionDefaultSort reads the default sort a collection's creator
// configured. An absent, empty, or unrecognized sort_config yields false,
// meaning source order.
func ParseCollectionDefaultSort(raw []byte, allowPersonalized bool) (QuerySort, bool) {
if len(bytes.TrimSpace(raw)) == 0 {
return QuerySort{}, false
}
var cfg collectionSortConfig
if err := json.Unmarshal(raw, &cfg); err != nil {
return QuerySort{}, false
}
return NormalizeCollectionSort(cfg.Field, cfg.Order, allowPersonalized)
}
// EncodeCollectionDefaultSort renders a validated default sort back into the
// sort_config JSON shape. An empty field encodes as `{}` — source order.
func EncodeCollectionDefaultSort(qs QuerySort, ok bool) (string, error) {
if !ok || strings.TrimSpace(qs.Field) == "" {
return "{}", nil
}
encoded, err := json.Marshal(collectionSortConfig(qs))
if err != nil {
return "", fmt.Errorf("encoding collection sort_config: %w", err)
}
return string(encoded), nil
}
func (q QueryDefinition) Normalize() QueryDefinition {
normalized := q
normalized.MediaScope = strings.ToLower(strings.TrimSpace(normalized.MediaScope))
-1
View File
@@ -2567,7 +2567,6 @@ func toVideoTrackRecords(tracks []models.VideoTrack) []VideoTrackRecord {
Codec: track.Codec,
DolbyVision: track.DolbyVision,
DVProfile: track.DVProfile,
DVLevel: track.DVLevel,
DVBLCompatID: track.DVBLCompatID,
DVELPresent: track.DVELPresent,
HDR10Plus: track.HDR10Plus,
+2 -5
View File
@@ -7,9 +7,9 @@ import (
"github.com/Silo-Server/silo-server/internal/models"
)
func TestToVideoTrackRecordsPreservesVideoMetadata(t *testing.T) {
func TestToVideoTrackRecordsPreservesColorRange(t *testing.T) {
got := toVideoTrackRecords([]models.VideoTrack{
{ColorRange: "tv", DVLevel: 6},
{ColorRange: "tv"},
{ColorRange: "pc"},
{ColorRange: "unknown"},
})
@@ -25,9 +25,6 @@ func TestToVideoTrackRecordsPreservesVideoMetadata(t *testing.T) {
got[2].ColorRange,
)
}
if got[0].DVLevel != 6 {
t.Fatalf("DVLevel = %d, want 6", got[0].DVLevel)
}
}
func TestCatalogSeedSearchUpsertIDsIncludesChangedItemsAndEmbeddings(t *testing.T) {
-1
View File
@@ -143,7 +143,6 @@ type VideoTrackRecord struct {
Codec string `json:"codec,omitempty"`
DolbyVision string `json:"dolby_vision,omitempty"`
DVProfile int `json:"dv_profile,omitempty"`
DVLevel int `json:"dv_level,omitempty"`
DVBLCompatID int `json:"dv_bl_compat_id,omitempty"`
DVELPresent bool `json:"dv_el_present,omitempty"`
HDR10Plus bool `json:"hdr10_plus,omitempty"`
+17 -3
View File
@@ -13,8 +13,11 @@ import (
"github.com/robfig/cron/v3"
)
const cloudflareURLMode = "cloudflare_token"
const chapterThumbnailSoftwareToneMapKey = "playback.chapter_thumbnail_software_tone_map_enabled"
const (
cloudflareURLMode = "cloudflare_token"
playbackSegmentRetentionSettingKey = "playback.segment_retention_seconds"
chapterThumbnailSoftwareToneMapKey = "playback.chapter_thumbnail_software_tone_map_enabled"
)
// adminSettingDefaults is the effective value shown by the Admin UI when no
// row exists in server_settings. Keep these values aligned with the runtime
@@ -50,7 +53,8 @@ var adminSettingDefaults = map[string]string{
"markers.lazy_playback": "false",
"playback.ffmpeg_path": "/usr/lib/jellyfin-ffmpeg/ffmpeg",
playbackTranscodeDirSettingKey: DefaultTranscodeDir,
"playback.transcode_dir": DefaultTranscodeDir,
playbackSegmentRetentionSettingKey: "600",
"playback.hw_accel": "auto",
"playback.transcode_enabled": "true",
"playback.local_transcode_fallback": "true",
@@ -301,6 +305,16 @@ func NormalizeAdminSetting(key, raw string) (string, error) {
return normalizeAdminInt(key, value, 1, 99)
case "transcode_throttle_seconds":
return normalizeAdminInt(key, value, 60, 86400)
case playbackSegmentRetentionSettingKey:
normalized, err := normalizeAdminInt(key, value, 0, 86400)
if err != nil {
return "", err
}
seconds, _ := strconv.Atoi(normalized)
if seconds != 0 && seconds < 120 {
return "", fmt.Errorf("%s must be 0 or between 120 and 86400", key)
}
return normalized, nil
case "ai.max_concurrent_jobs", "subtitle_ai.max_concurrent_jobs":
return normalizeAdminInt(key, value, 1, 1024)
case "subtitle_ai.batch_size":
+15
View File
@@ -225,6 +225,7 @@ func TestNormalizeAdminSettingRejectsInvalidValues(t *testing.T) {
{key: "theme.catalog_url", value: "http://raw.githubusercontent.com/Silo-Server/silo-themes/main/catalog.json"},
{key: "theme.catalog_url", value: "https://example.com/catalog.json"},
{key: "redis.url", value: "not-a-url"},
{key: "playback.segment_retention_seconds", value: "119"},
}
for _, tc := range tests {
t.Run(tc.key, func(t *testing.T) {
@@ -235,6 +236,20 @@ func TestNormalizeAdminSettingRejectsInvalidValues(t *testing.T) {
}
}
func TestNormalizeAdminSettingAcceptsSegmentRetentionBounds(t *testing.T) {
for _, value := range []string{"0", "120", "86400"} {
t.Run(value, func(t *testing.T) {
got, err := NormalizeAdminSetting("playback.segment_retention_seconds", value)
if err != nil {
t.Fatalf("NormalizeAdminSetting: %v", err)
}
if got != value {
t.Fatalf("normalized retention = %q, want %q", got, value)
}
})
}
}
func TestNormalizeAdminSettingAcceptsApprovedThemeCatalogURL(t *testing.T) {
got, err := NormalizeAdminSetting(
"theme.catalog_url",
+6 -21
View File
@@ -3,7 +3,6 @@ package config
import (
"errors"
"fmt"
"path/filepath"
"regexp"
"sort"
"strconv"
@@ -158,9 +157,10 @@ func (c MatcherConfig) TVSeriesRootQueueEnabled() bool {
// PlaybackConfig holds transcoding and playback settings.
type PlaybackConfig struct {
FFmpegPath string `yaml:"ffmpeg_path"`
TranscodeDir string `yaml:"transcode_dir"`
HWAccel string `yaml:"hw_accel"`
FFmpegPath string `yaml:"ffmpeg_path"`
TranscodeDir string `yaml:"transcode_dir"`
SegmentRetentionSeconds int `yaml:"segment_retention_seconds"`
HWAccel string `yaml:"hw_accel"`
// HWDevice is the GPU render device for hardware transcodes. A single
// path pins every GPU workload to that device; a comma-separated list
// (e.g. "/dev/dri/renderD128,/dev/dri/renderD129") balances workloads
@@ -408,26 +408,10 @@ var defaultJellyfinCompatServerID = uuid.NewSHA1(
[]byte("https://silo.local/jellycompat"),
).String()
const playbackTranscodeDirSettingKey = "playback.transcode_dir"
const downloadArtifactDirSettingKey = "download.artifact_dir"
// DefaultTranscodeDir is the fallback playback.transcode_dir; download
// artifacts default to a sibling directory (see EffectiveDownloadArtifactDir).
// artifacts default to a sibling directory (see downloads.effectiveArtifactDir).
const DefaultTranscodeDir = "/tmp/silo-transcode"
// EffectiveDownloadArtifactDir resolves the shared prepared-download artifact
// root. Keeping this path rule in config lets API and transcode-node processes
// independently derive the same destination from the same live settings.
func EffectiveDownloadArtifactDir(artifactDir, transcodeDir string) string {
if artifactDir != "" {
return artifactDir
}
if transcodeDir == "" {
transcodeDir = DefaultTranscodeDir
}
return filepath.Join(filepath.Dir(transcodeDir), "silo-download-artifacts")
}
const DefaultJellyfinCompatEmulatedServerVersion = "10.12.0"
const DefaultJellyfinWebVersion = "10.11.6"
const DefaultJellyfinWebInstallDir = "/var/lib/silo/compat/jellyfin-web"
@@ -496,6 +480,7 @@ func setDefaults() *configRaw {
Playback: PlaybackConfig{
FFmpegPath: "/usr/lib/jellyfin-ffmpeg/ffmpeg",
TranscodeDir: DefaultTranscodeDir,
SegmentRetentionSeconds: 600,
HWAccel: "auto",
ChapterThumbnailWorkers: 1,
ChapterThumbnailExecution: "local",
+10 -8
View File
@@ -3,9 +3,7 @@ package config
import (
"fmt"
"log/slog"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/google/uuid"
@@ -318,7 +316,15 @@ func LoadFromDB(m map[string]string) (*Config, error) {
// Playback
cfg.Playback.FFmpegPath = stringOr(m, "playback.ffmpeg_path", "/usr/lib/jellyfin-ffmpeg/ffmpeg")
cfg.Playback.TranscodeDir = stringOr(m, playbackTranscodeDirSettingKey, DefaultTranscodeDir)
cfg.Playback.TranscodeDir = stringOr(m, "playback.transcode_dir", DefaultTranscodeDir)
segmentRetentionSeconds, err := intOr(m, playbackSegmentRetentionSettingKey, 600)
if err != nil {
return nil, err
}
if segmentRetentionSeconds != 0 && (segmentRetentionSeconds < 120 || segmentRetentionSeconds > 86400) {
return nil, fmt.Errorf("%s must be 0 or between 120 and 86400", playbackSegmentRetentionSettingKey)
}
cfg.Playback.SegmentRetentionSeconds = segmentRetentionSeconds
cfg.Playback.HWAccel = stringOr(m, "playback.hw_accel", "auto")
cfg.Playback.HWDevice = stringOr(m, "playback.hw_device", "")
chapterThumbnailWorkers, err := intOr(m, "playback.chapter_thumbnail_workers", 1)
@@ -605,12 +611,8 @@ func LoadFromDB(m map[string]string) (*Config, error) {
if artifactMaxBytes < 0 {
return nil, fmt.Errorf("invalid value for %q: must be non-negative", "download.artifact_max_bytes")
}
artifactDir := strings.TrimSpace(stringOr(m, downloadArtifactDirSettingKey, ""))
if artifactDir != "" && !filepath.IsAbs(artifactDir) {
return nil, fmt.Errorf("invalid value for %q: must be an absolute path", downloadArtifactDirSettingKey)
}
cfg.Download.TranscodeEnabled = downloadTranscodeEnabled
cfg.Download.ArtifactDir = artifactDir
cfg.Download.ArtifactDir = stringOr(m, "download.artifact_dir", "")
cfg.Download.MaxConcurrentPrepares = maxConcurrentPrepares
cfg.Download.ArtifactMaxBytes = artifactMaxBytes
+18 -12
View File
@@ -35,18 +35,14 @@ func TestLoadFromDBMetadataPresignExpiryRejectsInvalidDuration(t *testing.T) {
}
}
func TestLoadFromDBDownloadArtifactDirRequiresAbsolutePath(t *testing.T) {
cfg, err := LoadFromDB(map[string]string{downloadArtifactDirSettingKey: "/mnt/silo-downloads"})
if err != nil {
t.Fatalf("LoadFromDB() with absolute artifact dir: %v", err)
}
if cfg.Download.ArtifactDir != "/mnt/silo-downloads" {
t.Fatalf("artifact dir = %q", cfg.Download.ArtifactDir)
}
_, err = LoadFromDB(map[string]string{downloadArtifactDirSettingKey: "relative/downloads"})
if err == nil || !strings.Contains(err.Error(), downloadArtifactDirSettingKey) {
t.Fatalf("relative artifact dir error = %v", err)
func TestLoadFromDBRejectsInvalidSegmentRetention(t *testing.T) {
for _, value := range []string{"-1", "119", "86401"} {
t.Run(value, func(t *testing.T) {
_, err := LoadFromDB(map[string]string{playbackSegmentRetentionSettingKey: value})
if err == nil || !strings.Contains(err.Error(), playbackSegmentRetentionSettingKey) {
t.Fatalf("LoadFromDB() error = %v, want retention bounds error", err)
}
})
}
}
@@ -201,6 +197,16 @@ func TestYAMLToSettingsMapJellyfinCompatEnabledDefaultsToLegacyListener(t *testi
}
}
func TestYAMLToSettingsMapPreservesExplicitlyDisabledSegmentRetention(t *testing.T) {
m := yamlSettingsMapFromString(t, `
playback:
segment_retention_seconds: 0
`)
if got := m[playbackSegmentRetentionSettingKey]; got != "0" {
t.Fatalf("segment retention = %q, want explicit disable", got)
}
}
func yamlSettingsMapFromString(t *testing.T, body string) map[string]string {
t.Helper()
path := t.TempDir() + "/silo.yaml"
+3 -7
View File
@@ -8,7 +8,7 @@ import "strings"
//
// Keys absent from this map and from restartRequiredPrefixes apply without a
// restart: they are either read live from the settings repo at request time
// (overlays, branding, markers, download limits, ...) or hot-reloaded through the
// (overlays, branding, markers, download.*, ...) or hot-reloaded through the
// nodeconfig watcher. When converting a frozen consumer to the live config,
// remove its key here — the admin UI restart banner is driven by this
// registry.
@@ -34,15 +34,11 @@ var restartRequiredKeys = map[string]bool{
// ffmpeg path and hwaccel live (new transcode sessions), but several
// startup-built consumers still freeze them (scanner ffprobe, chapter
// thumbnails, audiobook enricher) — keep restart-required until those
// convert. transcode_dir is also captured by dedicated transcode nodes for
// session and prepared-download storage. A configured download.artifact_dir is
// likewise captured by both API and transcode-node artifact managers. The
// chapter-thumbnail worker pool is sized at construction.
// convert. transcode_dir is fully live (only the playback handler reads
// it). The chapter-thumbnail worker pool is sized at construction.
"playback.ffmpeg_path": true,
"playback.hw_accel": true,
"playback.hw_device": true,
playbackTranscodeDirSettingKey: true,
downloadArtifactDirSettingKey: true,
"playback.chapter_thumbnail_workers": true,
// Scanner / matcher toggles captured at construction. Worker counts,
-2
View File
@@ -11,8 +11,6 @@ func TestRestartRequired(t *testing.T) {
{"server.listen", true},
{"auth.jwt_secret", true},
{"ratelimit.backend", true},
{playbackTranscodeDirSettingKey, true},
{downloadArtifactDirSettingKey, true},
// Prefix-covered namespaces.
{"database.max_connections", true},
{"userdb.backend", true},
+2 -1
View File
@@ -208,7 +208,8 @@ func YAMLToSettingsMap(path string) (map[string]string, error) {
// Playback
setIfNonEmpty(m, "playback.ffmpeg_path", raw.Playback.FFmpegPath)
setIfNonEmpty(m, playbackTranscodeDirSettingKey, raw.Playback.TranscodeDir)
setIfNonEmpty(m, "playback.transcode_dir", raw.Playback.TranscodeDir)
m[playbackSegmentRetentionSettingKey] = strconv.Itoa(raw.Playback.SegmentRetentionSeconds)
setIfNonEmpty(m, "playback.hw_accel", raw.Playback.HWAccel)
if raw.Playback.ChapterThumbnailWorkers != 0 {
m["playback.chapter_thumbnail_workers"] = strconv.Itoa(raw.Playback.ChapterThumbnailWorkers)
+1 -13
View File
@@ -259,10 +259,6 @@ var attrRegistry = map[string]map[string]attrValueType{
"bitrate_kbps": attrInteger,
"dropped_frames": attrInteger,
"audio_underruns": attrInteger,
"session_id": attrString,
"play_method": attrString,
"reason": attrString,
"position_ms": attrInteger,
},
"focus": {
"target": attrString,
@@ -273,17 +269,9 @@ var attrRegistry = map[string]map[string]attrValueType{
"path": attrString,
"status": attrInteger,
"duration_ms": attrInteger,
"outcome": attrString,
"error_code": attrString,
"attempt": attrInteger,
},
"lifecycle": {
"state": attrString,
"phase": attrString,
"duration_ms": attrInteger,
"outcome": attrString,
"reason": attrString,
"launch_type": attrString,
"state": attrString,
},
"crash": {
"fingerprint": attrString,
-436
View File
@@ -1,436 +0,0 @@
// Package downloadprepare defines the internal API used to create and relay a
// prepared download artifact on a dedicated transcode node.
package downloadprepare
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/Silo-Server/silo-server/internal/playback"
)
const (
ArtifactDirectoryName = "download-artifacts"
RelayReadIdleTimeout = 2 * time.Minute
)
var (
artifactIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]{1,128}$`)
ErrArtifactNotFound = errors.New("remote download artifact not found")
ErrRelayReadIdle = errors.New("remote download artifact read stalled")
)
func newHTTPClient(responseHeaderTimeout time.Duration) *http.Client {
baseTransport, ok := http.DefaultTransport.(*http.Transport)
if !ok {
baseTransport = &http.Transport{}
}
transport := baseTransport.Clone()
transport.ResponseHeaderTimeout = responseHeaderTimeout
return &http.Client{
Transport: transport,
CheckRedirect: func(_ *http.Request, _ []*http.Request) error {
return http.ErrUseLastResponse
},
}
}
var (
// Preparing a full file can legitimately take hours. Its lease-bound request
// context supplies cancellation, so imposing a response-header timeout here
// would abort every encode longer than that timeout before the node replies.
defaultPrepareHTTPClient = newHTTPClient(0)
defaultHTTPClient = newHTTPClient(60 * time.Second)
)
// Request is the environment-neutral portion of a prepared-file recipe. The
// transcode node supplies its own FFmpeg path, hardware mode, device list, and
// output path. ArtifactID is an opaque handle, never a caller-selected path.
type Request struct {
ArtifactID string `json:"artifact_id"`
InputPath string `json:"input_path"`
SourceVideoCodec string `json:"source_video_codec,omitempty"`
SourceVideoProfile string `json:"source_video_profile,omitempty"`
SourceVideoBitDepth int `json:"source_video_bit_depth,omitempty"`
SoftwareVideoDecode bool `json:"software_video_decode,omitempty"`
TargetCodecVideo string `json:"target_codec_video"`
TargetCodecAudio string `json:"target_codec_audio"`
TargetResolution string `json:"target_resolution,omitempty"`
TargetBitrateKbps int `json:"target_bitrate_kbps,omitempty"`
AudioTrackIndex int `json:"audio_track_index"`
TotalDuration float64 `json:"total_duration,omitempty"`
}
// Result identifies a completed artifact without exposing the node's local
// filesystem layout.
type Result struct {
ArtifactID string `json:"artifact_id"`
FileSize int64 `json:"file_size"`
}
func ValidArtifactID(id string) bool { return artifactIDPattern.MatchString(id) }
// NewRequest freezes the byte-affecting recipe while deliberately omitting
// environment-specific execution settings.
func NewRequest(artifactID string, opts playback.TranscodeOpts) Request {
return Request{
ArtifactID: artifactID,
InputPath: opts.InputPath,
SourceVideoCodec: opts.SourceVideoCodec,
SourceVideoProfile: opts.SourceVideoProfile,
SourceVideoBitDepth: opts.SourceVideoBitDepth,
SoftwareVideoDecode: opts.SoftwareVideoDecode,
TargetCodecVideo: opts.TargetCodecVideo,
TargetCodecAudio: opts.TargetCodecAudio,
TargetResolution: opts.TargetResolution,
TargetBitrateKbps: opts.TargetBitrateKbps,
AudioTrackIndex: opts.AudioTrackIndex,
TotalDuration: opts.TotalDuration,
}
}
// TranscodeOpts reconstructs the prepared-file options using the selected
// node's live execution settings.
func (r Request) TranscodeOpts(ffmpegPath, hwAccel, hwDevice string, sink playback.FFmpegLogSink) playback.TranscodeOpts {
return playback.TranscodeOpts{
InputPath: r.InputPath,
SourceVideoCodec: r.SourceVideoCodec,
SourceVideoProfile: r.SourceVideoProfile,
SourceVideoBitDepth: r.SourceVideoBitDepth,
SoftwareVideoDecode: r.SoftwareVideoDecode,
TargetCodecVideo: r.TargetCodecVideo,
TargetCodecAudio: r.TargetCodecAudio,
TargetResolution: r.TargetResolution,
TargetBitrateKbps: r.TargetBitrateKbps,
AudioTrackIndex: r.AudioTrackIndex,
SubtitleTrackIndex: -1,
FFmpegPath: ffmpegPath,
HWAccel: hwAccel,
HWDevice: hwDevice,
TotalDuration: r.TotalDuration,
NodeType: "transcode",
ExecutionMode: "download_prepare",
FFmpegLogSink: sink,
}
}
// RemotePreparer executes and manages artifacts on a selected transcode node.
type RemotePreparer interface {
Prepare(ctx context.Context, nodeURL, jwtSecret string, req Request) (Result, error)
Stat(ctx context.Context, nodeURL, jwtSecret, artifactID string) (Result, error)
Delete(ctx context.Context, nodeURL, jwtSecret, artifactID string) error
}
// HTTPPreparer implements RemotePreparer over bearer-protected node APIs. A
// full-file prepare has no transport timeout; its caller owns cancellation.
// Metadata and relay operations use a bounded response-header wait by default.
type HTTPPreparer struct {
Client *http.Client
ReadIdleTimeout time.Duration
}
func (p HTTPPreparer) Prepare(ctx context.Context, nodeURL, jwtSecret string, req Request) (Result, error) {
body, err := json.Marshal(req)
if err != nil {
return Result{}, fmt.Errorf("remote download prepare: marshal request: %w", err)
}
responseCtx, cancel := context.WithCancel(ctx)
httpReq, err := p.request(responseCtx, http.MethodPost, nodeURL, jwtSecret, "/downloads/prepare", bytes.NewReader(body))
if err != nil {
cancel()
return Result{}, fmt.Errorf("remote download prepare: %w", err)
}
httpReq.Header.Set("Content-Type", "application/json")
resp, err := p.prepareClient().Do(httpReq)
if err != nil {
cancel()
return Result{}, fmt.Errorf("remote download prepare: request: %w", err)
}
// The encode itself remains unbounded while waiting for response headers, but
// once the node starts its small JSON result body, each read must make progress.
resp.Body = newIdleReadCloser(resp.Body, p.readIdleTimeout(), cancel)
defer func() { _ = resp.Body.Close() }()
return decodeResult(resp, "remote download prepare")
}
func (p HTTPPreparer) Stat(ctx context.Context, nodeURL, jwtSecret, artifactID string) (Result, error) {
if !ValidArtifactID(artifactID) {
return Result{}, fmt.Errorf("remote download artifact stat: invalid artifact id")
}
httpReq, err := p.request(ctx, http.MethodHead, nodeURL, jwtSecret, "/downloads/artifacts/"+url.PathEscape(artifactID), nil)
if err != nil {
return Result{}, fmt.Errorf("remote download artifact stat: %w", err)
}
resp, err := p.client().Do(httpReq)
if err != nil {
return Result{}, fmt.Errorf("remote download artifact stat: request: %w", err)
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusNotFound {
return Result{}, ErrArtifactNotFound
}
if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
return Result{}, responseError(resp, "remote download artifact stat")
}
size, err := strconv.ParseInt(resp.Header.Get("Content-Length"), 10, 64)
if err != nil || size < 0 {
return Result{}, fmt.Errorf("remote download artifact stat: invalid content length")
}
return Result{ArtifactID: artifactID, FileSize: size}, nil
}
func (p HTTPPreparer) Delete(ctx context.Context, nodeURL, jwtSecret, artifactID string) error {
if !ValidArtifactID(artifactID) {
return fmt.Errorf("remote download artifact delete: invalid artifact id")
}
deleteCtx, cancel := context.WithCancel(ctx)
httpReq, err := p.request(deleteCtx, http.MethodDelete, nodeURL, jwtSecret, "/downloads/artifacts/"+url.PathEscape(artifactID), nil)
if err != nil {
cancel()
return fmt.Errorf("remote download artifact delete: %w", err)
}
resp, err := p.client().Do(httpReq)
if err != nil {
cancel()
return fmt.Errorf("remote download artifact delete: request: %w", err)
}
resp.Body = newIdleReadCloser(resp.Body, p.readIdleTimeout(), cancel)
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusNotFound {
return nil
}
if resp.StatusCode != http.StatusNoContent {
return responseError(resp, "remote download artifact delete")
}
return nil
}
// Open returns an authenticated streaming response for a GET or HEAD relay.
// The caller owns closing the body and copying only safe response headers.
func (p HTTPPreparer) Open(ctx context.Context, nodeURL, jwtSecret, artifactID, method string, sourceHeader http.Header) (*http.Response, error) {
if !ValidArtifactID(artifactID) {
return nil, fmt.Errorf("remote download artifact open: invalid artifact id")
}
if method != http.MethodGet && method != http.MethodHead {
return nil, fmt.Errorf("remote download artifact open: unsupported method %s", method)
}
relayCtx, cancel := context.WithCancel(ctx)
httpReq, err := p.request(relayCtx, method, nodeURL, jwtSecret, "/downloads/artifacts/"+url.PathEscape(artifactID), nil)
if err != nil {
cancel()
return nil, fmt.Errorf("remote download artifact open: %w", err)
}
for _, name := range []string{"Range", "If-Match", "If-Range", "If-None-Match", "If-Modified-Since", "If-Unmodified-Since"} {
if value := sourceHeader.Get(name); value != "" {
httpReq.Header.Set(name, value)
}
}
resp, err := p.client().Do(httpReq)
if err != nil {
cancel()
return nil, fmt.Errorf("remote download artifact open: request: %w", err)
}
resp.Body = newIdleReadCloser(resp.Body, p.readIdleTimeout(), cancel)
return resp, nil
}
// idleReadCloser bounds only time spent blocked in an upstream Read. Time
// between reads is deliberately ignored so local bandwidth throttling and slow
// clients cannot be mistaken for a stalled origin.
type idleReadCloser struct {
source io.ReadCloser
timeout time.Duration
readStart chan struct{}
readDone chan struct{}
stop chan struct{}
stopped chan struct{}
closeOnce sync.Once
timedOut atomic.Bool
cancel context.CancelFunc
}
func newIdleReadCloser(source io.ReadCloser, timeout time.Duration, cancel context.CancelFunc) io.ReadCloser {
if source == nil || timeout <= 0 {
return source
}
r := &idleReadCloser{
source: source,
timeout: timeout,
readStart: make(chan struct{}),
readDone: make(chan struct{}),
stop: make(chan struct{}),
stopped: make(chan struct{}),
cancel: cancel,
}
go r.watch()
return r
}
func (r *idleReadCloser) Read(p []byte) (int, error) {
if r.timedOut.Load() {
return 0, ErrRelayReadIdle
}
select {
case r.readStart <- struct{}{}:
case <-r.stop:
return 0, io.ErrClosedPipe
}
n, err := r.source.Read(p)
select {
case r.readDone <- struct{}{}:
case <-r.stop:
}
if r.timedOut.Load() {
return n, ErrRelayReadIdle
}
return n, err
}
func (r *idleReadCloser) Close() error {
r.closeOnce.Do(func() {
close(r.stop)
if r.cancel != nil {
r.cancel()
}
_ = r.source.Close()
<-r.stopped
})
return nil
}
func (r *idleReadCloser) watch() {
defer close(r.stopped)
timer := time.NewTimer(time.Hour)
if !timer.Stop() {
<-timer.C
}
reading := false
for {
select {
case <-r.readStart:
timer.Reset(r.timeout)
reading = true
case <-r.readDone:
if reading && !timer.Stop() {
select {
case <-timer.C:
default:
}
}
reading = false
case <-timer.C:
if reading {
r.timedOut.Store(true)
if r.cancel != nil {
r.cancel()
}
_ = r.source.Close()
reading = false
}
case <-r.stop:
if !timer.Stop() {
select {
case <-timer.C:
default:
}
}
return
}
}
}
func (p HTTPPreparer) request(ctx context.Context, method, nodeURL, jwtSecret, path string, body io.Reader) (*http.Request, error) {
base, err := url.Parse(strings.TrimSpace(nodeURL))
if err != nil || (base.Scheme != "http" && base.Scheme != "https") || base.Host == "" {
return nil, fmt.Errorf("invalid node URL")
}
base.Path = strings.TrimRight(base.Path, "/") + path
base.RawQuery = ""
base.Fragment = ""
req, err := http.NewRequestWithContext(ctx, method, base.String(), body)
if err != nil {
return nil, fmt.Errorf("build request: %w", err)
}
req.Header.Set("Authorization", "Bearer "+jwtSecret)
return req, nil
}
func (p HTTPPreparer) client() *http.Client {
if p.Client != nil {
return p.Client
}
return defaultHTTPClient
}
func (p HTTPPreparer) prepareClient() *http.Client {
if p.Client != nil {
return p.Client
}
return defaultPrepareHTTPClient
}
func (p HTTPPreparer) readIdleTimeout() time.Duration {
if p.ReadIdleTimeout > 0 {
return p.ReadIdleTimeout
}
return RelayReadIdleTimeout
}
// CopyResponseHeaders forwards only representation/range metadata that is safe
// across the transcode-node relay boundary.
func CopyResponseHeaders(dst, src http.Header) {
for _, name := range []string{
"Accept-Ranges", "Content-Length", "Content-Range",
"Content-Type", "ETag", "Last-Modified",
} {
if value := src.Get(name); value != "" {
dst.Set(name, value)
}
}
}
// RelayStatusAllowed identifies the complete set of normal ServeContent
// outcomes that an internal relay must preserve unchanged.
func RelayStatusAllowed(status int) bool {
return status >= http.StatusOK && status < http.StatusMultipleChoices ||
status == http.StatusNotModified ||
status == http.StatusPreconditionFailed ||
status == http.StatusRequestedRangeNotSatisfiable
}
func decodeResult(resp *http.Response, operation string) (Result, error) {
if resp.StatusCode == http.StatusNotFound {
return Result{}, ErrArtifactNotFound
}
if resp.StatusCode != http.StatusOK {
return Result{}, responseError(resp, operation)
}
var result Result
if err := json.NewDecoder(io.LimitReader(resp.Body, 64<<10)).Decode(&result); err != nil {
return Result{}, fmt.Errorf("%s: decode response: %w", operation, err)
}
if !ValidArtifactID(result.ArtifactID) || result.FileSize < 0 {
return Result{}, fmt.Errorf("%s: invalid response", operation)
}
return result, nil
}
func responseError(resp *http.Response, operation string) error {
message, err := io.ReadAll(io.LimitReader(resp.Body, 4096))
if err != nil {
return fmt.Errorf("%s: read node error response: %w", operation, err)
}
return fmt.Errorf("%s: node returned %d: %s", operation, resp.StatusCode, strings.TrimSpace(string(message)))
}
-234
View File
@@ -1,234 +0,0 @@
package downloadprepare
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestHTTPPreparerSendsAuthenticatedRecipe(t *testing.T) {
var got Request
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost || r.URL.Path != "/downloads/prepare" {
t.Fatalf("request = %s %s", r.Method, r.URL.Path)
}
if auth := r.Header.Get("Authorization"); auth != "Bearer secret" {
t.Fatalf("Authorization = %q", auth)
}
if err := json.NewDecoder(r.Body).Decode(&got); err != nil {
t.Fatal(err)
}
_ = json.NewEncoder(w).Encode(Result{ArtifactID: got.ArtifactID, FileSize: 123})
}))
defer server.Close()
want := Request{ArtifactID: "job-1", InputPath: "/media/movie.mkv", TargetCodecVideo: "h264", TargetCodecAudio: "aac"}
result, err := (HTTPPreparer{}).Prepare(context.Background(), server.URL+"/", "secret", want)
if err != nil {
t.Fatal(err)
}
if result != (Result{ArtifactID: "job-1", FileSize: 123}) {
t.Fatalf("result = %+v", result)
}
if got != want {
t.Fatalf("request = %+v, want %+v", got, want)
}
}
func TestHTTPPreparerReportsNodeFailure(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
http.Error(w, "mount unavailable", http.StatusUnprocessableEntity)
}))
defer server.Close()
_, err := (HTTPPreparer{}).Prepare(context.Background(), server.URL, "secret", Request{ArtifactID: "job-2"})
if err == nil {
t.Fatal("expected remote failure")
}
}
func TestHTTPPreparerManagesOpaqueArtifact(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/downloads/artifacts/artifact-1" {
t.Fatalf("path = %q", r.URL.Path)
}
if r.Header.Get("Authorization") != "Bearer secret" {
t.Fatalf("Authorization = %q", r.Header.Get("Authorization"))
}
switch r.Method {
case http.MethodHead:
w.Header().Set("Content-Length", "42")
case http.MethodDelete:
w.WriteHeader(http.StatusNoContent)
default:
t.Fatalf("method = %s", r.Method)
}
}))
defer server.Close()
client := HTTPPreparer{}
result, err := client.Stat(context.Background(), server.URL, "secret", "artifact-1")
if err != nil || result != (Result{ArtifactID: "artifact-1", FileSize: 42}) {
t.Fatalf("Stat = (%+v, %v)", result, err)
}
if err := client.Delete(context.Background(), server.URL, "secret", "artifact-1"); err != nil {
t.Fatal(err)
}
}
func TestHTTPPreparerDeleteStopsStalledErrorBodyRead(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
}
<-r.Context().Done()
}))
defer server.Close()
started := time.Now()
err := (HTTPPreparer{ReadIdleTimeout: 50 * time.Millisecond}).Delete(
context.Background(), server.URL, "secret", "artifact-stalled-delete",
)
if !errors.Is(err, ErrRelayReadIdle) {
t.Fatalf("Delete error = %v, want ErrRelayReadIdle", err)
}
if elapsed := time.Since(started); elapsed > time.Second {
t.Fatalf("stalled delete response took %s, want a bounded failure", elapsed)
}
}
func TestHTTPPreparerRejectsArtifactPathTraversal(t *testing.T) {
if _, err := (HTTPPreparer{}).Stat(context.Background(), "http://node", "secret", "../escape"); err == nil {
t.Fatal("expected invalid artifact id error")
}
}
func TestDefaultPrepareClientHasNoResponseHeaderDeadline(t *testing.T) {
transport, ok := defaultPrepareHTTPClient.Transport.(*http.Transport)
if !ok {
t.Fatalf("transport type = %T", defaultPrepareHTTPClient.Transport)
}
if transport.ResponseHeaderTimeout != 0 {
t.Fatalf("prepare response header timeout = %s, want none", transport.ResponseHeaderTimeout)
}
}
func TestHTTPPreparerPrepareStopsStalledResultBodyRead(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
}
<-r.Context().Done()
}))
defer server.Close()
started := time.Now()
_, err := (HTTPPreparer{ReadIdleTimeout: 50 * time.Millisecond}).Prepare(
context.Background(), server.URL, "secret", Request{ArtifactID: "artifact-stalled-result"},
)
if !errors.Is(err, ErrRelayReadIdle) {
t.Fatalf("Prepare error = %v, want ErrRelayReadIdle", err)
}
if elapsed := time.Since(started); elapsed > time.Second {
t.Fatalf("stalled result read took %s, want a bounded failure", elapsed)
}
}
func TestRelayStatusAllowedPreservesServeContentOutcomes(t *testing.T) {
for _, status := range []int{
http.StatusOK,
http.StatusPartialContent,
http.StatusNotModified,
http.StatusPreconditionFailed,
http.StatusRequestedRangeNotSatisfiable,
} {
if !RelayStatusAllowed(status) {
t.Errorf("status %d should be relayed", status)
}
}
for _, status := range []int{http.StatusTemporaryRedirect, http.StatusBadRequest, http.StatusInternalServerError} {
if RelayStatusAllowed(status) {
t.Errorf("status %d should be rejected", status)
}
}
}
func TestCopyResponseHeadersDoesNotExposeOriginArtifactFilename(t *testing.T) {
src := http.Header{
"Content-Disposition": {`attachment; filename="opaque-artifact.mp4"`},
"Content-Type": {"video/mp4"},
"Content-Length": {"42"},
}
dst := make(http.Header)
CopyResponseHeaders(dst, src)
if got := dst.Get("Content-Disposition"); got != "" {
t.Fatalf("Content-Disposition = %q, want omitted", got)
}
if dst.Get("Content-Type") != "video/mp4" || dst.Get("Content-Length") != "42" {
t.Fatalf("copied headers = %v", dst)
}
}
func TestHTTPPreparerOpenStopsStalledBodyRead(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "video/mp4")
w.WriteHeader(http.StatusOK)
if flusher, ok := w.(http.Flusher); ok {
flusher.Flush()
}
<-r.Context().Done()
}))
defer server.Close()
resp, err := (HTTPPreparer{ReadIdleTimeout: 50 * time.Millisecond}).Open(
context.Background(), server.URL, "secret", "artifact-stalled", http.MethodGet, nil,
)
if err != nil {
t.Fatal(err)
}
defer func() { _ = resp.Body.Close() }()
started := time.Now()
_, err = io.ReadAll(resp.Body)
if !errors.Is(err, ErrRelayReadIdle) {
t.Fatalf("ReadAll error = %v, want ErrRelayReadIdle", err)
}
if elapsed := time.Since(started); elapsed > time.Second {
t.Fatalf("stalled read took %s, want a bounded failure", elapsed)
}
}
func TestHTTPPreparerOpenAllowsContinuedBodyProgress(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
flusher, _ := w.(http.Flusher)
for _, chunk := range []string{"one", "two", "three"} {
_, _ = io.WriteString(w, chunk)
if flusher != nil {
flusher.Flush()
}
time.Sleep(10 * time.Millisecond)
}
}))
defer server.Close()
resp, err := (HTTPPreparer{ReadIdleTimeout: 50 * time.Millisecond}).Open(
context.Background(), server.URL, "secret", "artifact-progress", http.MethodGet, nil,
)
if err != nil {
t.Fatal(err)
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if string(body) != "onetwothree" {
t.Fatalf("body = %q", body)
}
}
+13 -5
View File
@@ -36,10 +36,6 @@ type Artifact struct {
AudioTrackIndex int
TargetBitrateKbps int
OutputPath string
OriginNodeID int
OriginNodeURL string
OriginNodeGroup string
OriginArtifactID string
FileSize int64
Status string
ErrorMessage string
@@ -61,6 +57,12 @@ func paramsHash(format, container, codecVideo, codecAudio, resolution string, au
return hex.EncodeToString(sum[:])
}
// defaultTranscodeDir is config's TranscodeDir default and roots
// prepared download artifacts when neither download.artifact_dir nor the
// transcode dir is configured. Keeping it absolute avoids writing artifacts
// relative to the process working directory.
const defaultTranscodeDir = config.DefaultTranscodeDir
// effectiveArtifactDir resolves where prepared artifacts are written: the
// configured download.artifact_dir when set, otherwise a dedicated directory
// alongside the transcode dir. The result is always rooted at a real volume,
@@ -71,7 +73,13 @@ func paramsHash(format, container, codecVideo, codecAudio, resolution string, au
// transcode dir, so an artifact dir nested under it would be wiped on the next
// transcode sweep.
func effectiveArtifactDir(artifactDir, transcodeDir string) string {
return config.EffectiveDownloadArtifactDir(artifactDir, transcodeDir)
if artifactDir != "" {
return artifactDir
}
if transcodeDir == "" {
transcodeDir = defaultTranscodeDir
}
return filepath.Join(filepath.Dir(transcodeDir), "silo-download-artifacts")
}
// artifactOutputPath derives a deterministic output path from
+7 -224
View File
@@ -11,8 +11,7 @@ import (
)
const artifactColumns = `id, media_file_id, format, params_hash, container, codec_video, codec_audio,
resolution, audio_track_index, target_bitrate_kbps, output_path,
origin_node_id, origin_node_url, origin_node_group, origin_artifact_id, file_size, status, error_message,
resolution, audio_track_index, target_bitrate_kbps, output_path, file_size, status, error_message,
attempts, max_attempts, lease_owner, lease_expires_at, next_retry_at,
created_at, completed_at, last_used_at`
@@ -22,18 +21,6 @@ type ArtifactRepository struct {
pool *pgxpool.Pool
}
// RemoteArtifactOrphan is a node-local cleanup candidate. The durable queue
// retains abandoned locators and verifies indeterminate readiness writes before
// deleting bytes, so transient failures cannot leak or delete a winning file.
type RemoteArtifactOrphan struct {
ID int64
DownloadArtifactID string
OriginNodeID int
OriginNodeURL string
OriginArtifactID string
Attempts int
}
// NewArtifactRepository creates an ArtifactRepository.
func NewArtifactRepository(pool *pgxpool.Pool) *ArtifactRepository {
return &ArtifactRepository{pool: pool}
@@ -44,8 +31,7 @@ func scanArtifact(row pgx.Row) (*Artifact, error) {
var leaseOwner *string
if err := row.Scan(
&a.ID, &a.MediaFileID, &a.Format, &a.ParamsHash, &a.Container, &a.CodecVideo, &a.CodecAudio,
&a.Resolution, &a.AudioTrackIndex, &a.TargetBitrateKbps, &a.OutputPath,
&a.OriginNodeID, &a.OriginNodeURL, &a.OriginNodeGroup, &a.OriginArtifactID, &a.FileSize, &a.Status, &a.ErrorMessage,
&a.Resolution, &a.AudioTrackIndex, &a.TargetBitrateKbps, &a.OutputPath, &a.FileSize, &a.Status, &a.ErrorMessage,
&a.Attempts, &a.MaxAttempts, &leaseOwner, &a.LeaseExpiresAt, &a.NextRetryAt,
&a.CreatedAt, &a.CompletedAt, &a.LastUsedAt,
); err != nil {
@@ -163,15 +149,14 @@ func (r *ArtifactRepository) Heartbeat(ctx context.Context, id, owner string, le
// another node — cannot flip a row it no longer owns. Returns false when the
// fence rejected the write (the lease was lost); the caller must then NOT flip
// linked downloads, leaving that to the current owner.
func (r *ArtifactRepository) MarkReady(ctx context.Context, id, owner, outputPath string, originNodeID int, originNodeURL, originNodeGroup, originArtifactID string, fileSize int64) (bool, error) {
func (r *ArtifactRepository) MarkReady(ctx context.Context, id, owner, outputPath string, fileSize int64) (bool, error) {
tag, err := r.pool.Exec(ctx,
`UPDATE download_artifacts
SET status = 'ready', output_path = $2, origin_node_id = $3, origin_node_url = $4,
origin_node_group = $5, origin_artifact_id = $6, file_size = $7, error_message = '',
SET status = 'ready', output_path = $2, file_size = $3, error_message = '',
completed_at = now(), last_used_at = now(),
lease_owner = NULL, lease_expires_at = NULL, next_retry_at = NULL
WHERE id = $1 AND lease_owner = $8 AND status = 'running'`,
id, outputPath, originNodeID, originNodeURL, originNodeGroup, originArtifactID, fileSize, owner,
WHERE id = $1 AND lease_owner = $4 AND status = 'running'`,
id, outputPath, fileSize, owner,
)
if err != nil {
return false, fmt.Errorf("marking artifact ready: %w", err)
@@ -252,8 +237,7 @@ func (r *ArtifactRepository) Requeue(ctx context.Context, id string) error {
tag, err := r.pool.Exec(ctx,
`UPDATE download_artifacts
SET status = 'queued', attempts = 0, error_message = '', next_retry_at = NULL,
lease_owner = NULL, lease_expires_at = NULL, completed_at = NULL,
origin_node_id = 0, origin_node_url = '', origin_node_group = '', origin_artifact_id = ''
lease_owner = NULL, lease_expires_at = NULL, completed_at = NULL
WHERE id = $1`,
id,
)
@@ -266,79 +250,6 @@ func (r *ArtifactRepository) Requeue(ctx context.Context, id string) error {
return nil
}
// RequeueRemote atomically transfers a ready remote locator into the cleanup
// queue and clears it from the artifact row. The locator can therefore never
// be deleted while the row still advertises it if either database write fails.
// applied is false when the row changed concurrently or no longer exists.
func (r *ArtifactRepository) RequeueRemote(ctx context.Context, artifact *Artifact) (linked []*Download, applied bool, err error) {
return r.requeueRemote(ctx, artifact, false)
}
// RequeueRemoteExactLocator additionally fences on the origin URL. Proxy miss
// reports use it so an older signed URL cannot requeue a row after an
// administrator has moved the same node/artifact locator to a new endpoint.
func (r *ArtifactRepository) RequeueRemoteExactLocator(ctx context.Context, artifact *Artifact) (linked []*Download, applied bool, err error) {
return r.requeueRemote(ctx, artifact, true)
}
func (r *ArtifactRepository) requeueRemote(ctx context.Context, artifact *Artifact, fenceURL bool) (linked []*Download, applied bool, err error) {
if artifact == nil {
return nil, false, nil
}
tx, err := r.pool.Begin(ctx)
if err != nil {
return nil, false, fmt.Errorf("beginning remote artifact requeue: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
query := `UPDATE download_artifacts
SET status = 'queued', attempts = 0, error_message = '', next_retry_at = NULL,
lease_owner = NULL, lease_expires_at = NULL, completed_at = NULL,
origin_node_id = 0, origin_node_url = '', origin_node_group = '', origin_artifact_id = ''
WHERE id = $1 AND status = 'ready' AND origin_node_id = $2 AND origin_artifact_id = $3`
args := []any{artifact.ID, artifact.OriginNodeID, artifact.OriginArtifactID}
if fenceURL {
query += ` AND origin_node_url = $4`
args = append(args, artifact.OriginNodeURL)
}
tag, err := tx.Exec(ctx, query, args...)
if err != nil {
return nil, false, fmt.Errorf("requeuing remote artifact: %w", err)
}
if tag.RowsAffected() == 0 {
return nil, false, nil
}
rows, err := tx.Query(ctx,
`UPDATE downloads
SET status = 'preparing', bytes_sent = 0, completed_at = NULL,
error_message = '', updated_at = now()
WHERE artifact_id = $1 AND status NOT IN ('cancelled', 'failed', 'revoked')
RETURNING `+downloadColumns,
artifact.ID,
)
if err != nil {
return nil, false, fmt.Errorf("resetting linked downloads for remote artifact requeue: %w", err)
}
linked, err = scanDownloads(rows)
rows.Close()
if err != nil {
return nil, false, fmt.Errorf("scanning reset downloads for remote artifact requeue: %w", err)
}
if _, err := tx.Exec(ctx,
`INSERT INTO download_artifact_orphans (download_artifact_id, origin_node_id, origin_node_url, origin_artifact_id)
VALUES ($1, $2, $3, $4)
ON CONFLICT (origin_node_id, origin_artifact_id) DO UPDATE
SET download_artifact_id = EXCLUDED.download_artifact_id,
origin_node_url = EXCLUDED.origin_node_url, next_retry_at = NULL`,
artifact.ID, artifact.OriginNodeID, artifact.OriginNodeURL, artifact.OriginArtifactID,
); err != nil {
return nil, false, fmt.Errorf("enqueueing remote artifact cleanup during requeue: %w", err)
}
if err := tx.Commit(ctx); err != nil {
return nil, false, fmt.Errorf("committing remote artifact requeue: %w", err)
}
return linked, true, nil
}
// TouchLastUsed bumps last_used_at for LRU accounting (called on serve).
func (r *ArtifactRepository) TouchLastUsed(ctx context.Context, id string) error {
_, err := r.pool.Exec(ctx, `UPDATE download_artifacts SET last_used_at = now() WHERE id = $1`, id)
@@ -348,26 +259,6 @@ func (r *ArtifactRepository) TouchLastUsed(ctx context.Context, id string) error
return nil
}
// RefreshRemoteLocator persists an enabled node's current URL/group while
// fencing against a concurrent requeue or replacement locator.
func (r *ArtifactRepository) RefreshRemoteLocator(ctx context.Context, artifact *Artifact) (bool, error) {
if artifact == nil {
return false, nil
}
tag, err := r.pool.Exec(ctx,
`UPDATE download_artifacts
SET origin_node_url = $2, origin_node_group = $3
WHERE id = $1 AND status = 'ready'
AND origin_node_id = $4 AND origin_artifact_id = $5`,
artifact.ID, artifact.OriginNodeURL, artifact.OriginNodeGroup,
artifact.OriginNodeID, artifact.OriginArtifactID,
)
if err != nil {
return false, fmt.Errorf("refreshing remote artifact locator: %w", err)
}
return tag.RowsAffected() == 1, nil
}
// ListReady returns ready artifacts ordered by least-recently-used first.
func (r *ArtifactRepository) ListReady(ctx context.Context) ([]*Artifact, error) {
rows, err := r.pool.Query(ctx,
@@ -455,111 +346,3 @@ func (r *ArtifactRepository) DeleteArtifact(ctx context.Context, id string) erro
}
return nil
}
// EnqueueRemoteOrphan durably records an abandoned attempt before its owning
// artifact lease or locator is discarded.
func (r *ArtifactRepository) EnqueueRemoteOrphan(ctx context.Context, downloadArtifactID string, nodeID int, nodeURL, artifactID string) error {
_, err := r.pool.Exec(ctx,
`INSERT INTO download_artifact_orphans (download_artifact_id, origin_node_id, origin_node_url, origin_artifact_id, next_retry_at)
VALUES ($1, $2, $3, $4, now() + interval '30 seconds')
ON CONFLICT (origin_node_id, origin_artifact_id) DO UPDATE
SET download_artifact_id = EXCLUDED.download_artifact_id,
origin_node_url = EXCLUDED.origin_node_url,
next_retry_at = EXCLUDED.next_retry_at`,
downloadArtifactID, nodeID, nodeURL, artifactID,
)
if err != nil {
return fmt.Errorf("enqueueing remote artifact cleanup: %w", err)
}
return nil
}
// ListRemoteOrphansDue interleaves due candidates across origins so a large
// unreachable-node backlog cannot starve cleanup for healthy origins while a
// single healthy origin can still use the full batch.
func (r *ArtifactRepository) ListRemoteOrphansDue(ctx context.Context, limit int) ([]RemoteArtifactOrphan, error) {
if limit <= 0 {
limit = 100
}
rows, err := r.pool.Query(ctx,
`WITH due AS (
SELECT id, download_artifact_id, origin_node_id, origin_node_url, origin_artifact_id, attempts,
row_number() OVER (
PARTITION BY origin_node_id, origin_node_url
ORDER BY attempts, created_at, id
) AS origin_rank,
created_at
FROM download_artifact_orphans
WHERE next_retry_at IS NULL OR next_retry_at <= now()
)
SELECT id, download_artifact_id, origin_node_id, origin_node_url, origin_artifact_id, attempts
FROM due
ORDER BY origin_rank, attempts, created_at, id
LIMIT $1`, limit)
if err != nil {
return nil, fmt.Errorf("listing remote artifact cleanup queue: %w", err)
}
defer rows.Close()
out := make([]RemoteArtifactOrphan, 0, limit)
for rows.Next() {
var orphan RemoteArtifactOrphan
if err := rows.Scan(&orphan.ID, &orphan.DownloadArtifactID, &orphan.OriginNodeID, &orphan.OriginNodeURL, &orphan.OriginArtifactID, &orphan.Attempts); err != nil {
return nil, fmt.Errorf("scanning remote artifact cleanup row: %w", err)
}
out = append(out, orphan)
}
return out, rows.Err()
}
// PrepareRemoteOrphanCleanup serializes cleanup against requeue and verifies
// that a locator from an indeterminate MarkReady result did not actually win
// the artifact row. owned locators have their stale cleanup row removed;
// abandoned locators are retry-leased before the caller performs remote I/O.
func (r *ArtifactRepository) PrepareRemoteOrphanCleanup(ctx context.Context, orphan RemoteArtifactOrphan) (owned, claimed bool, err error) {
tx, err := r.pool.Begin(ctx)
if err != nil {
return false, false, fmt.Errorf("beginning remote artifact cleanup claim: %w", err)
}
defer func() { _ = tx.Rollback(ctx) }()
var originNodeID int
var originArtifactID string
err = tx.QueryRow(ctx,
`SELECT origin_node_id, origin_artifact_id
FROM download_artifacts WHERE id = $1 FOR UPDATE`,
orphan.DownloadArtifactID,
).Scan(&originNodeID, &originArtifactID)
if err != nil && !errors.Is(err, pgx.ErrNoRows) {
return false, false, fmt.Errorf("checking remote artifact ownership: %w", err)
}
var present int64
if err := tx.QueryRow(ctx, `SELECT id FROM download_artifact_orphans WHERE id = $1 FOR UPDATE`, orphan.ID).Scan(&present); err != nil {
if errors.Is(err, pgx.ErrNoRows) {
return false, false, nil
}
return false, false, fmt.Errorf("locking remote artifact cleanup row: %w", err)
}
owned = originNodeID == orphan.OriginNodeID && originArtifactID == orphan.OriginArtifactID
if owned {
if _, err := tx.Exec(ctx, `DELETE FROM download_artifact_orphans WHERE id = $1`, orphan.ID); err != nil {
return false, false, fmt.Errorf("clearing owned remote artifact cleanup row: %w", err)
}
} else {
if _, err := tx.Exec(ctx,
`UPDATE download_artifact_orphans
SET attempts = attempts + 1, next_retry_at = now() + interval '2 minutes'
WHERE id = $1`, orphan.ID); err != nil {
return false, false, fmt.Errorf("claiming remote artifact cleanup row: %w", err)
}
}
if err := tx.Commit(ctx); err != nil {
return false, false, fmt.Errorf("committing remote artifact cleanup claim: %w", err)
}
return owned, true, nil
}
func (r *ArtifactRepository) DeleteRemoteOrphan(ctx context.Context, id int64) error {
if _, err := r.pool.Exec(ctx, `DELETE FROM download_artifact_orphans WHERE id = $1`, id); err != nil {
return fmt.Errorf("deleting remote artifact cleanup row: %w", err)
}
return nil
}
+3 -521
View File
@@ -9,9 +9,7 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"github.com/Silo-Server/silo-server/internal/downloadprepare"
"github.com/Silo-Server/silo-server/internal/idgen"
"github.com/Silo-Server/silo-server/internal/models"
)
func newArtifactTestRepo(t *testing.T) (*ArtifactRepository, *pgxpool.Pool, int) {
@@ -25,12 +23,6 @@ func newArtifactTestRepo(t *testing.T) (*ArtifactRepository, *pgxpool.Pool, int)
if present == nil {
t.Skip("download_artifacts migration has not been applied")
}
if err := pool.QueryRow(ctx, `SELECT to_regclass('public.download_artifact_orphans')::text`).Scan(&present); err != nil {
t.Fatalf("check download_artifact_orphans: %v", err)
}
if present == nil {
t.Skip("download_artifact_orphans migration has not been applied")
}
suffix := time.Now().UnixNano()
var folderID int
@@ -68,16 +60,6 @@ func newArtifact(t *testing.T, fileID int, hash string) *Artifact {
}
}
func remoteOrphansForArtifact(orphans []RemoteArtifactOrphan, artifactID string) []RemoteArtifactOrphan {
filtered := make([]RemoteArtifactOrphan, 0, 1)
for _, orphan := range orphans {
if orphan.DownloadArtifactID == artifactID {
filtered = append(filtered, orphan)
}
}
return filtered
}
// TestArtifactQueueClaimAndLeaseRecovery is the Phase 3 / invariant-3 acceptance
// test: a crash mid-encode (an expired lease) is recovered on the next sweep so
// the job re-enqueues and reaches ready, and concurrent workers never claim the
@@ -128,7 +110,7 @@ func TestArtifactQueueClaimAndLeaseRecovery(t *testing.T) {
if err != nil || claim2.ID != row.ID || claim2.Attempts != 2 {
t.Fatalf("reclaim ClaimNext = (%+v, %v), want attempts=2", claim2, err)
}
if applied, err := repo.MarkReady(ctx, row.ID, "worker-2", claim2.OutputPath, 0, "", "", "", 4242); err != nil || !applied {
if applied, err := repo.MarkReady(ctx, row.ID, "worker-2", claim2.OutputPath, 4242); err != nil || !applied {
t.Fatalf("MarkReady = (%v, %v), want (true, nil)", applied, err)
}
done, err := repo.GetByKey(ctx, fileID, "transcode", "hash-recovery")
@@ -199,7 +181,7 @@ func TestArtifactMarkFencedByOwner(t *testing.T) {
}
// A non-owner cannot mark the job ready or failed.
if applied, err := repo.MarkReady(ctx, row.ID, "owner-2", "/tmp/x.mp4", 0, "", "", "", 10); err != nil || applied {
if applied, err := repo.MarkReady(ctx, row.ID, "owner-2", "/tmp/x.mp4", 10); err != nil || applied {
t.Fatalf("MarkReady(non-owner) = (%v, %v), want (false, nil)", applied, err)
}
if _, applied, err := repo.MarkFailedOrRetry(ctx, row.ID, "owner-2", "boom", time.Second); err != nil || applied {
@@ -213,511 +195,11 @@ func TestArtifactMarkFencedByOwner(t *testing.T) {
}
// The real owner succeeds.
if applied, err := repo.MarkReady(ctx, row.ID, "owner-1", "/tmp/x.mp4", 0, "", "", "", 10); err != nil || !applied {
if applied, err := repo.MarkReady(ctx, row.ID, "owner-1", "/tmp/x.mp4", 10); err != nil || !applied {
t.Fatalf("MarkReady(owner) = (%v, %v), want (true, nil)", applied, err)
}
}
func TestArtifactRemoteLocatorRoundTripsAndRequeueClearsIt(t *testing.T) {
repo, _, fileID := newArtifactTestRepo(t)
ctx := context.Background()
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, "hash-remote-locator"))
if err != nil {
t.Fatal(err)
}
if _, err := repo.ClaimNext(ctx, "worker", time.Minute); err != nil {
t.Fatal(err)
}
if applied, err := repo.MarkReady(ctx, row.ID, "worker", "", 17, "http://transcode", "host-a", "artifact-opaque", 4242); err != nil || !applied {
t.Fatalf("MarkReady = (%v, %v)", applied, err)
}
ready, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
if ready.OutputPath != "" || ready.OriginNodeID != 17 || ready.OriginNodeURL != "http://transcode" || ready.OriginNodeGroup != "host-a" || ready.OriginArtifactID != "artifact-opaque" || ready.FileSize != 4242 {
t.Fatalf("ready artifact = %+v", ready)
}
ready.OriginNodeURL = "http://transcode-new"
ready.OriginNodeGroup = "host-new"
if applied, err := repo.RefreshRemoteLocator(ctx, ready); err != nil || !applied {
t.Fatalf("RefreshRemoteLocator = (%v, %v)", applied, err)
}
refreshed, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
if refreshed.OriginNodeURL != "http://transcode-new" || refreshed.OriginNodeGroup != "host-new" {
t.Fatalf("refreshed artifact = %+v", refreshed)
}
if err := repo.Requeue(ctx, row.ID); err != nil {
t.Fatal(err)
}
queued, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
if queued.OriginNodeID != 0 || queued.OriginNodeURL != "" || queued.OriginNodeGroup != "" || queued.OriginArtifactID != "" {
t.Fatalf("requeued artifact retained remote locator: %+v", queued)
}
}
func TestArtifactReadyPersistsRefreshedOriginLocator(t *testing.T) {
repo, _, fileID := newArtifactTestRepo(t)
ctx := context.Background()
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, "hash-refresh-origin-locator"))
if err != nil {
t.Fatal(err)
}
if _, err := repo.ClaimNext(ctx, "worker", time.Minute); err != nil {
t.Fatal(err)
}
if applied, err := repo.MarkReady(ctx, row.ID, "worker", "", 17, "http://old-url", "old-group", "artifact-refresh", 4242); err != nil || !applied {
t.Fatalf("MarkReady = (%v, %v)", applied, err)
}
manager := &ArtifactManager{
repo: repo,
preparer: &lifecycleTestPreparer{
resolvedURL: "http://new-url", resolvedGroup: "new-group",
},
}
resolved, err := manager.Ready(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
if resolved.OriginNodeURL != "http://new-url" || resolved.OriginNodeGroup != "new-group" {
t.Fatalf("resolved artifact = %+v", resolved)
}
persisted, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
if persisted.OriginNodeURL != "http://new-url" || persisted.OriginNodeGroup != "new-group" {
t.Fatalf("persisted artifact = %+v", persisted)
}
}
func TestArtifactReadyMapsRemovedOriginToInactiveWhenRequeueLosesFence(t *testing.T) {
repo, _, fileID := newArtifactTestRepo(t)
ctx := context.Background()
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, "hash-removed-origin-stale-fence"))
if err != nil {
t.Fatal(err)
}
if _, err := repo.ClaimNext(ctx, "worker", time.Minute); err != nil {
t.Fatal(err)
}
if applied, err := repo.MarkReady(ctx, row.ID, "worker", "", 17, "http://removed", "host-a", "artifact-removed", 4242); err != nil || !applied {
t.Fatalf("MarkReady = (%v, %v)", applied, err)
}
manager := &ArtifactManager{
repo: repo,
preparer: &lifecycleTestPreparer{
resolvedNode: 18,
resolveErr: ErrArtifactOriginRemoved,
},
}
_, err = manager.Ready(ctx, row.ID)
if !errors.Is(err, ErrDownloadNotActive) || !errors.Is(err, ErrArtifactOriginRemoved) {
t.Fatalf("Ready error = %v, want inactive removed-origin error", err)
}
}
func TestArtifactRecoveryContinuesAfterStaleLocatorRefresh(t *testing.T) {
repo, pool, fileID := newArtifactTestRepo(t)
ctx := context.Background()
ready := make([]*Artifact, 0, 2)
for i := range 2 {
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, fmt.Sprintf("hash-stale-locator-batch-%d", i)))
if err != nil {
t.Fatal(err)
}
if _, err := repo.ClaimNext(ctx, "worker", time.Minute); err != nil {
t.Fatal(err)
}
if applied, err := repo.MarkReady(ctx, row.ID, "worker", "", 17, "http://old-url", "host-a", fmt.Sprintf("artifact-%d", i), 4242); err != nil || !applied {
t.Fatalf("MarkReady(%d) = (%v, %v)", i, applied, err)
}
artifact, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
ready = append(ready, artifact)
}
// Simulate another worker replacing the first locator after this recovery
// pass read it. Its refresh must lose the fence without suppressing checks
// for the remaining artifacts on the still-healthy origin.
if _, err := pool.Exec(ctx,
`UPDATE download_artifacts SET origin_artifact_id = 'artifact-replaced' WHERE id = $1`,
ready[0].ID,
); err != nil {
t.Fatal(err)
}
preparer := &lifecycleTestPreparer{
resolvedURL: "http://new-url", resolvedGroup: "host-new",
stat: downloadprepare.Result{FileSize: 4242},
}
manager := &ArtifactManager{repo: repo, preparer: preparer}
manager.probeRemoteArtifactGroup(ctx, preparer, ready)
if len(preparer.statIDs) != 1 || preparer.statIDs[0] != ready[1].ID {
t.Fatalf("probed artifacts = %v, want only continuing artifact %s", preparer.statIDs, ready[1].ID)
}
persisted, err := repo.GetByID(ctx, ready[1].ID)
if err != nil {
t.Fatal(err)
}
if persisted.OriginNodeURL != "http://new-url" || persisted.OriginNodeGroup != "host-new" {
t.Fatalf("continued artifact locator = %+v", persisted)
}
}
func TestArtifactInvalidRemoteResultQueuesCleanup(t *testing.T) {
repo, pool, fileID := newArtifactTestRepo(t)
ctx := context.Background()
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, "hash-invalid-remote-result"))
if err != nil {
t.Fatal(err)
}
claimed, err := repo.ClaimNext(ctx, "worker", time.Minute)
if err != nil {
t.Fatal(err)
}
preparer := &lifecycleTestPreparer{prepared: PreparedArtifact{
OriginNodeID: 17, OriginNodeURL: "http://transcode", OriginNodeGroup: "host-a",
OriginArtifactID: "artifact-zero-byte", FileSize: 0,
}}
manager := &ArtifactManager{
repo: repo, owner: "worker", fileRepo: fakeFileResolver{file: &models.MediaFile{ID: fileID, FilePath: "/media/movie.mkv"}},
preparer: preparer,
}
manager.encodeOne(ctx, claimed)
queued, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
if queued.Status != ArtifactQueued {
t.Fatalf("artifact status = %q, want queued retry", queued.Status)
}
var orphanID int64
if err := pool.QueryRow(ctx,
`SELECT id FROM download_artifact_orphans
WHERE download_artifact_id = $1 AND origin_node_id = 17 AND origin_artifact_id = 'artifact-zero-byte'`,
row.ID,
).Scan(&orphanID); err != nil {
t.Fatalf("remote cleanup row: %v", err)
}
t.Cleanup(func() { _ = repo.DeleteRemoteOrphan(ctx, orphanID) })
}
func TestArtifactRecoveryDeletesWrongSizedRemoteBeforeRequeue(t *testing.T) {
repo, pool, fileID := newArtifactTestRepo(t)
ctx := context.Background()
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, "hash-remote-size-mismatch"))
if err != nil {
t.Fatal(err)
}
if _, err := repo.ClaimNext(ctx, "worker", time.Minute); err != nil {
t.Fatal(err)
}
if applied, err := repo.MarkReady(ctx, row.ID, "worker", row.OutputPath, 17, "http://transcode", "host-a", "artifact-truncated", 4242); err != nil || !applied {
t.Fatalf("MarkReady = (%v, %v)", applied, err)
}
preparer := &lifecycleTestPreparer{stat: downloadprepare.Result{ArtifactID: "artifact-truncated", FileSize: 42}}
manager := &ArtifactManager{
repo: repo,
downloads: NewRepository(pool),
preparer: preparer,
}
manager.recover(ctx)
got, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
if got.Status != ArtifactQueued || got.OriginArtifactID != "" {
t.Fatalf("recovered artifact = %+v, want queued without remote locator", got)
}
orphans, err := repo.ListRemoteOrphansDue(ctx, 10)
if err != nil {
t.Fatal(err)
}
orphans = remoteOrphansForArtifact(orphans, row.ID)
if len(orphans) != 1 || orphans[0].OriginNodeID != 17 || orphans[0].OriginArtifactID != "artifact-truncated" {
t.Fatalf("remote cleanup queue = %+v", orphans)
}
if preparer.deleted != "artifact-truncated" {
t.Fatalf("deleted remote artifact = %q, want artifact-truncated", preparer.deleted)
}
t.Cleanup(func() { _ = repo.DeleteRemoteOrphan(ctx, orphans[0].ID) })
}
func TestRemoteRecoveryBudgetBoundsUnreachableOrigins(t *testing.T) {
repo, pool, fileID := newArtifactTestRepo(t)
ctx := context.Background()
for i := range 5 {
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, fmt.Sprintf("hash-recovery-budget-%d-%d", time.Now().UnixNano(), i)))
if err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx,
`UPDATE download_artifacts
SET status = 'ready', file_size = 42, completed_at = now(),
origin_node_id = $2, origin_node_url = $3, origin_artifact_id = $4
WHERE id = $1`,
row.ID, 32000+i, fmt.Sprintf("http://unreachable-recovery-%d", i), fmt.Sprintf("artifact-blocked-%d", i),
); err != nil {
t.Fatal(err)
}
}
preparer := &lifecycleTestPreparer{statWait: true}
manager := &ArtifactManager{repo: repo, preparer: preparer, remoteRecoveryBudget: 50 * time.Millisecond}
begin := time.Now()
manager.recoverReadyArtifacts(ctx)
if elapsed := time.Since(begin); elapsed > time.Second {
t.Fatalf("recovery elapsed = %s, want bounded return", elapsed)
}
if started := preparer.statStarted.Load(); started == 0 || started > 4 {
t.Fatalf("started probes = %d, want one bounded wave of at most four", started)
}
}
func TestArtifactRemoteRequeueAtomicallyQueuesCleanup(t *testing.T) {
repo, pool, fileID := newArtifactTestRepo(t)
ctx := context.Background()
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, fmt.Sprintf("hash-remote-atomic-requeue-%d", time.Now().UnixNano())))
if err != nil {
t.Fatal(err)
}
if _, err := repo.ClaimNext(ctx, "worker", time.Minute); err != nil {
t.Fatal(err)
}
if applied, err := repo.MarkReady(ctx, row.ID, "worker", row.OutputPath, 23, "http://transcode-old", "host-a", "artifact-abandoned", 4242); err != nil || !applied {
t.Fatalf("MarkReady = (%v, %v)", applied, err)
}
ready, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
var userID int
if err := pool.QueryRow(ctx,
`INSERT INTO users (username, role, download_allowed) VALUES ($1, 'user', true) RETURNING id`,
fmt.Sprintf("requeue-user-%d", time.Now().UnixNano()),
).Scan(&userID); err != nil {
t.Fatal(err)
}
downloadID := fmt.Sprintf("requeue-download-%d", time.Now().UnixNano())
if err := NewRepository(pool).Create(ctx, &Download{
ID: downloadID, UserID: userID, MediaFileID: fileID,
ContentID: "requeue-content", Kind: KindQueued, Status: StatusCompleted,
Format: FormatTranscode, ArtifactID: row.ID, FileSize: ready.FileSize,
CreatedAt: time.Now(), UpdatedAt: time.Now(),
}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = pool.Exec(ctx, `DELETE FROM downloads WHERE id = $1`, downloadID)
_, _ = pool.Exec(ctx, `DELETE FROM users WHERE id = $1`, userID)
})
// An indeterminate MarkReady result may enqueue the locator even though its
// database write committed. Cleanup must recognize the winning locator and
// remove only the stale queue row, never the node-local bytes.
if err := repo.EnqueueRemoteOrphan(ctx, row.ID, ready.OriginNodeID, ready.OriginNodeURL, ready.OriginArtifactID); err != nil {
t.Fatal(err)
}
if _, err := pool.Exec(ctx, `UPDATE download_artifact_orphans SET next_retry_at = NULL WHERE origin_node_id = $1 AND origin_artifact_id = $2`, ready.OriginNodeID, ready.OriginArtifactID); err != nil {
t.Fatal(err)
}
orphans, err := repo.ListRemoteOrphansDue(ctx, 10)
orphans = remoteOrphansForArtifact(orphans, row.ID)
if err != nil || len(orphans) != 1 {
t.Fatalf("uncertain cleanup queue = %+v (%v)", orphans, err)
}
owned, claimed, err := repo.PrepareRemoteOrphanCleanup(ctx, orphans[0])
if err != nil || !claimed || !owned {
t.Fatalf("PrepareRemoteOrphanCleanup = (owned=%v claimed=%v err=%v)", owned, claimed, err)
}
if left, err := repo.ListRemoteOrphansDue(ctx, 10); err != nil || len(remoteOrphansForArtifact(left, row.ID)) != 0 {
t.Fatalf("owned cleanup rows left = %+v (%v)", left, err)
}
// The in-memory URL may already have followed an administrative edit; the
// transaction deliberately matches stable node/id fields and stores the
// refreshed URL as the cleanup target.
ready.OriginNodeURL = "http://transcode-new"
linked, applied, err := repo.RequeueRemote(ctx, ready)
if err != nil || !applied {
t.Fatalf("RequeueRemote = (%v, %v)", applied, err)
}
if len(linked) != 1 || linked[0].ID != downloadID || linked[0].Status != StatusPreparing {
t.Fatalf("reset linked downloads = %+v", linked)
}
queued, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
if queued.Status != ArtifactQueued || queued.OriginArtifactID != "" {
t.Fatalf("queued artifact = %+v", queued)
}
orphans, err = repo.ListRemoteOrphansDue(ctx, 10)
if err != nil {
t.Fatal(err)
}
orphans = remoteOrphansForArtifact(orphans, row.ID)
if len(orphans) != 1 || orphans[0].DownloadArtifactID != row.ID || orphans[0].OriginNodeURL != "http://transcode-new" || orphans[0].OriginArtifactID != "artifact-abandoned" {
t.Fatalf("remote cleanup queue = %+v", orphans)
}
t.Cleanup(func() { _ = repo.DeleteRemoteOrphan(ctx, orphans[0].ID) })
// A stale caller cannot enqueue a second cleanup after the row changed.
if _, applied, err := repo.RequeueRemote(ctx, ready); err != nil || applied {
t.Fatalf("stale RequeueRemote = (%v, %v), want false, nil", applied, err)
}
}
func TestProxyMissingReportFencesCompleteRemoteLocator(t *testing.T) {
repo, _, fileID := newArtifactTestRepo(t)
ctx := context.Background()
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, fmt.Sprintf("hash-proxy-missing-report-%d", time.Now().UnixNano())))
if err != nil {
t.Fatal(err)
}
if _, err := repo.ClaimNext(ctx, "worker", time.Minute); err != nil {
t.Fatal(err)
}
if applied, err := repo.MarkReady(ctx, row.ID, "worker", "", 23, "http://transcode-current", "host-a", "artifact-missing", 4242); err != nil || !applied {
t.Fatalf("MarkReady = (%v, %v)", applied, err)
}
manager := NewArtifactManager(repo, nil, nil, nil, "proxy-test", nil, nil)
stillReady, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
stale := *stillReady
stale.OriginNodeURL = "http://transcode-stale"
if applied, err := manager.requeueRemoteArtifactExactNow(ctx, &stale, "stale API miss"); err != nil || applied {
t.Fatalf("stale exact requeue = (%v, %v), want false, nil", applied, err)
}
if err := manager.ReportRemoteArtifactMissing(ctx, row.ID, "http://transcode-stale", "artifact-missing"); err != nil {
t.Fatal(err)
}
stillReady, err = repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
if stillReady.Status != ArtifactReady {
t.Fatalf("stale report changed artifact = %+v", stillReady)
}
if err := manager.ReportRemoteArtifactMissing(ctx, row.ID, "http://transcode-current", "artifact-missing"); err != nil {
t.Fatal(err)
}
queued, err := repo.GetByID(ctx, row.ID)
if err != nil {
t.Fatal(err)
}
if queued.Status != ArtifactQueued || queued.OriginArtifactID != "" {
t.Fatalf("artifact after authoritative missing report = %+v", queued)
}
orphans, err := repo.ListRemoteOrphansDue(ctx, 10)
if err != nil {
t.Fatal(err)
}
orphans = remoteOrphansForArtifact(orphans, row.ID)
if len(orphans) != 1 || orphans[0].DownloadArtifactID != row.ID || orphans[0].OriginArtifactID != "artifact-missing" {
t.Fatalf("remote cleanup queue = %+v", orphans)
}
t.Cleanup(func() { _ = repo.DeleteRemoteOrphan(ctx, orphans[0].ID) })
}
func TestRemoteCleanupBudgetBoundsUnreachableOrigins(t *testing.T) {
repo, _, fileID := newArtifactTestRepo(t)
ctx := context.Background()
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, fmt.Sprintf("hash-cleanup-budget-%d", time.Now().UnixNano())))
if err != nil {
t.Fatal(err)
}
suffix := time.Now().UnixNano()
originArtifactID := fmt.Sprintf("artifact-blocked-%d", suffix)
originURL := fmt.Sprintf("http://unreachable-%d", suffix)
if err := repo.EnqueueRemoteOrphan(ctx, row.ID, 31, originURL, originArtifactID); err != nil {
t.Fatal(err)
}
started := make(chan struct{})
preparer := &lifecycleTestPreparer{deleteStarted: started, deleteWait: true}
manager := &ArtifactManager{repo: repo, preparer: preparer, remoteCleanupBudget: 50 * time.Millisecond}
begin := time.Now()
manager.cleanupRemoteOrphans(ctx)
if elapsed := time.Since(begin); elapsed > time.Second {
t.Fatalf("cleanup elapsed = %s, want bounded return", elapsed)
}
select {
case <-started:
default:
t.Fatal("remote deletion was not attempted")
}
orphans, err := repo.ListRemoteOrphansDue(ctx, 100)
if err != nil {
t.Fatal(err)
}
for _, orphan := range orphans {
if orphan.DownloadArtifactID == row.ID {
t.Fatalf("timed-out orphan remained immediately due: %+v", orphan)
}
}
}
func TestListRemoteOrphansDueIsFairAcrossOrigins(t *testing.T) {
repo, pool, fileID := newArtifactTestRepo(t)
ctx := context.Background()
row, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, fmt.Sprintf("hash-orphan-fairness-%d", time.Now().UnixNano())))
if err != nil {
t.Fatal(err)
}
suffix := time.Now().UnixNano()
artifactIDs := []string{
fmt.Sprintf("fair-origin-a-first-%d", suffix),
fmt.Sprintf("fair-origin-a-second-%d", suffix),
fmt.Sprintf("fair-origin-b-%d", suffix),
}
for i, artifactID := range artifactIDs {
nodeID := 31001
nodeURL := "http://origin-a"
if i == 2 {
nodeID = 31002
nodeURL = "http://origin-b"
}
if err := repo.EnqueueRemoteOrphan(ctx, row.ID, nodeID, nodeURL, artifactID); err != nil {
t.Fatal(err)
}
}
t.Cleanup(func() {
_, _ = pool.Exec(ctx, `DELETE FROM download_artifact_orphans WHERE origin_artifact_id = ANY($1)`, artifactIDs)
})
if _, err := pool.Exec(ctx,
`UPDATE download_artifact_orphans
SET next_retry_at = NULL, created_at = '-infinity'
WHERE origin_artifact_id = ANY($1)`,
artifactIDs,
); err != nil {
t.Fatal(err)
}
due, err := repo.ListRemoteOrphansDue(ctx, 2)
if err != nil {
t.Fatal(err)
}
seen := make(map[int]int)
for _, orphan := range due {
if orphan.OriginNodeID == 31001 || orphan.OriginNodeID == 31002 {
seen[orphan.OriginNodeID]++
}
}
if seen[31001] != 1 || seen[31002] != 1 {
t.Fatalf("due candidates by origin = %+v, want one candidate for each origin", seen)
}
}
// TestHasActiveLinkCoversEphemeralRows pins the eviction guard: an ephemeral
// (device-less web) download row must protect its artifact from LRU cleanup
// exactly like a managed row does, and terminal rows must not.
-80
View File
@@ -4,14 +4,11 @@ import (
"context"
"encoding/json"
"strings"
"sync/atomic"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/config"
"github.com/Silo-Server/silo-server/internal/downloadprepare"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/playback"
)
func TestParamsHashStableAndDistinct(t *testing.T) {
@@ -52,83 +49,6 @@ func TestArtifactOutputPathDeterministic(t *testing.T) {
}
}
type lifecycleTestPreparer struct {
deleted string
prepared PreparedArtifact
resolveErr error
resolvedNode int
resolvedURL string
resolvedGroup string
stat downloadprepare.Result
statError error
statIDs []string
statStarted atomic.Int32
statWait bool
deleteErr error
deleteStarted chan struct{}
deleteWait bool
}
func (p *lifecycleTestPreparer) PrepareFile(context.Context, string, playback.TranscodeOpts, string) (PreparedArtifact, error) {
return p.prepared, nil
}
func (p *lifecycleTestPreparer) ResolveArtifact(_ context.Context, artifact *Artifact) error {
if p.resolvedNode != 0 {
artifact.OriginNodeID = p.resolvedNode
}
if p.resolvedURL != "" {
artifact.OriginNodeURL = p.resolvedURL
artifact.OriginNodeGroup = p.resolvedGroup
}
return p.resolveErr
}
func (p *lifecycleTestPreparer) StatArtifact(ctx context.Context, artifact *Artifact) (downloadprepare.Result, error) {
if p.statWait {
p.statStarted.Add(1)
select {
case <-ctx.Done():
return downloadprepare.Result{}, ctx.Err()
case <-time.After(2 * time.Second):
return downloadprepare.Result{}, context.DeadlineExceeded
}
}
p.statIDs = append(p.statIDs, artifact.ID)
return p.stat, p.statError
}
func (p *lifecycleTestPreparer) DeleteArtifact(ctx context.Context, artifact *Artifact) error {
p.deleted = artifact.OriginArtifactID
if p.deleteStarted != nil {
close(p.deleteStarted)
}
if p.deleteWait {
<-ctx.Done()
return ctx.Err()
}
return p.deleteErr
}
func TestArtifactManagerDeletesRemoteBytesThroughOwningNode(t *testing.T) {
preparer := &lifecycleTestPreparer{}
m := &ArtifactManager{preparer: preparer}
artifact := &Artifact{ID: "row-1", OriginNodeURL: "http://transcode", OriginArtifactID: "opaque-1"}
if !m.deleteArtifactBytes(context.Background(), artifact) {
t.Fatal("remote cleanup failed")
}
if preparer.deleted != "opaque-1" {
t.Fatalf("deleted = %q", preparer.deleted)
}
}
func TestRemoteArtifactRequeueRejectsNonPositiveNodeID(t *testing.T) {
manager := &ArtifactManager{repo: &ArtifactRepository{}}
_, err := manager.requeueRemoteArtifactNow(context.Background(), &Artifact{
ID: "row-1", OriginNodeID: -1, OriginNodeURL: "http://transcode", OriginArtifactID: "opaque-1",
}, "invalid node")
if err == nil {
t.Fatal("expected invalid remote locator error")
}
}
func TestEffectiveArtifactDir(t *testing.T) {
// Explicit config wins verbatim.
if got := effectiveArtifactDir("/data/artifacts", "/tmp/silo-transcode"); got != "/data/artifacts" {
+48 -444
View File
@@ -6,14 +6,10 @@ import (
"fmt"
"log/slog"
"os"
"strings"
"sync"
"time"
"github.com/google/uuid"
"github.com/Silo-Server/silo-server/internal/config"
"github.com/Silo-Server/silo-server/internal/downloadprepare"
"github.com/Silo-Server/silo-server/internal/idgen"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/playback"
@@ -25,52 +21,21 @@ const (
artifactMaxAttempts = 3
)
// PreparedArtifact describes either a local prepared file or an opaque artifact
// retained by a transcode node.
type PreparedArtifact struct {
OutputPath string
OriginNodeID int
OriginNodeURL string
OriginNodeGroup string
OriginArtifactID string
FileSize int64
}
func (a PreparedArtifact) Remote() bool {
return a.OriginNodeID > 0 && a.OriginNodeURL != "" && a.OriginArtifactID != ""
}
// EncodePreparer produces a single finalized artifact. The default
// implementation calls playback.PrepareFile and returns a local path; pooled
// implementations may return an opaque remote locator instead.
// EncodePreparer produces a single finalized file for an artifact. The default
// implementation calls playback.PrepareFile; tests substitute a fake.
type EncodePreparer interface {
PrepareFile(ctx context.Context, artifactID string, opts playback.TranscodeOpts, outputPath string) (PreparedArtifact, error)
PrepareFile(ctx context.Context, opts playback.TranscodeOpts, outputPath string) error
}
type playbackPreparer struct{}
func (playbackPreparer) PrepareFile(ctx context.Context, _ string, opts playback.TranscodeOpts, outputPath string) (PreparedArtifact, error) {
if err := playback.PrepareFile(ctx, opts, outputPath); err != nil {
return PreparedArtifact{}, err
}
stat, err := os.Stat(outputPath)
if err != nil {
return PreparedArtifact{}, fmt.Errorf("stat prepared artifact: %w", err)
}
return PreparedArtifact{OutputPath: outputPath, FileSize: stat.Size()}, nil
func (playbackPreparer) PrepareFile(ctx context.Context, opts playback.TranscodeOpts, outputPath string) error {
return playback.PrepareFile(ctx, opts, outputPath)
}
// NewPlaybackPreparer returns the production EncodePreparer (ffmpeg-backed).
func NewPlaybackPreparer() EncodePreparer { return playbackPreparer{} }
// remoteArtifactLifecycle is implemented by the pooled preparer so recovery
// and quota cleanup can manage node-local artifacts without filesystem access.
type remoteArtifactLifecycle interface {
ResolveArtifact(ctx context.Context, artifact *Artifact) error
StatArtifact(ctx context.Context, artifact *Artifact) (downloadprepare.Result, error)
DeleteArtifact(ctx context.Context, artifact *Artifact) error
}
// ArtifactNotifier publishes an event when a linked download changes state.
type ArtifactNotifier func(ctx context.Context, d *Download)
@@ -78,15 +43,13 @@ type ArtifactNotifier func(ctx context.Context, d *Download)
// jobs, drains them through a bounded worker pool with leased heartbeats, and
// recovers stranded jobs on startup.
type ArtifactManager struct {
repo *ArtifactRepository
downloads *Repository
fileRepo FileResolver
preparer EncodePreparer
owner string
liveCfg func() *config.Config
notify ArtifactNotifier
remoteRecoveryBudget time.Duration
remoteCleanupBudget time.Duration
repo *ArtifactRepository
downloads *Repository
fileRepo FileResolver
preparer EncodePreparer
owner string
liveCfg func() *config.Config
notify ArtifactNotifier
mu sync.Mutex
kick func()
@@ -136,37 +99,6 @@ func NewArtifactManager(
}
}
// ReportRemoteArtifactMissing fences a proxy-observed 404 against the signed
// database-row and node locator, then atomically requeues the artifact and its
// linked downloads. Stale tokens are harmless: the exact-locator transition
// only applies while the complete locator still owns the ready row.
func (m *ArtifactManager) ReportRemoteArtifactMissing(ctx context.Context, artifactID, originNodeURL, originArtifactID string) error {
if m == nil || m.repo == nil || strings.TrimSpace(artifactID) == "" ||
strings.TrimSpace(originNodeURL) == "" || !downloadprepare.ValidArtifactID(originArtifactID) {
return errors.New("remote artifact locator unavailable for missing report")
}
artifact, err := m.repo.GetByID(ctx, artifactID)
if err != nil {
if errors.Is(err, ErrNotFound) {
return nil
}
return err
}
if artifact.Status != ArtifactReady || artifact.OriginNodeURL != originNodeURL || artifact.OriginArtifactID != originArtifactID {
return nil
}
linked, applied, err := m.repo.RequeueRemoteExactLocator(ctx, artifact)
if err != nil || !applied {
return err
}
for _, download := range linked {
m.publish(ctx, download)
}
slog.WarnContext(ctx, "remote download artifact re-queued", "component", "downloads", "artifact_id", artifact.ID, "node", artifact.OriginNodeURL, "reason", "proxy observed remote output missing")
m.triggerDrain()
return nil
}
// SetKick wires a low-latency drain trigger (e.g. taskmanager RunTask) invoked
// when a new job is enqueued.
func (m *ArtifactManager) SetKick(kick func()) {
@@ -185,17 +117,6 @@ func (m *ArtifactManager) Ready(ctx context.Context, id string) (*Artifact, erro
if a.Status != ArtifactReady {
return nil, fmt.Errorf("artifact is %s: %w", a.Status, ErrDownloadNotActive)
}
if lifecycle, ok := m.preparer.(remoteArtifactLifecycle); ok && a.OriginArtifactID != "" {
if err := m.resolveRemoteArtifact(ctx, lifecycle, a); err != nil {
if !errors.Is(err, ErrArtifactOriginRemoved) {
return nil, err
}
if m.requeueRemoteArtifact(ctx, a, "origin node removed") {
return nil, fmt.Errorf("artifact origin was removed and preparation was requeued: %w", ErrDownloadNotActive)
}
return nil, fmt.Errorf("artifact origin was removed: %w", errors.Join(ErrDownloadNotActive, err))
}
}
_ = m.repo.TouchLastUsed(ctx, id)
return a, nil
}
@@ -293,26 +214,20 @@ func (m *ArtifactManager) triggerDrain() {
}
}
// RunOnce repairs queue state and drains pending work before probing ready
// remote artifacts. Network health checks therefore never block newly queued
// downloads at startup; any artifacts requeued by the bounded probe pass are
// drained in the second pass.
// RunOnce performs a startup-safe recovery sweep and then drains the queue
// until empty. It is safe to call concurrently across nodes; FOR UPDATE SKIP
// LOCKED prevents double-encoding.
func (m *ArtifactManager) RunOnce(ctx context.Context) error {
m.recoverQueueState(ctx)
if err := m.drain(ctx); err != nil {
return err
}
m.recoverReadyArtifacts(ctx)
m.recover(ctx)
return m.drain(ctx)
}
// recover is retained as the focused recovery entrypoint used by tests.
// recover repairs state a crash or lost lease can leave behind: it reclaims
// expired-lease running rows (back to queued, or failed when attempts are
// exhausted), reconciles linked downloads against terminal artifact states, and
// re-queues ready artifacts whose output file is missing on disk. Safe to run
// repeatedly (each step is idempotent).
func (m *ArtifactManager) recover(ctx context.Context) {
m.recoverQueueState(ctx)
m.recoverReadyArtifacts(ctx)
}
func (m *ArtifactManager) recoverQueueState(ctx context.Context) {
if _, err := m.repo.ReclaimExpiredLeases(ctx); err != nil {
slog.WarnContext(ctx, "download artifact lease reclaim failed", "component", "downloads", "error", err)
}
@@ -333,13 +248,9 @@ func (m *ArtifactManager) recoverQueueState(ctx context.Context) {
m.publish(ctx, d)
}
}
}
// recoverReadyArtifacts checks local paths directly and groups remote probes
// by origin. Origins run concurrently, but each origin is short-circuited after
// its first transient failure so one blackholed node costs one bounded timeout,
// never one timeout per artifact.
func (m *ArtifactManager) recoverReadyArtifacts(ctx context.Context) {
// Disk-presence sweep: stats every ready file, so it runs on the startup
// pass and then hourly rather than on every tick.
if !m.maintenanceDue(&m.lastDiskSweep) {
return
}
@@ -348,172 +259,21 @@ func (m *ArtifactManager) recoverReadyArtifacts(ctx context.Context) {
slog.WarnContext(ctx, "download artifact ready scan failed", "component", "downloads", "error", err)
return
}
remoteGroups := make(map[string][]*Artifact)
for _, a := range ready {
if a.OriginArtifactID != "" {
key := fmt.Sprintf("%d\x00%s", a.OriginNodeID, a.OriginNodeURL)
remoteGroups[key] = append(remoteGroups[key], a)
if a.OutputPath == "" {
continue
}
if a.OutputPath != "" {
if _, statErr := os.Stat(a.OutputPath); statErr != nil {
slog.WarnContext(ctx, "download artifact output missing, re-queuing", "component", "downloads", "artifact_id", a.ID, "path", a.OutputPath)
if err := m.repo.Requeue(ctx, a.ID); err != nil {
slog.WarnContext(ctx, "re-queue artifact failed", "component", "downloads", "artifact_id", a.ID, "error", err)
}
}
}
}
if len(remoteGroups) == 0 {
return
}
lifecycle, ok := m.preparer.(remoteArtifactLifecycle)
if !ok {
slog.WarnContext(ctx, "remote download artifacts cannot be checked", "component", "downloads")
return
}
budget := m.remoteRecoveryBudget
if budget <= 0 {
budget = defaultRemoteRecoveryBudget
}
recoveryCtx, cancel := context.WithTimeout(ctx, budget)
defer cancel()
const maxConcurrentOriginProbes = 4
sem := make(chan struct{}, maxConcurrentOriginProbes)
var wg sync.WaitGroup
for _, artifacts := range remoteGroups {
artifacts := artifacts
wg.Add(1)
go func() {
defer wg.Done()
select {
case sem <- struct{}{}:
defer func() { <-sem }()
case <-recoveryCtx.Done():
return
}
m.probeRemoteArtifactGroup(recoveryCtx, lifecycle, artifacts)
}()
}
wg.Wait()
}
func (m *ArtifactManager) probeRemoteArtifactGroup(ctx context.Context, lifecycle remoteArtifactLifecycle, artifacts []*Artifact) {
for _, a := range artifacts {
if err := m.resolveRemoteArtifact(ctx, lifecycle, a); err != nil {
if errors.Is(err, ErrArtifactOriginRemoved) {
m.requeueRemoteArtifact(ctx, a, "origin node removed")
if _, statErr := os.Stat(a.OutputPath); statErr != nil {
slog.WarnContext(ctx, "download artifact output missing, re-queuing", "component", "downloads", "artifact_id", a.ID, "path", a.OutputPath)
if err := m.repo.Requeue(ctx, a.ID); err != nil {
slog.WarnContext(ctx, "re-queue artifact failed", "component", "downloads", "artifact_id", a.ID, "error", err)
continue
}
if errors.Is(err, ErrDownloadNotActive) {
// The row changed concurrently; the origin itself is still healthy.
continue
}
slog.WarnContext(ctx, "remote download artifact resolution failed; skipping remaining origin batch", "component", "downloads", "artifact_id", a.ID, "error", err)
return
}
result, err := lifecycle.StatArtifact(ctx, a)
switch {
case errors.Is(err, ErrArtifactOriginRemoved):
m.requeueRemoteArtifact(ctx, a, "origin node removed")
case errors.Is(err, downloadprepare.ErrArtifactNotFound):
m.requeueRemoteArtifact(ctx, a, "remote output missing")
case err != nil:
// One node-level failure suppresses the rest of this origin's batch.
slog.WarnContext(ctx, "remote download artifact check failed; skipping remaining origin batch", "component", "downloads", "artifact_id", a.ID, "node", a.OriginNodeURL, "error", err)
return
case result.FileSize != a.FileSize:
m.requeueWrongSizedRemoteArtifact(ctx, lifecycle, a)
m.triggerDrain()
}
}
}
// requeueWrongSizedRemoteArtifact commits the durable fence before deleting
// bytes, but delays replacement work until the known-invalid locator has been
// quarantined. If deletion fails, the transaction's orphan row retains the
// exact locator for the regular retrying cleanup pass.
func (m *ArtifactManager) requeueWrongSizedRemoteArtifact(ctx context.Context, lifecycle remoteArtifactLifecycle, a *Artifact) {
applied, err := m.requeueRemoteArtifactWithFence(ctx, a, "remote output size mismatch", false, false)
if err != nil {
slog.WarnContext(ctx, "re-queue remote artifact failed", "component", "downloads", "artifact_id", a.ID, "error", err)
return
}
if !applied {
return
}
if err := lifecycle.DeleteArtifact(ctx, a); err != nil {
slog.WarnContext(ctx, "deleting rejected remote artifact failed", "component", "downloads", "artifact_id", a.ID, "node", a.OriginNodeURL, "error", err)
}
m.triggerDrain()
}
func (m *ArtifactManager) resolveRemoteArtifact(ctx context.Context, lifecycle remoteArtifactLifecycle, artifact *Artifact) error {
oldURL, oldGroup := artifact.OriginNodeURL, artifact.OriginNodeGroup
if err := lifecycle.ResolveArtifact(ctx, artifact); err != nil {
return err
}
if artifact.OriginNodeURL == oldURL && artifact.OriginNodeGroup == oldGroup {
return nil
}
applied, err := m.repo.RefreshRemoteLocator(ctx, artifact)
if err != nil {
return err
}
if !applied {
return fmt.Errorf("remote artifact changed while refreshing its origin: %w", ErrDownloadNotActive)
}
return nil
}
func (m *ArtifactManager) requeueRemoteArtifact(ctx context.Context, a *Artifact, reason string) bool {
applied, err := m.requeueRemoteArtifactNow(ctx, a, reason)
if err != nil {
slog.WarnContext(ctx, "re-queue remote artifact failed", "component", "downloads", "artifact_id", a.ID, "error", err)
return false
}
return applied
}
// requeueRemoteArtifactNow synchronously fences a stale remote locator, resets
// every linked download, and schedules the abandoned node-local file for
// cleanup. Request paths use the returned error so a failed state transition is
// never disguised as an ordinary missing catalog item.
func (m *ArtifactManager) requeueRemoteArtifactNow(ctx context.Context, a *Artifact, reason string) (bool, error) {
return m.requeueRemoteArtifactWithFence(ctx, a, reason, false, true)
}
func (m *ArtifactManager) requeueRemoteArtifactExactNow(ctx context.Context, a *Artifact, reason string) (bool, error) {
return m.requeueRemoteArtifactWithFence(ctx, a, reason, true, true)
}
func (m *ArtifactManager) requeueRemoteArtifactWithFence(ctx context.Context, a *Artifact, reason string, exactURL, triggerDrain bool) (bool, error) {
if m == nil || m.repo == nil || a == nil || a.ID == "" || a.OriginNodeID <= 0 || a.OriginArtifactID == "" {
return false, errors.New("remote artifact locator unavailable for requeue")
}
var linked []*Download
var applied bool
var err error
if exactURL {
linked, applied, err = m.repo.RequeueRemoteExactLocator(ctx, a)
} else {
linked, applied, err = m.repo.RequeueRemote(ctx, a)
}
if err != nil {
return false, err
}
if !applied {
return false, nil
}
for _, download := range linked {
m.publish(ctx, download)
}
slog.WarnContext(ctx, "remote download artifact re-queued", "component", "downloads", "artifact_id", a.ID, "node", a.OriginNodeURL, "reason", reason)
if triggerDrain {
m.triggerDrain()
}
return true, nil
}
// drain claims and encodes jobs through a bounded worker pool until the queue is
// empty or the context is canceled.
func (m *ArtifactManager) drain(ctx context.Context) error {
@@ -572,15 +332,7 @@ func (m *ArtifactManager) encodeOne(ctx context.Context, a *Artifact) {
}
opts := m.buildOpts(file, a)
// Each lease attempt owns a distinct node-local object. A worker that loses
// the readiness fence can therefore queue its object for deletion without
// racing the replacement worker's output on the same node.
remoteAttemptID := a.ID + "-" + uuid.NewString()
prepared, err := m.preparer.PrepareFile(hbCtx, remoteAttemptID, opts, a.OutputPath)
if err != nil {
if prepared.Remote() {
m.enqueueRemoteCleanup(ctx, a.ID, prepared, true)
}
if err := m.preparer.PrepareFile(hbCtx, opts, a.OutputPath); err != nil {
switch {
case ctx.Err() != nil:
// Parent shutting down: leave the job 'running'; its lease expires and
@@ -597,35 +349,20 @@ func (m *ArtifactManager) encodeOne(ctx context.Context, a *Artifact) {
}
}
remoteFieldsPresent := prepared.OriginNodeID != 0 || prepared.OriginNodeURL != "" || prepared.OriginNodeGroup != "" || prepared.OriginArtifactID != ""
if prepared.FileSize <= 0 || (remoteFieldsPresent && !prepared.Remote()) || (!prepared.Remote() && prepared.OutputPath == "") {
msg := "prepared artifact returned an invalid storage locator"
slog.WarnContext(ctx, "prepared artifact returned an invalid storage locator", "component", "downloads", "artifact_id", a.ID)
if prepared.Remote() {
m.enqueueRemoteCleanup(ctx, a.ID, prepared, true)
}
m.failJob(ctx, a, msg)
return
var size int64
if fi, statErr := os.Stat(a.OutputPath); statErr == nil {
size = fi.Size()
}
size := prepared.FileSize
// Fenced on lease ownership: if we lost the lease between encode and commit,
// applied is false and the current owner is responsible for flipping links —
// do not flip them here or we would race/duplicate that owner's work.
outputPath := prepared.OutputPath
if prepared.Remote() {
// Keep the deterministic API-local fallback path in the row so a later
// remote-missing requeue can safely fall back to integrated preparation.
outputPath = a.OutputPath
}
applied, err := m.repo.MarkReady(ctx, a.ID, m.owner, outputPath, prepared.OriginNodeID, prepared.OriginNodeURL, prepared.OriginNodeGroup, prepared.OriginArtifactID, size)
applied, err := m.repo.MarkReady(ctx, a.ID, m.owner, a.OutputPath, size)
if err != nil {
slog.ErrorContext(ctx, "marking artifact ready failed", "component", "downloads", "artifact_id", a.ID, "error", err)
m.cleanupRejectedPrepared(ctx, a.ID, prepared)
return
}
if !applied {
slog.WarnContext(ctx, "download artifact ready skipped; lease lost", "component", "downloads", "artifact_id", a.ID)
m.enqueueRemoteCleanup(ctx, a.ID, prepared, true)
return
}
flipped, err := m.downloads.MarkLinkedDownloadsReady(ctx, a.ID, size)
@@ -638,54 +375,6 @@ func (m *ArtifactManager) encodeOne(ctx context.Context, a *Artifact) {
}
}
func (m *ArtifactManager) cleanupRejectedPrepared(ctx context.Context, artifactID string, prepared PreparedArtifact) {
if !prepared.Remote() {
return
}
// Exec can report a transport error after PostgreSQL committed. Re-read the
// row before classifying this attempt as abandoned so cleanup never deletes
// the locator that actually won the readiness fence.
current, err := m.repo.GetByID(ctx, artifactID)
if err == nil {
if current.Status == ArtifactReady &&
current.OriginNodeID == prepared.OriginNodeID &&
current.OriginArtifactID == prepared.OriginArtifactID {
return
}
}
// A successful reread (or a definitively missing row) proves this locator
// lost. Other database errors leave MarkReady's outcome indeterminate, so a
// failed queue write must not fall back to deleting potentially-owned bytes.
safeImmediateDelete := err == nil || errors.Is(err, ErrNotFound)
m.enqueueRemoteCleanup(ctx, artifactID, prepared, safeImmediateDelete)
}
func (m *ArtifactManager) enqueueRemoteCleanup(ctx context.Context, artifactID string, prepared PreparedArtifact, safeImmediateDelete bool) bool {
if !prepared.Remote() {
return true
}
cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
defer cancel()
if err := m.repo.EnqueueRemoteOrphan(cleanupCtx, artifactID, prepared.OriginNodeID, prepared.OriginNodeURL, prepared.OriginArtifactID); err != nil {
slog.WarnContext(ctx, "persisting abandoned remote artifact failed", "component", "downloads", "artifact_id", prepared.OriginArtifactID, "node", prepared.OriginNodeURL, "error", err)
// A unique attempt ID permits best-effort immediate deletion when the
// caller knows the readiness write did not commit. An indeterminate write
// must retain the bytes if the verification queue is unavailable.
if safeImmediateDelete {
lifecycle, ok := m.preparer.(remoteArtifactLifecycle)
if !ok {
return false
}
artifact := &Artifact{OriginNodeID: prepared.OriginNodeID, OriginNodeURL: prepared.OriginNodeURL, OriginNodeGroup: prepared.OriginNodeGroup, OriginArtifactID: prepared.OriginArtifactID}
if deleteErr := lifecycle.DeleteArtifact(cleanupCtx, artifact); deleteErr == nil {
return true
}
}
return false
}
return true
}
func (m *ArtifactManager) failJob(ctx context.Context, a *Artifact, msg string) {
terminal, applied, err := m.repo.MarkFailedOrRetry(ctx, a.ID, m.owner, msg, backoffFor(a.Attempts))
if err != nil {
@@ -780,11 +469,9 @@ func (m *ArtifactManager) buildOpts(file *models.MediaFile, a *Artifact) playbac
// The server-disk *quota* is download.artifact_max_bytes (see the download
// limits & restrictions design); this sweep is not a quota.
const (
failedArtifactRetention = 24 * time.Hour
unlinkedArtifactRetention = 30 * 24 * time.Hour
ephemeralDownloadRetention = 7 * 24 * time.Hour
defaultRemoteRecoveryBudget = 15 * time.Second
defaultRemoteCleanupBudget = 15 * time.Second
failedArtifactRetention = 24 * time.Hour
unlinkedArtifactRetention = 30 * 24 * time.Hour
ephemeralDownloadRetention = 7 * 24 * time.Hour
)
// Cleanup runs the hygiene sweep, then evicts ready artifacts (LRU first) once
@@ -792,7 +479,6 @@ const (
// active download row (managed or ephemeral) — only artifacts whose links are
// all terminal are evictable.
func (m *ArtifactManager) Cleanup(ctx context.Context) error {
m.cleanupRemoteOrphans(ctx)
m.sweepStale(ctx)
budget := m.downloadConfig().ArtifactMaxBytes
if budget <= 0 {
@@ -821,8 +507,10 @@ func (m *ArtifactManager) Cleanup(ctx context.Context) error {
if active {
continue
}
if !m.deleteArtifactBytes(ctx, a) {
continue
if a.OutputPath != "" {
if err := os.Remove(a.OutputPath); err != nil && !os.IsNotExist(err) {
slog.WarnContext(ctx, "removing evicted artifact file failed", "component", "downloads", "artifact_id", a.ID, "error", err)
}
}
if err := m.repo.DeleteArtifact(ctx, a.ID); err != nil {
slog.WarnContext(ctx, "deleting evicted artifact row failed", "component", "downloads", "artifact_id", a.ID, "error", err)
@@ -834,68 +522,6 @@ func (m *ArtifactManager) Cleanup(ctx context.Context) error {
return nil
}
func (m *ArtifactManager) cleanupRemoteOrphans(ctx context.Context) {
lifecycle, ok := m.preparer.(remoteArtifactLifecycle)
if !ok {
return
}
budget := m.remoteCleanupBudget
if budget <= 0 {
budget = defaultRemoteCleanupBudget
}
cleanupCtx, cancel := context.WithTimeout(ctx, budget)
defer cancel()
orphans, err := m.repo.ListRemoteOrphansDue(cleanupCtx, 100)
if err != nil {
slog.WarnContext(ctx, "listing abandoned remote artifacts failed", "component", "downloads", "error", err)
return
}
groups := make(map[string][]RemoteArtifactOrphan)
for _, orphan := range orphans {
key := fmt.Sprintf("%d\x00%s", orphan.OriginNodeID, orphan.OriginNodeURL)
groups[key] = append(groups[key], orphan)
}
const maxConcurrentOriginDeletes = 4
sem := make(chan struct{}, maxConcurrentOriginDeletes)
var wg sync.WaitGroup
for _, group := range groups {
group := group
wg.Add(1)
go func() {
defer wg.Done()
select {
case sem <- struct{}{}:
defer func() { <-sem }()
case <-cleanupCtx.Done():
return
}
for _, orphan := range group {
owned, claimed, err := m.repo.PrepareRemoteOrphanCleanup(cleanupCtx, orphan)
if err != nil {
slog.WarnContext(ctx, "claiming abandoned remote artifact cleanup failed; skipping remaining origin batch", "component", "downloads", "artifact_id", orphan.OriginArtifactID, "error", err)
return
}
if !claimed || owned {
continue
}
artifact := &Artifact{
OriginNodeID: orphan.OriginNodeID, OriginNodeURL: orphan.OriginNodeURL,
OriginArtifactID: orphan.OriginArtifactID,
}
if err := lifecycle.DeleteArtifact(cleanupCtx, artifact); err != nil {
slog.WarnContext(ctx, "abandoned remote artifact cleanup failed; skipping remaining origin batch", "component", "downloads", "artifact_id", orphan.OriginArtifactID, "node", orphan.OriginNodeURL, "error", err)
return
}
if err := m.repo.DeleteRemoteOrphan(cleanupCtx, orphan.ID); err != nil {
slog.WarnContext(ctx, "clearing abandoned remote artifact cleanup row failed", "component", "downloads", "artifact_id", orphan.OriginArtifactID, "error", err)
return
}
}
}()
}
wg.Wait()
}
// sweepStale is the age-based hygiene pass: cold terminally-failed artifacts
// (with their leftover .part files), orphaned ready artifacts no download row
// references, and expired ephemeral download rows. Best-effort; every step
@@ -931,8 +557,13 @@ func (m *ArtifactManager) sweepStale(ctx context.Context) {
// removeArtifact deletes an artifact's output file, its .part leftover, and
// its row. Used by the hygiene sweep for rows nothing can serve again.
func (m *ArtifactManager) removeArtifact(ctx context.Context, a *Artifact, reason string) {
if !m.deleteArtifactBytes(ctx, a) {
return
if a.OutputPath != "" {
if err := os.Remove(a.OutputPath); err != nil && !os.IsNotExist(err) {
slog.WarnContext(ctx, "removing swept artifact file failed", "component", "downloads", "artifact_id", a.ID, "error", err)
}
if err := os.Remove(a.OutputPath + ".part"); err != nil && !os.IsNotExist(err) {
slog.WarnContext(ctx, "removing swept artifact partial failed", "component", "downloads", "artifact_id", a.ID, "error", err)
}
}
if err := m.repo.DeleteArtifact(ctx, a.ID); err != nil {
slog.WarnContext(ctx, "deleting swept artifact row failed", "component", "downloads", "artifact_id", a.ID, "error", err)
@@ -941,33 +572,6 @@ func (m *ArtifactManager) removeArtifact(ctx context.Context, a *Artifact, reaso
slog.InfoContext(ctx, "swept stale download artifact", "component", "downloads", "artifact_id", a.ID, "reason", reason, "bytes", a.FileSize)
}
func (m *ArtifactManager) deleteArtifactBytes(ctx context.Context, a *Artifact) bool {
if a.OriginArtifactID != "" {
lifecycle, ok := m.preparer.(remoteArtifactLifecycle)
if !ok {
slog.WarnContext(ctx, "remote artifact cleanup unavailable", "component", "downloads", "artifact_id", a.ID)
return false
}
if err := lifecycle.DeleteArtifact(ctx, a); err != nil {
slog.WarnContext(ctx, "removing remote artifact failed", "component", "downloads", "artifact_id", a.ID, "node", a.OriginNodeURL, "error", err)
return false
}
return true
}
if a.OutputPath == "" {
return true
}
if err := os.Remove(a.OutputPath); err != nil && !os.IsNotExist(err) {
slog.WarnContext(ctx, "removing artifact file failed", "component", "downloads", "artifact_id", a.ID, "error", err)
return false
}
if err := os.Remove(a.OutputPath + ".part"); err != nil && !os.IsNotExist(err) {
slog.WarnContext(ctx, "removing artifact partial failed", "component", "downloads", "artifact_id", a.ID, "error", err)
return false
}
return true
}
// backoffFor returns the retry delay for the next attempt after a failure.
func backoffFor(attempts int) time.Duration {
if attempts < 1 {
+7 -21
View File
@@ -78,16 +78,6 @@ func (bm *BandwidthManager) getUserLimiter(userID int) *rate.Limiter {
// ThrottledReader wraps an io.ReadSeeker with bandwidth limiting for the given user.
// If no limits are configured, the original reader is returned as-is.
func (bm *BandwidthManager) ThrottledReader(ctx context.Context, r io.ReadSeeker, userID int) io.ReadSeeker {
limited := bm.ThrottledStreamReader(ctx, r, userID)
if limited == r {
return r
}
return &throttledReadSeeker{Reader: limited, seeker: r}
}
// ThrottledStreamReader applies the same aggregate limits to a non-seekable
// stream, such as an authenticated response relayed from a transcode node.
func (bm *BandwidthManager) ThrottledStreamReader(ctx context.Context, r io.Reader, userID int) io.Reader {
if bm == nil {
return r
}
@@ -106,18 +96,10 @@ func (bm *BandwidthManager) ThrottledStreamReader(ctx context.Context, r io.Read
}
}
type throttledReadSeeker struct {
io.Reader
seeker io.Seeker
}
func (r *throttledReadSeeker) Seek(offset int64, whence int) (int64, error) {
return r.seeker.Seek(offset, whence)
}
// throttledReader applies bandwidth limiting to forward reads.
// throttledReader implements io.ReadSeeker with bandwidth limiting.
// Seek is delegated directly (not throttled), only forward reads are limited.
type throttledReader struct {
r io.Reader
r io.ReadSeeker
ctx context.Context
serverLim *rate.Limiter
userLim *rate.Limiter
@@ -157,6 +139,10 @@ func (t *throttledReader) Read(p []byte) (int, error) {
return n, err
}
func (t *throttledReader) Seek(offset int64, whence int) (int64, error) {
return t.r.Seek(offset, whence)
}
// BytesRead returns the total bytes read through this throttled reader.
func (t *throttledReader) BytesRead() int64 {
return t.bytesRead
-7
View File
@@ -79,13 +79,6 @@ var (
ErrInvalidSubtitleRef = errors.New("invalid subtitle reference")
ErrAssetNotFound = errors.New("download asset not found")
ErrFormatUnavailable = errors.New("requested download format is not available")
// ErrResponseCommitted reports a transfer failure after response headers
// were written. Handlers must not append an API error body, while service
// lifecycle code must still treat the transfer as incomplete.
ErrResponseCommitted = errors.New("download response already committed")
// ErrArtifactOriginRemoved means a durable remote locator points at a node
// no longer present in the enabled transcode pool.
ErrArtifactOriginRemoved = errors.New("download artifact origin node was removed")
// ErrQualityUnavailable means the requested quality is valid and permitted in
// principle but cannot be fulfilled by the current server wiring.
ErrQualityUnavailable = errors.New("requested download quality is not available")
-184
View File
@@ -1,184 +0,0 @@
package downloads
import (
"context"
"errors"
"fmt"
"log/slog"
"strings"
"github.com/Silo-Server/silo-server/internal/config"
"github.com/Silo-Server/silo-server/internal/downloadprepare"
"github.com/Silo-Server/silo-server/internal/nodepool"
"github.com/Silo-Server/silo-server/internal/playback"
)
// NodeAwarePreparer keeps artifact queue ownership central while executing the
// expensive FFmpeg process on a healthy transcode node when capacity permits.
// The node retains completed bytes behind an authenticated opaque-id endpoint;
// integrated installations and unavailable pools fall back to local work.
type NodeAwarePreparer struct {
local EncodePreparer
planner nodepool.TranscodeWorkPlanner
liveCfg func() *config.Config
remote downloadprepare.RemotePreparer
originLookup artifactOriginLookup
}
type artifactOriginLookup interface {
GetByID(ctx context.Context, id int) (*nodepool.Node, error)
}
func NewNodeAwarePreparer(local EncodePreparer, planner nodepool.TranscodeWorkPlanner, liveCfg func() *config.Config) *NodeAwarePreparer {
if local == nil {
local = playbackPreparer{}
}
return &NodeAwarePreparer{
local: local,
planner: planner,
liveCfg: liveCfg,
remote: downloadprepare.HTTPPreparer{},
}
}
// SetOriginLookup supplies the authoritative node record used when the active
// pool temporarily misses an enabled node, and to recover a changed URL after
// a disabled node has left that pool.
func (p *NodeAwarePreparer) SetOriginLookup(lookup artifactOriginLookup) {
p.originLookup = lookup
}
func (p *NodeAwarePreparer) PrepareFile(ctx context.Context, artifactID string, opts playback.TranscodeOpts, outputPath string) (PreparedArtifact, error) {
cfg := p.config()
jwtSecret := ""
if cfg != nil {
jwtSecret = strings.TrimSpace(cfg.Auth.JWTSecret)
}
if cfg == nil || jwtSecret == "" || p.remote == nil || p.planner == nil || !downloadprepare.ValidArtifactID(artifactID) {
return p.local.PrepareFile(ctx, artifactID, opts, outputPath)
}
node, release := p.planner.ReserveTranscodeWork("download-prepare-" + artifactID)
if node == nil {
return p.local.PrepareFile(ctx, artifactID, opts, outputPath)
}
slog.InfoContext(ctx, "dispatching download artifact prepare", "component", "downloads", "artifact_id", artifactID, "node", node.URL)
result, err := p.remote.Prepare(ctx, node.URL, jwtSecret, downloadprepare.NewRequest(artifactID, opts))
release()
if err == nil && result.ArtifactID == artifactID {
return remotePreparedArtifact(node, result), nil
}
if err == nil {
err = fmt.Errorf("remote download prepare returned artifact id %q, want %q", result.ArtifactID, artifactID)
}
if ctx.Err() != nil {
return remotePreparedArtifact(node, downloadprepare.Result{ArtifactID: artifactID}), ctx.Err()
}
// A completed encode can outlive a lost HTTP response. Probe the same opaque
// id before falling back so retry/recovery does not duplicate expensive work.
if recovered, statErr := p.remote.Stat(ctx, node.URL, jwtSecret, artifactID); statErr == nil && recovered.ArtifactID == artifactID {
slog.InfoContext(ctx, "recovered completed download artifact after lost response", "component", "downloads", "artifact_id", artifactID, "node", node.URL)
return remotePreparedArtifact(node, recovered), nil
} else if statErr == nil {
return remotePreparedArtifact(node, downloadprepare.Result{ArtifactID: artifactID}),
fmt.Errorf("remote download artifact recovery returned artifact id %q, want %q", recovered.ArtifactID, artifactID)
} else if !errors.Is(statErr, downloadprepare.ErrArtifactNotFound) {
slog.WarnContext(ctx, "remote download artifact recovery probe failed", "component", "downloads", "artifact_id", artifactID, "node", node.URL, "error", statErr)
// The POST may have completed even though its response was lost. If the
// follow-up probe is also indeterminate, retry the same opaque id later
// instead of falling back locally and orphaning completed remote bytes.
return remotePreparedArtifact(node, downloadprepare.Result{ArtifactID: artifactID}), errors.Join(err, fmt.Errorf("remote download artifact recovery probe: %w", statErr))
}
if ctx.Err() != nil {
return PreparedArtifact{}, ctx.Err()
}
slog.WarnContext(ctx, "remote download artifact prepare unavailable; falling back to local", "component", "downloads", "artifact_id", artifactID, "node", node.URL, "error", err)
return p.local.PrepareFile(ctx, artifactID, opts, outputPath)
}
func remotePreparedArtifact(node *nodepool.Node, result downloadprepare.Result) PreparedArtifact {
group := ""
if node.Group != nil {
group = *node.Group
}
return PreparedArtifact{
OriginNodeID: node.ID,
OriginNodeURL: strings.TrimRight(node.URL, "/"),
OriginNodeGroup: group,
OriginArtifactID: result.ArtifactID,
FileSize: result.FileSize,
}
}
func (p *NodeAwarePreparer) ResolveArtifact(ctx context.Context, artifact *Artifact) error {
if artifact == nil || artifact.OriginNodeID == 0 || p.planner == nil {
return ErrArtifactOriginRemoved
}
node, ok := p.planner.TranscodeNode(artifact.OriginNodeID)
if !ok || node == nil {
if p.originLookup != nil {
configured, err := p.originLookup.GetByID(ctx, artifact.OriginNodeID)
switch {
case err == nil && configured != nil && configured.Type == nodepool.NodeTypeTranscode:
applyArtifactOrigin(artifact, configured)
if configured.Enabled {
return nil
}
case err != nil && !errors.Is(err, nodepool.ErrNodeNotFound):
return fmt.Errorf("looking up artifact origin node: %w", err)
}
}
return ErrArtifactOriginRemoved
}
applyArtifactOrigin(artifact, node)
return nil
}
func applyArtifactOrigin(artifact *Artifact, node *nodepool.Node) {
artifact.OriginNodeURL = strings.TrimRight(node.URL, "/")
artifact.OriginNodeGroup = ""
if node.Group != nil {
artifact.OriginNodeGroup = *node.Group
}
}
func (p *NodeAwarePreparer) StatArtifact(ctx context.Context, artifact *Artifact) (downloadprepare.Result, error) {
if err := p.ResolveArtifact(ctx, artifact); err != nil {
return downloadprepare.Result{}, err
}
secret, err := p.remoteCredentials(artifact)
if err != nil {
return downloadprepare.Result{}, err
}
return p.remote.Stat(ctx, artifact.OriginNodeURL, secret, artifact.OriginArtifactID)
}
func (p *NodeAwarePreparer) DeleteArtifact(ctx context.Context, artifact *Artifact) error {
// Prefer the authoritative current URL, including for a disabled node. A
// deleted node has no newer record, so retain the last persisted URL as the
// best-effort cleanup target.
_ = p.ResolveArtifact(ctx, artifact)
secret, err := p.remoteCredentials(artifact)
if err != nil {
return err
}
return p.remote.Delete(ctx, artifact.OriginNodeURL, secret, artifact.OriginArtifactID)
}
func (p *NodeAwarePreparer) remoteCredentials(artifact *Artifact) (string, error) {
if artifact == nil || strings.TrimSpace(artifact.OriginNodeURL) == "" || !downloadprepare.ValidArtifactID(artifact.OriginArtifactID) {
return "", errors.New("remote artifact locator is incomplete")
}
cfg := p.config()
if cfg == nil || strings.TrimSpace(cfg.Auth.JWTSecret) == "" || p.remote == nil {
return "", errors.New("remote artifact credentials are unavailable")
}
return strings.TrimSpace(cfg.Auth.JWTSecret), nil
}
func (p *NodeAwarePreparer) config() *config.Config {
if p == nil || p.liveCfg == nil {
return nil
}
return p.liveCfg()
}
-325
View File
@@ -1,325 +0,0 @@
package downloads
import (
"context"
"errors"
"os"
"testing"
"github.com/Silo-Server/silo-server/internal/config"
"github.com/Silo-Server/silo-server/internal/downloadprepare"
"github.com/Silo-Server/silo-server/internal/nodepool"
"github.com/Silo-Server/silo-server/internal/playback"
)
type recordingEncodePreparer struct{ calls int }
func (p *recordingEncodePreparer) PrepareFile(_ context.Context, _ string, _ playback.TranscodeOpts, outputPath string) (PreparedArtifact, error) {
p.calls++
return PreparedArtifact{OutputPath: outputPath, FileSize: 8}, nil
}
type recordingRemotePreparer struct {
nodeURL string
secret string
request downloadprepare.Request
deleted string
deleteURL string
}
func (p *recordingRemotePreparer) Prepare(_ context.Context, nodeURL, secret string, req downloadprepare.Request) (downloadprepare.Result, error) {
p.nodeURL, p.secret, p.request = nodeURL, secret, req
return downloadprepare.Result{ArtifactID: req.ArtifactID, FileSize: 1234}, nil
}
func (p *recordingRemotePreparer) Stat(_ context.Context, _, _ string, artifactID string) (downloadprepare.Result, error) {
return downloadprepare.Result{ArtifactID: artifactID, FileSize: 1234}, nil
}
func (p *recordingRemotePreparer) Delete(_ context.Context, nodeURL, _ string, artifactID string) error {
p.deleted = artifactID
p.deleteURL = nodeURL
return nil
}
type staticArtifactOriginLookup struct {
node *nodepool.Node
err error
}
func (l staticArtifactOriginLookup) GetByID(context.Context, int) (*nodepool.Node, error) {
return l.node, l.err
}
type unavailableRemotePreparer struct{}
func (unavailableRemotePreparer) Prepare(context.Context, string, string, downloadprepare.Request) (downloadprepare.Result, error) {
return downloadprepare.Result{}, os.ErrNotExist
}
func (unavailableRemotePreparer) Stat(context.Context, string, string, string) (downloadprepare.Result, error) {
return downloadprepare.Result{}, downloadprepare.ErrArtifactNotFound
}
func (unavailableRemotePreparer) Delete(context.Context, string, string, string) error { return nil }
type responseLostRemotePreparer struct{}
func (responseLostRemotePreparer) Prepare(context.Context, string, string, downloadprepare.Request) (downloadprepare.Result, error) {
return downloadprepare.Result{}, context.DeadlineExceeded
}
func (responseLostRemotePreparer) Stat(_ context.Context, _, _ string, artifactID string) (downloadprepare.Result, error) {
return downloadprepare.Result{ArtifactID: artifactID, FileSize: 55}, nil
}
func (responseLostRemotePreparer) Delete(context.Context, string, string, string) error { return nil }
type indeterminateRemotePreparer struct{}
func (indeterminateRemotePreparer) Prepare(context.Context, string, string, downloadprepare.Request) (downloadprepare.Result, error) {
return downloadprepare.Result{}, context.DeadlineExceeded
}
func (indeterminateRemotePreparer) Stat(context.Context, string, string, string) (downloadprepare.Result, error) {
return downloadprepare.Result{}, os.ErrDeadlineExceeded
}
func (indeterminateRemotePreparer) Delete(context.Context, string, string, string) error { return nil }
type mismatchedRecoveryRemotePreparer struct{}
func (mismatchedRecoveryRemotePreparer) Prepare(context.Context, string, string, downloadprepare.Request) (downloadprepare.Result, error) {
return downloadprepare.Result{}, context.DeadlineExceeded
}
func (mismatchedRecoveryRemotePreparer) Stat(context.Context, string, string, string) (downloadprepare.Result, error) {
return downloadprepare.Result{ArtifactID: "unexpected-artifact", FileSize: 55}, nil
}
func (mismatchedRecoveryRemotePreparer) Delete(context.Context, string, string, string) error {
return nil
}
func TestNodeAwarePreparerUsesLeastLoadedHealthyNode(t *testing.T) {
group := "host-a"
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{
{URL: "http://busy", Enabled: true, Healthy: true, ActiveJobs: 3},
{ID: 17, URL: "http://idle", Enabled: true, Healthy: true, ActiveJobs: 1, Group: &group},
{URL: "http://unhealthy", Enabled: true, Healthy: false},
})
local := &recordingEncodePreparer{}
remote := &recordingRemotePreparer{}
cfg := &config.Config{}
cfg.Auth.JWTSecret = "secret"
p := NewNodeAwarePreparer(local, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), func() *config.Config { return cfg })
p.remote = remote
opts := playback.TranscodeOpts{InputPath: "/media/movie.mkv", TargetCodecVideo: "h264", TargetCodecAudio: "aac"}
prepared, err := p.PrepareFile(context.Background(), "artifact-1", opts, "/local/artifact-1.mp4")
if err != nil {
t.Fatal(err)
}
if local.calls != 0 {
t.Fatalf("local calls = %d, want 0", local.calls)
}
if remote.nodeURL != "http://idle" || remote.secret != "secret" || remote.request.ArtifactID != "artifact-1" || remote.request.InputPath != opts.InputPath {
t.Fatalf("remote call = node %q secret %q request %+v", remote.nodeURL, remote.secret, remote.request)
}
if prepared.OutputPath != "" || prepared.OriginNodeID != 17 || prepared.OriginNodeURL != "http://idle" || prepared.OriginNodeGroup != group || prepared.OriginArtifactID != "artifact-1" || prepared.FileSize != 1234 {
t.Fatalf("prepared = %+v", prepared)
}
}
func TestNodeAwarePreparerResolvesCurrentArtifactNodeURL(t *testing.T) {
group := "host-new"
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{{ID: 17, URL: "http://new-url", Group: &group, Enabled: true, Healthy: true}})
p := NewNodeAwarePreparer(nil, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), nil)
artifact := &Artifact{OriginNodeID: 17, OriginNodeURL: "http://old-url", OriginNodeGroup: "host-old", OriginArtifactID: "artifact-1"}
if err := p.ResolveArtifact(context.Background(), artifact); err != nil {
t.Fatal(err)
}
if artifact.OriginNodeURL != "http://new-url" || artifact.OriginNodeGroup != group {
t.Fatalf("artifact = %+v", artifact)
}
}
func TestNodeAwarePreparerUsesCurrentDisabledNodeURLForCleanup(t *testing.T) {
group := "host-new"
pool := nodepool.NewTranscodePool()
remote := &recordingRemotePreparer{}
cfg := &config.Config{}
cfg.Auth.JWTSecret = "secret"
p := NewNodeAwarePreparer(nil, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), func() *config.Config { return cfg })
p.remote = remote
p.SetOriginLookup(staticArtifactOriginLookup{node: &nodepool.Node{
ID: 17, Type: nodepool.NodeTypeTranscode, URL: "http://new-url", Group: &group, Enabled: false,
}})
artifact := &Artifact{OriginNodeID: 17, OriginNodeURL: "http://old-url", OriginNodeGroup: "host-old", OriginArtifactID: "artifact-1"}
if err := p.ResolveArtifact(context.Background(), artifact); !errors.Is(err, ErrArtifactOriginRemoved) {
t.Fatalf("ResolveArtifact error = %v, want ErrArtifactOriginRemoved", err)
}
if artifact.OriginNodeURL != "http://new-url" || artifact.OriginNodeGroup != group {
t.Fatalf("refreshed artifact = %+v", artifact)
}
if err := p.DeleteArtifact(context.Background(), artifact); err != nil {
t.Fatal(err)
}
if remote.deleteURL != "http://new-url" || remote.deleted != "artifact-1" {
t.Fatalf("cleanup target = %q %q", remote.deleteURL, remote.deleted)
}
}
func TestNodeAwarePreparerRecoversEnabledOriginMissingFromPool(t *testing.T) {
group := "host-new"
pool := nodepool.NewTranscodePool()
p := NewNodeAwarePreparer(nil, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), nil)
p.SetOriginLookup(staticArtifactOriginLookup{node: &nodepool.Node{
ID: 17, Type: nodepool.NodeTypeTranscode, URL: "http://new-url", Group: &group, Enabled: true,
}})
artifact := &Artifact{OriginNodeID: 17, OriginNodeURL: "http://old-url", OriginNodeGroup: "host-old", OriginArtifactID: "artifact-1"}
if err := p.ResolveArtifact(context.Background(), artifact); err != nil {
t.Fatalf("ResolveArtifact error = %v", err)
}
if artifact.OriginNodeURL != "http://new-url" || artifact.OriginNodeGroup != group {
t.Fatalf("refreshed artifact = %+v", artifact)
}
}
func TestNodeAwarePreparerReportsRemovedArtifactOrigin(t *testing.T) {
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{{ID: 17, URL: "http://disabled", Enabled: false, Healthy: true}})
p := NewNodeAwarePreparer(nil, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), nil)
artifact := &Artifact{OriginNodeID: 17, OriginNodeURL: "http://removed", OriginArtifactID: "artifact-1"}
if err := p.ResolveArtifact(context.Background(), artifact); !errors.Is(err, ErrArtifactOriginRemoved) {
t.Fatalf("ResolveArtifact error = %v, want ErrArtifactOriginRemoved", err)
}
}
func TestNodeAwarePreparerFallsBackLocallyWithoutEligibleCapacity(t *testing.T) {
limit := 1
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{{URL: "http://full", Enabled: true, Healthy: true, ActiveJobs: 1, MaxJobs: &limit}})
local := &recordingEncodePreparer{}
cfg := &config.Config{}
cfg.Auth.JWTSecret = "secret"
p := NewNodeAwarePreparer(local, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), func() *config.Config { return cfg })
p.remote = &recordingRemotePreparer{}
prepared, err := p.PrepareFile(context.Background(), "artifact-2", playback.TranscodeOpts{}, "/artifacts/job-2.mp4")
if err != nil || prepared.OutputPath == "" || local.calls != 1 {
t.Fatalf("prepared=%+v err=%v local calls=%d", prepared, err, local.calls)
}
}
func TestNodeAwarePreparerRetainsRequestedLocatorAfterMismatchedRecovery(t *testing.T) {
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{{ID: 17, URL: "http://idle", Enabled: true, Healthy: true}})
local := &recordingEncodePreparer{}
cfg := &config.Config{}
cfg.Auth.JWTSecret = "secret"
p := NewNodeAwarePreparer(local, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), func() *config.Config { return cfg })
p.remote = mismatchedRecoveryRemotePreparer{}
prepared, err := p.PrepareFile(context.Background(), "artifact-requested", playback.TranscodeOpts{}, "/local/job.mp4")
if err == nil {
t.Fatal("expected mismatched recovery error")
}
if !prepared.Remote() || prepared.OriginArtifactID != "artifact-requested" || prepared.FileSize != 0 {
t.Fatalf("prepared locator = %+v, want requested remote artifact", prepared)
}
if local.calls != 0 {
t.Fatalf("local calls = %d, want no fallback after indeterminate remote result", local.calls)
}
}
func TestNodeAwarePreparerFallsBackLocallyWithoutNodeCredentials(t *testing.T) {
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{{ID: 18, URL: "http://idle", Enabled: true, Healthy: true}})
local := &recordingEncodePreparer{}
p := NewNodeAwarePreparer(local, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), func() *config.Config { return &config.Config{} })
p.remote = &recordingRemotePreparer{}
if _, err := p.PrepareFile(context.Background(), "artifact-3", playback.TranscodeOpts{}, "/artifacts/job-3.mp4"); err != nil {
t.Fatal(err)
}
if local.calls != 1 {
t.Fatalf("local calls = %d, want 1", local.calls)
}
}
func TestNodeAwarePreparerUsesRemoteWithDefaultNodeLocalArtifactDir(t *testing.T) {
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{{ID: 19, URL: "http://idle", Enabled: true, Healthy: true}})
local := &recordingEncodePreparer{}
cfg := &config.Config{}
cfg.Auth.JWTSecret = "secret"
p := NewNodeAwarePreparer(local, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), func() *config.Config { return cfg })
p.remote = &recordingRemotePreparer{}
prepared, err := p.PrepareFile(context.Background(), "artifact-local", playback.TranscodeOpts{}, "/local/job.mp4")
if err != nil || !prepared.Remote() || local.calls != 0 {
t.Fatalf("prepared=%+v err=%v local calls=%d", prepared, err, local.calls)
}
}
func TestNodeAwarePreparerFallsBackLocallyWhenRemoteUnavailable(t *testing.T) {
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{{URL: "http://idle", Enabled: true, Healthy: true}})
local := &recordingEncodePreparer{}
cfg := &config.Config{}
cfg.Auth.JWTSecret = "secret"
p := NewNodeAwarePreparer(local, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), func() *config.Config { return cfg })
p.remote = unavailableRemotePreparer{}
prepared, err := p.PrepareFile(context.Background(), "artifact-4", playback.TranscodeOpts{}, "/local/job-4.mp4")
if err != nil || prepared.OutputPath == "" || local.calls != 1 {
t.Fatalf("prepared=%+v err=%v local calls=%d", prepared, err, local.calls)
}
}
func TestNodeAwarePreparerRecoversCompletedArtifactAfterResponseLoss(t *testing.T) {
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{{ID: 21, URL: "http://idle", Enabled: true, Healthy: true}})
local := &recordingEncodePreparer{}
cfg := &config.Config{}
cfg.Auth.JWTSecret = "secret"
p := NewNodeAwarePreparer(local, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), func() *config.Config { return cfg })
p.remote = responseLostRemotePreparer{}
prepared, err := p.PrepareFile(context.Background(), "artifact-5", playback.TranscodeOpts{}, "/local/job-5.mp4")
if err != nil || !prepared.Remote() || prepared.FileSize != 55 || local.calls != 0 {
t.Fatalf("prepared=%+v err=%v local calls=%d", prepared, err, local.calls)
}
}
func TestNodeAwarePreparerDoesNotFallBackAfterLeaseCancellation(t *testing.T) {
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{{URL: "http://idle", Enabled: true, Healthy: true}})
local := &recordingEncodePreparer{}
cfg := &config.Config{}
cfg.Auth.JWTSecret = "secret"
p := NewNodeAwarePreparer(local, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), func() *config.Config { return cfg })
p.remote = unavailableRemotePreparer{}
ctx, cancel := context.WithCancel(context.Background())
cancel()
_, err := p.PrepareFile(ctx, "artifact-6", playback.TranscodeOpts{}, "/local/job-6.mp4")
if !errors.Is(err, context.Canceled) {
t.Fatalf("error = %v, want context canceled", err)
}
if local.calls != 0 {
t.Fatalf("local calls = %d, want 0", local.calls)
}
}
func TestNodeAwarePreparerDoesNotFallBackWhenRecoveryProbeIsIndeterminate(t *testing.T) {
pool := nodepool.NewTranscodePool()
pool.SetNodes([]*nodepool.Node{{ID: 20, URL: "http://idle", Enabled: true, Healthy: true}})
local := &recordingEncodePreparer{}
cfg := &config.Config{}
cfg.Auth.JWTSecret = "secret"
p := NewNodeAwarePreparer(local, nodepool.NewPlanner(nodepool.NewProxyPool(), pool), func() *config.Config { return cfg })
p.remote = indeterminateRemotePreparer{}
if _, err := p.PrepareFile(context.Background(), "artifact-7", playback.TranscodeOpts{}, "/local/job-7.mp4"); err == nil {
t.Fatal("expected indeterminate remote error")
}
if local.calls != 0 {
t.Fatalf("local calls = %d, want 0", local.calls)
}
}
+1 -1
View File
@@ -289,7 +289,7 @@ func TestReconcileLinkedDownloads(t *testing.T) {
if _, err := arepo.ClaimNext(ctx, "w", time.Minute); err != nil {
t.Fatalf("claim ready artifact: %v", err)
}
if ok, err := arepo.MarkReady(ctx, readyArt.ID, "w", "/tmp/ready.mp4", 0, "", "", "", 4242); err != nil || !ok {
if ok, err := arepo.MarkReady(ctx, readyArt.ID, "w", "/tmp/ready.mp4", 4242); err != nil || !ok {
t.Fatalf("MarkReady = (%v, %v)", ok, err)
}
+30 -164
View File
@@ -16,7 +16,6 @@ import (
"github.com/Silo-Server/silo-server/internal/access"
"github.com/Silo-Server/silo-server/internal/catalog"
"github.com/Silo-Server/silo-server/internal/config"
"github.com/Silo-Server/silo-server/internal/downloadprepare"
"github.com/Silo-Server/silo-server/internal/idgen"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/playback"
@@ -75,24 +74,6 @@ type Capability struct {
MonitoringModes []string
}
// FileTarget is an already-authorized source or prepared artifact that may be
// served locally or delegated to a trusted proxy node. Callers must obtain it
// from the request-scoped resolver methods below; a path alone is never an
// authorization grant.
type FileTarget struct {
Path string
DownloadID string
MediaFileID int
ArtifactID string
OriginNodeID int
OriginNodeURL string
OriginNodeGroup string
OriginArtifactID string
// ProxyEligible is true when a proxy can read the path directly or relay the
// opaque artifact from its owning transcode node.
ProxyEligible bool
}
// Service orchestrates download permission checks, quota enforcement, quality
// policy, file resolution, and file serving for both ephemeral/account-level
// rows (DeviceID == "") and managed device-library entries (DeviceID set).
@@ -866,37 +847,26 @@ func (s *Service) List(ctx context.Context, userID int, profileID, deviceID stri
// ServeDirect validates permissions and serves a file directly for browser
// download. No persistent download record is created.
func (s *Service) ServeDirect(ctx context.Context, w http.ResponseWriter, r *http.Request, userID, fileID int, format string, filter catalog.AccessFilter) error {
target, err := s.ResolveDirectFile(ctx, userID, fileID, format, filter)
if err != nil {
if _, _, err := s.downloadConfigForUser(ctx, userID, ""); err != nil {
return err
}
return s.serveLocalFile(ctx, w, r, target.Path, userID)
}
// ResolveDirectFile authorizes a browser-style original download without
// writing response bytes. It is used by the API before minting a proxy token.
func (s *Service) ResolveDirectFile(ctx context.Context, userID, fileID int, format string, filter catalog.AccessFilter) (*FileTarget, error) {
cfg, _, err := s.downloadConfigForUser(ctx, userID, "")
if err != nil {
return nil, err
}
if format != "" && format != FormatOriginal {
return nil, ErrFormatUnavailable
return ErrFormatUnavailable
}
file, err := s.fileRepo.GetByID(ctx, fileID)
if err != nil {
return nil, fmt.Errorf("loading media file: %w", err)
return fmt.Errorf("loading media file: %w", err)
}
if file == nil || file.MissingSince != nil {
return nil, catalog.ErrItemNotFound
return catalog.ErrItemNotFound
}
if err := s.itemAccess.EnsureAccessible(ctx, file.ContentID, filter); err != nil {
return nil, err
return err
}
if !catalog.FileAllowedByAccess(file, filter) {
return nil, catalog.ErrItemNotFound
return catalog.ErrItemNotFound
}
return &FileTarget{Path: file.FilePath, MediaFileID: file.ID, ProxyEligible: proxyDeliveryAllowed(cfg)}, nil
return s.serveLocalFile(ctx, w, r, file.FilePath, userID)
}
// ServeFile serves a download's file. Managed entries (device header present)
@@ -904,19 +874,15 @@ func (s *Service) ResolveDirectFile(ctx context.Context, userID, fileID int, for
// before serving; ephemeral rows keep today's queued→downloading→completed
// behavior. The ephemeral path never serves a managed row, and vice versa.
func (s *Service) ServeFile(ctx context.Context, w http.ResponseWriter, r *http.Request, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter) error {
if deviceID != "" {
target, err := s.ResolveManagedFile(ctx, userID, profileID, deviceID, downloadID, filter)
if err != nil {
return err
}
return s.serveFileTarget(ctx, w, r, target, userID)
}
// Re-check policy — admin may have disabled downloads or revoked permission.
if _, _, err := s.downloadConfigForUser(ctx, userID, deviceID); err != nil {
return err
}
if deviceID != "" {
return s.serveManaged(ctx, w, r, userID, profileID, deviceID, downloadID, filter)
}
dl, err := s.repo.GetByID(ctx, downloadID)
if err != nil {
return err
@@ -959,31 +925,25 @@ func (s *Service) ServeFile(ctx context.Context, w http.ResponseWriter, r *http.
return nil
}
// ResolveManagedFile authorizes a managed entry on (user, profile, device),
// re-checks current policy and content access, and returns the source/artifact
// path for trusted proxy delegation.
func (s *Service) ResolveManagedFile(ctx context.Context, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter) (*FileTarget, error) {
cfg, _, err := s.downloadConfigForUser(ctx, userID, deviceID)
if err != nil {
return nil, err
}
// serveManaged authorizes a managed entry on (user, profile, device), re-checks
// per-profile content access (invariant 2), and streams the original source.
func (s *Service) serveManaged(ctx context.Context, w http.ResponseWriter, r *http.Request, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter) error {
if profileID == "" {
return nil, ErrProfileRequired
return ErrProfileRequired
}
dl, err := s.repo.GetManagedByID(ctx, downloadID, userID, profileID, deviceID)
if err != nil {
return nil, err
return err
}
if dl.Status == StatusRevoked {
return nil, fmt.Errorf("download is revoked: %w", ErrDownloadNotActive)
return fmt.Errorf("download is revoked: %w", ErrDownloadNotActive)
}
// A download id alone never authorizes access: re-check the requesting
// profile's content/library scope before serving any bytes.
if err := s.itemAccess.EnsureAccessible(ctx, dl.ContentID, filter); err != nil {
return nil, err
return err
}
proxyEligible := proxyDeliveryAllowed(cfg)
return s.resolveDownloadBytesTarget(ctx, dl, filter, proxyEligible, proxyEligible)
return s.serveDownloadBytes(ctx, w, r, dl, userID, filter)
}
// PatchStatus lets a client confirm a managed entry's local state
@@ -1085,28 +1045,20 @@ func (s *Service) resolveFile(ctx context.Context, req CreateRequest) (*models.M
// (catalog.FileAllowedByAccess): library scope and the profile's max playback
// quality can change after a row was registered.
func (s *Service) serveDownloadBytes(ctx context.Context, w http.ResponseWriter, r *http.Request, dl *Download, userID int, filter catalog.AccessFilter) error {
target, err := s.resolveDownloadBytesTarget(ctx, dl, filter, false, false)
if err != nil {
return err
}
return s.serveFileTarget(ctx, w, r, target, userID)
}
func (s *Service) resolveDownloadBytesTarget(ctx context.Context, dl *Download, filter catalog.AccessFilter, sourceProxyEligible, preparedProxyEligible bool) (*FileTarget, error) {
file, err := s.fileRepo.GetByID(ctx, dl.MediaFileID)
if err != nil {
return nil, fmt.Errorf("loading media file: %w", err)
return fmt.Errorf("loading media file: %w", err)
}
if file == nil {
return nil, catalog.ErrItemNotFound
return catalog.ErrItemNotFound
}
if dl.Format != FormatOriginal && dl.ArtifactID != "" {
if s.artifacts == nil {
return nil, ErrFormatUnavailable
return ErrFormatUnavailable
}
artifact, err := s.artifacts.Ready(ctx, dl.ArtifactID)
if err != nil {
return nil, err
return err
}
// The quality ceiling applies to what is actually served — the prepared
// artifact's resolution, not the source's (a 720p transcode of a 4K
@@ -1116,34 +1068,17 @@ func (s *Service) resolveDownloadBytesTarget(ctx context.Context, dl *Download,
served.Resolution = artifact.Resolution
}
if !catalog.FileAllowedByAccess(&served, filter) {
return nil, catalog.ErrItemNotFound
return catalog.ErrItemNotFound
}
return &FileTarget{
Path: artifact.OutputPath,
DownloadID: dl.ID,
MediaFileID: file.ID,
ArtifactID: artifact.ID,
OriginNodeID: artifact.OriginNodeID,
OriginNodeURL: artifact.OriginNodeURL,
OriginNodeGroup: artifact.OriginNodeGroup,
OriginArtifactID: artifact.OriginArtifactID,
ProxyEligible: preparedProxyEligible,
}, nil
return s.serveLocalFile(ctx, w, r, artifact.OutputPath, userID)
}
if file.MissingSince != nil {
return nil, catalog.ErrItemNotFound
return catalog.ErrItemNotFound
}
if !catalog.FileAllowedByAccess(file, filter) {
return nil, catalog.ErrItemNotFound
return catalog.ErrItemNotFound
}
return &FileTarget{Path: file.FilePath, DownloadID: dl.ID, MediaFileID: file.ID, ProxyEligible: sourceProxyEligible}, nil
}
// Download bandwidth settings are aggregate guarantees enforced by the API
// process. Until proxy nodes share a distributed limiter, keep capped transfers
// local rather than silently multiplying the configured limit per proxy node.
func proxyDeliveryAllowed(cfg config.DownloadConfig) bool {
return cfg.ServerBandwidthBPS <= 0 && cfg.UserBandwidthBPS <= 0
return s.serveLocalFile(ctx, w, r, file.FilePath, userID)
}
func (s *Service) serveLocalFile(ctx context.Context, w http.ResponseWriter, r *http.Request, path string, userID int) error {
@@ -1161,7 +1096,8 @@ func (s *Service) serveLocalFile(ctx context.Context, w http.ResponseWriter, r *
return fmt.Errorf("stat file: %w", err)
}
w.Header().Set("Content-Disposition", attachmentDisposition(path))
filename := sanitizeFilename(filepath.Base(path))
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
w.Header().Set("Content-Type", playback.MimeFromExtension(path))
var reader io.ReadSeeker = f
@@ -1173,76 +1109,6 @@ func (s *Service) serveLocalFile(ctx context.Context, w http.ResponseWriter, r *
return nil
}
func attachmentDisposition(path string) string {
filename := sanitizeFilename(filepath.Base(path))
return fmt.Sprintf(`attachment; filename="%s"`, filename)
}
func (s *Service) serveFileTarget(ctx context.Context, w http.ResponseWriter, r *http.Request, target *FileTarget, userID int) error {
if target == nil {
return catalog.ErrItemNotFound
}
if target.OriginArtifactID == "" {
return s.serveLocalFile(ctx, w, r, target.Path, userID)
}
secret := ""
if s.artifacts != nil && s.artifacts.liveCfg != nil {
if cfg := s.artifacts.liveCfg(); cfg != nil {
secret = strings.TrimSpace(cfg.Auth.JWTSecret)
}
}
if secret == "" {
return errors.New("remote artifact credentials unavailable")
}
client := downloadprepare.HTTPPreparer{}
resp, err := client.Open(ctx, target.OriginNodeURL, secret, target.OriginArtifactID, r.Method, r.Header)
if err != nil {
return err
}
defer func() { _ = resp.Body.Close() }()
if resp.StatusCode == http.StatusNotFound {
artifact := &Artifact{
ID: target.ArtifactID,
OriginNodeID: target.OriginNodeID,
OriginNodeURL: target.OriginNodeURL,
OriginNodeGroup: target.OriginNodeGroup,
OriginArtifactID: target.OriginArtifactID,
}
if _, err := s.artifacts.requeueRemoteArtifactExactNow(ctx, artifact, "remote output missing"); err != nil {
return err
}
return fmt.Errorf("remote artifact was missing and preparation was requeued: %w", ErrDownloadNotActive)
}
if !downloadprepare.RelayStatusAllowed(resp.StatusCode) {
return fmt.Errorf("remote artifact node returned %d", resp.StatusCode)
}
// Preserve an origin-provided disposition, but supply the same sanitized
// attachment filename as local delivery when the node omits one.
if strings.TrimSpace(target.Path) != "" {
w.Header().Set("Content-Disposition", attachmentDisposition(target.Path))
}
downloadprepare.CopyResponseHeaders(w.Header(), resp.Header)
w.WriteHeader(resp.StatusCode)
if r.Method == http.MethodHead {
return nil
}
var reader io.Reader = resp.Body
if s.bandwidth != nil {
reader = s.bandwidth.ThrottledStreamReader(ctx, reader, userID)
}
if _, err := io.Copy(w, reader); err != nil {
// Headers and a partial body are already committed, so returning an error
// normally makes an API handler append JSON to the media response. Return a
// committed-response sentinel so lifecycle state records the failed transfer
// while the handler leaves the partial media response untouched.
if ctx.Err() == nil {
slog.WarnContext(ctx, "remote download artifact relay interrupted", "component", "downloads", "artifact_id", target.OriginArtifactID, "error", err)
}
return fmt.Errorf("%w: relaying remote artifact: %w", ErrResponseCommitted, err)
}
return nil
}
// pickBestFile selects the highest-resolution file from a list.
func pickBestFile(files []*models.MediaFile) *models.MediaFile {
if len(files) == 1 {
-19
View File
@@ -1,19 +0,0 @@
package downloads
import (
"testing"
"github.com/Silo-Server/silo-server/internal/config"
)
func TestProxyDeliveryAllowedPreservesAggregateBandwidthLimits(t *testing.T) {
if !proxyDeliveryAllowed(config.DownloadConfig{}) {
t.Fatal("uncapped downloads should be proxy eligible")
}
if proxyDeliveryAllowed(config.DownloadConfig{ServerBandwidthBPS: 1}) {
t.Fatal("server-capped downloads must stay API-local")
}
if proxyDeliveryAllowed(config.DownloadConfig{UserBandwidthBPS: 1}) {
t.Fatal("user-capped downloads must stay API-local")
}
}
+2 -2
View File
@@ -17,8 +17,8 @@ import (
// test never drains the queue, so it is never invoked.
type stubQuotaPreparer struct{}
func (stubQuotaPreparer) PrepareFile(_ context.Context, _ string, _ playback.TranscodeOpts, outputPath string) (PreparedArtifact, error) {
return PreparedArtifact{OutputPath: outputPath}, nil
func (stubQuotaPreparer) PrepareFile(_ context.Context, _ playback.TranscodeOpts, _ string) error {
return nil
}
// TestArtifactQuotaCheckedBeforeEnqueue pins two quota invariants for
-204
View File
@@ -1,204 +0,0 @@
package downloads
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/config"
)
func TestServiceRelaysRemoteArtifactOnEstablishedRoute(t *testing.T) {
const secret = "artifact-secret"
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/downloads/artifacts/artifact-1" || r.Header.Get("Authorization") != "Bearer "+secret {
t.Errorf("origin request = %s %s auth=%q", r.Method, r.URL.Path, r.Header.Get("Authorization"))
http.Error(w, "unexpected origin request", http.StatusBadRequest)
return
}
if r.Header.Get("Range") != "bytes=1-3" {
t.Errorf("Range = %q", r.Header.Get("Range"))
http.Error(w, "unexpected range", http.StatusBadRequest)
return
}
w.Header().Set("Content-Length", "3")
w.Header().Set("Content-Range", "bytes 1-3/5")
w.Header().Set("Content-Type", "video/mp4")
w.Header().Set("Content-Disposition", `attachment; filename="artifact-1.mp4"`)
w.WriteHeader(http.StatusPartialContent)
_, _ = w.Write([]byte("123"))
}))
defer origin.Close()
cfg := &config.Config{}
cfg.Auth.JWTSecret = secret
svc := &Service{artifacts: &ArtifactManager{liveCfg: func() *config.Config { return cfg }}}
req := httptest.NewRequest(http.MethodGet, "/api/v1/downloads/dl/file", nil)
req.Header.Set("Range", "bytes=1-3")
rr := httptest.NewRecorder()
err := svc.serveFileTarget(req.Context(), rr, req, &FileTarget{
Path: `/artifacts/Movie "Final".mp4`,
OriginNodeURL: origin.URL,
OriginArtifactID: "artifact-1",
}, 7)
if err != nil {
t.Fatal(err)
}
if rr.Code != http.StatusPartialContent || rr.Body.String() != "123" || rr.Header().Get("Content-Range") != "bytes 1-3/5" ||
rr.Header().Get("Content-Disposition") != `attachment; filename="Movie _Final_.mp4"` {
t.Fatalf("response status=%d body=%q range=%q disposition=%q", rr.Code, rr.Body.String(), rr.Header().Get("Content-Range"), rr.Header().Get("Content-Disposition"))
}
}
func TestServiceRequeuesReadyArtifactWhenRemoteFileIsMissing(t *testing.T) {
repo, pool, fileID := newArtifactTestRepo(t)
ctx := context.Background()
artifact, _, err := repo.EnsureQueued(ctx, newArtifact(t, fileID, fmt.Sprintf("hash-missing-remote-%d", time.Now().UnixNano())))
if err != nil {
t.Fatal(err)
}
origin := httptest.NewServer(http.NotFoundHandler())
defer origin.Close()
const (
secret = "artifact-secret"
originNodeID = 424242
originArtifactID = "artifact-missing"
)
if _, err := pool.Exec(ctx,
`UPDATE download_artifacts
SET status = 'ready', file_size = 23, completed_at = now(),
origin_node_id = $2, origin_node_url = $3, origin_node_group = 'host-a', origin_artifact_id = $4
WHERE id = $1`,
artifact.ID, originNodeID, origin.URL, originArtifactID,
); err != nil {
t.Fatal(err)
}
var userID int
if err := pool.QueryRow(ctx,
`INSERT INTO users (username, role, download_allowed) VALUES ($1, 'user', true) RETURNING id`,
fmt.Sprintf("missing-remote-user-%d", time.Now().UnixNano()),
).Scan(&userID); err != nil {
t.Fatal(err)
}
downloadID := fmt.Sprintf("missing-remote-download-%d", time.Now().UnixNano())
downloadRepo := NewRepository(pool)
if err := downloadRepo.Create(ctx, &Download{
ID: downloadID, UserID: userID, MediaFileID: fileID,
ContentID: "missing-remote-content", Kind: KindQueued, Status: StatusCompleted,
Format: FormatTranscode, ArtifactID: artifact.ID, FileSize: 23,
CreatedAt: time.Now(), UpdatedAt: time.Now(),
}); err != nil {
t.Fatal(err)
}
t.Cleanup(func() {
_, _ = pool.Exec(ctx,
`DELETE FROM download_artifact_orphans
WHERE download_artifact_id = $1 AND origin_node_id = $2 AND origin_artifact_id = $3`,
artifact.ID, originNodeID, originArtifactID,
)
_, _ = pool.Exec(ctx, `DELETE FROM downloads WHERE id = $1`, downloadID)
_, _ = pool.Exec(ctx, `DELETE FROM users WHERE id = $1`, userID)
})
cfg := &config.Config{}
cfg.Auth.JWTSecret = secret
svc := &Service{artifacts: &ArtifactManager{
repo: repo,
liveCfg: func() *config.Config { return cfg },
}}
req := httptest.NewRequest(http.MethodGet, "/api/v1/downloads/dl/file", nil)
err = svc.serveFileTarget(req.Context(), httptest.NewRecorder(), req, &FileTarget{
ArtifactID: artifact.ID,
OriginNodeID: originNodeID,
OriginNodeURL: origin.URL,
OriginNodeGroup: "host-a",
OriginArtifactID: originArtifactID,
}, 7)
if !errors.Is(err, ErrDownloadNotActive) {
t.Fatalf("serveFileTarget error = %v, want ErrDownloadNotActive", err)
}
queued, err := repo.GetByID(ctx, artifact.ID)
if err != nil {
t.Fatal(err)
}
if queued.Status != ArtifactQueued || queued.OriginNodeID != 0 || queued.OriginArtifactID != "" {
t.Fatalf("artifact after missing remote response = %+v", queued)
}
linked, err := downloadRepo.GetByID(ctx, downloadID)
if err != nil {
t.Fatal(err)
}
if linked.Status != StatusPreparing || linked.BytesSent != 0 {
t.Fatalf("linked download after missing remote response = %+v", linked)
}
var cleanupURL string
if err := pool.QueryRow(ctx,
`SELECT origin_node_url FROM download_artifact_orphans
WHERE download_artifact_id = $1 AND origin_node_id = $2 AND origin_artifact_id = $3`,
artifact.ID, originNodeID, originArtifactID,
).Scan(&cleanupURL); err != nil {
t.Fatalf("remote cleanup row: %v", err)
}
if cleanupURL != origin.URL {
t.Fatalf("remote cleanup URL = %q, want %q", cleanupURL, origin.URL)
}
}
func TestServiceDoesNotAppendAPIErrorAfterRemoteBodyIsCommitted(t *testing.T) {
const secret = "artifact-secret"
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
w.Header().Set("Content-Length", "10")
w.WriteHeader(http.StatusOK)
_, _ = io.WriteString(w, "123")
}))
defer origin.Close()
cfg := &config.Config{}
cfg.Auth.JWTSecret = secret
svc := &Service{artifacts: &ArtifactManager{liveCfg: func() *config.Config { return cfg }}}
req := httptest.NewRequest(http.MethodGet, "/api/v1/downloads/dl/file", nil)
rr := httptest.NewRecorder()
err := svc.serveFileTarget(req.Context(), rr, req, &FileTarget{
OriginNodeURL: origin.URL,
OriginArtifactID: "artifact-1",
}, 7)
if !errors.Is(err, ErrResponseCommitted) {
t.Fatalf("serveFileTarget error = %v, want ErrResponseCommitted", err)
}
if rr.Code != http.StatusOK || rr.Body.String() != "123" {
t.Fatalf("response status=%d body=%q", rr.Code, rr.Body.String())
}
}
func TestServiceRelaysRemotePreconditionFailure(t *testing.T) {
const secret = "artifact-secret"
origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if got := r.Header.Get("If-Match"); got != `"old-etag"` {
t.Errorf("origin If-Match = %q", got)
http.Error(w, "unexpected condition", http.StatusBadRequest)
return
}
w.Header().Set("ETag", `"new-etag"`)
w.WriteHeader(http.StatusPreconditionFailed)
}))
defer origin.Close()
cfg := &config.Config{}
cfg.Auth.JWTSecret = secret
svc := &Service{artifacts: &ArtifactManager{liveCfg: func() *config.Config { return cfg }}}
req := httptest.NewRequest(http.MethodGet, "/api/v1/downloads/dl/file", nil)
req.Header.Set("If-Match", `"old-etag"`)
rr := httptest.NewRecorder()
if err := svc.serveFileTarget(req.Context(), rr, req, &FileTarget{
OriginNodeURL: origin.URL,
OriginArtifactID: "artifact-1",
}, 7); err != nil {
t.Fatal(err)
}
if rr.Code != http.StatusPreconditionFailed || rr.Header().Get("ETag") != `"new-etag"` {
t.Fatalf("response status=%d etag=%q body=%q", rr.Code, rr.Header().Get("ETag"), rr.Body.String())
}
}
-50
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"strings"
"time"
)
type matcherRepository interface {
@@ -12,7 +11,6 @@ type matcherRepository interface {
MatchMediaByTitleYear(ctx context.Context, kind, title string, year int) ([]mediaLookupRow, error)
MatchEpisodeByExternalID(ctx context.Context, column, value string) ([]mediaLookupRow, error)
MatchEpisodeBySeries(ctx context.Context, seriesID string, seasonNumber, episodeNumber int) (*Match, error)
MatchEpisodesBySeries(ctx context.Context, seriesID string, seasonNumber *int, watchedAt *time.Time) ([]Match, error)
}
type Matcher struct {
@@ -36,54 +34,6 @@ func (m *Matcher) Match(ctx context.Context, record Record) (*Match, string, err
}
}
// MatchLeaves resolves a provider watched marker to concrete playable items.
// Movies and episodes have one leaf. Whole-show and whole-season markers must
// expand to the local episodes that existed when the provider marker was
// recorded; Silo derives series/season completion from those episode leaves.
func (m *Matcher) MatchLeaves(ctx context.Context, record Record) ([]Match, string, error) {
switch record.Kind {
case KindMovie, KindEpisode:
match, reason, err := m.Match(ctx, record)
if err != nil || match == nil {
return nil, reason, err
}
return []Match{*match}, "", nil
case KindSeries, KindSeason:
seriesRecord := record
var seasonNumber *int
if record.Kind == KindSeason {
seriesRecord = Record{
Kind: KindSeries,
Title: record.SeriesTitle,
Year: record.SeriesYear,
IMDbID: record.SeriesIMDbID,
TMDBID: record.SeriesTMDBID,
TVDBID: record.SeriesTVDBID,
}
season := record.SeasonNumber
seasonNumber = &season
}
seriesRecord.Kind = KindSeries
series, reason, err := m.matchSeries(ctx, seriesRecord)
if err != nil || series == nil {
return nil, reason, err
}
matches, err := m.repo.MatchEpisodesBySeries(ctx, series.MediaItemID, seasonNumber, record.LastPlayedAt)
if err != nil {
return nil, "", err
}
if len(matches) == 0 {
if seasonNumber != nil {
return nil, fmt.Sprintf("no available episodes matched for season %d", *seasonNumber), nil
}
return nil, "no available episodes matched for series", nil
}
return matches, "", nil
default:
return nil, "unsupported item kind", nil
}
}
func (m *Matcher) matchMovie(ctx context.Context, record Record) (*Match, string, error) {
match, reason, err := m.matchMedia(ctx, KindMovie, record.TMDBID, record.IMDbID, "", record.Title, record.Year, false)
if err != nil {
-69
View File
@@ -4,7 +4,6 @@ import (
"context"
"fmt"
"testing"
"time"
)
type matcherRepoStub struct {
@@ -12,15 +11,11 @@ type matcherRepoStub struct {
mediaByExternal map[string][]mediaLookupRow
mediaByTitleYear map[string][]mediaLookupRow
episodeBySeries *Match
episodesBySeries []Match
episodeBySeriesCalls int
episodeBySeriesID string
episodeBySeriesSeason int
episodeBySeriesEpisode int
episodesBySeriesID string
episodesBySeriesSeason *int
episodesBySeriesAt *time.Time
mediaByExternalCalls int
}
@@ -45,70 +40,6 @@ func (s *matcherRepoStub) MatchEpisodeBySeries(_ context.Context, seriesID strin
return s.episodeBySeries, nil
}
func (s *matcherRepoStub) MatchEpisodesBySeries(_ context.Context, seriesID string, seasonNumber *int, watchedAt *time.Time) ([]Match, error) {
s.episodesBySeriesID = seriesID
if seasonNumber != nil {
season := *seasonNumber
s.episodesBySeriesSeason = &season
}
if watchedAt != nil {
at := *watchedAt
s.episodesBySeriesAt = &at
}
return s.episodesBySeries, nil
}
func TestMatcherMatchLeavesExpandsWholeShowToEpisodes(t *testing.T) {
t.Parallel()
watchedAt := time.Date(2025, time.October, 15, 20, 0, 0, 0, time.UTC)
repo := &matcherRepoStub{
mediaByExternal: map[string][]mediaLookupRow{
"series:tmdb_id:1396": {{ContentID: "series-1", Title: "Breaking Bad", Year: 2008}},
},
episodesBySeries: []Match{
{MediaItemID: "episode-1", Kind: KindEpisode},
{MediaItemID: "episode-2", Kind: KindEpisode},
},
}
matches, reason, err := NewMatcher(repo).MatchLeaves(context.Background(), Record{
Kind: KindSeries, Title: "Breaking Bad", Year: 2008, TMDBID: "1396", LastPlayedAt: &watchedAt,
})
if err != nil || reason != "" {
t.Fatalf("MatchLeaves error = %v, reason = %q", err, reason)
}
if len(matches) != 2 || matches[0].MediaItemID != "episode-1" || matches[1].MediaItemID != "episode-2" {
t.Fatalf("matches = %#v", matches)
}
if repo.episodesBySeriesID != "series-1" || repo.episodesBySeriesSeason != nil ||
repo.episodesBySeriesAt == nil || !repo.episodesBySeriesAt.Equal(watchedAt) {
t.Fatalf("unexpected expansion scope: id=%q season=%v watched_at=%v",
repo.episodesBySeriesID, repo.episodesBySeriesSeason, repo.episodesBySeriesAt)
}
}
func TestMatcherMatchLeavesPreservesSpecialsSeasonZero(t *testing.T) {
t.Parallel()
repo := &matcherRepoStub{
mediaByExternal: map[string][]mediaLookupRow{
"series:tvdb_id:75978": {{ContentID: "series-1", Title: "Family Guy", Year: 1999}},
},
episodesBySeries: []Match{{MediaItemID: "special-1", Kind: KindEpisode}},
}
matches, reason, err := NewMatcher(repo).MatchLeaves(context.Background(), Record{
Kind: KindSeason, SeriesTitle: "Family Guy", SeriesYear: 1999,
SeriesTVDBID: "75978", SeasonNumber: 0,
})
if err != nil || reason != "" || len(matches) != 1 {
t.Fatalf("matches = %#v, reason = %q, err = %v", matches, reason, err)
}
if repo.episodesBySeriesSeason == nil || *repo.episodesBySeriesSeason != 0 {
t.Fatalf("season scope = %v, want explicit season 0", repo.episodesBySeriesSeason)
}
}
func TestMatcherMatchEpisode_UsesExternalIDsWithoutSeasonEpisodeNumbers(t *testing.T) {
t.Parallel()
-39
View File
@@ -1050,45 +1050,6 @@ func (r *Repository) MatchEpisodeBySeries(ctx context.Context, seriesID string,
return &match, nil
}
func (r *Repository) MatchEpisodesBySeries(
ctx context.Context,
seriesID string,
seasonNumber *int,
watchedAt *time.Time,
) ([]Match, error) {
rows, err := r.pool.Query(ctx, `
SELECT e.content_id, COALESCE(e.title, ''), COALESCE(series.year, 0)
FROM episodes e
LEFT JOIN media_items series ON series.content_id = e.series_id
WHERE e.series_id = $1
AND ($2::integer IS NULL OR e.season_number = $2)
AND ($3::timestamptz IS NULL OR e.air_date IS NULL OR e.air_date <= ($3::timestamptz AT TIME ZONE 'UTC')::date)
AND EXISTS (
SELECT 1 FROM episode_libraries el WHERE el.episode_id = e.content_id
)
ORDER BY e.season_number, e.episode_number, e.content_id`,
seriesID, seasonNumber, watchedAt,
)
if err != nil {
return nil, fmt.Errorf("matching episodes by series: %w", err)
}
defer rows.Close()
matches := make([]Match, 0)
for rows.Next() {
var match Match
if err := rows.Scan(&match.MediaItemID, &match.Title, &match.Year); err != nil {
return nil, fmt.Errorf("scanning series episode match: %w", err)
}
match.Kind = KindEpisode
matches = append(matches, match)
}
if err := rows.Err(); err != nil {
return nil, fmt.Errorf("iterating series episode matches: %w", err)
}
return matches, nil
}
func scanSource(scanner interface{ Scan(dest ...any) error }) (*Source, error) {
var source Source
if err := scanner.Scan(
-1
View File
@@ -21,7 +21,6 @@ const (
KindMovie = "movie"
KindSeries = "series"
KindSeason = "season"
KindEpisode = "episode"
)
+21
View File
@@ -12,6 +12,7 @@ package httpstream
import (
"context"
"errors"
"fmt"
"io"
"net"
"net/http"
@@ -171,6 +172,26 @@ func (s *RollingDeadlineWriter) BytesWritten() int64 {
return s.bytesWritten
}
// CompletedFullResponse reports whether the complete representation was
// accepted by the transport without a write error. The request context is not
// consulted because net/http may cancel it after the response reaches the
// client but before the handler performs post-response accounting. ServeContent
// returns 206 for a whole-file Range request such as "bytes=0-", so byte count
// alone is not enough to distinguish that transfer from a partial range.
func (s *RollingDeadlineWriter) CompletedFullResponse(fullSize int64) bool {
if fullSize <= 0 || s.bytesWritten != fullSize || s.firstWriteErr != nil {
return false
}
switch s.statusCode {
case http.StatusOK:
return true
case http.StatusPartialContent:
return s.Header().Get("Content-Range") == fmt.Sprintf("bytes 0-%d/%d", fullSize-1, fullSize)
default:
return false
}
}
// Outcome classifies the first write failure, or a canceled request when no
// write failure was surfaced by the transport.
func (s *RollingDeadlineWriter) Outcome(ctx context.Context) StreamOutcome {
@@ -271,6 +271,42 @@ func TestServeContentReadFromOutcomeCompletedAndCounted(t *testing.T) {
}
}
func TestCompletedFullResponse(t *testing.T) {
const body = "complete segment"
tests := []struct {
name string
rangeHeader string
method string
cancelAfterWrite bool
want bool
}{
{name: "ordinary get", method: http.MethodGet, want: true},
{name: "completed get with canceled context", method: http.MethodGet, cancelAfterWrite: true, want: true},
{name: "open ended full range", method: http.MethodGet, rangeHeader: "bytes=0-", want: true},
{name: "explicit full range", method: http.MethodGet, rangeHeader: "bytes=0-15", want: true},
{name: "partial range", method: http.MethodGet, rangeHeader: "bytes=1-", want: false},
{name: "head", method: http.MethodHead, want: false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rr := httptest.NewRecorder()
sw := newRollingDeadlineWriter(rr, time.Second, 0)
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
req := httptest.NewRequest(tt.method, "/segment.ts", nil).WithContext(ctx)
req.Header.Set("Range", tt.rangeHeader)
http.ServeContent(sw, req, "segment.ts", time.Time{}, strings.NewReader(body))
if tt.cancelAfterWrite {
cancel()
}
if got := sw.CompletedFullResponse(int64(len(body))); got != tt.want {
t.Fatalf("CompletedFullResponse = %v, want %v (status=%d range=%q bytes=%d)",
got, tt.want, sw.StatusCode(), rr.Header().Get("Content-Range"), sw.BytesWritten())
}
})
}
}
// TestReadFromPreservesCompletion exercises the io.ReaderFrom path used by
// http.ServeContent (sendfile) under a server WriteTimeout shorter than the
// transfer, with a source large enough to require multiple bounded slices.
@@ -526,15 +526,6 @@ func (s *progressCountingStore) GetSeriesPlaybackPreference(context.Context, str
func (s *progressCountingStore) DeleteSeriesPlaybackPreference(context.Context, string, string) error {
panic("unused")
}
func (s *progressCountingStore) SetCollectionSortPreference(context.Context, userstore.CollectionSortPreference) error {
panic("unused")
}
func (s *progressCountingStore) GetCollectionSortPreference(context.Context, string, string, string) (*userstore.CollectionSortPreference, error) {
panic("unused")
}
func (s *progressCountingStore) ClearCollectionSortPreference(context.Context, string, string, string) error {
panic("unused")
}
func (s *progressCountingStore) GetLibraryPlaybackPreference(context.Context, string, int) (*userstore.LibraryPlaybackPreference, error) {
panic("unused")
}
+27 -15
View File
@@ -184,6 +184,7 @@ type PlaybackHandler struct {
FFmpegPath string
HWAccel string
TranscodeDir string
SegmentRetentionSeconds func() int
// tm is the shared transcode-session lifecycle (live map, reconstruct) — the
// same type the native handler uses, so jellycompat gets the reconstruct cap
// and node-affinity rule for free. The reconstruction recipe is carried in the
@@ -268,36 +269,40 @@ func NewPlaybackHandler(
transcodeDir := filepath.Join(os.TempDir(), "silo-transcode")
ffmpegPath := ""
hwAccel := ""
segmentRetentionSeconds := 600
if cfg != nil {
if cfg.Playback.TranscodeDir != "" {
transcodeDir = cfg.Playback.TranscodeDir
}
ffmpegPath = cfg.Playback.FFmpegPath
hwAccel = cfg.Playback.HWAccel
segmentRetentionSeconds = cfg.Playback.SegmentRetentionSeconds
}
h := &PlaybackHandler{
cfg: cfg,
content: content,
codec: codec,
deviceProfiles: deviceProfiles,
playbackStore: playbackStore,
sessionMgr: sessionMgr,
fileResolver: fileResolver,
storeProvider: storeProvider,
FFmpegPath: ffmpegPath,
HWAccel: hwAccel,
TranscodeDir: transcodeDir,
tm: playback.NewTranscodeManager(),
cfg: cfg,
content: content,
codec: codec,
deviceProfiles: deviceProfiles,
playbackStore: playbackStore,
sessionMgr: sessionMgr,
fileResolver: fileResolver,
storeProvider: storeProvider,
FFmpegPath: ffmpegPath,
HWAccel: hwAccel,
TranscodeDir: transcodeDir,
SegmentRetentionSeconds: func() int { return segmentRetentionSeconds },
tm: playback.NewTranscodeManager(),
}
// Wire the shared transcode manager with closures so it reads the handler's
// (late-set) JWTSecret lazily, matching the native handler.
h.tm.JWTSecretFn = func() string { return h.JWTSecret }
h.tm.Config = func() playback.TranscodeRuntimeConfig {
return playback.TranscodeRuntimeConfig{
TranscodeDir: h.TranscodeDir,
FFmpegPath: h.FFmpegPath,
HWAccel: h.HWAccel,
TranscodeDir: h.TranscodeDir,
FFmpegPath: h.FFmpegPath,
HWAccel: h.HWAccel,
SegmentRetentionSeconds: h.segmentRetentionSeconds(),
}
}
if reg, ok := sessionMgr.(interface {
@@ -337,6 +342,13 @@ func NewPlaybackHandler(
return h
}
func (h *PlaybackHandler) segmentRetentionSeconds() int {
if h.SegmentRetentionSeconds != nil {
return h.SegmentRetentionSeconds()
}
return 600
}
// CleanupOrphanedTranscodes removes stale per-session transcode dirs, sparing
// those whose recipe card still exists. Delegates to the shared manager.
func (h *PlaybackHandler) CleanupOrphanedTranscodes() (int, error) {
@@ -0,0 +1,22 @@
package jellycompat
import (
"testing"
"github.com/Silo-Server/silo-server/internal/config"
)
func TestPlaybackHandlerReadsLiveSegmentRetention(t *testing.T) {
cfg := &config.Config{Playback: config.PlaybackConfig{SegmentRetentionSeconds: 300}}
handler := NewPlaybackHandler(cfg, nil, nil, nil, nil, nil, nil, nil)
if got := handler.tm.Config().SegmentRetentionSeconds; got != 300 {
t.Fatalf("configured segment retention = %d, want 300", got)
}
liveRetention := 300
handler.SegmentRetentionSeconds = func() int { return liveRetention }
liveRetention = 120
if got := handler.tm.Config().SegmentRetentionSeconds; got != 120 {
t.Fatalf("reloaded segment retention = %d, want 120", got)
}
}
+7
View File
@@ -103,6 +103,13 @@ func NewRouter(deps Dependencies) chi.Router {
}
userDataHandler := NewUserDataHandler(deps.ContentService, deps.UserDataService, deps.IDCodec, deps.Config)
playbackHandler := NewPlaybackHandler(deps.Config, deps.ContentService, deps.IDCodec, deps.DeviceProfiles, deps.PlaybackStore, deps.SessionMgr, deps.FileResolver, deps.UserStoreProvider)
startupSegmentRetention := playbackHandler.SegmentRetentionSeconds
playbackHandler.SegmentRetentionSeconds = func() int {
if cfg := deps.CurrentConfig(); cfg != nil {
return cfg.Playback.SegmentRetentionSeconds
}
return startupSegmentRetention()
}
if deps.DB != nil {
playbackHandler.profileStaler = recommendations.NewRepo(deps.DB)
}
+20 -14
View File
@@ -21,6 +21,7 @@ import (
"github.com/go-chi/chi/v5"
"github.com/Silo-Server/silo-server/internal/httpstream"
"github.com/Silo-Server/silo-server/internal/models"
"github.com/Silo-Server/silo-server/internal/nodepool"
"github.com/Silo-Server/silo-server/internal/playback"
@@ -418,8 +419,8 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
}
}
segmentFile := name + "." + ext
segmentPath, err := transcodeSession.GetSegment(segmentFile)
segmentName := name + "." + ext
segmentLease, err := transcodeSession.OpenSegment(segmentName)
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) {
segNum, parseErr := playback.ParseSegmentNumber(name)
if parseErr == nil {
@@ -430,7 +431,7 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
lastProducedAgeMS = now.Sub(decision.Progress.LastProducedAt).Milliseconds()
}
slog.InfoContext(r.Context(), "transcode segment missing", "component", "jellycompat",
"segment", segmentFile,
"segment", segmentName,
"requested_segment", segNum,
"produced_head", decision.Progress.ProducedHead,
"last_requested_segment", decision.Progress.LastRequestedSegment,
@@ -444,7 +445,7 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
)
if decision.Wait {
slog.InfoContext(r.Context(), "transcode segment wait", "component", "jellycompat",
"segment", segmentFile,
"segment", segmentName,
"requested_segment", segNum,
"produced_head", decision.Progress.ProducedHead,
"last_requested_segment", decision.Progress.LastRequestedSegment,
@@ -456,10 +457,10 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
"session", playSession.UpstreamSessionID,
"playback_session_id", playSession.UpstreamSessionID,
)
segmentPath, err = transcodeSession.WaitForSegment(segmentFile, decision.WaitTimeout)
segmentLease, err = transcodeSession.WaitForOpenSegment(segmentName, decision.WaitTimeout)
if err != nil && errors.Is(err, playback.ErrSegmentNotFound) {
slog.InfoContext(r.Context(), "transcode segment wait timeout", "component", "jellycompat",
"segment", segmentFile,
"segment", segmentName,
"requested_segment", segNum,
"produced_head", decision.Progress.ProducedHead,
"last_requested_segment", decision.Progress.LastRequestedSegment,
@@ -479,7 +480,7 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
if seekErr != nil && !errors.Is(seekErr, playback.ErrManifestNotReady) {
slog.ErrorContext(r.Context(), "resolve transcode seek target", "component", "jellycompat",
"error", seekErr,
"segment", segmentFile,
"segment", segmentName,
"play_session", playSessionID,
"session", playSession.UpstreamSessionID,
"playback_session_id", playSession.UpstreamSessionID,
@@ -498,7 +499,7 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
if ok {
slog.InfoContext(r.Context(), "transcode seek restart", "component", "jellycompat",
"segment", segmentFile,
"segment", segmentName,
"requested_segment", segNum,
"produced_head", decision.Progress.ProducedHead,
"last_requested_segment", decision.Progress.LastRequestedSegment,
@@ -519,14 +520,14 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
seekSeconds,
segNum,
); restartErr == nil {
segmentPath, err = transcodeSession.WaitForSegment(segmentFile, 30*time.Second)
segmentLease, err = transcodeSession.WaitForOpenSegment(segmentName, 30*time.Second)
}
}
}
} else if transcodeSession.IsRunning() {
// Non-numbered segment (e.g. init.mp4 for fMP4 HLS).
// Wait briefly — the init segment is written almost immediately.
segmentPath, err = transcodeSession.WaitForSegment(segmentFile, 10*time.Second)
segmentLease, err = transcodeSession.WaitForOpenSegment(segmentName, 10*time.Second)
}
}
if err != nil {
@@ -535,11 +536,15 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
return
}
if segNum, parseErr := playback.ParseSegmentNumber(name); parseErr == nil {
transcodeSession.ReportSegmentDownloaded(segNum)
defer func() { _ = segmentLease.Close() }()
sw := httpstream.NewRollingDeadlineWriter(w)
http.ServeContent(sw, r, segmentLease.Info.Name(), segmentLease.Info.ModTime(), segmentLease.File)
if r.Method == http.MethodGet &&
sw.CompletedFullResponse(segmentLease.Info.Size()) {
if segNum, parseErr := playback.ParseSegmentNumber(name); parseErr == nil {
transcodeSession.ReportSegmentDownloadedForGeneration(segNum, segmentLease.Generation)
}
}
http.ServeFile(w, r, segmentPath)
}
// hlsSegmentErrorResponse maps a segment-retrieval error to a Jellyfin-faithful
@@ -1669,6 +1674,7 @@ func (h *PlaybackHandler) ensureTranscodeSession(ctx context.Context, playSessio
TotalDuration: float64(source.Version.Duration),
FastStart: true,
}
opts.SegmentRetentionSeconds = h.segmentRetentionSeconds()
if source.TranscodeAudio {
opts.TargetCodecVideo = "copy"
}
-1
View File
@@ -292,7 +292,6 @@ type VideoTrack struct {
Codec string `json:"codec,omitempty"`
DolbyVision string `json:"dolby_vision,omitempty"`
DVProfile int `json:"dv_profile,omitempty"`
DVLevel int `json:"dv_level,omitempty"`
DVBLCompatID int `json:"dv_bl_compat_id,omitempty"`
DVELPresent bool `json:"dv_el_present,omitempty"`
DVEnhancementLayer string `json:"dv_enhancement_layer,omitempty"` // none, mel, fel, unknown
+9 -155
View File
@@ -4,8 +4,6 @@ import (
"context"
"sync"
"time"
"github.com/google/uuid"
)
// Plan is the result of a node selection for one playback session.
@@ -22,22 +20,6 @@ type SessionPlanner interface {
PlanSession(sessionID, currentTranscodeURL string, needsTranscode bool, estBitrateKbps int) Plan
}
// DownloadPlanner selects proxy nodes for unbounded file delivery. Downloads
// have no predictable bitrate, so implementations must not admit them onto a
// proxy with a configured bandwidth cap.
type DownloadPlanner interface {
PlanDownload(sessionID string, preferredGroup ...string) Plan
ReleaseSession(sessionID string)
}
// TranscodeWorkPlanner reserves capacity for non-streaming GPU work such as a
// prepared download. The returned release function must be called after the
// remote operation ends or falls back locally.
type TranscodeWorkPlanner interface {
ReserveTranscodeWork(workID string) (*Node, func())
TranscodeNode(nodeID int) (*Node, bool)
}
// reservation bridges the gap between assigning a session to a node and the
// node's health reports reflecting that session.
//
@@ -109,65 +91,6 @@ func (p *Planner) PlanSession(sessionID, currentTranscodeURL string, needsTransc
return p.PlanSessionWith(sessionID, currentTranscodeURL, needsTranscode, estBitrateKbps, nil)
}
// TranscodeNode returns the current pool record for a persistent node id. It
// lets durable artifact locators follow an administrator-edited node URL/group.
func (p *Planner) TranscodeNode(nodeID int) (*Node, bool) {
if p == nil || p.transcodes == nil || nodeID == 0 {
return nil, false
}
for _, node := range p.transcodes.Nodes() {
if node != nil && node.ID == nodeID && node.Enabled {
return node, true
}
}
return nil, false
}
// PlanDownload picks a healthy proxy for an unbounded file transfer. A
// configured proxy bandwidth cap cannot be reserved accurately without a known
// transfer rate, so capped proxies are excluded instead of being oversubscribed
// during the egress meter's convergence window.
func (p *Planner) PlanDownload(sessionID string, preferredGroup ...string) Plan {
if p == nil || p.proxies == nil || sessionID == "" {
return Plan{}
}
p.mu.Lock()
defer p.mu.Unlock()
now := p.now()
p.pruneReservations(now)
delete(p.reserved, sessionID)
group := ""
if len(preferredGroup) > 0 {
group = preferredGroup[0]
}
var candidates, fallback []*Node
for _, node := range p.proxies.Nodes() {
if node == nil || !node.Enabled || !node.Healthy || !p.underCap(node, now) {
continue
}
if node.MaxBandwidthKbps != nil && *node.MaxBandwidthKbps > 0 {
continue
}
fallback = append(fallback, node)
if group != "" && node.Group != nil && *node.Group == group {
candidates = append(candidates, node)
}
}
if group == "" || len(candidates) == 0 {
candidates = fallback
}
if len(candidates) == 0 {
return Plan{}
}
rrKey := "download:" + group
proxy := candidates[p.rr[rrKey]%len(candidates)]
p.rr[rrKey]++
p.reserved[sessionID] = &reservation{proxyURL: proxy.URL, createdAt: now}
return Plan{ProxyNode: proxy}
}
// PlanSessionWith behaves like PlanSession but restricts transcode-node
// selection to nodes accepted by eligible (nil accepts every node). Capability
// -aware playback planning uses it so a recipe that only some pooled nodes
@@ -193,27 +116,24 @@ func (p *Planner) PlanSessionWith(sessionID, currentTranscodeURL string, needsTr
}
proxies := p.proxies.Nodes()
transcodes := p.transcodes.Nodes()
// Group health is computed over the full pool before any narrowing:
// eligibility restricts what may be selected, not co-location semantics.
groupHealthy := groupHealth(proxies, transcodes)
if eligible != nil {
filtered := make([]*Node, 0, len(transcodes))
for _, node := range transcodes {
if eligible(node) {
filtered = append(filtered, node)
}
}
transcodes = filtered
}
var plan Plan
if needsTranscode {
if eligible != nil {
transcodes = filterNodes(transcodes, eligible)
}
plan.TranscodeNode = p.pickTranscode(transcodes, proxies, groupHealthy, currentTranscodeURL, estBitrateKbps, now)
if plan.TranscodeNode != nil {
plan.ProxyNode = p.pickProxy(proxies, groupHealthy, plan.TranscodeNode.Group, estBitrateKbps, now)
}
} else {
// A proxy-only plan has no transcode node, so the predicate applies to
// the proxy: it is the node that will execute the recipe. Filtering
// before selection means a capability mismatch skips to a capable
// sibling instead of abandoning the pool after one round-robin pick.
if eligible != nil {
proxies = filterNodes(proxies, eligible)
}
plan.ProxyNode = p.pickProxy(proxies, groupHealthy, nil, estBitrateKbps, now)
}
@@ -231,36 +151,6 @@ func (p *Planner) PlanSessionWith(sessionID, currentTranscodeURL string, needsTr
return plan
}
// filterNodes returns the nodes accepted by keep, preserving pool order so
// round-robin cursors stay meaningful across selections.
func filterNodes(nodes []*Node, keep func(*Node) bool) []*Node {
filtered := make([]*Node, 0, len(nodes))
for _, node := range nodes {
if keep(node) {
filtered = append(filtered, node)
}
}
return filtered
}
// ProxyNodeURLs lists the URLs of every enabled pooled proxy node, healthy or
// not, mirroring TranscodeNodeURLs. Capability planning wants the deployment's
// toolchain; an unreachable node excludes itself when its capability fetch
// fails.
func (p *Planner) ProxyNodeURLs() []string {
if p == nil || p.proxies == nil {
return nil
}
nodes := p.proxies.Nodes()
urls := make([]string, 0, len(nodes))
for _, node := range nodes {
if node != nil && node.URL != "" {
urls = append(urls, node.URL)
}
}
return urls
}
// TranscodeNodeURLs lists the URLs of every enabled pooled transcode node,
// healthy or not: capability planning wants the deployment's toolchain, and
// an unreachable node excludes itself when its capability fetch fails. An
@@ -290,42 +180,6 @@ func (p *Planner) ReleaseSession(sessionID string) {
p.mu.Unlock()
}
// ReserveTranscodeWork selects the least-loaded healthy transcode node while
// sharing the same health-bridging reservation accounting as playback. Unlike
// a playback session it does not require a proxy partner: the completed file
// is written to the configured shared artifact store and served later.
func (p *Planner) ReserveTranscodeWork(workID string) (*Node, func()) {
if p == nil || p.transcodes == nil || workID == "" {
return nil, func() {}
}
p.mu.Lock()
now := p.now()
p.pruneReservations(now)
reservationID := workID + "-" + uuid.NewString()
var best *Node
for _, node := range p.transcodes.Nodes() {
if node == nil || !node.Enabled || !node.Healthy || !p.underCap(node, now) {
continue
}
if best == nil || p.effectiveJobs(node, now) < p.effectiveJobs(best, now) {
best = node
}
}
if best != nil {
p.reserved[reservationID] = &reservation{transcodeURL: best.URL, createdAt: now}
}
p.mu.Unlock()
if best == nil {
return nil, func() {}
}
var once sync.Once
return best, func() {
once.Do(func() { p.ReleaseSession(reservationID) })
}
}
// groupHealth reports, for every group label present in either pool, whether
// all of its enabled members are healthy. Pools only hold enabled nodes, so
// disabled nodes never count against a group.
-162
View File
@@ -1,7 +1,6 @@
package nodepool
import (
"strconv"
"testing"
"time"
)
@@ -510,164 +509,3 @@ func TestUnknownBitrateAdmittedBelowCap(t *testing.T) {
t.Fatalf("unknown-bitrate stream should be rejected at cap, got %+v", got)
}
}
func TestReserveTranscodeWorkSharesCapacityReservations(t *testing.T) {
capOne := 1
transcodes := NewTranscodePool()
transcodes.SetNodes([]*Node{
{URL: "http://transcode-a", Enabled: true, Healthy: true, MaxJobs: &capOne},
{URL: "http://transcode-b", Enabled: true, Healthy: true, MaxJobs: &capOne},
})
planner := NewPlanner(NewProxyPool(), transcodes)
first, releaseFirst := planner.ReserveTranscodeWork("download-1")
if first == nil || first.URL != "http://transcode-a" {
t.Fatalf("first node = %+v", first)
}
second, releaseSecond := planner.ReserveTranscodeWork("download-2")
if second == nil || second.URL != "http://transcode-b" {
t.Fatalf("second node = %+v", second)
}
if third, _ := planner.ReserveTranscodeWork("download-3"); third != nil {
t.Fatalf("third node = %+v, want no capacity", third)
}
releaseFirst()
third, releaseThird := planner.ReserveTranscodeWork("download-3")
if third == nil || third.URL != "http://transcode-a" {
t.Fatalf("third node after release = %+v", third)
}
releaseSecond()
releaseThird()
}
func TestReserveTranscodeWorkOverlappingAttemptsReleaseOnlyTheirOwnReservation(t *testing.T) {
capTwo := 2
transcodes := NewTranscodePool()
transcodes.SetNodes([]*Node{{URL: "http://transcode-a", Enabled: true, Healthy: true, MaxJobs: &capTwo}})
planner := NewPlanner(NewProxyPool(), transcodes)
first, releaseFirst := planner.ReserveTranscodeWork("same-artifact")
second, releaseSecond := planner.ReserveTranscodeWork("same-artifact")
if first == nil || second == nil {
t.Fatalf("overlapping reservations = first %+v second %+v", first, second)
}
if third, _ := planner.ReserveTranscodeWork("third-attempt"); third != nil {
t.Fatalf("third reservation = %+v, want full node", third)
}
releaseFirst()
third, releaseThird := planner.ReserveTranscodeWork("third-attempt")
if third == nil {
t.Fatal("first release did not free its own reservation")
}
if fourth, _ := planner.ReserveTranscodeWork("fourth-attempt"); fourth != nil {
t.Fatalf("second overlapping reservation was lost: fourth = %+v", fourth)
}
releaseSecond()
releaseThird()
}
func TestPlanDownloadSkipsBandwidthCappedProxiesAndReservesJobCapacity(t *testing.T) {
bandwidthCap := 100_000
jobCap := 1
proxies := NewProxyPool()
proxies.SetNodes([]*Node{
{URL: "http://bandwidth-capped", Enabled: true, Healthy: true, MaxBandwidthKbps: &bandwidthCap},
{URL: "http://uncapped", Enabled: true, Healthy: true, MaxJobs: &jobCap},
})
planner := NewPlanner(proxies, NewTranscodePool())
first := planner.PlanDownload("download-1")
if first.ProxyNode == nil || first.ProxyNode.URL != "http://uncapped" {
t.Fatalf("first plan = %+v", first)
}
if second := planner.PlanDownload("download-2"); second.ProxyNode != nil {
t.Fatalf("second plan = %+v, want job-cap rejection", second)
}
planner.ReleaseSession("download-1")
if second := planner.PlanDownload("download-2"); second.ProxyNode == nil {
t.Fatal("download was not admitted after reservation release")
}
}
func TestPlanDownloadPrefersArtifactOriginGroup(t *testing.T) {
groupA, groupB := "host-a", "host-b"
proxies := NewProxyPool()
proxies.SetNodes([]*Node{
{URL: "http://proxy-a", Group: &groupA, Enabled: true, Healthy: true},
{URL: "http://proxy-b", Group: &groupB, Enabled: true, Healthy: true},
})
planner := NewPlanner(proxies, NewTranscodePool())
plan := planner.PlanDownload("download-grouped", groupB)
if plan.ProxyNode == nil || plan.ProxyNode.URL != "http://proxy-b" {
t.Fatalf("plan = %+v, want proxy-b", plan)
}
}
func TestPlanDownloadFallsBackWhenOriginGroupHasNoProxy(t *testing.T) {
group := "host-a"
proxies := NewProxyPool()
proxies.SetNodes([]*Node{{URL: "http://proxy-a", Group: &group, Enabled: true, Healthy: true}})
planner := NewPlanner(proxies, NewTranscodePool())
plan := planner.PlanDownload("download-fallback", "host-missing")
if plan.ProxyNode == nil || plan.ProxyNode.URL != "http://proxy-a" {
t.Fatalf("plan = %+v, want proxy-a fallback", plan)
}
}
// A proxy-only plan applies the eligibility predicate to the proxy: it is the
// node that executes the recipe. Without this, one incapable round-robin pick
// would abandon a pool that still holds a capable sibling.
func TestPlanSessionWithFiltersProxiesForProxyOnlyPlans(t *testing.T) {
proxies := NewProxyPool()
proxies.SetNodes([]*Node{
{ID: 1, URL: "http://proxy-old", Enabled: true, Healthy: true},
{ID: 2, URL: "http://proxy-new", Enabled: true, Healthy: true},
})
planner := NewPlanner(proxies, NewTranscodePool())
// Only the upgraded proxy can run the recipe; every selection must land
// there regardless of where the round-robin cursor happens to be.
for i := 0; i < 4; i++ {
plan := planner.PlanSessionWith("session-"+strconv.Itoa(i), "", false, 0, func(n *Node) bool {
return n.URL == "http://proxy-new"
})
if plan.ProxyNode == nil || plan.ProxyNode.URL != "http://proxy-new" {
t.Fatalf("selection %d = %#v, want the capable proxy", i, plan.ProxyNode)
}
}
}
func TestPlanSessionWithReturnsNoProxyWhenNoneAreCapable(t *testing.T) {
proxies := NewProxyPool()
proxies.SetNodes([]*Node{{ID: 1, URL: "http://proxy-old", Enabled: true, Healthy: true}})
planner := NewPlanner(proxies, NewTranscodePool())
plan := planner.PlanSessionWith("session-none", "", false, 0, func(*Node) bool { return false })
if plan.ProxyNode != nil {
t.Fatalf("proxy = %#v, want none when the pool cannot execute the recipe", plan.ProxyNode)
}
// An empty plan must not leave a reservation pinning capacity.
if _, reserved := planner.reserved["session-none"]; reserved {
t.Fatal("an unsatisfiable plan left a reservation behind")
}
}
func TestProxyNodeURLsListsEnabledProxies(t *testing.T) {
proxies := NewProxyPool()
proxies.SetNodes([]*Node{
{ID: 1, URL: "http://proxy-1", Enabled: true, Healthy: true},
{ID: 2, URL: "http://proxy-2", Enabled: true, Healthy: false},
})
planner := NewPlanner(proxies, NewTranscodePool())
// Unhealthy nodes are still listed: capability planning wants the
// deployment's toolchain, and an unreachable node excludes itself when its
// capability fetch fails.
urls := planner.ProxyNodeURLs()
if len(urls) != 2 {
t.Fatalf("proxy urls = %v, want both pooled proxies", urls)
}
}
+1 -1
View File
@@ -26,7 +26,7 @@ type SessionInfo struct {
UserID string `json:"user_id,omitempty"`
MediaItemID string `json:"media_item_id,omitempty"`
MediaTitle string `json:"media_title,omitempty"`
Type string `json:"type"` // "direct_play", "remux", "transcode", "download_prepare", "download"
Type string `json:"type"` // "direct_play", "remux", "transcode"
CodecVideo string `json:"codec_video,omitempty"`
CodecAudio string `json:"codec_audio,omitempty"`
Resolution string `json:"resolution,omitempty"`
+2 -133
View File
@@ -39,28 +39,12 @@ func (p *interestTrackingProvider) ForUser(ctx context.Context, userID int) (use
return store, err
}
tracked := &interestTrackingStore{UserStore: store, userID: userID, system: p.system, updater: p.system.Interest}
// Preserve the interface upgrades callers probe for. Both are conditional
// on the backing store: advertising a capability it does not have would
// send callers down a fast path that can only fail.
registry, hasDevices := store.(userstore.DeviceRegistry)
rollup, hasRollup := store.(userstore.SeriesEpisodeRollupStore)
switch {
case hasDevices && hasRollup:
return &interestTrackingStoreWithDevicesAndRollup{
interestTrackingStore: tracked,
DeviceRegistry: registry,
SeriesEpisodeRollupStore: rollup,
}, nil
case hasDevices:
// Preserve the DeviceRegistry interface upgrade some callers probe for.
if registry, ok := store.(userstore.DeviceRegistry); ok {
return &interestTrackingStoreWithDevices{
interestTrackingStore: tracked,
DeviceRegistry: registry,
}, nil
case hasRollup:
return &interestTrackingStoreWithRollup{
interestTrackingStore: tracked,
SeriesEpisodeRollupStore: rollup,
}, nil
}
return tracked, nil
}
@@ -81,48 +65,11 @@ type interestTrackingStoreWithDevices struct {
userstore.DeviceRegistry
}
// interestTrackingStoreWithRollup adds the series-rollup capability only when
// the backing store actually has it. Unlike the other capabilities, this one
// cannot be forwarded unconditionally: the per-user SQLite backend has no
// catalog tables and genuinely cannot answer the query, and callers treat
// "implements the interface" as "can do this". Advertising it anyway would
// send every jellycompat series request down the fast path to fail, logging a
// warning each time before falling back — turning an expected capability
// absence into recurring noise on successful requests.
type interestTrackingStoreWithRollup struct {
*interestTrackingStore
userstore.SeriesEpisodeRollupStore
}
type interestTrackingStoreWithDevicesAndRollup struct {
*interestTrackingStore
userstore.DeviceRegistry
userstore.SeriesEpisodeRollupStore
}
var _ userstore.SettingValueCompareAndSetter = (*interestTrackingStore)(nil)
var _ userstore.SettingMutationTransactioner = (*interestTrackingStore)(nil)
var _ userstore.SettingValueCompareAndSetter = (*interestTrackingStoreWithDevices)(nil)
var _ userstore.SettingMutationTransactioner = (*interestTrackingStoreWithDevices)(nil)
// Optional store capabilities must survive the decorator. Embedding the
// UserStore interface promotes only that interface's methods, so each of these
// needs an explicit forward below; the assertions make a missing one a compile
// error instead of a silent production slowdown.
//
// SeriesEpisodeRollupStore is deliberately absent here: it is conditional on
// the backing store, so it lives on the wrapper types above rather than being
// forwarded unconditionally.
var _ userstore.WatchedBatchWriter = (*interestTrackingStore)(nil)
var _ userstore.VisibleHistoryAdder = (*interestTrackingStore)(nil)
var _ userstore.HistoryVisibilityStore = (*interestTrackingStore)(nil)
var _ userstore.WatchedBatchWriter = (*interestTrackingStoreWithDevices)(nil)
var _ userstore.VisibleHistoryAdder = (*interestTrackingStoreWithDevices)(nil)
var _ userstore.HistoryVisibilityStore = (*interestTrackingStoreWithDevices)(nil)
var _ userstore.SeriesEpisodeRollupStore = (*interestTrackingStoreWithRollup)(nil)
var _ userstore.SeriesEpisodeRollupStore = (*interestTrackingStoreWithDevicesAndRollup)(nil)
var _ userstore.DeviceRegistry = (*interestTrackingStoreWithDevicesAndRollup)(nil)
// WithPreferenceSettingsTransaction preserves the optional atomic-settings
// capability of the wrapped store. Preference writes do not affect interest
// signals, so the transaction can pass through unchanged; keeping the method
@@ -351,84 +298,6 @@ func (s *interestTrackingStore) RemoveHistoryItems(ctx context.Context, profileI
return err
}
// --- Optional store capabilities.
//
// This decorator embeds the userstore.UserStore *interface*, which promotes
// only the methods declared on that interface. Any optional capability the
// backing store implements is invisible through the wrapper unless it is
// forwarded explicitly here — and because callers reach these via type
// assertion with a working fallback, a missing forward is silent: no error, no
// test failure, just a much slower path in production. Marking a series
// watched regressed exactly this way. When adding a new optional capability to
// userstore, forward it here and extend the compile-time assertions below.
// MarkWatchedBatch forwards the transactional batch mark-watched write, whose
// whole purpose is that a large series commits as one unit. Falling through to
// the per-target loop would restore the partial-write window it removes.
func (s *interestTrackingStore) MarkWatchedBatch(
ctx context.Context,
profileID string,
targets []userstore.MarkWatchedTarget,
entries []userstore.WatchHistoryEntry,
) ([]userstore.WatchHistoryEntry, error) {
writer, ok := s.UserStore.(userstore.WatchedBatchWriter)
if !ok {
// The helper runs against the inner store, so this decorator's own
// MarkWatched hook never fires; queue here or the recompute is lost.
// A mid-loop error still leaves earlier targets written, so queue
// regardless of err — a redundant queue only costs one recompute,
// while a missing one leaves profile_series_interest stale until some
// unrelated mutation or the rebuild task touches the series.
written, err := userstore.MarkWatchedBatch(ctx, s.UserStore, profileID, targets, entries)
s.queueTargetMutations(profileID, targets)
return written, err
}
written, err := writer.MarkWatchedBatch(ctx, profileID, targets, entries)
// The batch write is one transaction: on error nothing landed, so there is
// nothing to recompute.
if err == nil {
s.queueItemMutations(profileID, written)
}
return written, err
}
// queueItemMutations records one interest mutation per written entry, matching
// what the per-target path queues through MarkWatched.
func (s *interestTrackingStore) queueItemMutations(profileID string, entries []userstore.WatchHistoryEntry) {
for _, entry := range entries {
s.updater.QueueItemMutation(s.userID, profileID, entry.MediaItemID)
}
}
// queueTargetMutations queues by requested target rather than written entry,
// for the non-transactional path where a partial write may have occurred.
func (s *interestTrackingStore) queueTargetMutations(profileID string, targets []userstore.MarkWatchedTarget) {
for _, target := range targets {
s.updater.QueueItemMutation(s.userID, profileID, target.MediaItemID)
}
}
// AddVisibleHistory forwards the watermark-aware history insert. The generic
// fallback needs two round-trips (timestamp lookup, then insert) to do the
// same job.
func (s *interestTrackingStore) AddVisibleHistory(ctx context.Context, entry userstore.WatchHistoryEntry) (userstore.WatchHistoryEntry, error) {
adder, ok := s.UserStore.(userstore.VisibleHistoryAdder)
if !ok {
return userstore.AddVisibleHistory(ctx, s.UserStore, entry)
}
return adder.AddVisibleHistory(ctx, entry)
}
// VisibleHistoryTimestamps forwards the batched watermark lookup; without it
// callers assume a single wall-clock timestamp for every item.
func (s *interestTrackingStore) VisibleHistoryTimestamps(ctx context.Context, profileID string, mediaItemIDs []string, at time.Time) (map[string]string, error) {
visibility, ok := s.UserStore.(userstore.HistoryVisibilityStore)
if !ok {
return userstore.VisibleHistoryTimestamps(ctx, s.UserStore, profileID, mediaItemIDs, at)
}
return visibility.VisibleHistoryTimestamps(ctx, profileID, mediaItemIDs, at)
}
func (s *interestTrackingStore) DeleteHistoryBySource(ctx context.Context, profileID string, mediaItemIDs []string, source userstore.WatchHistorySource) error {
err := s.UserStore.DeleteHistoryBySource(ctx, profileID, mediaItemIDs, source)
if err == nil {
@@ -100,231 +100,3 @@ func TestInterestTrackingStorePreservesSettingCapabilities(t *testing.T) {
t.Fatalf("wrapped receipt = %+v (%v)", receipt, err)
}
}
// TestInterestTrackingStorePreservesWatchStateCapabilities pins the optional
// watch-state capabilities through the wrapper. These are type-asserted at
// their call sites (userstore.MarkWatchedBatch, AddVisibleHistory,
// VisibleHistoryTimestamps, jellycompat's rollup), and the decorator embeds
// the UserStore *interface* — which promotes only the methods in that
// interface. A capability the decorator does not forward explicitly is
// therefore invisible in production, and the caller silently degrades to its
// slow per-item fallback with no error and no test failure.
func TestInterestTrackingStorePreservesWatchStateCapabilities(t *testing.T) {
db, err := sql.Open("sqlite3", ":memory:")
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
if err := userdb.InitSchema(db); err != nil {
t.Fatalf("init schema: %v", err)
}
inner := userdb.NewSQLiteUserStore(db)
provider := WrapUserStoreProvider(
preferenceTransactionTestProvider{store: inner},
&System{},
)
wrapped, err := provider.ForUser(context.Background(), 1)
if err != nil {
t.Fatalf("ForUser: %v", err)
}
// Every capability the bare store advertises must survive wrapping,
// otherwise the wrapped store is a silent performance downgrade.
for _, capability := range []struct {
name string
has func(userstore.UserStore) bool
}{
{"WatchedBatchWriter", func(s userstore.UserStore) bool {
_, ok := s.(userstore.WatchedBatchWriter)
return ok
}},
{"VisibleHistoryAdder", func(s userstore.UserStore) bool {
_, ok := s.(userstore.VisibleHistoryAdder)
return ok
}},
{"HistoryVisibilityStore", func(s userstore.UserStore) bool {
_, ok := s.(userstore.HistoryVisibilityStore)
return ok
}},
} {
if !capability.has(inner) {
t.Fatalf("test setup: bare store does not implement %s", capability.name)
}
if !capability.has(wrapped) {
t.Errorf("interest-tracking wrapper dropped %s; callers silently fall back to the per-item path",
capability.name)
}
}
// SeriesEpisodeRollupStore is the exception: it must be advertised only
// when the backing store can actually answer it. The per-user SQLite store
// has no catalog tables, so claiming the capability would send every
// jellycompat series request down a fast path that can only fail and log.
if _, ok := userstore.UserStore(inner).(userstore.SeriesEpisodeRollupStore); ok {
t.Fatal("test setup: SQLite store unexpectedly implements SeriesEpisodeRollupStore")
}
if _, ok := wrapped.(userstore.SeriesEpisodeRollupStore); ok {
t.Error("wrapper advertises SeriesEpisodeRollupStore for a store that cannot perform the rollup")
}
// The forwarded batch write must actually reach the backing store.
writer, ok := wrapped.(userstore.WatchedBatchWriter)
if !ok {
t.Fatal("wrapper dropped WatchedBatchWriter; cannot verify pass-through")
}
if err := wrapped.CreateProfile(context.Background(), userstore.Profile{ID: "p1", Name: "Test"}); err != nil {
t.Fatalf("CreateProfile: %v", err)
}
watchedAt := time.Date(2026, 8, 15, 12, 0, 0, 0, time.UTC).Format(time.RFC3339)
written, err := writer.MarkWatchedBatch(context.Background(), "p1",
[]userstore.MarkWatchedTarget{
{MediaItemID: "ep-1", DurationSeconds: 1200},
{MediaItemID: "ep-2", DurationSeconds: 1500},
},
[]userstore.WatchHistoryEntry{
{ProfileID: "p1", MediaItemID: "ep-1", WatchedAt: watchedAt, DurationSeconds: 1200, Completed: true, Source: userstore.WatchHistorySourceManual},
{ProfileID: "p1", MediaItemID: "ep-2", WatchedAt: watchedAt, DurationSeconds: 1500, Completed: true, Source: userstore.WatchHistorySourceManual},
},
)
if err != nil {
t.Fatalf("MarkWatchedBatch through wrapper: %v", err)
}
if len(written) != 2 {
t.Fatalf("MarkWatchedBatch returned %d entries, want 2", len(written))
}
for _, id := range []string{"ep-1", "ep-2"} {
progress, err := wrapped.GetProgress(context.Background(), "p1", id)
if err != nil || progress == nil || !progress.Completed {
t.Fatalf("GetProgress(%s) = %+v (%v), want completed", id, progress, err)
}
}
}
// storeWithoutBatchWriter wraps a UserStore and hides the WatchedBatchWriter
// capability, standing in for a backend that has not implemented it.
type storeWithoutBatchWriter struct {
userstore.UserStore
}
// TestInterestTrackingStoreQueuesMutationsOnBatchFallback covers the branch
// taken when the backing store lacks WatchedBatchWriter. The decorator hands
// the work to the generic helper against s.UserStore — the *inner* store — so
// the decorator's own MarkWatched hook never runs and nothing queues an
// interest recompute. Without this, marking a series watched on such a backend
// updates progress and history but leaves profile_series_interest stale until
// some unrelated mutation or the rebuild task happens to touch the series.
func TestInterestTrackingStoreQueuesMutationsOnBatchFallback(t *testing.T) {
db, err := sql.Open("sqlite3", ":memory:")
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
if err := userdb.InitSchema(db); err != nil {
t.Fatalf("init schema: %v", err)
}
inner := storeWithoutBatchWriter{UserStore: userdb.NewSQLiteUserStore(db)}
if _, ok := userstore.UserStore(inner).(userstore.WatchedBatchWriter); ok {
t.Fatal("test setup: inner store must not implement WatchedBatchWriter")
}
updater := &InterestUpdater{pending: map[interestMutation]int{}}
store := &interestTrackingStore{
UserStore: inner,
userID: 1,
system: &System{},
updater: updater,
}
if err := store.CreateProfile(context.Background(), userstore.Profile{ID: "p1", Name: "Test"}); err != nil {
t.Fatalf("CreateProfile: %v", err)
}
watchedAt := time.Date(2026, 8, 15, 12, 0, 0, 0, time.UTC).Format(time.RFC3339)
if _, err := store.MarkWatchedBatch(context.Background(), "p1",
[]userstore.MarkWatchedTarget{
{MediaItemID: "ep-1", DurationSeconds: 1200},
{MediaItemID: "ep-2", DurationSeconds: 1500},
},
[]userstore.WatchHistoryEntry{
{ProfileID: "p1", MediaItemID: "ep-1", WatchedAt: watchedAt, DurationSeconds: 1200, Completed: true, Source: userstore.WatchHistorySourceManual},
{ProfileID: "p1", MediaItemID: "ep-2", WatchedAt: watchedAt, DurationSeconds: 1500, Completed: true, Source: userstore.WatchHistorySourceManual},
},
); err != nil {
t.Fatalf("MarkWatchedBatch (fallback path): %v", err)
}
updater.mu.Lock()
queued := make(map[string]struct{}, len(updater.pending))
for mutation := range updater.pending {
queued[mutation.itemID] = struct{}{}
}
updater.mu.Unlock()
for _, itemID := range []string{"ep-1", "ep-2"} {
if _, ok := queued[itemID]; !ok {
t.Errorf("no interest mutation queued for %s on the batch fallback path; interest goes stale", itemID)
}
}
}
// rollupCapableStore is a UserStore that also implements the series rollup,
// standing in for the Postgres backend.
type rollupCapableStore struct {
userstore.UserStore
called bool
}
func (s *rollupCapableStore) SeriesEpisodeWatchCounts(_ context.Context, _ string, seriesIDs []string) (map[string]userstore.SeriesWatchCounts, error) {
s.called = true
counts := make(map[string]userstore.SeriesWatchCounts, len(seriesIDs))
for _, seriesID := range seriesIDs {
counts[seriesID] = userstore.SeriesWatchCounts{TotalEpisodes: 3, WatchedCount: 2}
}
return counts, nil
}
// TestInterestTrackingStoreForwardsRollupWhenSupported is the other half of the
// conditional: a backend that can do the rollup must keep advertising it
// through the wrapper, and calls must reach it. Losing this would silently
// push jellycompat back onto the per-episode rollup for every series.
func TestInterestTrackingStoreForwardsRollupWhenSupported(t *testing.T) {
db, err := sql.Open("sqlite3", ":memory:")
if err != nil {
t.Fatalf("open sqlite: %v", err)
}
t.Cleanup(func() { _ = db.Close() })
if err := userdb.InitSchema(db); err != nil {
t.Fatalf("init schema: %v", err)
}
inner := &rollupCapableStore{UserStore: userdb.NewSQLiteUserStore(db)}
provider := WrapUserStoreProvider(preferenceTransactionTestProvider{store: inner}, &System{})
wrapped, err := provider.ForUser(context.Background(), 1)
if err != nil {
t.Fatalf("ForUser: %v", err)
}
rollup, ok := wrapped.(userstore.SeriesEpisodeRollupStore)
if !ok {
t.Fatal("wrapper dropped SeriesEpisodeRollupStore for a store that supports it")
}
counts, err := rollup.SeriesEpisodeWatchCounts(context.Background(), "p1", []string{"series-1"})
if err != nil {
t.Fatalf("SeriesEpisodeWatchCounts: %v", err)
}
if !inner.called {
t.Error("rollup call did not reach the backing store")
}
if counts["series-1"].WatchedCount != 2 {
t.Errorf("counts = %+v, want WatchedCount 2", counts["series-1"])
}
// The other capabilities must still survive alongside the rollup.
if _, ok := wrapped.(userstore.WatchedBatchWriter); !ok {
t.Error("rollup-capable wrapper dropped WatchedBatchWriter")
}
if _, ok := wrapped.(userstore.SettingValueCompareAndSetter); !ok {
t.Error("rollup-capable wrapper dropped SettingValueCompareAndSetter")
}
}
+23 -71
View File
@@ -2,6 +2,7 @@ package notifications
import (
"context"
"errors"
"fmt"
"log/slog"
"sync"
@@ -9,6 +10,7 @@ import (
"github.com/Silo-Server/silo-server/internal/access"
"github.com/Silo-Server/silo-server/internal/userstore"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
)
@@ -125,34 +127,18 @@ func (u *InterestUpdater) flush(ctx context.Context) {
profileID string
seriesID string
}
// Resolve every queued item to its series in one query. Marking or
// unmarking a whole series queues one mutation per episode, and those
// thousands of items collapse to a single series — resolving them
// one-at-a-time cost thousands of round-trips before the dedupe below
// could discard them.
itemIDs := make([]string, 0, len(batch))
for mutation := range batch {
itemIDs = append(itemIDs, mutation.itemID)
}
seriesByItem, resolveErr := u.resolveSeriesIDs(ctx, itemIDs)
if resolveErr != nil {
u.logger.WarnContext(ctx, "interest series resolution failed",
"item_count", len(itemIDs), "error", resolveErr)
}
seen := make(map[recomputeKey]struct{}, len(batch))
for mutation, failures := range batch {
if ctx.Err() != nil {
return // shutting down; the rebuild task repairs anything dropped
}
if resolveErr != nil {
// The lookup failed for the whole batch, so no item can be
// classified as "not a series". Requeue rather than silently
// dropping interest updates.
seriesID, ok, err := u.resolveSeriesID(ctx, mutation.itemID)
if err != nil {
u.logger.WarnContext(ctx, "interest series resolution failed",
"item_id", mutation.itemID, "error", err)
requeue[mutation] = failures + 1
continue
}
seriesID, ok := seriesByItem[mutation.itemID]
if !ok {
continue // movies and unknown items have no series interest
}
@@ -163,7 +149,7 @@ func (u *InterestUpdater) flush(ctx context.Context) {
seen[key] = struct{}{}
recomputeCtx, cancel := context.WithTimeout(ctx, interestRecomputeTime)
err := u.RecomputeSeries(recomputeCtx, mutation.userID, mutation.profileID, seriesID)
err = u.RecomputeSeries(recomputeCtx, mutation.userID, mutation.profileID, seriesID)
cancel()
if err != nil {
u.logger.WarnContext(ctx, "interest recompute failed",
@@ -192,60 +178,26 @@ func (u *InterestUpdater) flush(ctx context.Context) {
u.mu.Unlock()
}
// resolveSeriesIDs maps many media item IDs (series, season, or episode) to
// their series content IDs in one query. Items with no series — movies and
// unknown IDs — are absent from the result, which is what lets the caller keep
// treating "missing" as "no series interest". Semantics match resolveSeriesID
// per item; only the number of round-trips differs.
func (u *InterestUpdater) resolveSeriesIDs(ctx context.Context, itemIDs []string) (map[string]string, error) {
result := make(map[string]string, len(itemIDs))
ids := make([]string, 0, len(itemIDs))
seen := make(map[string]struct{}, len(itemIDs))
for _, itemID := range itemIDs {
if itemID == "" {
continue
}
if _, dup := seen[itemID]; dup {
continue
}
seen[itemID] = struct{}{}
ids = append(ids, itemID)
}
if len(ids) == 0 {
return result, nil
}
// One branch per item kind, unioned, mirroring the single-item lookup.
rows, err := u.pool.Query(ctx, `
SELECT content_id, series_id FROM episodes WHERE content_id = ANY($1::text[])
// resolveSeriesID maps a media item ID (series, season, or episode) to its
// series content ID. Returns ok=false for movies and unknown items.
func (u *InterestUpdater) resolveSeriesID(ctx context.Context, itemID string) (string, bool, error) {
var seriesID string
err := u.pool.QueryRow(ctx, `
SELECT series_id FROM episodes WHERE content_id = $1
UNION ALL
SELECT content_id, series_id FROM seasons WHERE content_id = ANY($1::text[])
SELECT series_id FROM seasons WHERE content_id = $1
UNION ALL
SELECT content_id, content_id FROM media_items WHERE content_id = ANY($1::text[]) AND type = 'series'`,
ids,
)
SELECT content_id FROM media_items WHERE content_id = $1 AND type = 'series'
LIMIT 1`,
itemID,
).Scan(&seriesID)
if errors.Is(err, pgx.ErrNoRows) {
return "", false, nil
}
if err != nil {
return nil, err
return "", false, err
}
defer rows.Close()
for rows.Next() {
var itemID, seriesID string
if err := rows.Scan(&itemID, &seriesID); err != nil {
return nil, err
}
if seriesID == "" {
continue
}
// An ID matching more than one branch keeps the first row, matching
// the single-item query's UNION ALL ... LIMIT 1 ordering.
if _, exists := result[itemID]; !exists {
result[itemID] = seriesID
}
}
if err := rows.Err(); err != nil {
return nil, err
}
return result, nil
return seriesID, seriesID != "", nil
}
// RecomputeSeries rebuilds the (profile, series) interest rows from
@@ -1,85 +0,0 @@
package notifications
import (
"context"
"os"
"testing"
"github.com/jackc/pgx/v5/pgxpool"
)
// TestResolveSeriesIDsBatchesLookups pins the batched item→series resolution
// used by the interest flush loop. Marking or unmarking a whole series queues
// one mutation per episode, and those all collapse to a single series; the
// previous per-item lookup issued one query each, which is what made a
// large-series unmark spend minutes in `episodes` lookups.
//
// Uses temp tables shadowing episodes/seasons/media_items, with the pool
// pinned to one connection so every query sees them.
func TestResolveSeriesIDsBatchesLookups(t *testing.T) {
dsn := os.Getenv("SILO_TEST_DATABASE_URL")
if dsn == "" {
t.Skip("set SILO_TEST_DATABASE_URL to run the DB-backed series resolution test")
}
ctx := context.Background()
config, err := pgxpool.ParseConfig(dsn)
if err != nil {
t.Fatalf("parse db config: %v", err)
}
config.MaxConns = 1
pool, err := pgxpool.NewWithConfig(ctx, config)
if err != nil {
t.Fatalf("connect db: %v", err)
}
t.Cleanup(pool.Close)
for _, ddl := range []string{
`CREATE TEMP TABLE episodes (content_id text PRIMARY KEY, series_id text NOT NULL)`,
`CREATE TEMP TABLE seasons (content_id text PRIMARY KEY, series_id text NOT NULL)`,
`CREATE TEMP TABLE media_items (content_id text PRIMARY KEY, type text NOT NULL)`,
} {
if _, err := pool.Exec(ctx, ddl); err != nil {
t.Fatalf("create temp table: %v", err)
}
}
if _, err := pool.Exec(ctx, `
INSERT INTO episodes (content_id, series_id) VALUES ('ep-1','series-1'),('ep-2','series-1'),('ep-3','series-2');
INSERT INTO seasons (content_id, series_id) VALUES ('season-1','series-1');
INSERT INTO media_items (content_id, type) VALUES ('series-1','series'),('movie-1','movie');
`); err != nil {
t.Fatalf("seed: %v", err)
}
updater := &InterestUpdater{pool: pool}
got, err := updater.resolveSeriesIDs(ctx, []string{
"ep-1", "ep-2", "ep-3", "season-1", "series-1", "movie-1", "missing", "", "ep-1",
})
if err != nil {
t.Fatalf("resolveSeriesIDs: %v", err)
}
for itemID, wantSeries := range map[string]string{
"ep-1": "series-1",
"ep-2": "series-1",
"ep-3": "series-2",
"season-1": "series-1",
"series-1": "series-1",
} {
if got[itemID] != wantSeries {
t.Errorf("resolveSeriesIDs[%s] = %q, want %q", itemID, got[itemID], wantSeries)
}
}
// Movies, unknown IDs, and blanks must be absent rather than mapped: the
// flush loop reads "missing" as "no series interest" and skips them.
for _, itemID := range []string{"movie-1", "missing", ""} {
if seriesID, ok := got[itemID]; ok {
t.Errorf("resolveSeriesIDs[%q] = %q, want absent", itemID, seriesID)
}
}
if empty, err := updater.resolveSeriesIDs(ctx, nil); err != nil || len(empty) != 0 {
t.Fatalf("resolveSeriesIDs(nil) = %v (%v), want empty", empty, err)
}
}
+1 -4
View File
@@ -53,9 +53,6 @@ func SourceDescriptorFromFileV3(file *models.MediaFile, audioIndex int) SourceDe
source.DynamicRange = normalizeDynamicRangeV3(track)
source.HDR10Plus = track.HDR10Plus || strings.Contains(strings.ToLower(track.VideoRangeType), "hdr10+")
source.DVProfile = track.DVProfile
if track.DVLevel >= 1 && track.DVLevel <= 13 {
source.DVLevel = track.DVLevel
}
source.DVBLCompatID = track.DVBLCompatID
source.VideoCopyUnsafe = videoCopyUnsafeFile(file)
switch EnhancementLayerV3(strings.ToLower(track.DVEnhancementLayer)) {
@@ -205,7 +202,7 @@ func outputRangeEligibleV3(source SourceDescriptorV3, request StartRequestV3) (b
claims.DolbyVisionReason = "profile_7_enhancement_layer_unknown"
return false, claims
}
if hdr != nil && hdrSupportsDolbyVisionSourceV3(*hdr, source) {
if hdr != nil && containsIntV3(hdr.DolbyVisionProfiles, source.DVProfile) {
claims.DolbyVision = true
claims.DolbyVisionReason = "native_profile_supported"
return true, claims
+1 -49
View File
@@ -121,7 +121,6 @@ var (
string(playback.ReplanOperationSeekFailureRecoveryV3),
string(playback.ReplanOperationTrackChangeV3),
string(playback.ReplanOperationQualityChangeV3),
string(playback.ReplanOperationOutputChangeV3),
}
// The two operations ReplanRequestV3.Validate rejects without a failure
// classification. The schema expresses the same rule as a conditional.
@@ -161,25 +160,6 @@ func TestClientProgressPersistenceRequiresExplicitStartPosition(t *testing.T) {
}
}
func TestIntentOnlyReplansRejectFailureEvidence(t *testing.T) {
schemas := compileSchemasV3(t)
for _, operation := range []playback.ReplanOperationV3{
playback.ReplanOperationTrackChangeV3,
playback.ReplanOperationQualityChangeV3,
playback.ReplanOperationOutputChangeV3,
} {
t.Run(string(operation), func(t *testing.T) {
request := decodeJSONValue(t, mustReadFile(t, filepath.Join(goldenRootV3, "replan_request.json"))).(map[string]any)
request["operation"] = string(operation)
request["quality_preference"] = "720p"
request["failure"] = map[string]any{"classification": "decoder_failure"}
if err := schemas["replan-request.schema.json"].Validate(request); err == nil {
t.Fatal("intent-only replan with failure satisfied the schema")
}
})
}
}
// TestGoldenFixturesSatisfyTheSchemas closes the loop the schemas exist for: a
// schema that no longer accepts what cmd/playbackfixtures emits is a schema
// that describes a server nobody runs.
@@ -233,7 +213,7 @@ func TestConformanceMatrixEmbeddedWireBodiesSatisfySchemas(t *testing.T) {
request := scenario["request"].(map[string]any)
validate(name, "replan-request.schema.json", request)
switch request["operation"] {
case string(playback.ReplanOperationTrackChangeV3), string(playback.ReplanOperationQualityChangeV3), string(playback.ReplanOperationOutputChangeV3), string(playback.ReplanOperationSeekReanchorV3):
case string(playback.ReplanOperationTrackChangeV3), string(playback.ReplanOperationQualityChangeV3), string(playback.ReplanOperationSeekReanchorV3):
if failure, ok := request["failure"]; ok {
t.Errorf("scenario %q intent-only replan carries failure = %#v", name, failure)
}
@@ -512,34 +492,6 @@ func TestRequestSchemaRequiredFieldsMatchValidators(t *testing.T) {
assertStringsEqual(t, "route_event.required", schemaStrings(t, routeEvent, "required"), []string{"protocol_version", "playback_attempt_id", "event"})
}
// TestRequestSchemasShareIdenticalDefs pins the one thing two copies of a
// definition can never be trusted to keep on their own. Start and replan
// deserialize the *same* Go types — ClientCodecCapabilitiesV3 and
// ClientPlaybackContextV3 — through the same validator, so any $def they both
// declare must be byte-identical. Adding a field to one schema and not the
// other compiles, validates, and ships a contract that lies about one of the
// two endpoints; this is what catches it.
func TestRequestSchemasShareIdenticalDefs(t *testing.T) {
start := schemaValue(t, mustReadObject(t, filepath.Join(schemaRootV3, "v3", "start-request.schema.json")), "$defs").(map[string]any)
replan := schemaValue(t, mustReadObject(t, filepath.Join(schemaRootV3, "v3", "replan-request.schema.json")), "$defs").(map[string]any)
shared := make([]string, 0, len(start))
for name := range start {
if _, ok := replan[name]; ok {
shared = append(shared, name)
}
}
sort.Strings(shared)
if !slices.Contains(shared, "client_playback_context") || !slices.Contains(shared, "client_capabilities") {
t.Fatalf("shared $defs = %v, want the client contract definitions in both request schemas", shared)
}
for _, name := range shared {
if !reflect.DeepEqual(start[name], replan[name]) {
t.Errorf("$defs.%s differs between start-request and replan-request; both deserialize the same Go type", name)
}
}
}
func compileSchemasV3(t *testing.T) map[string]*jsonschema.Schema {
t.Helper()
+1 -3
View File
@@ -108,9 +108,7 @@ func resolveCopySeekAnchor(
interval := strconv.FormatFloat(requestedSeekSeconds, 'f', 6, 64) + "%+0.001"
cmd := exec.CommandContext(ctx, ffprobePathFromFFmpeg(ffmpegPath),
"-v", "error",
// Match the remux transport's 0:V:0 map. Uppercase V excludes attached
// pictures, thumbnails, and cover art from the video stream ordinal.
"-select_streams", "V:0",
"-select_streams", "v:0",
"-read_intervals", interval,
"-show_entries", "packet=pts_time,dts_time,flags",
"-of", "json",
@@ -17,12 +17,10 @@ func TestResolveCopySeekAnchorUsesKeyPacketTimestamp(t *testing.T) {
dir := t.TempDir()
ffmpegPath := filepath.Join(dir, "ffmpeg")
ffprobePath := filepath.Join(dir, "ffprobe")
argsPath := filepath.Join(dir, "ffprobe-args")
if err := os.WriteFile(ffmpegPath, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil {
t.Fatalf("write fake ffmpeg: %v", err)
}
probe := `#!/bin/sh
printf '%s\n' "$@" > "` + argsPath + `"
printf '%s' '{"packets":[{"pts_time":"N/A","dts_time":"14.500000","flags":"K__"}]}'
`
if err := os.WriteFile(ffprobePath, []byte(probe), 0o755); err != nil {
@@ -36,13 +34,6 @@ printf '%s' '{"packets":[{"pts_time":"N/A","dts_time":"14.500000","flags":"K__"}
if anchor != 14.5 || segment != 7 {
t.Fatalf("resolved anchor = %v, segment = %d; want 14.5, 7", anchor, segment)
}
args, err := os.ReadFile(argsPath)
if err != nil {
t.Fatalf("read ffprobe args: %v", err)
}
if !strings.Contains(string(args), "-select_streams\nV:0\n") {
t.Fatalf("ffprobe did not select the remux video stream:\n%s", args)
}
}
func TestResolveCopySeekAnchorCoalescesMatchingConcurrentProbes(t *testing.T) {

Some files were not shown because too many files have changed in this diff Show More