Compare commits
26
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d01b5de3bd | ||
|
|
2bbe60219f | ||
|
|
5b72176142 | ||
|
|
cd9e358bbc | ||
|
|
40f117bd2d | ||
|
|
12f5d5f8ed | ||
|
|
fc65dcaf16 | ||
|
|
4288c7c5c6 | ||
|
|
e5bf0155ad | ||
|
|
b1502d70f5 | ||
|
|
ecb4555eec | ||
|
|
8a9cf04a41 | ||
|
|
70064823a6 | ||
|
|
7adedf6429 | ||
|
|
c24d839681 | ||
|
|
09de4f08ec | ||
|
|
561e00e4ca | ||
|
|
f3e1fd3b38 | ||
|
|
e35efddac7 | ||
|
|
fca9f38a33 | ||
|
|
4b2031fdf8 | ||
|
|
6ae1ea30c8 | ||
|
|
0cc706e066 | ||
|
|
53c9313b57 | ||
|
|
0217cce2df | ||
|
|
22ffaad911 |
@@ -95,6 +95,9 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/secret"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
"github.com/Silo-Server/silo-server/internal/server"
|
||||
"github.com/Silo-Server/silo-server/internal/streamenforcer"
|
||||
"github.com/Silo-Server/silo-server/internal/streammonitor"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
"github.com/Silo-Server/silo-server/internal/taskmanager"
|
||||
taskrepository "github.com/Silo-Server/silo-server/internal/taskmanager/repository"
|
||||
@@ -102,6 +105,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/taskmanager/triggers"
|
||||
"github.com/Silo-Server/silo-server/internal/telemetry"
|
||||
"github.com/Silo-Server/silo-server/internal/transcodenode"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
"github.com/Silo-Server/silo-server/internal/usercollections"
|
||||
"github.com/Silo-Server/silo-server/internal/userdb"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
@@ -668,9 +672,43 @@ func main() {
|
||||
tracker.Cleanup(cleanupCtx)
|
||||
}()
|
||||
|
||||
// Kill switch: edges consult this per request. Central writes revocations
|
||||
// (over-cap, admin terminate, account revocation) which propagate here via
|
||||
// Redis pub/sub + poll. Shares the offload Redis.
|
||||
revStore := streamrevoke.New(streamrevoke.Options{Redis: redisClient, Bus: eventBus})
|
||||
revStore.StartSync(appCtx)
|
||||
|
||||
var handler http.Handler
|
||||
if mode == "proxy" {
|
||||
srv := proxy.NewServer(watcher, tracker)
|
||||
srv.SetRevocationStore(revStore)
|
||||
// Resolve the viewer's address through the operator's trusted-proxy
|
||||
// boundary, as the native surface does. Without this the edge records
|
||||
// the connecting peer, which behind an ingress or load balancer is the
|
||||
// same address for every viewer — collapsing the admin session view and
|
||||
// any per-viewer analysis to one indistinguishable client.
|
||||
edgeResolver := clientip.NewResolver(nil)
|
||||
if cidrs, cidrErr := clientip.LoadTrustedCIDRs(appCtx, settingsRepo); cidrErr != nil {
|
||||
// Fail closed on trust, not on startup: an empty trusted list means
|
||||
// forwarding headers are ignored entirely, so a spoofed header can
|
||||
// never be believed. The edge falls back to the connecting peer.
|
||||
slog.Warn("edge client-IP trust list unavailable; ignoring forwarding headers",
|
||||
"component", "proxy", "error", cidrErr)
|
||||
} else {
|
||||
edgeResolver.UpdateTrustedCIDRs(cidrs)
|
||||
}
|
||||
// Keep the boundary current: the setting is hot-reloadable on central,
|
||||
// so the edge must not need a restart to follow it.
|
||||
watcher.OnChange(func(_, _ *config.Config) {
|
||||
cidrs, loadErr := clientip.LoadTrustedCIDRs(context.Background(), settingsRepo)
|
||||
if loadErr != nil {
|
||||
slog.Warn("edge client-IP trust list reload failed",
|
||||
"component", "proxy", "error", loadErr)
|
||||
return
|
||||
}
|
||||
edgeResolver.UpdateTrustedCIDRs(cidrs)
|
||||
})
|
||||
srv.SetClientIPResolver(edgeResolver)
|
||||
handler = srv.Handler()
|
||||
} else {
|
||||
srv := transcodenode.NewServer(watcher, tracker)
|
||||
@@ -682,6 +720,7 @@ func main() {
|
||||
// Reclaim orphaned transcode dirs at boot and hourly thereafter, bound
|
||||
// to appCtx so it stops on shutdown.
|
||||
srv.StartOrphanSweeper(appCtx)
|
||||
srv.SetRevocationStore(revStore)
|
||||
handler = srv.Handler()
|
||||
}
|
||||
|
||||
@@ -745,6 +784,34 @@ func main() {
|
||||
defer func() { _ = apiRedisClient.Close() }()
|
||||
}
|
||||
|
||||
// One process-local registry is shared by every download-class serving
|
||||
// surface and the admin view. It is intentionally separate from live streams.
|
||||
maxUserTransfers, maxUserTransfersErr := transfers.ParseMaxPerUser(settings[transfers.MaxPerUserSetting])
|
||||
if maxUserTransfersErr != nil {
|
||||
slog.Warn("invalid max user concurrent transfers setting; using default",
|
||||
"component", "transfers",
|
||||
"setting", transfers.MaxPerUserSetting,
|
||||
"value", settings[transfers.MaxPerUserSetting],
|
||||
"error", maxUserTransfersErr,
|
||||
)
|
||||
maxUserTransfers, _ = transfers.ParseMaxPerUser("")
|
||||
}
|
||||
transferMonitoring := newTransferRegistryComposition(maxUserTransfers)
|
||||
|
||||
// Central-side kill switch: written by the async enforcer, admin terminate,
|
||||
// and account revocation; edges enforce it via Redis pub/sub + poll. Memory-
|
||||
// only when Redis is absent (single-node integrated). The Postgres durable
|
||||
// mirror lets the kill list survive a server restart AND a Redis flush, so a
|
||||
// restart-resilient stream (PR #174) cannot be reconstructed and re-served
|
||||
// after being killed. NewPostgresDurableStore returns a true nil interface
|
||||
// when the pool is nil, degrading cleanly to memory/Redis-only.
|
||||
streamRevocation := streamrevoke.New(streamrevoke.Options{
|
||||
Redis: apiRedisClient,
|
||||
Bus: eventBus,
|
||||
Durable: streamrevoke.NewPostgresDurableStore(pool),
|
||||
})
|
||||
streamRevocation.StartSync(appCtx)
|
||||
|
||||
// Assigned below once the trusted-proxy config is seeded; captured by the
|
||||
// OnServerSettingUpdated closure, which only runs on admin requests after
|
||||
// startup completes.
|
||||
@@ -765,6 +832,7 @@ func main() {
|
||||
SecretCipher: dataCipher,
|
||||
EventBus: eventBus,
|
||||
RedisClient: apiRedisClient,
|
||||
RevocationStore: streamRevocation,
|
||||
LogStreamHub: logStreamHub,
|
||||
RealtimeHub: realtimeHub,
|
||||
EventsHub: eventsHub,
|
||||
@@ -798,6 +866,7 @@ func main() {
|
||||
configWatcher.RequestReload()
|
||||
},
|
||||
}
|
||||
transferMonitoring.wireAPI(&deps)
|
||||
accessGroupStore := access.NewGroupStore(pool)
|
||||
audiobooksService := audiobooks.New(&audiobooksSettingsAdapter{repo: settingsRepo})
|
||||
absCompatEnabled, err := audiobooksService.ABSCompatEnabled(appCtx)
|
||||
@@ -1677,6 +1746,7 @@ func main() {
|
||||
// Build the reconciler early enough that playback handlers can trigger
|
||||
// immediate session syncs after start/stop events.
|
||||
nodeIdentity := resolveNodeIdentity()
|
||||
instanceID := nodeIdentity + ":" + uuid.NewString()
|
||||
|
||||
var reconciler *worker.Reconciler
|
||||
var heartbeatWriter *worker.HeartbeatWriter
|
||||
@@ -1702,6 +1772,64 @@ func main() {
|
||||
}
|
||||
deps.SessionSyncer = reconciler
|
||||
|
||||
// Async brain: off the hot path, trims over-cap streams by writing
|
||||
// revocations the edges enforce. Source is the authoritative monitoring
|
||||
// picture — Redis for multi-node, the in-process session manager for
|
||||
// integrated. Fails open on limit-lookup errors.
|
||||
// Union of (a) the in-process session manager — authoritative for streams
|
||||
// this node serves directly, integrated or not — and (b) the edge Redis
|
||||
// records — authoritative for offloaded/edge-served streams. Deduped by
|
||||
// session id. Using the union (not "Redis when present, else manager")
|
||||
// fixes the case where integrated mode has Redis configured but writes no
|
||||
// silo:sessions:* keys, which would otherwise blind the enforcer.
|
||||
// The mapping (Session → monitoring record, carrying route + client identity
|
||||
// + progress) is shared with the admin session list via streammonitor so there is
|
||||
// one Session→record definition. NodeName marks the serving host so
|
||||
// integrated streams aren't shown node-less.
|
||||
localSource := streammonitor.NewFuncSource(func(ctx context.Context) ([]nodesessions.SessionInfo, error) {
|
||||
return streammonitor.LiveLocalSessions(sessionMgr, nodeIdentity), nil
|
||||
})
|
||||
var monitorSource streammonitor.Source = localSource
|
||||
if apiRedisClient != nil {
|
||||
publisher := nodesessions.NewPublisher(apiRedisClient, instanceID, nodeIdentity)
|
||||
publisher.Start(appCtx, 10*time.Second, func() []nodesessions.SessionInfo {
|
||||
return streammonitor.LiveLocalSessions(sessionMgr, nodeIdentity)
|
||||
})
|
||||
defer func() {
|
||||
cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cleanupCancel()
|
||||
publisher.Cleanup(cleanupCtx)
|
||||
}()
|
||||
monitorSource = streammonitor.NewMultiSource(localSource, streammonitor.NewRedisSource(apiRedisClient))
|
||||
}
|
||||
// Resolve caps through the session manager so the enforcer and synchronous
|
||||
// admission share one provider (installed by the API wiring) and can never
|
||||
// disagree about a user's effective, group-merged cap.
|
||||
overCapRevocationTTL, ttlErr := streamenforcer.ParseRevocationTTL(
|
||||
settings[streamenforcer.RevocationTTLSetting],
|
||||
)
|
||||
if ttlErr != nil {
|
||||
slog.Warn("invalid over-cap revocation TTL; using default",
|
||||
"setting", streamenforcer.RevocationTTLSetting,
|
||||
"value", settings[streamenforcer.RevocationTTLSetting],
|
||||
"default", streamenforcer.DefaultRevocationTTL,
|
||||
"error", ttlErr)
|
||||
overCapRevocationTTL = streamenforcer.DefaultRevocationTTL
|
||||
}
|
||||
enforcer := streamenforcer.NewWithRevocationTTL(monitorSource, func(ctx context.Context, userID int) (int, error) {
|
||||
limits, err := sessionMgr.EffectiveLimits(ctx, userID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return limits.MaxStreams, nil
|
||||
}, streamRevocation, 0, overCapRevocationTTL)
|
||||
if apiRedisClient != nil {
|
||||
enforcer.SetCoordinator(streamenforcer.NewRedisCoordinator(
|
||||
apiRedisClient, instanceID, streamenforcer.DefaultInterval,
|
||||
))
|
||||
}
|
||||
enforcer.Start(appCtx)
|
||||
|
||||
nodeURL := fmt.Sprintf("http://%s%s", nodeIdentity, cfg.Server.Listen)
|
||||
heartbeatWriter = worker.NewHeartbeatWriter(deps.DB, nodeIdentity, mode, nodeURL)
|
||||
}
|
||||
@@ -2227,7 +2355,9 @@ func main() {
|
||||
Detail: absDetailSvc,
|
||||
SessionMgr: sessionMgr,
|
||||
SessionSyncer: deps.SessionSyncer,
|
||||
Revocation: streamRevocation,
|
||||
}
|
||||
transferMonitoring.wireABS(&absHDeps)
|
||||
absH := audiobooksService.BuildABSHandler(absHDeps)
|
||||
deps.ABSHandler = absH
|
||||
}
|
||||
@@ -2340,6 +2470,12 @@ func main() {
|
||||
if compatServer != nil {
|
||||
compatServer.SessionStore().DeleteByUserID(userID)
|
||||
}
|
||||
// Also revoke this user's live stream credentials so a session revocation
|
||||
// (disable, password change, permission change) kills in-flight playback
|
||||
// at the edge within one propagation/poll interval, not just compat login.
|
||||
if err := streamRevocation.RevokeUser(ctx, userID, "user_sessions_revoked"); err != nil {
|
||||
slog.Warn("revoke user streams failed", "user_id", userID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
distFS, fsErr := fs.Sub(siloweb.DistFS, "dist")
|
||||
@@ -2478,6 +2614,7 @@ func main() {
|
||||
RecipeNodeStore: noderecipe.NewStore(apiRedisClient, 0),
|
||||
SessionSyncer: deps.SessionSyncer,
|
||||
}
|
||||
transferMonitoring.wireJellycompat(&compatDeps)
|
||||
|
||||
// Wire direct dependencies when DB is available.
|
||||
if deps.DB != nil {
|
||||
@@ -2512,6 +2649,7 @@ func main() {
|
||||
compatDeps.DetailSvc = detailSvc
|
||||
compatDeps.FolderRepo = folderRepo
|
||||
compatDeps.SessionMgr = sessionMgr
|
||||
compatDeps.RevocationStore = streamRevocation
|
||||
compatDeps.UserStoreProvider = userStoreProvider
|
||||
compatDeps.WatchCompletionObserver = deps.WatchCompletionObserver
|
||||
compatDeps.SettingsRepo = settingsRepo
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"github.com/Silo-Server/silo-server/internal/api"
|
||||
"github.com/Silo-Server/silo-server/internal/audiobooks"
|
||||
"github.com/Silo-Server/silo-server/internal/jellycompat"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
// transferRegistryComposition owns the one process-local registry and keeps
|
||||
// independently constructed protocol surfaces from accidentally diverging.
|
||||
type transferRegistryComposition struct {
|
||||
registry *transfers.Registry
|
||||
}
|
||||
|
||||
func newTransferRegistryComposition(maxPerUser int) *transferRegistryComposition {
|
||||
return &transferRegistryComposition{registry: transfers.NewWithOptions(transfers.Options{MaxPerUser: &maxPerUser})}
|
||||
}
|
||||
|
||||
func (c *transferRegistryComposition) wireAPI(deps *api.Dependencies) {
|
||||
deps.TransferRegistry = c.registry
|
||||
}
|
||||
|
||||
func (c *transferRegistryComposition) wireJellycompat(deps *jellycompat.Dependencies) {
|
||||
deps.TransferRegistry = c.registry
|
||||
}
|
||||
|
||||
func (c *transferRegistryComposition) wireABS(deps *audiobooks.ABSHandlerDeps) {
|
||||
deps.Transfers = c.registry
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/api"
|
||||
"github.com/Silo-Server/silo-server/internal/audiobooks"
|
||||
"github.com/Silo-Server/silo-server/internal/jellycompat"
|
||||
)
|
||||
|
||||
func TestTransferRegistryCompositionSharesOneInstance(t *testing.T) {
|
||||
composition := newTransferRegistryComposition(24)
|
||||
var apiDeps api.Dependencies
|
||||
var compatDeps jellycompat.Dependencies
|
||||
var absDeps audiobooks.ABSHandlerDeps
|
||||
|
||||
composition.wireAPI(&apiDeps)
|
||||
composition.wireJellycompat(&compatDeps)
|
||||
composition.wireABS(&absDeps)
|
||||
|
||||
if apiDeps.TransferRegistry == nil {
|
||||
t.Fatal("API/native/admin transfer registry is nil")
|
||||
}
|
||||
if got := apiDeps.TransferRegistry.MaxPerUser(); got != 24 {
|
||||
t.Fatalf("max per-user transfers = %d, want 24", got)
|
||||
}
|
||||
if apiDeps.TransferRegistry != compatDeps.TransferRegistry {
|
||||
t.Fatal("jellycompat received a different transfer registry")
|
||||
}
|
||||
if apiDeps.TransferRegistry != absDeps.Transfers {
|
||||
t.Fatal("ABS received a different transfer registry")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,628 @@
|
||||
# Playback Paths — Monitoring & Kill-Switch Coverage Matrix
|
||||
|
||||
> Reference checklist for stream monitoring + revocation kill-switch coverage.
|
||||
> Check every change to playback/monitoring/revocation against this. As-built
|
||||
> companion to the design plan
|
||||
> [`2026-07-04-stream-monitoring-and-kill-switch.md`](../superpowers/plans/2026-07-04-stream-monitoring-and-kill-switch.md)
|
||||
> (that doc is the intent; this one is the shipped coverage). Status as of
|
||||
> 2026-07-05: monitoring, enforcement, and the async over-cap enforcer are all
|
||||
> shipped across two code commits — the *monitoring* commit (`feat(playback):
|
||||
> server-observed stream monitoring`) and the *kill-switch* commit
|
||||
> (`feat(playback): stream kill switch + async over-cap enforcer`). GAP-1..GAP-4
|
||||
> are all RESOLVED (see Findings): GAP-4 (kill list did not survive a restart) was
|
||||
> closed by wiring a durable Postgres mirror, and GAP-3 (in-flight cut for
|
||||
> integrated direct-play) was closed by adding `Unwrap()` to the native/compat
|
||||
> middleware writers so the cut reaches the socket. A post-review hardening pass
|
||||
> then closed GAP-5..GAP-9 found by adversarial re-review: user-kill cutoff
|
||||
> semantics, compat manifest/subtitle/mint-bypass coverage, terminate-by-id,
|
||||
> in-flight download cut, admin-list dedupe, and the metered-writer sendfile chain
|
||||
> (see Findings). That hardening is folded into the two code commits above — the
|
||||
> branch is organized as monitoring → kill-switch → this docs commit, not as a
|
||||
> running series of fix commits. (This doc references sibling commits by role, not
|
||||
> SHA — squash/rebase rewrites hashes, so a SHA citation goes stale the moment the
|
||||
> branch is amended.)
|
||||
>
|
||||
> **Update 2026-07-29 — two-model review round (Claude Opus 5 + Codex gpt-5.6-sol).**
|
||||
> A second adversarial pass over the rebased branch found **six open gaps**, none of
|
||||
> which the branch's own tests caught: GAP-10 (ABS in-flight cuts no-op because
|
||||
> `statusRecorder` lacks `Unwrap`), GAP-11 (ebook/comic/PDF serving is neither
|
||||
> observed nor killable), GAP-12 (the rolling write deadline can erase a revocation
|
||||
> cut), GAP-13 (`mergeStreams` discards edge `BytesServed`), GAP-14
|
||||
> (transfer-registry saturation serves unmonitored), GAP-15 (edge transcode liveness
|
||||
> is request-observed, not byte-observed). GAP-10 and GAP-11 in particular mean the
|
||||
> previous blanket claim that enforcement holds "on every serve surface" was **wrong**.
|
||||
>
|
||||
> **Update 2026-07-30 — serve-path batch landed.** **GAP-10, GAP-11, GAP-12 and
|
||||
> GAP-13 are RESOLVED**; each is scored inline below. GAP-12 was pulled forward from
|
||||
> the revocation batch because it makes the GAP-10 fix inert (see its entry).
|
||||
>
|
||||
> **Update 2026-07-30 — tracker-lifecycle batch landed.** **GAP-15 is RESOLVED**
|
||||
> (edge visibility is now separate from served liveness, and only bytes written from a
|
||||
> 200/206 upstream response advance `LastServedAt`). Also closed in that batch: the
|
||||
> overlapping-request defect that let one finishing request delete another live
|
||||
> request's record, the async-tracking race that could leave a permanently
|
||||
> non-expiring ghost session, and the protocol-v3 logical-vs-transport identity split
|
||||
> that counted one stream twice — all three were sources of a **wrong over-cap
|
||||
> count**, which is why they precede the revocation batch.
|
||||
>
|
||||
> **Update 2026-07-31 — liveness/replica batch landed.** **GAP-14 is RESOLVED**:
|
||||
> download-class pours that cannot be monitored are refused (429/503 + `Retry-After`)
|
||||
> instead of served blind, and a per-user concurrent-transfer cap
|
||||
> (`playback.max_user_concurrent_transfers`, default 24) makes registry saturation
|
||||
> unreachable by one actor. Decision **A5** landed with it, so the client-progress
|
||||
> caveat below is now closed. Decision **A6** landed too: integrated sessions are
|
||||
> published to the shared `silo:sessions:` picture under a per-process instance id and
|
||||
> one Redis-elected evaluator runs each pass, so replicas are no longer mutually blind
|
||||
> — but **synchronous admission is still per-process**, so cross-replica excess is
|
||||
> trimmed asynchronously rather than refused at the door.
|
||||
>
|
||||
> Monitoring projection is **not** uniformly async, and the claim has been corrected
|
||||
> rather than the code rushed: the first Redis write per session is synchronous, later
|
||||
> updates ride the refresh tick. An ordered, lifecycle-aware projection queue is
|
||||
> deliberately deferred rather than shipping a lossy or reorderable one — a naive
|
||||
> fire-and-forget projection is what caused the ghost-session defect above.
|
||||
>
|
||||
> One claim elsewhere in this doc was **not true** and is now resolved: monitoring
|
||||
> genuinely does "never trust client progress" as of the liveness batch — the
|
||||
> `LastActivityAt` fallback is gone from the projection *and* from reaping, with the
|
||||
> single deliberate exception that a **paused** session holding an open, ping-checked
|
||||
> realtime/WebSocket connection is exempt from reaping (an observed connection, not a
|
||||
> reported position; it preserves the issue #243 fix). Do not restore the old
|
||||
> fallback.
|
||||
>
|
||||
> Note also that every "cut" in this document means **cut within ~5s**, not
|
||||
> immediately: the in-flight watcher polls on a 5s interval.
|
||||
|
||||
## Dimensions
|
||||
|
||||
- **Server layouts:** (A) integrated, no Redis · (B) integrated, with Redis ·
|
||||
(C) multi-node, with Redis (proxy/transcode edge nodes).
|
||||
- **Playback types:** direct play · remux · transcode (HLS).
|
||||
- **Routes:** native silo `/api/v1` · jellycompat (`:8096`) ·
|
||||
Audiobookshelf-compatible (`/abs`, `/api`, and public feed/session paths).
|
||||
|
||||
## Key runtime facts (why cells collapse)
|
||||
|
||||
- **Integrated (A, B) serves bytes locally.** Node selection returns an empty
|
||||
plan when no edge nodes exist, and both native and jellycompat fall through to
|
||||
in-process serving (`nodepool/planner.go:248`, `jellycompat/handlers_playback.go:306`
|
||||
— the `proxyNode == nil` guard in `buildProxyRedirectURL`). A vs B differ only
|
||||
in *revocation propagation* (memory-only vs Redis) and *monitor source*
|
||||
(FuncSource vs MultiSource) — the **serve + enforcement points are identical**.
|
||||
So A ≡ B for this matrix; treated as one column "Integrated".
|
||||
- **Multi-node (C) serves bytes on edges.** Native playback hands the client a
|
||||
proxy-node URL at session start (`playback.go:1575`); jellycompat 302-redirects
|
||||
to a proxy node (`streams.go:120`). Edge serving = `internal/proxy` + `internal/transcodenode`.
|
||||
- **jellycompat shares the SessionManager.** A jellycompat play starts a real
|
||||
`playback.SessionManager` session (`streams.go:1127`), so the monitor/enforcer
|
||||
see it exactly like native. The compat `PlaybackSessionStore` is separate
|
||||
bookkeeping (PlaySessionId → recipe), not liveness/enforcement.
|
||||
- **Local transcode fallback:** in multi-node, if no transcode node is available,
|
||||
jellycompat falls back to LOCAL transcode (`streams.go:277`,
|
||||
`LocalTranscodeFallbackAllowed`) — i.e. the "Integrated jellycompat" serve path
|
||||
can also occur under layout C.
|
||||
|
||||
## The matrix
|
||||
|
||||
Legend: ✅ covered · ⚠️ covered with caveat · ❌ gap.
|
||||
|
||||
### Serving handler (where the bytes come from)
|
||||
|
||||
| Route | Type | Integrated (A/B) | Multi-node (C) |
|
||||
|---|---|---|---|
|
||||
| native | direct | `StreamHandler.HandleStream` → `ServeDirectPlay` (`stream.go:141`) | proxy `handleDirectPlay` (`proxy/server.go:171`) |
|
||||
| native | remux | `HandleStream` → `ServeRemux` (`stream.go:157`) | proxy `handleRemux` |
|
||||
| native | transcode | `HandleGetTranscodeSegment` local → `ServeFile` (`playback.go:2860`) | proxy → transcode node `handleSegment` |
|
||||
| jellycompat | direct | `HandleVideoStream` → `ServeDirectPlay` (`streams.go:153`) | 302 → proxy `handleDirectPlay` |
|
||||
| jellycompat | remux | `HandleVideoStream` → `ServeRemux` (`streams.go:151`) | 302 → proxy `handleRemux` |
|
||||
| jellycompat | transcode | `HandleHLSSegment` → `ServeFile` (`streams.go:543`) | 302 → proxy → transcode node |
|
||||
| ABS | authenticated file/download | `handleFileStream` → `ServeDirectPlay` | same integrated handler |
|
||||
| ABS | public playback track | `handlePublicTrack` → `ServeDirectPlay` | same integrated handler |
|
||||
| ABS | public RSS feed file | `handlePublicFeedFile` → `ServeFile` | same integrated handler |
|
||||
| ebook | original file | `serveEbookInline` → `ServeContent` (`ebook_reader.go:626`) | same integrated handler |
|
||||
| ebook | converted EPUB | `serveConvertedEpub` → `ServeContent` (`ebook_convert_serve.go:160`) | same integrated handler |
|
||||
|
||||
### Monitoring (server-observed existence, reported to central)
|
||||
|
||||
The monitoring model separates **existence** (must be server-observed, never
|
||||
gated by a client report — the hidden-stream defense) from **timing** (client
|
||||
progress is fine, especially for native). A disguised re-streamer that pulls
|
||||
bytes but withholds/falsifies progress must still be counted.
|
||||
|
||||
| Route | Type | Integrated (A/B) — existence signal | Multi-node (C) — existence signal |
|
||||
|---|---|---|---|
|
||||
| native | direct | ✅ transport-count shield for the whole pour (`stream.go:136`) | ✅ edge tracker, byte-observed (`sessionByteWriter`) |
|
||||
| native | remux | ✅ transport-count shield (`stream.go:146`) | ✅ edge tracker, byte-observed |
|
||||
| native | transcode | ✅ per-segment transport marker around `ServeFile` (`playback.go:2860`) | ✅ edge tracker (Touch + AddBytes/segment) |
|
||||
| jellycompat | direct | ✅ transport-count shield (`streams.go:129`) | ✅ edge tracker, owner+route carried |
|
||||
| jellycompat | remux | ✅ transport-count shield | ✅ edge tracker, owner+route carried |
|
||||
| jellycompat | transcode | ✅ per-segment transport marker around `ServeFile` (`streams.go:543`) | ✅ edge tracker, owner+route carried |
|
||||
| ABS | authenticated file/download | ✅ separate in-memory transfer record; cap-exempt | same process only |
|
||||
| ABS | public playback track | ✅ native-session transport marker + metered bytes | same |
|
||||
| ABS | public RSS feed file | ✅ separate in-memory transfer record; cap-exempt | same process only |
|
||||
| ebook | original / converted EPUB | ✅ transfer record + meter (cap-exempt per A4) | same |
|
||||
|
||||
- **GAP-11 — RESOLVED (serve-path batch).** The
|
||||
`/api/v1/ebooks/{content_id}/files/{file_id}/read` route group applied only
|
||||
`apimw.RequireProfile`, and both serve paths wrapped in `RollingDeadlineWriter`
|
||||
alone: no metered writer, no transfer record, no `Refuse`, no `WatchAndCut`. Not a
|
||||
text-file-sized path — `ebookReaderFormat` accepts `epub, pdf, mobi, azw, azw3, cbz,
|
||||
cbr, fb2, fbz`, and CBZ/CBR archives and scanned PDFs routinely run 100 MB–1 GB+, so
|
||||
a revoked user kept pulling the whole comic library at full speed while the admin view
|
||||
showed nothing.
|
||||
|
||||
Now metered, registered in the transfer registry, refused on entry and cut in flight,
|
||||
following the ABS file-handler idiom. **Cap-exempt per decision A4** — admission is
|
||||
untouched and reading consumes no video stream slot.
|
||||
|
||||
⚠️ **Correction to the earlier follow-up guidance, which said to "reuse
|
||||
`guardRevocationCut`" here — do not.** That wrapper keys on a `session_id` URL param
|
||||
this route does not have (it would pass `""`) and takes identity from
|
||||
`streamRequestIdentity`, which looks for a `?st=` stream token this route never
|
||||
carries. It compiles and silently guards nothing. The identity has to come from the
|
||||
profile/auth context instead.
|
||||
- **GAP-13 — RESOLVED (serve-path batch).** `streammonitor.mergeStreams` picked the
|
||||
record with the later `LastServedAt` *wholesale* and backfilled only
|
||||
`UserID`/`ProfileID`/`MediaFileID`, `Route`, `ClientIP`, `ClientName`, `HWAccel` and
|
||||
`Position` — **`BytesServed` was not merged**, so when the central record was fresher
|
||||
a stream whose bytes were all poured at an edge reported `0`. Now merged as a **max,
|
||||
not a sum**: the two records are two observers of one pour, so summing would
|
||||
double-count. `DedupeSessionInfos` had the identical hole and feeds the admin view;
|
||||
fixed there too.
|
||||
- **GAP-14 — RESOLVED (liveness/replica batch).** The transfer registry is capped at
|
||||
`defaultMaxEntries = 10_000` (`transfers/registry.go`). Past that, `Begin` returned
|
||||
`ErrRegistryFull` and every call site logged at **Debug** and served anyway, with
|
||||
byte updates for the unregistered pour discarded. Since there is no connection cap
|
||||
anywhere (abuse matrix E28) and the compat surface is unthrottled (E25/E27), an
|
||||
attacker could hold the registry at its ceiling and blind download-class monitoring
|
||||
for every other user.
|
||||
|
||||
**Decision A7 shipped: fail closed, plus a per-user cap.** A per-user concurrent
|
||||
cap (`playback.max_user_concurrent_transfers`, default 24, checked *before* the
|
||||
global limit) makes saturation unreachable by one actor, and a pour that cannot be
|
||||
registered is now refused — 429 `transfer_limit_exceeded` or 503
|
||||
`monitoring_unavailable`, both with `Retry-After` — rather than served unobserved.
|
||||
Note the site count: there are **five** call sites, not the four previously listed
|
||||
here; `api/handlers/ebook_reader.go` was missing. Saturation is warned at the
|
||||
registry (rate-limited per user, carrying route and user id) rather than once per
|
||||
rejected request, which an actor parked at its cap could turn into log
|
||||
amplification.
|
||||
- **Existence is server-observed on every cell** *except the ebook rows above*.
|
||||
Integrated: a session is
|
||||
unreapable while `activeTransportCount > 0`, and every byte-serving path now
|
||||
holds that marker — direct/remux for the whole pour, transcode for each segment
|
||||
serve (the segment markers were added to close a hidden-stream hole where a slow
|
||||
single-segment drain past the 45s grace could be reaped mid-serve; the compat
|
||||
HLS path previously refreshed liveness *only* from the client's progress POST,
|
||||
a direct violation, now also transport-marked). Edge: a Redis record exists for
|
||||
the whole connection (direct/remux) or while segments are pulled (transcode),
|
||||
advanced only by real bytes. **No path requires a client progress report to stay
|
||||
visible or counted.**
|
||||
- **Timing is server-observed; the client is trusted only for position.**
|
||||
`Session.LastServedAt` is deliberately distinct from `LastActivityAt` and is
|
||||
advanced only by server-observed bytes or transport begin/end. As of decision A5
|
||||
client progress feeds **neither** the enforcer's victim ordering **nor** reaping:
|
||||
idleness is measured from `LastServedAt`, falling back only to `StartedAt` behind a
|
||||
bounded never-served grace, and a never-served session projects an *empty*
|
||||
`LastServedAt` so it sorts as the stalest over-cap victim. The one deliberate
|
||||
exception is a **paused** session holding an open, ping-checked realtime/WebSocket
|
||||
connection, which stays exempt from reaping — an observed connection, not a reported
|
||||
position (issue #243). Integrated `BytesServed` and
|
||||
`LastServedAt` are mapped alongside the edge equivalents. On the central side this
|
||||
holds exactly: `Session.LastServedAt` is written only by `BeginTransport`,
|
||||
`EndTransport` and `AddServedBytes` (`playback/session.go:1132,1151,1169`) — no
|
||||
progress-report path touches it.
|
||||
**GAP-15 — RESOLVED.** The proxy now creates transcode visibility separately
|
||||
from served liveness. Only bytes actually written from a 200/206 upstream
|
||||
response advance `LastServedAt`; zero-byte responses, upstream failures, and
|
||||
non-success error bodies do not.
|
||||
- Report-to-central: **C** = Redis-backed (edge writes `silo:sessions:*`,
|
||||
`streammonitor.RedisSource` reads). The first projection write for a session is
|
||||
synchronous so visibility is established before the request returns; later
|
||||
liveness and byte updates are asynchronous on the refresh tick. Consequently,
|
||||
a slow Redis adds latency to the first request of a stream. An ordered,
|
||||
lifecycle-aware projection queue is deliberately deferred rather than making
|
||||
this path lossy or reorderable. **A/B** = in-process SessionManager IS central;
|
||||
read via `streammonitor.FuncSource`. MultiSource unions both, deduped by
|
||||
canonical logical session identity when available.
|
||||
- Owner + attribution: the transcode node's start record now carries owner + route
|
||||
+ client (threaded via `TranscodeStartRequest`), and `streammonitor.mergeStreams`
|
||||
additionally backfills any missing owner/route/client from another record for the
|
||||
same session — so an ownerless-but-freshest node record can never bucket a stream
|
||||
under user 0 (which the enforcer skips) or drop route/client from the view.
|
||||
|
||||
### Monitoring data model (what central sees per stream)
|
||||
|
||||
First-class monitoring goal: see **all** active streams with enough to act on.
|
||||
Captured per live stream (`streammonitor.LiveStream` / `nodesessions.SessionInfo`):
|
||||
|
||||
| Field | Source | Notes |
|
||||
|---|---|---|
|
||||
| existence, session id | both | primary; server-observed |
|
||||
| user id, profile id | token claims / Session | primary; owner attribution |
|
||||
| media file id | claims / Session | primary; human title is a display-time lookup (follow-up) |
|
||||
| play method (direct/remux/transcode) | `Type` | primary |
|
||||
| **route (native/jellycompat)** | `Route` — `Session.Origin` (integrated) or token `Origin` claim (edge) | primary; native and jellycompat share the SessionManager and `Type`, so route is a distinct field seeded at the two `WithClientInfo` sites and carried to edges in the token |
|
||||
| node serving | NodeName/NodeURL | integrated stamps the local host |
|
||||
| client ip / client name | `Session.ClientIP/ClientName` (integrated) or edge request + `ClientName` claim | secondary. **Not usable as a re-streaming fingerprint as-is:** it is one value per session, stamped at session creation (integrated) or overwritten per request (edge), so concurrent viewers of one session collapse to a single address. A restream heuristic needs per-request observation at serve time — see abuse-matrix correction #9 |
|
||||
| position | `Session.Position` | secondary timing |
|
||||
| bytes served | edge `AddBytes` / integrated `SessionMeteredWriter` | server-observed throughput signal |
|
||||
| hw/sw, resolution, codecs | Session / node record | secondary |
|
||||
|
||||
- **Admin visibility:** `HandleListSessions` unions the Redis edge records with the
|
||||
in-process integrated sessions (`LiveLocalSessions`), so a single-node integrated
|
||||
deployment is no longer blind (previously Redis-only). The union is deduped by
|
||||
session id (`streammonitor.DedupeSessionInfos`, mirroring `mergeStreams`) so a
|
||||
stream tracked by both the central manager and the edge serving it shows as ONE
|
||||
row — matching the enforcer's count. A `node_id` filter targets
|
||||
an edge, so integrated sessions appear only in the unfiltered listing.
|
||||
- **Download visibility is a separate sibling collection.** The same unfiltered
|
||||
admin response includes `transfers`, sourced from a bounded process-local
|
||||
registry. Transfer rows contain a unique per-pour id, optional download-row
|
||||
correlation id, owner/profile, media file, route, client metadata, timestamps,
|
||||
and server-observed bytes. A `node_id` filter excludes them because they belong
|
||||
to this API process, not an edge. The byte timestamp advances in coarse metered
|
||||
chunks (1 MiB or final close), and multi-replica deployments see only the
|
||||
registry of the process answering the request.
|
||||
- **Downloads are an intentional exemption — with asymmetries.** `/downloads/*`
|
||||
(native), compat `/Items/{id}/Download`, and ABS authenticated file/download
|
||||
and RSS feed serves transfer full media without a live-stream session and do NOT
|
||||
count against the live-stream cap. They now create separate, in-memory transfer
|
||||
records while a pour is active. Native download quotas run only when a download
|
||||
row is created; neither `ServeDownload` nor `ServeDirect` gates a pour, and
|
||||
compat/ABS routes have no shared download quota. All routes arm the shared
|
||||
`WatchAndCut`, and the same hook deletes every compat login so reconnects need
|
||||
re-auth. **Arming it was previously not the same as it working:** on ABS routes the
|
||||
cut could not reach the socket at all (GAP-10) and on the native download routes it
|
||||
was undone by the rolling write deadline (GAP-12). Both are now fixed, so "a per-user
|
||||
stream revocation cuts an in-flight download pour" holds on every one of these routes
|
||||
— within one ~5s watch tick.
|
||||
|
||||
### Kill switch (revocation enforced on the serve path)
|
||||
|
||||
| Route | Type | Integrated (A/B) | Multi-node (C) |
|
||||
|---|---|---|---|
|
||||
| native | direct | ✅ `guardRevocationCut` (Refuse + in-flight cut†) | ✅ `verifyToken` + `cutOnRevocation` (in-flight cut) |
|
||||
| native | remux | ✅ `guardRevocationCut` (Refuse + cut†) | ✅ verifyToken + cutOnRevocation |
|
||||
| native | transcode | ✅ `guardRevocation` (refused within one segment) | ✅ proxy verifyToken + node `refuseIfRevoked` + reconstruct guard‡ |
|
||||
| jellycompat | direct | ✅ `Refuse` + in-flight cut† (GAP-1/3 fixed) | ✅ via proxy; ✅ local fallback guarded |
|
||||
| jellycompat | remux | ✅ `Refuse` + in-flight cut† | ✅ via proxy; ✅ local fallback guarded |
|
||||
| jellycompat | transcode | ✅ `Refuse` per segment (GAP-1 fixed) | ✅ via proxy/node; ✅ local fallback guarded |
|
||||
| ABS | authenticated file/download | ✅ bearer JWT `iat` owner cutoff refuses on entry + in-flight cut† (GAP-10 fixed) | same |
|
||||
| ABS | public playback track | ✅ native session id + session `StartedAt` owner cutoff + cut† (GAP-10 fixed) | same |
|
||||
| ABS | public RSS feed file | ✅ feed `CreatedAt` owner cutoff refuses on entry + cut† (GAP-10 fixed) | same |
|
||||
| ebook | original file / converted EPUB | ✅ `Refuse` on entry + in-flight cut† (GAP-11 fixed) | same |
|
||||
| native | subtitles | ✅ `guardRevocationCut` (Refuse + cut†) | ✅ verifyToken + cutOnRevocation |
|
||||
| jellycompat | subtitles | ✅ `Refuse` + cut† (delivery only — extraction is buffered) | same |
|
||||
|
||||
† In-flight cut uses `streamrevoke.Store.WatchAndCut` (SetWriteDeadline, checked on
|
||||
entry then every `Options.WatchInterval` — **5s in production**, so a cut lands within
|
||||
~5s, not instantly). Works at the edge (the proxy's metered writer implements
|
||||
`Unwrap`) **and** on the native/compat integrated surfaces: the native
|
||||
(`statusWriter`, `requestStatusWriter`) and compat (`loggingResponseWriter`,
|
||||
`compatImageProxyTagResponseWriter`, `debugResponseWriter`) middleware writers
|
||||
implement `Unwrap()`, so `http.NewResponseController` reaches the socket instead of
|
||||
no-oping (see GAP-3). chi's own `middleware.Compress` wrapper also implements
|
||||
`Unwrap()` (chi v5.2.5 `middleware/compress.go:374`), so the globally-mounted
|
||||
compressor does not break the chain either. ABS's `statusRecorder` now implements it
|
||||
too (GAP-10), and a **latch** on the request context stops any
|
||||
`RollingDeadlineWriter` downstream of the cut from re-arming the socket (GAP-12).
|
||||
|
||||
**The audit that produced that list originally missed the ABS surface — see GAP-10.**
|
||||
Any new middleware that wraps a byte-serving route must implement `Unwrap()`, and the check
|
||||
belongs in a test that exercises the *mounted* router, not the handler in isolation.
|
||||
|
||||
**GAP-10 — RESOLVED (serve-path batch).** Every ABS route is wrapped by `accessLog`
|
||||
(`audiobooks/abs/handler.go:334`), whose `statusRecorder`
|
||||
(`audiobooks/abs/access_log.go`) implemented `Write`, `WriteHeader`, `Hijack` and
|
||||
`Flush` but **not** `Unwrap()`. The chain `SessionMeteredWriter → statusRecorder`
|
||||
dead-ended, `http.NewResponseController(...).SetWriteDeadline` returned
|
||||
`ErrNotSupported`, and `WatchAndCut` discarded that error and silently stopped
|
||||
watching — a multi-GB ABS pour started before a `RevokeUser` ran to completion.
|
||||
Fixed by adding `Unwrap()`. `WatchAndCut` now **logs** a failed `SetWriteDeadline`
|
||||
(once per watcher) instead of discarding it, so the next wrapper of this shape is
|
||||
loud rather than invisible.
|
||||
|
||||
The old test passed throughout because `audiobooks/abs/revocation_test.go` called the
|
||||
handlers directly and never saw the middleware. The replacement drives the **mounted**
|
||||
router over a **real socket**; it and the compile-time `Unwrap` assertion were both
|
||||
confirmed to fail when `Unwrap()` is removed again, so this cannot regress silently.
|
||||
|
||||
**GAP-12 — RESOLVED (serve-path batch).** Was scoped to Batch 3, but pulled forward
|
||||
because it makes the GAP-10 fix *inert*: adding `Unwrap()` is what makes
|
||||
`SetWriteDeadline` start succeeding, which is also what makes
|
||||
`RollingDeadlineWriter.bump()` start succeeding — and `bump()` pushed the deadline
|
||||
back out to `now + StallWindow` (180s) once `bumpStep` (15s) had elapsed, plus once
|
||||
more from its own constructor. Fixing GAP-10 alone would have left the ABS kill switch
|
||||
broken, just differently.
|
||||
|
||||
The obvious fix does not work and was rejected: the rolling writer is constructed
|
||||
*inside* `ServeDirectPlay`/`ServeRemux` and **wraps** the writer `WatchAndCut` holds,
|
||||
so it sits *above* the watcher. `Unwrap()` walks toward the socket, so the watcher can
|
||||
never reach it by writer introspection. The cut has to travel by a side channel.
|
||||
|
||||
Fixed with `httpstream.CutLatch`, carried on the request context and consulted by
|
||||
`bump()`. Once latched the writer never extends the deadline again. `bump()` re-checks
|
||||
the latch *after* setting a future deadline so a concurrent cut cannot be lost to the
|
||||
check/set race, and `WatchAndCut` keeps re-applying the deadline each tick rather than
|
||||
returning after the first cut, as belt-and-braces for any topology the latch misses.
|
||||
|
||||
**Known bound, by design:** the watcher polls, so a revoked pour keeps delivering for
|
||||
up to one watch interval — **5s in production** — before the cut lands. On a fast link
|
||||
that is a meaningful amount of data. Read every "cut" claim below as "cut within ~5s",
|
||||
not "cut immediately". `Options.WatchInterval` makes the interval injectable so
|
||||
real-socket tests do not have to wait it out.
|
||||
|
||||
‡ The transcode node's serve-path check is session-only (`refuseIfRevoked` passes
|
||||
`userID = 0`), so a per-*user* kill is enforced by the fronting proxy's `verifyToken`
|
||||
(which passes `claims.UserID`), not at the node's segment serve. The node's
|
||||
reconstruct guard *does* use the real `claims.UserID`, so a rebuild-after-restart of
|
||||
a user-killed session is blocked.
|
||||
|
||||
### Restart durability (kill survives a process restart / Redis loss)
|
||||
|
||||
This axis exists because the branch sits on top of PR #174 (restart-resilient
|
||||
playback): a session killed before a restart is **reconstructed** afterward from a
|
||||
durable recipe card, so if the kill list does not also survive the restart the
|
||||
reconstructed stream is silently re-served. The kill list must be at least as
|
||||
durable as the thing it kills.
|
||||
|
||||
| Deployment | Session survives restart (PR #174) | Kill survives restart | Kill survives Redis flush |
|
||||
|---|---|---|---|
|
||||
| A — integrated, no Redis | ✅ recipe card (PG) | ✅ durable PG mirror | ✅ (never used Redis) |
|
||||
| B — integrated, with Redis | ✅ | ✅ durable PG mirror (+ Redis warm) | ✅ durable PG mirror |
|
||||
| C — multi-node, with Redis | ✅ | ✅ central durable PG mirror; edges re-warm from Redis | ✅ central re-warms edges from PG→Redis on next write/poll |
|
||||
|
||||
\* Steady-state guarantee. Transient boot caveat: if the durable warm exhausts its
|
||||
bounded retry **and** Redis is empty, the kill list is empty until the first poll
|
||||
tick (≤60s). This fails *open* by design — see the "Warm is async-tolerant" bullet
|
||||
below.
|
||||
|
||||
- **Hot path unchanged.** `IsRevoked` is still a pure in-memory map read. Postgres
|
||||
is touched only on write (`Upsert`), on warm/reconcile (`List` at
|
||||
`StartSync` and on the poll tick), and on trim (`Prune`) — never per request.
|
||||
- **Central-side only.** The durable mirror is wired into the integrated/api
|
||||
`streamrevoke.Store` (`cmd/silo/main.go`), not the edge/proxy/transcode nodes,
|
||||
which have no app DB and enforce via Redis pub/sub + poll.
|
||||
- **Bounded growth.** Rows are keyed by `(kind, id)`, so the async enforcer
|
||||
re-revoking every pass UPSERTs one row rather than accumulating; expired rows are
|
||||
physically reclaimed by `Prune` on the poll tick. `List` filters active rows
|
||||
with `unrevoked_at IS NULL` and returns live tombstones separately.
|
||||
- **Expiry and cutoff merge independently.** Both the in-memory `applyLocal` and
|
||||
durable `Upsert` keep the later expiry while advancing `RevokedAt` (and its
|
||||
reason) to the later cutoff. A newer user cutoff with a shorter requested
|
||||
horizon therefore cannot be suppressed by an older long-lived record.
|
||||
- **Warm is async-tolerant.** By product decision, a just-reconstructed revoked
|
||||
stream may serve a few seconds before the durable warm/next Refuse tick cuts it;
|
||||
no blocking startup ordering is required. Edge caveat: if the boot durable warm
|
||||
exhausts its bounded retry **and** Redis is empty, the kill list is empty until
|
||||
the first poll tick (≤60s). This fails *open* by design (a kill switch cannot
|
||||
fail closed without blocking all playback when the DB is briefly unavailable).
|
||||
- **Edge Redis-outage fails open.** Edge nodes (proxy/transcode) have no durable
|
||||
store and learn *new* kills only from Redis pub/sub + SCAN. While Redis is
|
||||
unreachable, already-cached kills persist but a kill issued *during* the outage
|
||||
does not reach the edge until Redis recovers — enforcement fails open there.
|
||||
- **Kills are operator-visible and undoable.** Admins can list, create, and
|
||||
remove entries with `GET/POST /api/v1/admin/streams/revocations` and
|
||||
`DELETE /api/v1/admin/streams/revocations/{kind}/{id}`. `Unrevoke` removes the
|
||||
local and Redis copies, durably changes the Postgres row into a bounded
|
||||
tombstone, and publishes an unrevocation event. The tombstone survives restart
|
||||
and prevents a delayed reconcile or stale replica from resurrecting the removed
|
||||
kill, while a later legitimate revoke clears it. If unrevocation publish
|
||||
fails, other processes retain the kill until expiry: the residual failure is
|
||||
safe (dead rather than unexpectedly live) and is reported as a warning.
|
||||
- **Durable self-heal compares expiry.** The maintenance pass re-upserts a local
|
||||
entry when the durable copy is missing or expires earlier, repairing a failed
|
||||
longer mirror instead of treating mere row presence as healthy.
|
||||
|
||||
## Findings
|
||||
|
||||
> GAP-1..GAP-3 all shipped in the kill-switch commit (see the Status banner for
|
||||
> how commits are referenced by role rather than SHA).
|
||||
|
||||
**GAP-1 — RESOLVED.** jellycompat LOCAL serving now consults the
|
||||
shared `Store.Refuse` in `HandleVideoStream` (direct/remux) and `HandleHLSSegment`
|
||||
(per segment), closing the integrated / local-transcode-fallback hole.
|
||||
|
||||
**GAP-2 — RESOLVED.** `buildProxyRedirectURL` now stamps
|
||||
`uid`/`pid`/`mfid` onto the jellycompat proxy-redirect token (looked up from the
|
||||
upstream SessionManager session), so edge monitoring/kill attribution matches
|
||||
native. `AuthUserID = 0` no longer occurs for jellycompat.
|
||||
|
||||
**GAP-3 — RESOLVED.** The in-flight cut is shared as
|
||||
`streamrevoke.Store.WatchAndCut` and applied to native `/stream`
|
||||
(`guardRevocationCut`) and jellycompat `HandleVideoStream`. The original caveat —
|
||||
`SetWriteDeadline` might not reach the socket through the native/compat middleware
|
||||
chain — was **confirmed** as a guaranteed integrated no-op (four middleware writers
|
||||
wrapped `w` without `Unwrap()`) and then fixed by adding `Unwrap()` to
|
||||
`statusWriter`, `requestStatusWriter`, `loggingResponseWriter`,
|
||||
`compatImageProxyTagResponseWriter`, and `debugResponseWriter`. The cut now reaches
|
||||
the socket in integrated mode; if it ever no-ops again the stream still stops on
|
||||
its next request via `Refuse` (no regression). See VERIFY-1.
|
||||
|
||||
**GAP-4 — RESOLVED.** The kill list did not survive a process restart or a Redis
|
||||
flush, but PR #174's restart-resilient playback *does* reconstruct the session — so
|
||||
a stream revoked before a restart was silently re-served afterward. This was
|
||||
universal in deployment A (integrated, no Redis: the kill list was pure RAM) and
|
||||
occurred on any Redis loss in B/C. Fixed by wiring a concrete Postgres
|
||||
`DurableStore` (`internal/streamrevoke/durable_postgres.go`, table
|
||||
`stream_revocations`) into the central `streamrevoke.Store`: `Revoke` mirrors to
|
||||
Postgres, `StartSync` warms from it on boot, and the poll tick re-warms (heals a
|
||||
Redis flush) and `Prune`s expired rows. The `defaultTTL` (24h) is held `>=` the
|
||||
recipe-card `MaxTokenTTL` (24h) as an invariant so a kill cannot expire before its
|
||||
session can be reconstructed, and expiry is **monotonic** on every write (in-memory
|
||||
`applyLocal` + durable `Upsert` `GREATEST`). The async enforcer uses a
|
||||
revoke-if-absent write with a default TTL derived from `playback.MaxTokenTTL`, so
|
||||
repeated evaluation neither reopens the token after five minutes nor slides a
|
||||
false positive forever. Hot path
|
||||
(`IsRevoked`) stays an in-memory read.
|
||||
|
||||
**GAP-5 — RESOLVED (post-review).** User-kind revocations were a blanket ban:
|
||||
any admin user edit (via `OnUserSessionsRevoked`) 403'd that user's playback for
|
||||
24h even after re-login, unrevocably. Fixed with cutoff semantics — see
|
||||
"user-ID kills" above.
|
||||
|
||||
**GAP-6 — RESOLVED (post-review).** Compat coverage holes: `HandleMasterManifest`
|
||||
/ `HandleHLSManifest` had no revocation check and kept (or, post-restart,
|
||||
re-spawned) a killed session's ffmpeg via the ensure path; `HandleVideoStream`
|
||||
checked revocation only after `ensureUpstreamPlayback`, which can replace an
|
||||
unreconstructable killed session with a fresh id that passed the check (kill
|
||||
dodged by re-hitting the URL); `HandleSubtitleStream` ignored kills entirely.
|
||||
All four now `Refuse` up front.
|
||||
|
||||
**GAP-7 — RESOLVED (post-review).** Admin terminate 404'd before revoking when
|
||||
the session was absent from the central in-memory manager — exactly the
|
||||
edge-served / post-restart / progress-withholding streams the kill switch
|
||||
exists for (the admin list showed them via the Redis union; terminate couldn't
|
||||
touch them). Terminate now writes the revocation FIRST, keyed on the session id
|
||||
alone, and answers `202 {status: "revoked"}` when there is no local session for
|
||||
the cooperative command.
|
||||
|
||||
**GAP-8 — RESOLVED (post-review).** The admin session list double-counted every
|
||||
edge-served stream (central manager row + edge Redis row, no dedupe), unlike
|
||||
the enforcer's merged picture. Now deduped by session id with owner/attribution
|
||||
carry-forward (`streammonitor.DedupeSessionInfos`).
|
||||
|
||||
**GAP-9 — RESOLVED (post-review).** The "restore sendfile" commit was dead code
|
||||
end-to-end: every stream route runs inside `meterEgress`, and
|
||||
`meteredResponseWriter` hid `io.ReaderFrom`, so `sessionByteWriter.ReadFrom`'s
|
||||
fast path never fired and all direct-play/remux bytes went through a userspace
|
||||
copy. `meteredResponseWriter` now forwards `ReadFrom` (metering the returned
|
||||
total); a chain test locks the full production writer stack onto the sendfile
|
||||
path. Related tracker fixes in the same pass: `Touch`/`Track` preserve the
|
||||
first-seen `StartedAt` (was reset per segment/range request, corrupting the
|
||||
enforcer's tie-break and the admin start time), `AddBytes` no longer recreates
|
||||
entries for pruned sessions (slow permanent leak), the proxy→node segment pour
|
||||
attributes bytes incrementally (a slow drain can't go invisible mid-segment or
|
||||
post bytes to a dead record), and the transcode node marks serve activity so
|
||||
its record's `LastServedAt` is no longer frozen at start time.
|
||||
|
||||
**GAP-3 (original, superseded) — no in-flight cut for native/compat LOCAL direct-play.**
|
||||
The original finding: `guardRevocation` refused the next request/reconnect but could
|
||||
not hang up an in-flight `ServeFile` on the central process for a single long GET.
|
||||
Now closed — see GAP-3 above (shared `WatchAndCut` + middleware `Unwrap()`). Even
|
||||
if the cut degrades, admin terminate also fires the realtime command + session stop,
|
||||
so a killed direct-play stops on its next request at worst.
|
||||
|
||||
## Open verification items (check before prod)
|
||||
|
||||
- [x] **VERIFY-1 — in-flight cut reaches the socket on native + jellycompat — RESOLVED.**
|
||||
`streamrevoke.Store.WatchAndCut` cuts a revoked long-GET direct-play/remux by
|
||||
setting a zero write deadline via `http.NewResponseController(w)`, which walks the
|
||||
writer chain via `Unwrap()`. This was **confirmed broken** in integrated mode:
|
||||
`statusWriter`/`requestStatusWriter` (native) and `loggingResponseWriter`/
|
||||
`compatImageProxyTagResponseWriter`/`debugResponseWriter` (compat) wrapped `w`
|
||||
without `Unwrap()`, so `SetWriteDeadline` returned `ErrNotSupported` and the cut
|
||||
no-oped. Fixed by adding `Unwrap() http.ResponseWriter` to all five (pattern from
|
||||
`activitylog/middleware.go:186`). A live smoke test is still worthwhile: start a
|
||||
native and a jellycompat single-long-GET direct-play, admin-terminate it, confirm
|
||||
the connection drops within ~5s rather than only on the next request.
|
||||
- [x] **VERIFY-2 — download routes vs the stream cap — RESOLVED (documented exemption).**
|
||||
Native, compat, and ABS download-class routes remain intentionally exempt from
|
||||
the live-stream cap but are visible in the sibling, process-local `transfers`
|
||||
array. Native concurrency/period checks happen only at row creation, not at
|
||||
serve time. A per-user revocation is a cutoff: it refuses/cuts older pours but
|
||||
deliberately allows a new authenticated pour started after the cutoff.
|
||||
- [x] **VERIFY-4 — transcode buffer-ahead evasion — RESOLVED.** Unpaused
|
||||
transcodes use a bounded 10-minute integrated grace measured from the same
|
||||
activity resolution advanced by server-observed bytes/transports; paused
|
||||
transcodes retain the longer 30-minute paused grace. Edge transcode records
|
||||
use a 180-second idle window consistently in `ActiveCount`, `Snapshot`, and
|
||||
refresh. This deliberately avoids a local ffmpeg liveness probe, which cannot
|
||||
represent offloaded or completed copy-mode transcodes.
|
||||
- [ ] **VERIFY-3 — multi-replica central enforcement.** The async enforcer runs on
|
||||
every integrated/api process. Two *integrated* replicas behind a load balancer
|
||||
each see only their own in-process `FuncSource` (integrated mode writes no
|
||||
`silo:sessions:*` records), so a user split across both can exceed `max_streams`
|
||||
undetected; multiple *api* replicas each run an independent enforcer and revoke
|
||||
the same victims (idempotent, but redundant write load). A single-leader election
|
||||
for the enforcer, or having integrated nodes publish their local sessions to the
|
||||
shared Redis picture, would close this.
|
||||
|
||||
## Design assessment (shared vs duplicated code)
|
||||
|
||||
**Kill DECISION is well-centralized — keep it.** All revocation state and the
|
||||
`IsRevoked(sessionID, userID)` check live in one place (`internal/streamrevoke`).
|
||||
Session kills and user kills use the SAME store and the SAME check — `RevokeSession`
|
||||
and `RevokeUser` write into one `items` map; `IsRevoked` tests both keys. No
|
||||
duplication at the decision layer.
|
||||
|
||||
**Kill ENFORCEMENT — DONE.** The "extract sid/uid → refuse (403)"
|
||||
step is now a single shared method `streamrevoke.Store.Refuse(w, sessionID, userID)
|
||||
bool`. All four serve surfaces call it (proxy `verifyToken`, transcode node
|
||||
`refuseIfRevoked`, native `guardRevocation`, jellycompat serve handlers); each only
|
||||
owns its token extraction (path `{token}`, query `st`, or compat `PlaySessionId`).
|
||||
The in-flight connection-cut is the shared `WatchAndCut` (SetWriteDeadline), used by
|
||||
both the edge (`cutOnRevocation`) and the native/compat long-pour paths. ONE section
|
||||
per concern, as intended.
|
||||
|
||||
**Monitoring WRITE side is two implementations by necessity — leave it.** Edges
|
||||
have no SessionManager; the integrated process has no tracker. `nodesessions.Tracker`
|
||||
(edge) and `SessionManager` (integrated) are different runtimes, not duplicated
|
||||
logic, and they already converge at the single read layer (`streammonitor`,
|
||||
unioned by MultiSource). Forcing one implementation would be worse.
|
||||
|
||||
**user-ID kills — plumbing check (as asked).** Two distinct layers, correctly
|
||||
separate:
|
||||
- *Auth/login revocation* (pre-existing): `OnUserSessionsRevoked` revokes native
|
||||
auth sessions (`auth.SessionRepository.RevokeAllByUser`) and drops compat login
|
||||
tokens (`SessionStore.DeleteByUserID`). This governs "can this user authenticate",
|
||||
not "stop this byte stream".
|
||||
- *Stream revocation* (new): `streamRevocation.RevokeUser` writes a `KindUser`
|
||||
entry into the SAME `streamrevoke` store as session kills. Session-level and
|
||||
user-level STREAM kills already share all plumbing (one store, one `IsRevoked`).
|
||||
- **A user kill is a CUTOFF, not a ban (GAP-5 fix).** `IsRevoked` matches a
|
||||
`KindUser` entry only when the stream's credential predates the revocation:
|
||||
the stream token's `iat` on token-bearing surfaces (edge proxy, transcode
|
||||
node, native `?st=`), the access-token `iat` on native authenticated routes,
|
||||
and the compat login's stable `CreatedAt` on normal Jellyfin sessions. An
|
||||
unknown (zero) credential time never matches (fail open). API keys have no
|
||||
issue time, so a user cutoff cannot cut an API-key-owned pour; this is an
|
||||
accepted, logged limitation. Per-login logout cuts remain out of scope until
|
||||
stream credentials carry per-login identity. This is what makes it safe for
|
||||
`OnUserSessionsRevoked` — which fires
|
||||
on ANY admin edit of password/role/enabled/permissions/quality — to also
|
||||
write a stream kill: without the cutoff, a routine permission tweak 403'd
|
||||
the user's playback for the full 24h TTL after re-login unless an operator
|
||||
explicitly removed the kill.
|
||||
- These two layers should NOT be merged — conflating "may log in" with "this
|
||||
stream must die" would couple auth and playback. They are chained in the same
|
||||
hook, which is the right seam.
|
||||
|
||||
## Recommended follow-up order
|
||||
|
||||
GAP-1, GAP-2, GAP-3, and GAP-4 are all shipped (see Findings). Already-shipped
|
||||
hardening not to re-do: `Revoke` propagation uses `context.WithoutCancel`, so an
|
||||
aborted admin request can no longer strand a kill in central memory only.
|
||||
Remaining work:
|
||||
|
||||
1. **Operator config follow-up:** the over-cap lifetime is now validated as
|
||||
`playback.over_cap_revocation_ttl` and defaults to the reconstructable token
|
||||
lifetime. It is startup-bound and affects only future revocations after the
|
||||
required restart: monotonic expiry cannot shorten an existing kill.
|
||||
Poll/default admin-revocation settings remain separate follow-up work.
|
||||
2. **Media title enrichment:** resolve `MediaFileID → title` at admin display time
|
||||
so the view shows *what* is being watched, not just a numeric id (also enables a
|
||||
distinct-title re-streaming heuristic).
|
||||
3. **Multi-replica enforcement (VERIFY-3, explicitly deferred):** single-leader election for the async
|
||||
enforcer, or have integrated nodes publish local sessions to the shared Redis
|
||||
picture so the cap holds across replicas.
|
||||
4. **Invariant guard test:** `defaultTTL >= playback.MaxTokenTTL` is coupled only by
|
||||
a prose comment (streamrevoke can't import playback — import cycle). Add a test
|
||||
in a package that can import both, so raising `MaxTokenTTL` can't silently
|
||||
violate it.
|
||||
5. **Transcode-node ghost records (explicitly deferred):** the node's session-backed `Track` record is
|
||||
refreshed forever until an explicit stop/cleanup; if central dies or loses the
|
||||
session without the stop reaching the node, an owner-attributed ghost persists
|
||||
in Redis indefinitely — permanently +1 in the user's live count (the enforcer's
|
||||
freshness ordering trims the ghost first, so real streams survive, but the
|
||||
enforcer re-revokes it every pass and the admin view overcounts). Needs a
|
||||
node-side idle sweep tied to real serve activity (`MarkServed` now provides the
|
||||
signal).
|
||||
6. **Download controls phase 2:** add per-download kill and a standing per-user
|
||||
download block (a user revocation is only a cutoff and permits newer
|
||||
credentials/requests). Separately unify native, compat, and ABS under a shared
|
||||
download quota and rolling volume budget. Also resolve the pre-existing native
|
||||
completion semantics (`ServeContent` cannot report write failure), correlate
|
||||
ABS playback sessions with duplicate `abs_file_stream` transfer rows, and make
|
||||
transfer visibility multi-replica rather than process-local.
|
||||
@@ -0,0 +1,778 @@
|
||||
# Stream & API Abuse — Coverage Matrix
|
||||
|
||||
> **What this is.** A red-team assessment of how the `feat/sauron-async-enforcer`
|
||||
> branch (server-observed stream monitoring + revocation kill switch + async
|
||||
> over-cap enforcer) — layered on the current `main` — reacts to ~29 concrete
|
||||
> abuse stories. Each story is scored on two axes the branch itself separates:
|
||||
>
|
||||
> - **Detection** — does the server *see* the abuse and *label it correctly*
|
||||
> (right user, right stream, right kind)?
|
||||
> - **Enforcement** — can the server *stop* it, and *how fast*?
|
||||
>
|
||||
> **Companion docs:** the intent lives in
|
||||
> [`../superpowers/plans/2026-07-04-stream-monitoring-and-kill-switch.md`](../superpowers/plans/2026-07-04-stream-monitoring-and-kill-switch.md);
|
||||
> the as-built coverage-by-path lives in
|
||||
> [`playback-paths-monitoring-kill-matrix.md`](playback-paths-monitoring-kill-matrix.md).
|
||||
> This doc is the *adversarial* companion: it assumes a motivated abuser and asks
|
||||
> where the design ends.
|
||||
>
|
||||
> **Scope note.** The branch is scoped to **concurrent playback-stream abuse**. A
|
||||
> large fraction of real-world "abuse" (library enumeration load, bulk ripping,
|
||||
> API request floods, CPU exhaustion, re-broadcasting) lives *outside* that scope.
|
||||
> Where that is the case this doc says so plainly and points at the subsystem that
|
||||
> would actually own the defense (`internal/ratelimit`, `internal/downloads`,
|
||||
> catalog browse, node scheduling) — several of which have real gaps of their own.
|
||||
|
||||
## Legend
|
||||
|
||||
| Symbol | Detection | Enforcement |
|
||||
|---|---|---|
|
||||
| ✅ | Seen and correctly attributed | Stopped (and how fast) |
|
||||
| ⚠️ | Seen but partial / mislabeled / lossy | Partial, slow, best-effort, or config-dependent |
|
||||
| ❌ | Invisible to the server | Cannot stop |
|
||||
|
||||
Enforcement latency budget for the async path is **~120s** = enforcer tick
|
||||
(`DefaultInterval = 30s`, `streamenforcer/enforcer.go`) + revocation propagation
|
||||
(pub/sub push immediate; ≤60s poll safety net, `streamrevoke/store.go`) + in-flight
|
||||
cut (5s `WatchAndCut` ticker for long pours; next segment for HLS). Synchronous
|
||||
admission refusal (`ErrTooManyStreams`, `playback/session.go`) is **immediate**.
|
||||
|
||||
---
|
||||
|
||||
## Corrections the investigations turned up (read first)
|
||||
|
||||
The branch docs oversell several things. The stories below are scored against the
|
||||
**code**, not the plan.
|
||||
|
||||
> **Round 2 — 2026-07-29, two-model review (Claude Opus 5 + Codex gpt-5.6-sol).**
|
||||
> Corrections 4–8 below were found by a second adversarial pass over the rebased
|
||||
> branch and are **open defects, not doc drift**. They matter because they hit the
|
||||
> two stated priorities directly: #1 "see all bytes, no invisible streams"
|
||||
> (corrections 5, 7, 8) and #2 "kill any stream" (corrections 4, 6). Affected rows in
|
||||
> the summary matrix have been downgraded accordingly.
|
||||
|
||||
1. **The async enforcer uses the effective, group-merged limit.** An earlier
|
||||
audit found it reading raw `users.max_streams`, which is zero ("inherit") for
|
||||
standard accounts. That is no longer true: its limit function calls
|
||||
`SessionManager.EffectiveLimits`, the same group-merged policy used by
|
||||
admission. Standard users therefore receive the Default Group cap instead
|
||||
of being treated as unlimited.
|
||||
|
||||
2. **The re-streaming heuristic does not exist.** The plan describes shipping
|
||||
`OverCapRule` + a disabled `RestreamHeuristicRule` behind a pluggable `Rule`
|
||||
interface. In code, `streamenforcer/enforcer.go` is a single hardcoded over-cap
|
||||
loop — no rule interface, no restream logic, not even a disabled stub. The
|
||||
fingerprints a heuristic *would* need (`ClientIP`, `ClientName`, `MediaFileID`,
|
||||
`BytesServed`) are captured in `streammonitor` but nothing consumes them for
|
||||
detection.
|
||||
|
||||
3. **There is no server-wide or per-node transcode/CPU cap.** Only a per-*user*
|
||||
`max_transcodes` (default 2, `<=0` = unlimited) checked at admission. The one
|
||||
ffmpeg semaphore (`reconstructSem`, sized to `NumCPU`) guards **only** the
|
||||
restart-reconstruct path in `transcodenode/server.go`, **not** fresh starts.
|
||||
|
||||
> **Update 2026-07-30 — serve-path batch landed.** Corrections **4, 5, 6 and 7 below
|
||||
> are now FIXED** (GAP-10, GAP-11, GAP-12, GAP-13); they are kept here with their
|
||||
> original wording because the summary-matrix rows and the follow-up list still
|
||||
> reference them, and because the *reason* each was invisible to the test suite is the
|
||||
> durable lesson. Correction **8 (GAP-14) is now also fixed** by the liveness/replica
|
||||
> batch under decision A7. See the follow-up list at the end for what is left.
|
||||
>
|
||||
> **Update 2026-07-30 — tracker-lifecycle batch landed.** GAP-15 is also fixed, along
|
||||
> with three defects that all produced a **wrong over-cap count**: overlapping edge
|
||||
> requests deleting a live record, an async-tracking race that could strand a
|
||||
> permanently non-expiring ghost session, and the protocol-v3 logical-vs-transport
|
||||
> identity split that counted one stream twice (masked until now because fresh v3
|
||||
> starts sent no owner at all, so the record landed under user 0, which the enforcer
|
||||
> skips — silently exempting the stream from the cap). These precede the revocation
|
||||
> batch on purpose: decision A1 removes the self-healing that limits the damage of a
|
||||
> miscount, so the count must be trustworthy first.
|
||||
>
|
||||
> Both claims this document previously flagged as false are now true, with named
|
||||
> bounds. The kill switch does "keep a stream dead": since the revocation batch an
|
||||
> over-cap kill lasts the token's full reconstructable life instead of reopening after
|
||||
> 5m. Monitoring does "never trust client progress": since the liveness batch the
|
||||
> `LastActivityAt` fallback is gone from both the projection and reaping — the single
|
||||
> deliberate exception being that a **paused** session holding an open, ping-checked
|
||||
> realtime/WebSocket connection is exempt from reaping (an observed connection, not a
|
||||
> reported position). Still read every "cut" as "cut within ~5s" — the in-flight
|
||||
> watcher polls on a 5s interval.
|
||||
|
||||
4. **The ABS in-flight kill switch does not work.** *(FIXED — `Unwrap()` added; the
|
||||
replacement test drives the mounted router over a real socket and was confirmed to
|
||||
fail without the fix.)* Every ABS route is wrapped by
|
||||
`accessLog` (`audiobooks/abs/handler.go:334`), whose `statusRecorder`
|
||||
(`audiobooks/abs/access_log.go:83`) implements `Write`, `WriteHeader` and `Hijack`
|
||||
but **not** `Unwrap()`. `WatchAndCut` walks
|
||||
`SessionMeteredWriter → statusRecorder` and dead-ends, so
|
||||
`SetWriteDeadline` returns `ErrNotSupported` — an error `WatchAndCut` discards
|
||||
(`streamrevoke/store.go:187`). The watcher then silently stops. A multi-GB
|
||||
audiobook pour started before a `RevokeUser` runs to completion.
|
||||
`audiobooks/abs/revocation_test.go` calls handlers directly, bypassing the
|
||||
middleware, so the test suite reports this as working. (GAP-10)
|
||||
|
||||
5. **Ebook, comic and PDF serving is invisible and un-killable.** *(FIXED — now
|
||||
metered, registered in the transfer registry, refused on entry and cut in flight;
|
||||
cap-exempt per decision A4. Note the fix does **not** reuse `guardRevocationCut` as
|
||||
originally suggested — that wrapper keys on a `session_id` param and `?st=` token
|
||||
this route does not carry, so it would have silently guarded nothing.)* The
|
||||
`/api/v1/ebooks/{content_id}/files/{file_id}/read` group (`api/router.go:2417`)
|
||||
carries only `apimw.RequireProfile`. Both `serveEbookInline`
|
||||
(`ebook_reader.go:626`) and `serveConvertedEpub` (`ebook_convert_serve.go:160`)
|
||||
pour whole files with no meter, no transfer record, no `Refuse` and no
|
||||
`WatchAndCut`. `ebookReaderFormat` (`ebook_reader.go:675`) accepts `cbz`/`cbr`
|
||||
comic archives and `pdf` — routinely 100 MB–1 GB+ each. (GAP-11)
|
||||
|
||||
6. **The rolling write deadline can erase a revocation cut.** *(FIXED — a `CutLatch`
|
||||
on the request context stops `bump()` from re-arming a cut socket, and the watcher
|
||||
keeps re-applying. Pulled forward from the revocation batch because it makes
|
||||
correction 4's fix inert: adding `Unwrap()` is what makes `SetWriteDeadline` start
|
||||
working, which is also what makes `bump()` start working.)* `WatchAndCut` sets the
|
||||
socket deadline to *now* once, then returns. On routes wrapped in
|
||||
`httpstream.RollingDeadlineWriter` — native managed and direct downloads
|
||||
(`api/handlers/downloads.go:447`) — `bump()`
|
||||
(`httpstream/rolling_deadline.go:95`) runs before every write slice and, once the
|
||||
15s `bumpStep` has elapsed, pushes the deadline back out to `now + 180s`. Nothing
|
||||
re-arms the watcher. The cut is therefore reliable against a *stalled* pour and
|
||||
unreliable against a *fast-draining* one — weakest against the ripping case it
|
||||
exists to stop. (GAP-12)
|
||||
|
||||
7. **Merged snapshots discard edge byte totals.** *(FIXED — merged as a max, not a sum,
|
||||
since the two records are two observers of one pour; `DedupeSessionInfos` had the
|
||||
same hole and was fixed too.)* `streammonitor.mergeStreams` takes
|
||||
the record with the later `LastServedAt` wholesale and backfills owner, route,
|
||||
client, HWAccel and position — but **not `BytesServed`**. When the central record
|
||||
is fresher, a stream whose bytes were all poured at an edge reports `0` bytes to
|
||||
the operator. (GAP-13)
|
||||
|
||||
8. **Transfer-registry saturation serves unmonitored.** The registry caps at
|
||||
`defaultMaxEntries = 10_000` (`transfers/registry.go:16`); past that `Begin`
|
||||
returns `ErrRegistryFull` and all four call sites log at **Debug** and serve
|
||||
anyway, discarding byte updates (`registry.go:122`). With no connection cap
|
||||
anywhere (E28) and an unthrottled compat surface (E25/E27), an attacker can pin
|
||||
the registry at its ceiling and blind download-class monitoring for everyone
|
||||
else. (GAP-14)
|
||||
|
||||
One thing the docs *under*-sell: `main` already ships a real general API rate
|
||||
limiter (`internal/ratelimit/`), enabled by default — but it is mounted only on
|
||||
the authenticated native surface + specific auth endpoints, and the entire
|
||||
Jellyfin-compat surface is unthrottled (see Category E).
|
||||
|
||||
---
|
||||
|
||||
## Summary matrix
|
||||
|
||||
### Category A — Concurrency-cap abuse (the branch's core competency)
|
||||
|
||||
| # | Abuse story | Detection | Enforcement | One-line verdict |
|
||||
|---|---|---|---|---|
|
||||
| 1 | User opens N+1 concurrent **transcodes** (over cap) | ✅ | ✅ immediate | Admission refuses the N+1 start synchronously |
|
||||
| 2 | User opens N+1 concurrent **direct-play** streams | ✅ | ✅ immediate | Same admission gate; method-agnostic |
|
||||
| 3 | **Shared account**, many devices streaming at once | ✅ | ✅ / ⚠️ | Admission caps per-user on one process; cross-replica excess is now seen and trimmed by the enforcer within ~120s (#7) |
|
||||
| 4 | **Buffer-ahead then pause** fetches >45s to duck the cap | ✅ | ✅ | Unpaused transcodes retain a 10m integrated / 180s edge server-observed grace; paused sessions now age from the last **served byte**, not the last client ping |
|
||||
| 5 | **Withhold/falsify progress** to hide a stream | ✅ | ✅ | Existence is byte-observed, and since A5 progress no longer confers liveness or reprieves reaping — a progress-only phantom ages out and sorts as the stalest over-cap victim |
|
||||
| 6 | **Rapid session churn** between enforcer ticks | ⚠️ | ⚠️ | Admission blocks over-cap starts; no rate limit on `/start` |
|
||||
| 7 | Split streams across **multiple integrated replicas** | ✅ | ⚠️ ~120s | Integrated sessions are published to the shared `silo:sessions:` picture and one elected evaluator trims the excess. **Admission is still per-process**, so the over-cap starts are admitted and then trimmed, not refused |
|
||||
| 8 | **Standard user** exceeds cap via the edge/multi-node path | ✅ | ✅ ~120s | Enforcer resolves the group-merged effective cap and trims the excess |
|
||||
|
||||
### Category B — Admin control / kill switch
|
||||
|
||||
| # | Abuse story | Detection | Enforcement | One-line verdict |
|
||||
|---|---|---|---|---|
|
||||
| 9 | Admin **terminates** a specific abusive live session | ✅ | ✅ | Terminate writes revocation first, keyed on sid; 202 even if no local session |
|
||||
| 10 | Admin **bans a user** (revoke all their streams) | ✅ | ✅ | Cutoff refuses every *next* request; in-flight cut now lands on every surface within ~5s (GAP-10/GAP-12 fixed) |
|
||||
| 11 | Killed stream **reconnects** with the same token | ✅ | ✅ | Revocation outlives token; every reconnect `Refuse`d |
|
||||
| 12 | Killed stream **survives a server restart** | ✅ | ✅ | Durable Postgres mirror re-warms the kill list; monotonic expiry |
|
||||
| 13 | Kill issued **during an edge Redis outage** | ⚠️ | ⚠️ | Edge fails **open**: new kills don't reach edge until Redis recovers |
|
||||
|
||||
### Category C — Re-streaming / redistribution
|
||||
|
||||
| # | Abuse story | Detection | Enforcement | One-line verdict |
|
||||
|---|---|---|---|---|
|
||||
| 14 | Re-broadcast **one** Silo stream to many external viewers | ❌ | ❌ | Under-cap = untouched; no restream heuristic exists (correction #2) |
|
||||
| 15 | **Many** concurrent Silo streams feeding a restream service | ⚠️ | ✅ ~120s | Caught only as raw over-count, not labeled re-streaming |
|
||||
| 16 | **Mint & hoard** many 24h tokens, fan out later | ❌ | ❌ / ⚠️ | No mint cap, signature-only, stop doesn't revoke; caught only if fan-out becomes over-cap |
|
||||
|
||||
### Category D — Ripping / bulk data exfiltration
|
||||
|
||||
| # | Abuse story | Detection | Enforcement | One-line verdict |
|
||||
|---|---|---|---|---|
|
||||
| 17 | Rip whole library via **native download** endpoints | ✅ | ⚠️ | Pours are always monitored or refused (429/503) since A7, and a per-user concurrent cap bounds one actor; creation-time gate only; **still no volume cap** |
|
||||
| 18 | Rip via **compat `/Items/{id}/Download`** (Infuse) | ✅ | ⚠️ | Pour is monitored or refused, and the per-user concurrent cap applies; **still no volume quota** |
|
||||
| 18a | Rip via authenticated **ABS file/download** route | ✅ | ⚠️ | Pour visible, cuttable in flight (GAP-10), and now monitored-or-refused with a per-user concurrent cap (A7); still **no volume quota** |
|
||||
| 18b | Pull an **ABS public playback track** | ✅ | ✅ | Native session id is monitored/metered; session and owner kills land |
|
||||
| 18c | Pull a **public ABS RSS feed file** | ✅ | ⚠️ | Active pour visible; owner cutoff uses feed creation time and the in-flight cut now lands (GAP-10 fixed). Closing a feed still does not cut its current pour; that needs a separately designed, collision-safe feed capability revocation id and is deferred. |
|
||||
| 18d | Rip via the **ebook/comic/PDF reader** route | ✅ | ⚠️ | Metered, transfer-rowed, refused on entry and cut in flight (GAP-11); also fails closed under A7 — this was the fifth `Begin` call site the batch handoff missed. Cap-exempt per A4; still no volume quota |
|
||||
| 19 | Rip via **sequential direct-play GETs** (one at a time) | ⚠️ | ❌ | Cap counts concurrency, not volume; stays at 1 forever. A7's per-user cap is also concurrency-based and does not touch this |
|
||||
| 20 | Admin **stops an in-progress rip** mid-transfer | — | ✅ | `WatchAndCut` now lands on every download-class route within ~5s (GAP-10/GAP-12 fixed); admin must still issue the revoke |
|
||||
|
||||
### Category E — API / DB load abuse (non-stream; branch is orthogonal)
|
||||
|
||||
| # | Abuse story | Detection | Enforcement | One-line verdict |
|
||||
|---|---|---|---|---|
|
||||
| 21 | **Infuse pulls the whole library list on every app start** | ❌ | ⚠️ | Per-page capped at 1000; full browse is uncached & unthrottled |
|
||||
| 22 | **Thundering herd** — many clients cold-cache full enumeration | ❌ | ⚠️ | Latest/hub rails singleflight; general browse does not |
|
||||
| 23 | Client **pages a 100k+ library as fast as possible** | ❌ | ❌ | Per-page COUNT + rising OFFSET; no page-rate limit, no cache |
|
||||
| 24 | **Expensive sorted/filtered views** over the whole library | ❌ | ⚠️ | Some sorts index-optimized; `DateCreated`/filters = full top-N + COUNT, uncapped |
|
||||
| 25 | Badly-coded client **retry storm** (thousands req/s) | ✅ | ⚠️ | Native authed surface throttled (per-IP 120rps); compat & `/auth/refresh` not |
|
||||
| 26 | Tight-loop polling **`/auth/refresh`** | ✅ | ❌ | No rate-limit middleware on that route |
|
||||
| 27 | **Credential stuffing** via compat `/Users/AuthenticateByName` | ✅ | ❌ | Compat router mounts no limiter; native login is capped 20/min |
|
||||
| 28 | Many concurrent **HTTP connections / slowloris** | ❌ | ❌ | No connection cap / `LimitListener`; only Read/Write/Idle timeouts |
|
||||
| 29 | Spawn many concurrent **transcodes to exhaust CPU** | ⚠️ | ❌ | Per-user `max_transcodes` only; no per-node/server-wide ffmpeg cap |
|
||||
|
||||
---
|
||||
|
||||
## Detailed stories
|
||||
|
||||
### Category A — Concurrency-cap abuse
|
||||
|
||||
**A1. N+1 concurrent transcodes (over cap).**
|
||||
The (N+1)th `StartSession` returns `ErrTooManyStreams` *synchronously* at admission
|
||||
(`playback/session.go`, `inlineAdmissionErrorLocked`), using the group-merged
|
||||
effective cap (5 for a standard user). The client never gets a session. The async
|
||||
enforcer resolves the same effective cap as its redundant backstop.
|
||||
**Detection ✅ / Enforcement ✅ (immediate).**
|
||||
|
||||
**A2. N+1 concurrent direct-play streams.**
|
||||
Direct-play sessions are `Track`-backed and counted identically at admission — the
|
||||
`MaxStreams` check is method-agnostic. Same immediate refusal. **✅ / ✅.**
|
||||
|
||||
**A3. Shared account, many devices at once.**
|
||||
All sessions carry the same `AuthUserID`; `streammonitor.ByUser` groups them and
|
||||
`Client*` fields expose the fan-out. The per-user cap bounds simultaneous streams
|
||||
to the effective `MaxStreams`. **Caveat:** the cap is per-process (see A7), so
|
||||
spreading devices across integrated replicas multiplies the allowance.
|
||||
**Detection ✅ / Enforcement ✅ on one process, ⚠️ across replicas.**
|
||||
|
||||
**A4. Buffer-ahead then pause (VERIFY-4 resolved).**
|
||||
An unpaused transcode now has a bounded 10-minute integrated grace, driven by
|
||||
server-observed `LastServedAt`, and ephemeral edge transcode records use a
|
||||
180-second idle window consistently in count, snapshot, and refresh. A genuinely
|
||||
paused session still receives the longer 30-minute paused grace. This covers
|
||||
local, completed copy-mode, and offloaded transcodes without probing local ffmpeg.
|
||||
**Detection ✅ / Enforcement ✅.**
|
||||
|
||||
**A5. Withhold/falsify progress reports.**
|
||||
This *fails* as an evasion. Existence and liveness are **byte-observed**
|
||||
(`BeginTransport`/`EndTransport` bracket every pour; `AddBytes`/`MarkServed` set
|
||||
`LastServedAt`), never derived from client progress. Falsifying `Position` only
|
||||
corrupts a secondary display field and, at worst, changes which of the user's
|
||||
*own* over-cap sessions `selectVictims` trims first. **Detection ✅ / Enforcement ✅.**
|
||||
*Two round-2 caveats, both bounded:* (a) at the **edge**, `touchTranscodeSession`
|
||||
fires before the node is proxied (`proxy/server.go:308,317`), so hammering dead
|
||||
segment URLs advances `LastServedAt` without serving a byte — liveness there is
|
||||
request-observed, not byte-observed (GAP-15). Because the enforcer groups by user,
|
||||
this only lets an attacker choose *which of its own* streams gets trimmed. (b) the
|
||||
merged snapshot could report `0` bytes for an edge-served stream (GAP-13) — **fixed**;
|
||||
`BytesServed` is now merged as a max in both `mergeStreams` and `DedupeSessionInfos`,
|
||||
so the byte signal is sound as well as the existence signal.
|
||||
|
||||
**A6. Rapid session churn between ticks.**
|
||||
No rate limit exists on `/start` or `/transcode/start` (the `ratelimit`
|
||||
middleware is wired only to auth endpoints). In integrated mode synchronous
|
||||
admission still refuses over-cap starts instantly, so churn can't exceed the cap
|
||||
locally. The gap is the multi-node/edge picture, where admission may have run on a
|
||||
different node and the enforcer only reconciles every ~120s.
|
||||
**Detection ⚠️ / Enforcement ⚠️ (integrated fast, edge lags).**
|
||||
|
||||
**A7. Split across multiple integrated replicas.**
|
||||
`activeCountLocked` counts only the local `SessionManager` map — there is no
|
||||
distributed admission counter. Two integrated replicas each admit up to the cap
|
||||
independently. The enforcer is the only cross-replica backstop, but integrated
|
||||
replicas write no `silo:sessions:*` records, so a remote enforcer cannot see them.
|
||||
**Detection ⚠️ / Enforcement ⚠️.** (VERIFY-3.)
|
||||
|
||||
**A8. Standard user exceeds the cap on the edge path.**
|
||||
The enforcer resolves `SessionManager.EffectiveLimits`, so a raw
|
||||
`users.max_streams = 0` inherits the group cap rather than meaning unlimited.
|
||||
The edge monitoring picture is grouped by resolved owner and excess victims are
|
||||
revoked on the async pass. **✅ / ✅ (~120s).**
|
||||
|
||||
### Category B — Admin control / kill switch
|
||||
|
||||
**B9. Admin terminates a specific session.**
|
||||
`terminate` writes the revocation **first**, keyed on the session id alone, then
|
||||
issues the cooperative realtime command; it answers `202 {status:"revoked"}` even
|
||||
when the session is absent from the central in-memory manager (GAP-7) — exactly the
|
||||
edge-served / post-restart / progress-withholding streams the kill switch exists
|
||||
for. **Detection ✅ / Enforcement ✅.**
|
||||
|
||||
**B10. Admin bans a user.**
|
||||
`RevokeUser` writes a `KindUser` entry; `IsRevoked` matches it for any of the
|
||||
user's sessions whose credential predates the revocation (cutoff semantics, GAP-5),
|
||||
so in-flight pours are cut (`WatchAndCut`) and reconnects `Refuse`d, while a
|
||||
later legitimate re-login is not permanently banned. **✅ / ✅.**
|
||||
|
||||
**B11. Killed stream reconnects with the same token.**
|
||||
The revocation entry outlives the token TTL, so every reconnect with the same
|
||||
credential is refused. "Stays dead." **✅ / ✅.**
|
||||
|
||||
**B12. Killed stream survives a restart.**
|
||||
The durable Postgres mirror (`streamrevoke/durable_postgres.go`, table
|
||||
`stream_revocations`) is warmed on boot and re-armed on the poll tick. Revocation
|
||||
expiry and cutoff merge independently, and durable unrevocation tombstones prevent
|
||||
restart or stale-replica resurrection. This closes GAP-4, where PR #174's
|
||||
restart-resilient playback would
|
||||
otherwise reconstruct and re-serve a killed stream. **✅ / ✅.** *Transient boot
|
||||
caveat:* if the durable warm exhausts its bounded retry **and** Redis is empty, the
|
||||
kill list is empty until the first poll tick (≤60s) — fails **open** by design.
|
||||
|
||||
**B13. Kill issued during an edge Redis outage.**
|
||||
Edge nodes have no durable store and learn *new* kills only from Redis pub/sub +
|
||||
SCAN. While Redis is unreachable, already-cached kills persist but a kill issued
|
||||
*during* the outage does not reach the edge until Redis recovers. **Detection ⚠️ /
|
||||
Enforcement ⚠️ (fails open at the edge).**
|
||||
|
||||
### Category C — Re-streaming / redistribution
|
||||
|
||||
**C14. Re-broadcast one Silo stream to many external viewers. — NO DEFENSE.**
|
||||
The classic Stremio/Plex-share abuse: one account pulls a single stream and a proxy
|
||||
fans it out to hundreds. It stays at concurrency 1, so admission and the enforcer
|
||||
never fire, and the re-streaming heuristic that would catch it *does not exist*
|
||||
(correction #2). **Correction #9 (2026-07-31): the fingerprints are NOT ready to
|
||||
consume, contrary to what this document and the architecture plan previously said.**
|
||||
`ClientIP` is a *single* value per session, not a set: integrated mode stamps it once at
|
||||
session creation and no media request ever updates it
|
||||
(`internal/api/handlers/stream.go` never touches `ClientIP`), while the edge tracker
|
||||
overwrites `records[sessionID]` on every `Track`/`EnsureEphemeral`
|
||||
(`internal/nodesessions/tracker.go`). `mergeStreams` then collapses multi-node records
|
||||
to one winner and only backfills an IP when the winner has none. So five devices pulling
|
||||
one session still present as one address, and a polling consumer cannot see fan-out at
|
||||
all. Building the heuristic requires collecting bounded viewer observations **at
|
||||
authenticated media-serve time**, not consuming the existing snapshot. Note also that C14
|
||||
proper — a downstream proxy re-broadcasting one pulled stream — is invisible to any
|
||||
IP-based signal by construction: Silo sees exactly one address. (The edge's own address
|
||||
resolution was separately blind behind an ingress and is now fixed — `edgeClientIP`
|
||||
resolves through the `clientip.trusted_proxies` boundary — but that only makes viewers
|
||||
distinguishable; it does not create the per-session address *set* the heuristic needs.)
|
||||
**Detection ❌ / Enforcement ❌.**
|
||||
|
||||
**C15. Many concurrent streams feeding a restream service (over cap).**
|
||||
This *is* caught — but only as a raw over-count, not labeled as re-streaming.
|
||||
`ByUser` counts sessions per uid; the enforcer revokes victims beyond the limit
|
||||
within ~120s (and admission refuses new over-cap starts immediately). Relies on
|
||||
correct owner attribution (jellycompat/edge records must carry the owner or they
|
||||
bucket under user 0 and are skipped — mitigated by `mergeStreams` owner-adoption).
|
||||
**Detection ⚠️ (as over-cap, not as restream) / Enforcement ✅ ~120s.**
|
||||
|
||||
**C16. Mint & hoard 24h tokens, fan out later.**
|
||||
Stream tokens are signature-only 24h JWTs (`streamtoken/token.go`) with no `jti`,
|
||||
no one-time-use, no per-user mint counter. A normal Stop does **not** write a
|
||||
revocation, so a token for a stopped session stays cryptographically valid for its
|
||||
full 24h. Hoarding is free and invisible at mint time. Fan-out is caught *only* if
|
||||
the aggregate live use trips the per-user over-cap enforcer — i.e. a low-concurrency
|
||||
fan-out (few tokens, each widely re-streamed) collapses back into C14 and evades
|
||||
everything. **Detection ❌ (at mint) / Enforcement ❌ (unless it becomes over-cap).**
|
||||
|
||||
### Category D — Ripping / bulk data exfiltration
|
||||
|
||||
**D17. Rip via native download endpoints.**
|
||||
Download record creation is bounded by `download.max_concurrent_per_user`
|
||||
(default **3**, `internal/downloads/limiter.go`). Serving and re-fetching an
|
||||
existing row is not gated. `download.max_per_period` and both bandwidth
|
||||
knobs default to **0 = unlimited** — and the bandwidth managers are *rate* throttles
|
||||
(token buckets), never *volume* caps. The concurrency check runs at record
|
||||
creation, so a `create → complete → create` loop pulls the entire library 3 files
|
||||
at a time forever. Active pours and server-observed bytes are visible through the
|
||||
process-local admin `transfers` array, but disappear at completion and are not a
|
||||
cumulative signal. **No total-volume cap exists anywhere.** **Detection ⚠️ /
|
||||
Enforcement ⚠️ (creation-time gate only).**
|
||||
|
||||
**D18. Rip via compat `/Items/{id}/Download`. — WIDEST HOLE.**
|
||||
`HandleDownload` (`jellycompat/streams.go`) requires only a compat session and the
|
||||
per-item library-access filter, then serves the raw file with full Range support.
|
||||
**No download row, no quota, no bandwidth throttle, and no byte cap.** An
|
||||
Infuse-style client can GET/Range every original file back-to-back. The active
|
||||
pour and served bytes are visible in the process-local admin transfer registry,
|
||||
but there is no durable history or automated consumer. **Detection ⚠️ /
|
||||
Enforcement ❌.**
|
||||
|
||||
**D18d. Rip via the ebook/comic/PDF reader route. — SECOND-WIDEST HOLE.**
|
||||
`GET /api/v1/ebooks/{content_id}/files/{file_id}/read` (`api/router.go:2420`) is
|
||||
guarded only by `apimw.RequireProfile`. `serveEbookInline` (`ebook_reader.go:626`)
|
||||
opens the original file and hands it to `http.ServeContent` wrapped in nothing but a
|
||||
`RollingDeadlineWriter`; `serveConvertedEpub` (`ebook_convert_serve.go:160`) does the
|
||||
same for the converted EPUB. There is **no metered writer, no transfer registry
|
||||
entry, no `Refuse`, and no `WatchAndCut`** — none of the `guardRevocation*` wrappers
|
||||
used by `/stream/{session_id}` (`api/router.go:2541`) are applied. Full Range support
|
||||
is inherited from `ServeContent`. The accepted formats include `cbz`, `cbr` and `pdf`
|
||||
(`ebook_reader.go:675`), so this is a large-file path, not a text path. A user
|
||||
revoked for ripping previously kept pulling the entire comic/ebook library at full
|
||||
speed while the admin transfer list showed nothing and `BytesServed` never moved.
|
||||
|
||||
**FIXED (GAP-11):** the route now registers a transfer record, meters its bytes,
|
||||
refuses a revoked reader on entry and cuts an in-flight pour within ~5s. It stays
|
||||
**cap-exempt** per decision A4, so reading never consumes a video stream slot, and
|
||||
there is still no *volume* quota — a patient single-threaded rip remains possible.
|
||||
**Detection ✅ / Enforcement ⚠️.**
|
||||
|
||||
**D19. Rip via sequential direct-play GETs.**
|
||||
Playing items one at a time keeps `activeCountLocked == 1`, always under the cap.
|
||||
Because the cap counts concurrency and never cumulative bytes, sequential
|
||||
direct-play of every raw file is a fully-permitted rip. Each play *is* an
|
||||
attributable session, but nothing correlates sequential sessions into "this user is
|
||||
ripping." **Detection ⚠️ / Enforcement ❌.**
|
||||
|
||||
**D20. Admin stops an in-progress rip mid-transfer.**
|
||||
`streamrevoke.Store.WatchAndCut` (checks on entry +
|
||||
every 5s, forces the socket via `SetWriteDeadline`) and arms it on native downloads
|
||||
(`api/handlers/downloads.go`) and compat download/direct-play
|
||||
(`jellycompat/streams.go`). So a **user revocation now cuts an in-flight download**
|
||||
— best-effort (no-op if the writer chain lacks write-deadline support, then stops on
|
||||
next request). The admin still has to *decide* to revoke; nothing auto-detects the
|
||||
rip. **Enforcement ⚠️.**
|
||||
|
||||
### Category E — API / DB load abuse (branch is orthogonal)
|
||||
|
||||
**E21. Infuse pulls the whole library list on every app start.**
|
||||
Per-page size is hard-capped (`compatBrowseMaxLimit = 1000`,
|
||||
`jellycompat/content_direct.go`; native 100), so no single call dumps 100k. But the
|
||||
general paged browse (`handleBrowseItems` → `BrowsePage`) is **uncached and
|
||||
unthrottled** — every app-start re-runs fresh SQL, and full-page requests add a
|
||||
per-page `COUNT` over the filtered set. The `Latest`/hub rails *are* cached (15-min
|
||||
shared resolved-list cache + singleflight), but that's a rail, not the full list.
|
||||
Nothing attributes listing load per client/device. **Detection ❌ / Enforcement ⚠️
|
||||
(first page cheap & capped; full enumeration pays full price each start).**
|
||||
|
||||
**E22. Thundering herd on cold cache.**
|
||||
For `Latest`/hub rails, `singleflight` collapses concurrent identical builds into
|
||||
one DB hit. For the **general browse there is no singleflight and no cache** — N
|
||||
concurrent full-library enumerations run N independent SQL workloads, bounded only
|
||||
by the shared pool (`userdb.pool_max_open`, default 500), beyond which requests
|
||||
queue/time out rather than shed gracefully. **Detection ❌ / Enforcement ⚠️ (only
|
||||
the cached rails are protected).**
|
||||
|
||||
**E23. Page a 100k+ library as fast as possible.**
|
||||
Each page is capped at 1000 rows, but nothing caps the number of pages, the page
|
||||
rate, or aggregate cost; deep pages incur a per-page filtered `COUNT` and rising
|
||||
`OFFSET` cost. No rate limit on `/Items` (compat router mounts no throttle).
|
||||
**Detection ❌ / Enforcement ❌ (beyond the per-page size cap).**
|
||||
|
||||
**E24. Expensive sorted/filtered views over the whole library.**
|
||||
Some sorts are index-optimized (`recently_added` walks a dedicated index; the
|
||||
multi-library case fans into per-library index walks + in-memory merge). But
|
||||
arbitrary client sorts map through `mapSortBy` — `SortBy=DateCreated` "forces a
|
||||
full-library top-N heapsort" (code comment), and genre/name-prefix/person filters
|
||||
add WHERE/JOIN predicates with no cost cap beyond the LIMIT, plus a full-filtered
|
||||
`COUNT` per page when `include_total` is on (default). No query-cost estimator, no
|
||||
cache for filtered browses, no rate limit. **Detection ❌ / Enforcement ⚠️
|
||||
(query-shape-dependent).**
|
||||
|
||||
**E25. Badly-coded client retry storm.**
|
||||
`main` *does* ship `internal/ratelimit/` (token-bucket, default-on): global 1000
|
||||
rps + per-IP 120 rps, mounted inside the **authenticated native group**
|
||||
(`api/router.go`, after `RequireAuth`). So a storm against an authed native
|
||||
endpoint is shed per-IP. But the limiter runs *after* auth, the entire
|
||||
**jellycompat surface is unthrottled**, `/auth/refresh` has no limiter, the Redis
|
||||
backend **fails open** on error, and the default memory backend is per-process (so
|
||||
"global 1000 rps" is really per-instance). Attribution + telemetry exist
|
||||
(Prometheus by path/status, access log with `client_ip`). **Detection ✅ /
|
||||
Enforcement ⚠️ (native authed only).**
|
||||
|
||||
**E26. Tight-loop polling `/auth/refresh`.**
|
||||
Registered as a plain public route (`api/router.go`) with **no** `AuthEndpointHandler`
|
||||
and **outside** the authenticated group — it hits neither the global, per-IP, nor
|
||||
per-endpoint limiter. A tight refresh loop is unthrottled. **Detection ✅ (logged) /
|
||||
Enforcement ❌.**
|
||||
|
||||
**E27. Credential stuffing via compat `/Users/AuthenticateByName`.**
|
||||
Native `/api/v1/auth/login` is capped at 20/min/IP via `AuthEndpointHandler`. The
|
||||
Jellyfin-compat login has **zero throttling** — the compat router mounts no limiter
|
||||
— and there is no account-level lockout/backoff anywhere. **Detection ✅ /
|
||||
Enforcement ❌ (compat path wide open).**
|
||||
|
||||
**E28. Many concurrent HTTP connections / slowloris.**
|
||||
No connection cap: no `netutil.LimitListener`, no per-IP connection limit, no
|
||||
handler semaphore. The `http.Server` sets only Read/Write/Idle timeouts (and the
|
||||
compat + ABS servers even set `WriteTimeout: 0`). The rate limiter counts
|
||||
*requests*, not *connections*. **Detection ❌ / Enforcement ❌.**
|
||||
|
||||
**E29. Spawn many concurrent transcodes to exhaust CPU.**
|
||||
Only the per-user `max_transcodes` (default 2, `<=0` = unlimited) is checked, at
|
||||
admission. There is **no per-node or server-wide ffmpeg/CPU cap** on fresh starts
|
||||
(the `reconstructSem` NumCPU semaphore guards only the restart-reconstruct path).
|
||||
Node scheduling has an optional `MaxJobs`/`MaxBandwidthKbps` per node
|
||||
(`nodepool`), but both default to unlimited. N users at a high per-user cap — or one
|
||||
user with `max_transcodes <= 0` — can saturate the box. **Detection ⚠️ (sessions
|
||||
visible, no CPU/process signal) / Enforcement ❌ (no aggregate cap).**
|
||||
|
||||
---
|
||||
|
||||
## What the branch genuinely delivers vs. what it does not
|
||||
|
||||
**Delivers (and it's solid):**
|
||||
- Authoritative, **byte-observed** stream existence on the *playback* surfaces that a
|
||||
client cannot hide by lying about or withholding progress (A5) — with the two
|
||||
bounded caveat GAP-15 noted there, and now **including** ebooks (GAP-11 fixed) and
|
||||
subtitle pours. GAP-13 is fixed, so merged byte totals are trustworthy too, and
|
||||
GAP-14 is fixed: a download-class pour that cannot be monitored is now **refused**
|
||||
rather than served blind.
|
||||
- A durable, restart-surviving, reconnect-proof **kill switch** for a *specific
|
||||
session* or a *user*, via one shared `Refuse` + `WatchAndCut` (B9–B12, D20).
|
||||
`Refuse` — the next-request half — holds everywhere it is mounted. The
|
||||
`WatchAndCut` half was weaker than previously documented — it no-op'd on all ABS
|
||||
routes (GAP-10), was racy on rolling-deadline routes (GAP-12) and was absent from
|
||||
ebooks (GAP-11) — and **all three are now fixed**, so it lands on every mounted
|
||||
surface. Two honest bounds remain: the cut takes up to one **~5s** watch tick, and it
|
||||
does **not** "keep a stream dead" — an over-cap kill still reopens after 5m until
|
||||
decision A1 lands.
|
||||
- Immediate **synchronous admission** refusal of over-cap starts, and an async
|
||||
over-cap reconciler for the multi-node picture (A1, A2, C15) — *when a per-user
|
||||
cap is set*. Since A6, integrated sessions are published to the shared
|
||||
`silo:sessions:` picture and a single elected evaluator trims cross-replica excess.
|
||||
|
||||
**Does not cover (by scope or by gap):**
|
||||
- **Distributed synchronous admission.** A6 gave every replica the same *monitoring*
|
||||
picture, but `StartSession`'s cap check still reads only the local `SessionManager`.
|
||||
Streams spread across replicas are all admitted and then trimmed asynchronously
|
||||
within the ~120s budget, not refused at the door.
|
||||
- **Under-cap re-streaming** (C14) and **token hoarding/fan-out** (C16) — no
|
||||
detection, no enforcement; the heuristic that was scoped for this is unimplemented.
|
||||
- **Bulk ripping** by download (D17/D18) or sequential direct-play (D19) — the cap
|
||||
is a *concurrency* gate, not a *volume* quota; compat download is entirely
|
||||
unquota'd.
|
||||
- **Library-enumeration DB load** (E21–E24) — orthogonal subsystem; per-page capped
|
||||
but per-client-uncached, unthrottled, and unattributed.
|
||||
- **API floods on the compat surface, `/auth/refresh`, and connection exhaustion**
|
||||
(E25–E28) — rate limiter has real coverage gaps.
|
||||
- **CPU/transcode exhaustion** (E29) — no aggregate cap.
|
||||
|
||||
## Settled design decisions (2026-07-30)
|
||||
|
||||
These were open forks blocking the remaining batches. All are now decided, so follow-up
|
||||
issues inherit them rather than re-litigating.
|
||||
|
||||
| # | Question | Decision |
|
||||
|---|---|---|
|
||||
| **A1** | Over-cap enforcement model | **Long revocation** matching the token's reconstructable lifetime, so an over-cap kill stops reopening every 5m. **Must not ship before the tracker-lifecycle batch** — it removes the self-healing property that currently limits the damage of a wrong count, and every known source of a wrong count is in that batch. |
|
||||
| **A2** | Revocation state model | **Durable tombstones** — an independent `RevokedAt`/`ExpiresAt` merge *plus* a durable tombstone so an un-ban survives a restart and cannot be re-`Upsert`ed by a stale replica. One Goose migration. |
|
||||
| **A3** | Credential identity for the user cutoff | **Presented credential time.** Native access uses token `iat`; normal Jellyfin compatibility sessions use their stable login `CreatedAt`, not the refreshed bridged token. API keys and valid JWTs without `iat` use zero and deliberately fail open, so a user cutoff cannot cut their pours. Per-login logout cuts stay out of scope (they need authorization-generation/per-login identity in the stream credential). |
|
||||
| **A4** | What counts as a "stream" | **Observe + make killable, keep cap-exempt** for the ABS bare file route and ebook/comic/PDF reading. Neither consumes a video stream slot. **Implemented.** |
|
||||
| **A5** | Liveness source of truth | **Separate server-observed liveness entirely.** Client progress never feeds enforcement or reaping; the `LastActivityAt` fallback for `LastServedAt` is removed, and reaping measures from `LastServedAt` (falling back only to `StartedAt`) behind a configurable never-served grace. Paused sessions holding an open realtime/WebSocket connection stay exempt — an observed, ping-checked connection is server-observed, unlike a reported progress position, and this preserves the issue #243 fix. **Implemented.** |
|
||||
| **A6** | Multi-replica visibility | **Publish every integrated stream to Redis** so all replica enforcers share one picture, keyed by a per-process instance id. A renewable Redis lease elects one evaluator per tick so replicas cannot trim from divergent snapshots. Fixes **monitoring** only — synchronous admission stays per-process. **Implemented.** |
|
||||
| **A7** | Fail-open vs fail-closed monitoring | **Fail closed + connection cap.** Download-class pours are refused (429/503 + `Retry-After`) rather than served unmonitored, and a per-user concurrent-transfer cap (`playback.max_user_concurrent_transfers`, default 24) makes registry saturation unreachable by one actor. **Implemented.** |
|
||||
| **A8** | Scope | **Split.** The serve-path, capability and docs work lands first; tracker lifecycle → revocation state → liveness/replicas → re-stream heuristic follow in that dependency order. |
|
||||
|
||||
A new finding recorded while implementing A4, for the A3 batch: `streamRequestIdentity`
|
||||
verifies a stream token's signature but never checks that the token's `SessionID`
|
||||
matches the `session_id` in the URL, so a valid token for one session can supply
|
||||
identity on another session's route. It belongs with A3 rather than being fragmented
|
||||
across two batches.
|
||||
|
||||
## Recommended follow-ups (prioritized by exposure)
|
||||
|
||||
**Round-2 defects come first — these are open bugs in shipped behavior, not
|
||||
enhancements.** In rough fix-cost order:
|
||||
|
||||
0a. ~~**Add `Unwrap()` to `abs.statusRecorder`**~~ — **DONE.** Plus `WatchAndCut` now
|
||||
logs (once per watcher) rather than discarding a failed `SetWriteDeadline`, and the
|
||||
replacement test drives the *mounted* router over a real socket. Verified
|
||||
non-vacuous: it fails if `Unwrap()` is removed again. (GAP-10)
|
||||
0b. ~~**Wire the ebook routes into the stream kill switch and the transfer registry**~~
|
||||
— **DONE**, cap-exempt per decision A4. ⚠️ **The original advice in this line —
|
||||
"reuse `guardRevocationCut`" — was wrong and was not followed.** That wrapper reads
|
||||
`chi.URLParam(r, "session_id")`, which the ebook route does not have (it would pass
|
||||
`""`), and takes identity from `streamRequestIdentity`, which looks for a `?st=`
|
||||
stream token the route never carries. It compiles and silently guards nothing. The
|
||||
ebook fix takes identity from the profile/auth context and otherwise follows the ABS
|
||||
file-handler idiom. (GAP-11)
|
||||
0c. ~~**Make the revocation cut survive the rolling deadline**~~ — **DONE** via a
|
||||
`CutLatch` on the request context. Note the sticky-flag-on-`RollingDeadlineWriter`
|
||||
option suggested here could not be implemented as written: the rolling writer is
|
||||
constructed *inside* `ServeDirectPlay`/`ServeRemux` and wraps the writer the watcher
|
||||
holds, so it sits *above* the watcher and `Unwrap()` (which walks toward the socket)
|
||||
can never reach it. Hence the context side channel. The re-applying watcher was also
|
||||
kept, as belt-and-braces. (GAP-12)
|
||||
0d. ~~**Merge `BytesServed` as a max, not wholesale**~~ — **DONE** in both
|
||||
`mergeStreams` and `DedupeSessionInfos` (GAP-13). **GAP-15 is also DONE:** edge
|
||||
transcode visibility is created before proxying, while liveness and byte totals
|
||||
advance only for bytes actually written from a 200/206 upstream response.
|
||||
0e. **Monitoring projection is not uniformly asynchronous.** The first Redis write
|
||||
for each edge session remains synchronous so the record is visible before the
|
||||
request returns; subsequent liveness and byte projection happens asynchronously
|
||||
on the refresh tick. A slow Redis therefore adds latency to the first request of
|
||||
a stream. The proposed ordered, bounded projection queue is deliberately deferred
|
||||
until its startup, drain, cleanup ordering, backpressure, and refresh interaction
|
||||
can be designed together.
|
||||
0f. ~~**Decide the registry-saturation policy explicitly**~~ (GAP-14) — **DONE (A7).**
|
||||
Download-class pours fail *closed* once the registry is full (429/503 with
|
||||
`Retry-After`), and a per-user concurrent-transfer cap
|
||||
(`playback.max_user_concurrent_transfers`, default 24) makes saturation
|
||||
unreachable by a single actor. Five call sites, not four — `ebook_reader.go` was
|
||||
missing from the original list. Note this does **not** close E28: there is still
|
||||
no server-wide connection cap, only a per-user transfer cap.
|
||||
0g. ~~**Bound the kill-list propagation lock.**~~ **DONE.** Redis and pub/sub run
|
||||
before the durable mirror, and detached propagation/startup warm use bounded
|
||||
contexts. `opMu` deliberately remains held across propagation so a same-process
|
||||
unrevoke cannot interleave with an older mirror write. The cross-replica race on
|
||||
the Redis mirror is **now also closed**: `mirrorToRedis` merges server-side in Lua
|
||||
with the same monotonic semantics as `applyLocal`, comparing exact `(sec, nsec)`
|
||||
pairs and merging *before* deciding to delete, so a lapsed incoming revocation
|
||||
cannot remove a live stronger one. Falls back to the previous plain `SET` (logged
|
||||
once) if the script cannot run.
|
||||
|
||||
Then, as originally scoped:
|
||||
|
||||
1. **Download controls phase 2:** add per-download kill and a standing per-user
|
||||
download block. The existing user revocation is a cutoff, not a ban: it cuts
|
||||
older pours but deliberately allows new ones.
|
||||
2. **Unify download quotas and add a per-user *volume* budget** (D17/D19): cover
|
||||
native serving, compat, and ABS with a rolling bytes-per-period cap that spans
|
||||
downloads and direct-play, since concurrency caps cannot bound a rip.
|
||||
Also deferred: correct native completion semantics after partial/write-failed
|
||||
responses, correlate duplicate ABS playback-session and `abs_file_stream`
|
||||
views, and aggregate transfer visibility across API replicas.
|
||||
3. **Extend rate limiting to the compat surface + `/auth/refresh`** and add a
|
||||
connection cap / `LimitListener` (E25–E28).
|
||||
4. **Add a per-node concurrent-transcode cap** and wire `nodepool.MaxJobs` defaults
|
||||
(E29).
|
||||
5. **Implement the restream heuristic** (C16 fan-out; **not** C14): the fingerprints do
|
||||
**not** already flow in usable form — see correction #9. A distinct-viewer-IP rule
|
||||
needs per-request observation at media-serve time. The edge trust-boundary half is
|
||||
**done**: `internal/proxy/server.go`'s `edgeClientIP` now resolves through the
|
||||
operator's `clientip.trusted_proxies` boundary like the native surface, so viewers
|
||||
behind an ingress are distinguishable and a forged `X-Forwarded-For` from an untrusted
|
||||
peer is ignored. Tracked as issue #522.
|
||||
Decisions are now settled: distinct viewer IPs per session as the signal, Redis
|
||||
with a ~24h TTL when configured and in-memory otherwise, detection **on by
|
||||
default and alert-only**, with auto-kill behind an operator setting that defaults
|
||||
to off. Note the privacy posture explicitly — this retains per-viewer IP history
|
||||
on every deployment.
|
||||
6. **Per-client listing-load accounting** (E21–E24): attribute browse cost per
|
||||
device and add caching/singleflight to the general paged browse, not just the
|
||||
Latest/hub rails.
|
||||
|
||||
## Accepted limitations from the liveness/replica batch (2026-07-31)
|
||||
|
||||
Deliberate, recorded decisions. Changing any of them is a new decision, not a bugfix.
|
||||
|
||||
- **Paused sessions age from the last served byte, not the last client ping.** A paused
|
||||
session holding an open realtime/WebSocket connection stays exempt from reaping; one
|
||||
whose client only POSTs progress is reaped after the 30m paused grace and resumes via
|
||||
the restart-resilient reconstruct path. This is the A5 boundary: an observed,
|
||||
ping-checked connection counts as liveness, a reported position does not.
|
||||
- **A never-served session is retained for a bounded grace** (`DefaultUnservedSessionGrace`,
|
||||
2m) so a slow first byte is not reaped instantly. It is a separate configurable knob
|
||||
rather than a floor on `SetLivenessGracePeriods`, which would otherwise silently
|
||||
override an operator's shorter window.
|
||||
- **`last_served_at` is omitted for integrated sessions that have never served.** The
|
||||
admin session list shares the enforcer's projection, so it no longer reports a client
|
||||
progress timestamp as a serve time. The field is unchanged when real bytes exist.
|
||||
- **Synchronous admission remains per-process.** See "does not cover" above.
|
||||
- **The integrated session publisher waits one 10s tick before its first publish**, so a
|
||||
freshly-started replica's sessions are briefly absent from the shared picture. Bounded
|
||||
and self-correcting; that replica's own enforcer still sees them through the local
|
||||
source.
|
||||
- **The enforcer lease is advisory.** A Redis error means every replica evaluates, which
|
||||
is the pre-A6 behavior — failing to coordinate must never mean failing to enforce.
|
||||
- **The Redis revocation merge degrades to a plain `SET`** if the Lua script cannot run
|
||||
(logged once). Behavior then matches pre-A6.
|
||||
- **Transfer-saturation rejections are logged at Warn inside the registry**, rate-limited
|
||||
per user and carrying route and user id — not once per rejected request at each call
|
||||
site, which an actor parked at its cap could turn into a log-amplification vector.
|
||||
- **`playback.max_user_concurrent_transfers` set too low will 429 legitimate
|
||||
multi-connection downloaders**, which routinely open 4-8 sockets per file and consume
|
||||
one transfer id per concurrent Range request.
|
||||
- **The Lua-executing tests skip without `SILO_TEST_REDIS_ADDR`**, and this repository
|
||||
runs no `go test` job in CI, so that evidence is local-only until a Redis service is
|
||||
added to CI.
|
||||
|
||||
## AI-use disclosure
|
||||
|
||||
This assessment was produced with AI assistance, from a read-only audit of the
|
||||
`feat/sauron-async-enforcer` branch and current `main`. No code behavior was changed.
|
||||
Findings cite the code as of the audit; the corrections above are where the branch's
|
||||
own plan/coverage docs overstate what the shipped code does.
|
||||
|
||||
Round 1 (corrections 1–3) was produced with Claude. Round 2 (corrections 4–8, dated
|
||||
2026-07-29) was a **two-model** pass: Claude Opus 5 and Codex `gpt-5.6-sol` reviewed
|
||||
the rebased branch independently, and every finding recorded here was re-verified
|
||||
against the code by hand before being written down. Corrections 4, 6, 7 and the
|
||||
`opMu` follow-up (0f) originated with Codex; corrections 5 and 8 were found by both
|
||||
models independently. Two Codex claims were **rejected** on verification and are
|
||||
deliberately not recorded above: that chi's `middleware.Compress` wrapper breaks the
|
||||
`Unwrap` chain (chi v5.2.5 implements `Unwrap()` at `middleware/compress.go:374`),
|
||||
and that client progress reports advance the central `Session.LastServedAt` (only
|
||||
`BeginTransport`/`EndTransport`/`AddServedBytes` write it — the real mechanism is the
|
||||
edge `Touch`, recorded as GAP-15).
|
||||
|
||||
Round 3 (2026-07-30) implemented corrections 4–7 as a **cross-model relay**: Claude
|
||||
Opus 5 (`claude-opus-5[1m]`) planned, reconciled and reviewed; Codex `gpt-5.6-sol` at
|
||||
medium effort adversarially reviewed the plan, implemented it, and took one remediation
|
||||
round. Involvement classification: **AI-assisted implementation with human-directed
|
||||
scope and AI cross-review**; every design fork (A1–A8) was decided by the maintainer,
|
||||
not by either model.
|
||||
|
||||
Adversarial findings and their resolution, recorded because the disclosure standard
|
||||
requires it:
|
||||
|
||||
- Codex's plan review found the decisive issue: fixing GAP-10 alone is **inert**,
|
||||
because the same `Unwrap()` that makes `SetWriteDeadline` work also makes
|
||||
`RollingDeadlineWriter.bump()` work, and `bump()` erases the cut. GAP-12 was pulled
|
||||
forward from a later batch as a result. **Accepted.**
|
||||
- Codex correctly refuted two claims in the Claude plan: that native subtitle URLs carry
|
||||
a `?st=` stream token (they do not — `subtitleStreamURL` emits only `file_id`), and
|
||||
that the `transfers` capability belonged on `/admin/sessions/capabilities` (it belongs
|
||||
to `/admin/node-sessions`, which is separately gated). It also correctly refuted a
|
||||
claimed "latent nil bug" in the ABS transfer defer — `Registry.End` guards a nil
|
||||
receiver. **All accepted; the plan was corrected before implementation.**
|
||||
- Claude's review of the Codex implementation found five defects, all confirmed by
|
||||
reading or by running the tests: a **failing real-socket ebook test** (Codex's sandbox
|
||||
could not run `httptest` listeners, so every real-socket test it wrote was
|
||||
unexecuted); **GAP-12 left unfixed on the proxy edge remux path**; Warn-log spam every
|
||||
5s from the now-perpetual watcher; `HandleVideoStream` still using three separate
|
||||
`time.Now()` values — the exact race Codex itself had raised and fixed only in the
|
||||
subtitle path; and subtitle metering counting error-response bodies. **All five fixed
|
||||
in one remediation round and re-verified here.**
|
||||
- Claude's Batch-5 capability code was then reviewed by Codex, which found that the
|
||||
vocabulary drift test was not genuinely bidirectional — it sampled rejected strings,
|
||||
so a kind newly *accepted* by the parser could go unadvertised. **Accepted:** the
|
||||
parser and the advertisement now read one shared map.
|
||||
|
||||
Verification note: `pnpm` is absent on the host used for this round, so `make build`,
|
||||
`pnpm run lint` and `pnpm run format:check` could not be run locally and must come from
|
||||
CI. No frontend files were changed. The `internal/access` and `internal/jellycompat`
|
||||
test packages **do not compile on `main`** (stale `UserStore` doubles missing
|
||||
`GetOnboardingState`), so the compat changes in this round are **verified by reading
|
||||
only** — CI cannot exercise them either until that is fixed separately.
|
||||
|
||||
Round 5 (2026-07-31) implemented the liveness/replica batch (A5, A6, A7) as the same
|
||||
cross-model relay: Claude Opus 5 planned and reviewed, Codex `gpt-5.6-sol` at medium
|
||||
effort adversarially reviewed the plan and implemented it in three commit-sized passes,
|
||||
and Claude re-ran every verification independently.
|
||||
|
||||
- Codex's plan review returned 14 findings. The load-bearing ones, all **accepted**:
|
||||
the proposed Lua merge's `ttl <= 0 ⇒ DEL` could delete a *stronger* concurrent
|
||||
revocation, so the script must merge before deciding to delete; the repository's Redis
|
||||
test doubles are hand-written go-redis `ProcessHook` fakes that **cannot execute Lua**,
|
||||
so the originally-proposed merge tests would have proven nothing; a hardcoded
|
||||
never-served grace would silently override `SetLivenessGracePeriods`' documented
|
||||
contract and break four existing tests; and the per-user cap option could not express
|
||||
both "unset ⇒ default" and "explicit 0 ⇒ unlimited" as a plain `int`.
|
||||
- Two review claims were **rejected** on verification. That the new setting required an
|
||||
Admin UI field (and therefore frontend changes): `playback.over_cap_revocation_ttl`,
|
||||
added by Batch 5 on this same branch, has identical backend plumbing and is
|
||||
deliberately absent from `PlaybackSettings.tsx`'s `KEYS` list, so backend-only matches
|
||||
the branch's own precedent. And that the setting should be parsed near the enforcer
|
||||
wiring: the registry is constructed far earlier, so it is parsed at the construction
|
||||
site from the settings map already loaded at startup.
|
||||
- Claude's own planning found a conflict the decision record did not anticipate: A5 says
|
||||
client progress must never feed reaping, but `session_paused_grace_test.go` encodes
|
||||
issue #243, where reaping a paused transcode froze clients. Resolved by keying the
|
||||
paused exemption on an open, **ping-checked** realtime/WebSocket connection — a
|
||||
server-observed connection rather than a client-reported position — which satisfies
|
||||
both.
|
||||
- Codex's sandbox **blocks loopback TCP**, so the six real-Redis tests it wrote never
|
||||
executed there; it reported this explicitly rather than claiming they passed. Claude
|
||||
ran them against a real Redis 8.6.2 (`EVAL` + `cjson` confirmed) and then
|
||||
mutation-checked them by bypassing the Lua merge back to the old unconditional `SET`:
|
||||
four of five failed, confirming they genuinely exercise the merge.
|
||||
- Claude's review found one defect Codex missed: `TestHandleListSessions...` asserted
|
||||
`last_served_at == LastActivityAt`, encoding the very fallback A5 removes. It surfaced
|
||||
only when running `./internal/api/...`, which the first verification pass skipped.
|
||||
Fixed, and the test now also asserts the field reappears once bytes are served.
|
||||
|
||||
Verification note for this round: the Lua-executing tests are gated on
|
||||
`SILO_TEST_REDIS_ADDR` and were run locally against Redis 8.6.2. This repository has
|
||||
**no CI job that runs `go test`** at all, so all Go test evidence here is local. `pnpm`
|
||||
remains absent on this host; no frontend files were changed in this round.
|
||||
@@ -0,0 +1,248 @@
|
||||
# Stream Monitoring & Kill-Switch Implementation Plan
|
||||
|
||||
> **Status: IMPLEMENTED (with a minor operator-config follow-up open)** on branch `feat/sauron-async-enforcer`. All three phases shipped — Phase 1 (authoritative server-side monitoring), Phase 2 (revocation kill switch + edge/native/jellycompat enforcement), and Phase 3 (async over-cap enforcer). The one still-open item is operator ergonomics, not a phase: the `auth.stream_revocation_poll` / `auth.stream_revocation_ttl` keys are not yet wired through the settings pipeline (defaults 60s / 24h are hardcoded) — see "As-built deltas" and the coverage matrix's follow-up list. A durable Postgres mirror of the kill list was added **after** this plan was written, so kills now survive a server restart or a Redis flush (this plan's "optional Postgres durability" is no longer optional — it closes the gap where restart-resilient playback could reconstruct and re-serve an already-killed stream). For the **as-built** coverage across server layout × playback type × route, the resolved gap findings (GAP-1..GAP-4), and the open verification items, see the companion reference: [`playback-paths-monitoring-kill-matrix.md`](../../architecture/playback-paths-monitoring-kill-matrix.md). The task checkboxes below are retained as the original plan of record.
|
||||
|
||||
> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. **Do not edit this plan file while implementing it.**
|
||||
|
||||
**Goal:** Gain control over stream abuse (concurrency-cap violations, admin termination, Stremio-style re-streaming) **without** putting the central database on the per-segment hot path and **without** changing the client-facing token protocol (the later-added `Origin` route claim is server-signed and opaque to clients — they only echo the token, so the wire protocol clients see is unchanged). Do it with two primitives plus an off-hot-path decision loop: (1) authoritative server-side monitoring that never trusts the client, and (2) a kill switch that stops any stream within ~120s and keeps it dead.
|
||||
|
||||
**Architecture:** Split the problem into *detection* and *enforcement*, and keep both off the hot path.
|
||||
|
||||
- **Detection = authoritative monitoring (no client trust).** Bytes can only reach a viewer by flowing through the edge (segment serving) or a held-open direct-play socket that the edge itself is writing. So the server *inherently* knows every live stream — it is the one sending the bytes. We make the existing `nodesessions` records the source of truth, refreshed by *real serve activity* (segment served / active transport / open connection), enriched with `last_served_at` and bytes, and mirror the same signal into integrated mode. No client progress report is trusted for liveness. A stream that "plays quietly" is still pulling segments (or holding a socket) and is therefore visible; one that stops pulling ages out on its own.
|
||||
- **Enforcement = a revocation kill switch.** A small revocation set lives in Redis (multi-node) or in-memory (integrated), mirrored to Postgres for durability (shipped as a wired part of the kill path, not an optional add-on — it is what lets a kill survive a restart/Redis flush). Edges cache it in memory, refreshed by pub/sub push (near-instant) plus a ≤60s poll (safety net). The **only** hot-path addition is a local in-memory set-membership check per segment — no per-segment Redis or Postgres. On a hit the edge refuses the request and cuts the connection; for direct-play the edge periodically re-checks during the long GET and aborts. A kill "stays dead" because the revocation entry outlives the token and every reconnect is refused.
|
||||
- **Decision = an async brain, fully off the hot path.** A central background evaluator reads the live monitoring picture and the per-user limits (`users.max_streams`), and decides what to kill: over-cap victims, admin-terminated sessions, and (pluggable) abuse heuristics. Every reason — exceeded limit, admin terminate, Stremio abuse — collapses to the *same* action: write a revocation. Enforcement follows within ~120s. The delay is acceptable for a media server.
|
||||
|
||||
The client-facing token **wire protocol is unchanged**. Its `sid`/`uid`/`pid` claims are the revocation keys; the server may add opaque server-signed claims (e.g. the later-added `Origin` route claim) that clients only echo. Nothing about mint sites, segment URLs, HLS manifests, or client behavior changes. This is why the approach is low-risk relative to a short-ticket redesign (see "Background — why not short tickets").
|
||||
|
||||
**Tech Stack:** Go, chi routers (proxy/transcode edge + native api), `redis/go-redis/v9`, Postgres via pgx + Goose migrations, the existing `nodeconfig.Watcher` settings pipeline, the `cache` package Redis client + `EventBus` pub/sub.
|
||||
|
||||
---
|
||||
|
||||
## Commands
|
||||
|
||||
Commands assume the repository root is the cwd.
|
||||
|
||||
- Build: `make build`
|
||||
- Backend only: `go build ./... && go vet ./...`
|
||||
- Lint: `make lint`
|
||||
- New migration: `make migrate-create NAME=revoked_stream_sessions`
|
||||
- Local services: `docker compose up -d postgres redis`
|
||||
|
||||
---
|
||||
|
||||
## Background — what we learned (why this shape)
|
||||
|
||||
Audited integration points this plan builds on (do not re-derive):
|
||||
|
||||
- **Stream token** is a stateless signed JWT, 24h TTL, verified at the edge by signature only, no store, no revocation today (`internal/streamtoken/token.go`; mints at `internal/api/handlers/playback.go:517,1601,2159` and `internal/jellycompat/handlers_playback.go:317`). Claims carry `sid` (`SessionID`), `uid`, `pid`, `mfid` — our revocation keys.
|
||||
- **Edge already sees every segment and already writes Redis session records.** Proxy `touchTranscodeSession` → `nodesessions.Tracker.Touch` on every manifest/segment (`internal/proxy/server.go:192`; `internal/nodesessions/tracker.go:137`), 60s TTL, refreshed every 30s. Direct-play/remux are tracked for the request lifetime (`proxy/server.go:143-145,156-158`). Records already carry `AuthUserID`/`ProfileID`/`MediaFileID` (`tracker.go:41-43`) but **no** `last_served_at` or bytes yet.
|
||||
- **Edge egress is already metered** but only as a node aggregate (`internal/proxy/egress.go`), not per session.
|
||||
- **Edge Redis is mandatory** and constructed once (`cmd/silo/main.go:576-580,605`), reused for the recipe store (`main.go:623`) — reuse the same client for a revocation store via a `SetRevocationStore(...)` wired exactly like `SetRecipeStore`.
|
||||
- **Central Redis** is `apiRedisClient` (`main.go:672`, may be nil on Redis-less single-node), injected as `deps.RedisClient` (`internal/api/router.go:135`). Central and edges point at the **same** Redis in multi-node (existing `silo:sessions:*` write/read contract proves it).
|
||||
- **Config auto-propagates to edges** via `nodeconfig.Watcher` reading `server_settings` (`internal/nodeconfig/watcher.go`); follow the `jellyfin_compat.session_ttl` precedent (`internal/config/config.go:217,233`; `internal/config/db_loader.go:381`) for any new key.
|
||||
- **Pub/sub already exists** on `cache.ChannelAdmin` and the watcher subscribes to it (`watcher.go:83-93`) — reuse this channel/pattern to push revocations to edges near-instantly.
|
||||
- **Revocation hook exists but is compat-only.** `OnUserSessionsRevoked` (`router.go:162`, `admin.go:101`, impl `cmd/silo/main.go:2047`) currently only deletes Jellyfin-compat sessions. Admin terminate (`internal/api/handlers/admin_playback_control.go:63` `HandleTerminateSession` → `handleSessionCommand`) currently only sends a realtime command + `stopPlaybackSessionByID` fallback; it does **not** revoke the stream credential at the edge. These are the exact hooks to extend.
|
||||
- **Concurrency counting already exists in-process** (`internal/playback/session.go` `SessionManager`, per-user `activeCountLocked`, `SessionLimitProvider` reading `users.max_streams`) but only for the local API process and only at admission. The async brain reuses the *limit provider* but computes the live count from the monitoring picture instead of the in-process map, so it works across nodes.
|
||||
|
||||
**Why not short tickets (the rejected alternative):** turning the 24h token into a short-lived play ticket would enforce the cap continuously, but (a) Silo's `master.m3u8` is a **VOD** playlist fetched once (`internal/playback/transcode.go:1231` `GenerateFullManifest`), so short tickets embedded in segment URLs would expire mid-movie and 401 the rest of the stream — the "renewal rides on playlist refresh" premise does not hold; (b) flipping the global token TTL is a break-all-playback change that cannot be behaviorally verified without real clients. The monitor-and-kill approach accepts a ~120s enforcement delay in exchange for zero client-protocol change and zero hot-path DB/Redis.
|
||||
|
||||
---
|
||||
|
||||
## File Structure
|
||||
|
||||
### Phase 1 — Authoritative monitoring (no client trust)
|
||||
|
||||
- Modify `internal/nodesessions/tracker.go`
|
||||
- Add `LastServedAt string` and `BytesServed int64` to `SessionInfo`.
|
||||
- Refresh `LastServedAt` (and accumulate `BytesServed`) on real serve activity, not just first touch. Keep the Redis write throttled (only re-`SET` when the record materially changes or on the 30s refresh) so this stays cheap.
|
||||
- Add a `Snapshot(ctx) ([]SessionInfo, error)` helper (edge-local view) if useful for `/status`.
|
||||
- Modify `internal/proxy/server.go`
|
||||
- Feed real bytes-served per session into the tracker (wrap the segment/direct/remux response writers, or reuse the `egressMeter` write hook keyed by `claims.SessionID`).
|
||||
- Ensure direct-play/remux held-open connections keep `LastServedAt` fresh periodically (a ticking update while `io.Copy`/`ServeFile` runs), so a long quiet pour is never invisible.
|
||||
- Modify `internal/proxy/egress.go`
|
||||
- Allow the metered writer to report per-session deltas to a callback (so the tracker can attribute bytes), in addition to the existing node-aggregate rate.
|
||||
- Modify `internal/transcodenode/server.go`
|
||||
- Mirror the same `last_served_at`/bytes tracking on the node's own segment serve path (`handleSegment`).
|
||||
- Modify `internal/api/handlers/stream.go` and `internal/api/handlers/playback.go`
|
||||
- Integrated mode: on the native segment/direct serve paths, record server-side serve activity (last-served/bytes) into the same monitoring surface (in-memory for single-node; the tracker if Redis present). Stop relying on client progress reports for liveness. Reuse `BeginTransport`/`EndTransport` (`internal/playback/session.go:577,593`) plus a serve-activity touch.
|
||||
- Modify `internal/api/handlers/nodes.go`
|
||||
- Extend `HandleListSessions` (`:303`) to surface `last_served_at`/bytes so the admin view and the brain see the authoritative picture.
|
||||
- Create `internal/streammonitor/monitor.go`
|
||||
- A thin central-side aggregator that returns a normalized "live streams" snapshot: read Redis `silo:sessions:*` (multi-node) or the in-process `SessionManager`/tracker (integrated), keyed by `uid`, with `sid`, method, `last_served_at`, bytes, node. This is the single input to the async brain and to admin monitoring.
|
||||
- Create `internal/streammonitor/monitor_test.go`
|
||||
- Table tests over synthetic session records: liveness aging, per-user grouping, multi-node de-dup by `sid`.
|
||||
|
||||
### Phase 2 — Kill switch (revocation) + enforcement
|
||||
|
||||
- Create `internal/streamrevoke/store.go`
|
||||
- Interface `Store` with `Revoke(ctx, key RevKey, reason string, until time.Time) error`, `IsRevoked(key RevKey) bool` (local, hot-path, in-memory), `List(ctx) ([]Revocation, error)`, `StartSync(ctx)`.
|
||||
- `RevKey` addresses a session (`sid`) or a user (`uid`); a user revocation matches all that user's sessions.
|
||||
- In-memory backend (integrated): a `map`/set behind an RWMutex with TTL entries; `Revoke` writes locally.
|
||||
- Redis backend (multi-node): `silo:revoked:sess:{sid}` / `silo:revoked:user:{uid}` keys with TTL = token remaining life (default 24h); `StartSync` warms an in-memory cache from a `SCAN` on start, subscribes to `cache.ChannelAdmin` for push updates, and polls every ≤60s as a safety net. `IsRevoked` reads only the in-memory cache (no per-call Redis).
|
||||
- Postgres durability mirror (see migration), wired central-side, so kills survive a Redis flush/restart; repopulate Redis + cache on startup.
|
||||
- Create `internal/streamrevoke/store_test.go`
|
||||
- Cover: local cache hit/miss, TTL expiry, user-key matches session, pub/sub push applies, poll reconciles, durability repopulation.
|
||||
- Create `migrations/sql/<timestamp>_revoked_stream_sessions.sql` (via `make migrate-create NAME=revoked_stream_sessions`)
|
||||
- Table `revoked_stream_sessions(kind text, key text, reason text, revoked_at timestamptz, expires_at timestamptz, primary key(kind,key))` for durable kills; index on `expires_at` for pruning.
|
||||
- Modify `cmd/silo/main.go`
|
||||
- Construct the revocation store on both sides: edge (reuse `redisClient` from `main.go:576`, wire `srv.SetRevocationStore(...)` next to `SetRecipeStore` at `:623` and the proxy equivalent); central (reuse `apiRedisClient` `:672`, fall back to in-memory when nil). Call `StartSync` for edges.
|
||||
- Extend the `OnUserSessionsRevoked` closure (`:2047`) to also `Revoke(user:{uid})` in the store, so account-level revocation kills streams too.
|
||||
- Modify `internal/proxy/server.go`
|
||||
- In `verifyToken` (`:126`) / each serve handler: after signature verify, `if revStore.IsRevoked(sid or uid) { close + 403 }`. For direct-play/remux, run a periodic re-check during the long serve (a deadline/ticker goroutine bound to the request context) that aborts `io.Copy`/`ServeFile` and closes the connection when revoked.
|
||||
- Modify `internal/transcodenode/server.go`
|
||||
- Same revocation check on `handleSegment`/`handleManifest`; refuse + stop feeding a revoked session (and tear down its ffmpeg via the existing session teardown, but ALSO keep the revocation so `reconstructFromToken` refuses to rebuild it — otherwise the node self-reconstructs the kill away, `server.go:620`).
|
||||
- Modify `internal/api/handlers/admin_playback_control.go`
|
||||
- In `handleSessionCommand` for `terminate` (`:63`): in addition to the realtime command + `stopPlaybackSessionByID`, call `revStore.Revoke(sess:{sid})` so termination sticks at the edge regardless of client cooperation. Add an explicit `stop`-vs-`terminate` semantic: `terminate` revokes; `stop` remains a cooperative request.
|
||||
- Modify `internal/api/router.go`
|
||||
- Inject the revocation store into the handlers that need it (`Dependencies` field, like `RedisClient`).
|
||||
- Modify `internal/config/config.go`, `internal/config/db_loader.go`, `internal/config/yaml_import.go`
|
||||
- Add `auth.stream_revocation_poll` (default `60s`) and `auth.stream_revocation_ttl` (default `24h`) following the `jellyfin_compat.session_ttl` precedent. Decide restart-required vs hot-reload (`internal/config/restart_keys.go`); prefer hot-reload.
|
||||
|
||||
### Phase 3 — Async decision engine (the brain)
|
||||
|
||||
- Create `internal/playback/enforcer.go` (or `internal/worker/stream_enforcer.go`, following the `internal/worker/cleanup.go` ticker precedent)
|
||||
- A background loop (central only) that every N seconds: pulls the live snapshot from `streammonitor`, groups by `uid`, reads each user's limit via the existing `SessionLimitProvider` / `users.max_streams`, and for any user over cap selects victims (default: most-recently-started beyond the cap) and calls `revStore.Revoke(sess:{sid}, "over_cap")`.
|
||||
- Pluggable rule interface `Rule(snapshot) []KillDecision` so admin/abuse rules compose. Ship two rules: `OverCapRule` and a stub `RestreamHeuristicRule` (documented, disabled by default) that flags sustained N-distinct-title concurrency / throughput anomalies per `uid`.
|
||||
- Structured logging + metrics for every kill: who, why, which rule.
|
||||
- Create `internal/playback/enforcer_test.go`
|
||||
- Deterministic tests: over-cap victim selection, no-op when under cap, idempotent re-revocation, limit-provider error → fail-open (do not kill).
|
||||
- Modify `cmd/silo/main.go`
|
||||
- Start the enforcer loop on central (integrated/api), wire it to `streammonitor` + the limit provider + `revStore`.
|
||||
- Modify `internal/api/handlers/admin_playback_control.go` / a new admin endpoint
|
||||
- Optional: expose the current kill decisions / recently revoked sessions for the admin UI ("why was this killed").
|
||||
|
||||
---
|
||||
|
||||
## Implementation Tasks
|
||||
|
||||
### Phase 1 — Authoritative monitoring
|
||||
|
||||
- [ ] Add `LastServedAt` + `BytesServed` to `nodesessions.SessionInfo`; refresh on real serve activity with a throttled Redis write.
|
||||
- [ ] Wrap edge serve writers (proxy segment/direct/remux) to attribute per-session bytes + keep `LastServedAt` fresh during long pours; extend `egressMeter` with a per-session delta callback.
|
||||
- [ ] Mirror serve-activity tracking on the transcode node (`handleSegment`).
|
||||
- [ ] Integrated mode: record server-side serve activity on native serve paths; stop trusting client progress for liveness.
|
||||
- [ ] Add `internal/streammonitor` aggregator returning the normalized per-`uid` live snapshot (Redis and in-process backends) + tests.
|
||||
- [ ] Surface `last_served_at`/bytes in `HandleListSessions` for the admin view.
|
||||
- [ ] `go build ./... && go vet ./...`; manual check that a playing stream appears with a moving `last_served_at` and disappears within the TTL after it stops — with the client sending no progress reports.
|
||||
|
||||
### Phase 2 — Kill switch
|
||||
|
||||
- [ ] Create `internal/streamrevoke` store: interface + in-memory backend + Redis backend (SCAN warm-up, pub/sub push, ≤60s poll, in-memory hot-path cache) + tests.
|
||||
- [ ] Add durable `revoked_stream_sessions` migration and the Postgres mirror + startup repopulation.
|
||||
- [ ] Wire the store into edges (`SetRevocationStore` on proxy + transcode Server, `StartSync`) and central (reuse `apiRedisClient`, in-memory fallback).
|
||||
- [ ] Enforce on the edge hot path: local `IsRevoked` check per segment (refuse + close); periodic re-check + abort for direct-play/remux long pours.
|
||||
- [ ] Ensure the transcode node does NOT self-reconstruct a revoked session (`reconstructFromToken` consults the store).
|
||||
- [ ] Extend `OnUserSessionsRevoked` to revoke `user:{uid}`; make admin `terminate` revoke `sess:{sid}` (keep `stop` cooperative).
|
||||
- [ ] Add config keys (`auth.stream_revocation_poll`, `auth.stream_revocation_ttl`) via the settings pipeline.
|
||||
- [ ] `go build ./... && go vet ./...`; manual check: revoke a live session → it stops within ~one poll interval and a reconnect with the same token is refused (stays dead); revoke a user → all their streams stop.
|
||||
|
||||
### Phase 3 — Async brain
|
||||
|
||||
- [ ] Create the enforcer loop + pluggable `Rule` interface; implement `OverCapRule` + stub `RestreamHeuristicRule`; fail-open on limit-provider errors.
|
||||
- [ ] Start the enforcer on central; wire snapshot + limit provider + revocation store.
|
||||
- [ ] Structured logging/metrics for every kill; optional admin "recent kills" endpoint.
|
||||
- [ ] `enforcer_test.go` deterministic tests.
|
||||
- [ ] `go build ./... && go vet ./...`; manual check: start N+1 concurrent streams for a capped user → within ~120s the over-cap stream(s) are killed and stay dead; under-cap users are untouched.
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
- Unit: `internal/streammonitor` (aggregation/aging), `internal/streamrevoke` (cache/TTL/pub-sub/poll/durability), `internal/playback` enforcer (victim selection, fail-open).
|
||||
- Integration (manual, needs `docker compose up -d postgres redis` + a real client): verify authoritative monitoring with a client that sends no progress; verify kill-within-poll-interval + stays-dead for transcode, direct-play, and remux; verify multi-node (central issues kill, edge enforces) via the shared Redis.
|
||||
- Regression: existing playback is unaffected when nothing is revoked (the only hot-path addition is a local set lookup that returns false).
|
||||
|
||||
## Risks / follow-ups
|
||||
|
||||
- **Enforcement latency (~≤120s)** is intentional and accepted; document it so operators expect a short window before a kill takes effect.
|
||||
- **Direct-play long GET** requires a periodic in-serve revocation check + connection cut; a naive single-GET client with no resume may see an error rather than a clean stop — acceptable for a killed (abusive/over-cap) stream.
|
||||
- **Redis flush/restart** would drop in-flight revocations unless the Postgres mirror is in place; keep the mirror for durability (reliability-first).
|
||||
- **Integrated single-node without Redis** uses the in-memory revocation store and in-process monitoring; the durable mirror still applies. No cross-node concern there.
|
||||
- **Abuse heuristics** (`RestreamHeuristicRule`) ship disabled; tune against real traffic before enabling to avoid false kills.
|
||||
- **Bytes attribution** on the edge is best-effort for monitoring/heuristics; it must never gate the hot path.
|
||||
|
||||
## As-built deltas from this plan
|
||||
|
||||
The shipped implementation follows this plan; a few names/shapes differ (the coverage matrix is authoritative for the current state):
|
||||
|
||||
- The durable mirror landed as table `stream_revocations(kind, id, reason, revoked_at, expires_at)` (PK `(kind,id)`) via `internal/streamrevoke/durable_postgres.go`, and is **wired central-side and non-optional** — warmed on boot (bounded retry) and re-armed into Redis on the poll tick so multi-node edges reconverge after a Redis flush.
|
||||
- Enforcement is centralized in two shared helpers `streamrevoke.Store.Refuse` (403) and `WatchAndCut` (in-flight connection cut) used by every serve surface, rather than per-handler ad-hoc checks.
|
||||
- **Expiry is monotonic on every write.** `applyLocal` and the durable `Upsert` keep whichever copy expires later (`GREATEST`), so the async enforcer's short 5m self-heal TTL can never shorten a longer admin `RevokeSession` on the same session key (which would otherwise reopen the restart-resurrection window the durable mirror closes).
|
||||
- **In-flight cut required a middleware fix.** `WatchAndCut` uses `http.NewResponseController(w).SetWriteDeadline`, which walks `Unwrap()`. The native and jellycompat middleware `ResponseWriter` wrappers lacked `Unwrap()`, so the integrated cut was a guaranteed no-op until `Unwrap()` was added to `statusWriter`, `requestStatusWriter`, `loggingResponseWriter`, `compatImageProxyTagResponseWriter`, and `debugResponseWriter`.
|
||||
- **Ownership carry-forward in the monitor.** `streammonitor.mergeStreams` recovers a resolved `UserID`/`ProfileID`/`MediaFileID` (and backfills route/client) across the same-session dedupe, so the transcode node's ownerless start record can't bucket a stream under user 0 and exempt it from the cap.
|
||||
- **Existence vs timing, made explicit.** Monitoring separates *existence* (server-observed on every path — the transport-count shield integrated, byte-observed at the edge — so a hidden stream that withholds progress is still counted) from *timing* (client progress is trusted, native fully). The transcode serve paths (native `playback.go` and compat `HandleHLSSegment`) gained a per-segment transport marker so a slow single-segment drain can't be reaped mid-serve and the compat HLS path no longer depends on the client's progress POST for liveness. A buffer-ahead idle gap >45s is a remaining reaper follow-up (VERIFY-4).
|
||||
- **First-class monitoring fields.** Added a `Route` (native/jellycompat) dimension — seeded from `Session.Origin` and carried to edges via a token `Origin` claim — plus client IP/name and position, all surfaced through `SessionInfo`/`LiveStream`. The transcode node's start record is now owner+route+client attributed via `TranscodeStartRequest`. The admin session list unions the in-process integrated sessions (`LiveLocalSessions`) so single-node deployments aren't Redis-blind. Downloads remain an explicit cap exemption (separate quota).
|
||||
- `auth.stream_revocation_poll` / `auth.stream_revocation_ttl` operator config keys are not yet wired (defaults 60s / 24h are hardcoded) — remaining follow-up.
|
||||
|
||||
### Later hardening pass (post-plan)
|
||||
|
||||
A follow-up audit against the two goals — *no invisible streams* and *every stream
|
||||
killable* — found four more gaps this plan did not anticipate. All are shipped; the
|
||||
coverage matrix remains authoritative.
|
||||
|
||||
- **Integrated byte accounting.** `BytesServed` was edge-only, so a single-node install
|
||||
reported `bytes_served: 0` for every stream, and integrated `LastServedAt` was mapped
|
||||
from `LastActivityAt` — which a client progress report advances, letting a client
|
||||
influence which of its own streams the enforcer trims. `Session` now carries
|
||||
`BytesServed` and a distinct `LastServedAt` advanced only by server-observed events, fed
|
||||
by one shared `playback.SessionMeteredWriter` wired at every integrated pour (it forwards
|
||||
`io.ReaderFrom` to keep sendfile and implements `Unwrap()` to keep the cut reaching the
|
||||
socket).
|
||||
- **VERIFY-4 closed.** An unpaused transcode now gets a bounded 10m grace measured from
|
||||
that server-observed clock, plus a 180s idle window for transcode records at the edge
|
||||
tracker. This was chosen over an ffmpeg liveness probe, which sees only *local* processes
|
||||
and reports false once a copy-mode encode completes ahead of playback. Paused sessions
|
||||
keep their load-bearing 30m grace.
|
||||
- **The kill list gained an operator surface.** `Store.List()` existed but was reachable
|
||||
from nowhere. Added `GET`/`POST`/`DELETE /api/v1/admin/streams/revocations` plus
|
||||
`Store.Unrevoke` — the first way to undo a kill, since expiry is deliberately monotonic.
|
||||
Unrevoke is tombstone-guarded so a concurrent poll reconcile cannot resurrect the row,
|
||||
and `maintain` now shares the revoke/unrevoke lock so its durable self-heal cannot
|
||||
re-`Upsert` a just-deleted kill.
|
||||
- **The Audiobookshelf surface was outside the design entirely** — three byte-serving
|
||||
routes, none consulting the kill switch, one of them (`/feed/{slug}/file/{ino}`)
|
||||
unauthenticated. All three now enforce revocation with their real credential-issue time
|
||||
(ABS bearer JWTs are stateless and survive `OnUserSessionsRevoked`, so request-entry time
|
||||
would have meant a user kill never refused a later ABS request).
|
||||
|
||||
Still deferred, and deliberately so: multi-replica enforcement (VERIFY-3), the
|
||||
transcode-node ghost-record sweep, and a shared quota for the three download-class routes.
|
||||
|
||||
### Post-plan hardening round 2 — 2026-07-30 (serve-path batch)
|
||||
|
||||
A two-model adversarial review found six gaps the branch's own tests did not catch
|
||||
(GAP-10..GAP-15, all recorded in the companion coverage matrices). Four are now closed;
|
||||
the fixes changed two things this plan asserted:
|
||||
|
||||
- **"Every byte-serving path is observed" was false**, and is now closer to true.
|
||||
Ebook/comic/PDF reading had no meter, no transfer record, no `Refuse` and no in-flight
|
||||
cut (GAP-11); it now has all four, cap-exempt per decision A4. Subtitle pours on all
|
||||
three surfaces were entry-gated only and now carry a transport span, a meter and a
|
||||
watcher. The no-proxy remote transcode hop now meters its media bodies. It is still not
|
||||
literally *every* path: transfer-registry saturation (GAP-14) can still blind
|
||||
download-class monitoring, and edge transcode liveness is still request-observed
|
||||
(GAP-15).
|
||||
- **"The in-flight cut works" was false on the ABS surface and racy elsewhere.** ABS's
|
||||
`statusRecorder` lacked `Unwrap()`, so `SetWriteDeadline` returned `ErrNotSupported`
|
||||
and the cut was a silent no-op on every ABS route (GAP-10) — and the existing test
|
||||
passed throughout because it called handlers directly, bypassing the middleware that
|
||||
broke it. Fixing that alone was **not sufficient**: the same `Unwrap()` makes
|
||||
`RollingDeadlineWriter.bump()` start working, and `bump()` pushed the cut deadline back
|
||||
out (GAP-12). Both are fixed, the latter with a request-context `CutLatch`, because the
|
||||
rolling writer is built *inside* the serve helpers and wraps the writer the watcher
|
||||
holds — so it cannot be reached by `Unwrap()`, which walks the other way.
|
||||
|
||||
**A bound this plan never stated:** the in-flight watcher polls, so a cut lands within
|
||||
one interval (**5s in production**), not instantly. Every "cuts the stream" claim in this
|
||||
plan should be read that way.
|
||||
|
||||
Two of this plan's other claims remain **not yet true** and are tracked to later batches:
|
||||
an over-cap kill still reopens after 5m (decision A1), and monitoring still consults
|
||||
client-progress `LastActivityAt` as a liveness fallback, so "never trusts client
|
||||
progress" awaits decision A5.
|
||||
|
||||
## AI-use disclosure
|
||||
|
||||
This plan was drafted with AI assistance (Claude), based on a read-only audit of the current codebase. No behavior was changed by writing it. The Status banner and As-built deltas section were added after implementation.
|
||||
|
||||
The 2026-07-30 hardening round was a cross-model relay: Claude Opus 5 (`claude-opus-5[1m]`) planned, reconciled and reviewed; Codex `gpt-5.6-sol` at medium effort adversarially reviewed the plan, implemented it, and took one remediation round. Every design fork was decided by the maintainer. The adversarial findings on both sides and their resolution are recorded in [`stream-abuse-matrix.md`](../../architecture/stream-abuse-matrix.md) under "AI-use disclosure"; commands assume the repository root is the cwd.
|
||||
@@ -0,0 +1,308 @@
|
||||
# Abuse Cold-Enforcer — Architecture Decision & Plan
|
||||
|
||||
> **Branch:** `abuse-cold-enforcer` (based on `origin/main`).
|
||||
> **Name meaning:** "cold" = every decision stays **off the hot path**; "abuse" =
|
||||
> the scope widens from concurrency-only (the `feat/sauron-async-enforcer` branch)
|
||||
> to the full set of abuse classes catalogued in
|
||||
> [`../../architecture/stream-abuse-matrix.md`](../../architecture/stream-abuse-matrix.md).
|
||||
> Commands assume the repository root is the cwd.
|
||||
>
|
||||
> **Inputs learned from:** the `feat/sauron-async-enforcer` branch (kill-switch
|
||||
> implementation), its plan
|
||||
> [`2026-07-04-stream-monitoring-and-kill-switch.md`](2026-07-04-stream-monitoring-and-kill-switch.md),
|
||||
> its coverage matrix
|
||||
> [`../../architecture/playback-paths-monitoring-kill-matrix.md`](../../architecture/playback-paths-monitoring-kill-matrix.md),
|
||||
> and the adversarial assessment
|
||||
> [`../../architecture/stream-abuse-matrix.md`](../../architecture/stream-abuse-matrix.md).
|
||||
|
||||
---
|
||||
|
||||
> **DECISION UPDATE (2026-07-07, supersedes §2.3 and P2 below).** After adversarial
|
||||
> review, the two-tier lease was **dropped**. Everything a lease provides is already
|
||||
> latent in the deny-list pipeline: enforcer revocations carry a short self-healing
|
||||
> TTL and are re-asserted every tick (reversibility + continuous re-evaluation), and
|
||||
> the one genuine lease benefit — surviving a sustained outage — is captured by
|
||||
> **reason-scoped revocation TTLs** (deterministic violations like a byte-budget
|
||||
> breach write a durable revocation until the period rolls; over-cap keeps the short
|
||||
> self-healing TTL; admin kills stay 24h + durable). Fail-closed enforcement was
|
||||
> judged wrong for its only distinctive feeder (untuned heuristics — false positive +
|
||||
> Redis blip would kill legitimate playback), and a second enforcement mechanism with
|
||||
> a second failure posture was judged more fragile than the coverage it buys (one
|
||||
> matrix row, B13, during a >5m Redis outage). **The architecture is the single
|
||||
> pipeline: `streammonitor` → enforcer tick → `streamrevoke` → in-memory `IsRevoked`,
|
||||
> with policy checks for *new* activity at synchronous admission.** Pillars 2–4 and
|
||||
> phases P0/P1/P3/P4 stand unchanged; P1's enforcement action becomes a durable
|
||||
> revocation instead of a Tier-1 flag.
|
||||
|
||||
## 1. Non-negotiables (inherited, do not regress)
|
||||
|
||||
1. **Off the hot path.** The per-segment / per-byte serve path may add at most one
|
||||
local in-memory lookup. No per-request DB or Redis on the serve path. (This is
|
||||
the "cold" in cold-enforcer.)
|
||||
2. **No client-facing protocol change.** Clients (Jellyfin, Infuse, Silo native)
|
||||
keep echoing the same stable stream token. Any short-lived state lives
|
||||
**server-side**, keyed by the token's `sid`/`uid`. This constraint is what
|
||||
killed the naïve short-ticket idea in the prior plan; §3 shows how to keep the
|
||||
ticket *benefits* without violating it.
|
||||
3. **Reliability first / predictable under failure.** A brain/Redis hiccup must not
|
||||
silently break legitimate playback. Failure posture must be a tunable knob, not
|
||||
an accident.
|
||||
4. **Reuse, don't rewrite.** The sauron branch's *authoritative, byte-observed
|
||||
monitoring* (`streammonitor`, `nodesessions`) and its *durable, reconnect-proof
|
||||
revocation* (`streamrevoke`) are good primitives. Keep them; change how the
|
||||
*decision* is made and *widen what is decided*.
|
||||
|
||||
---
|
||||
|
||||
## 2. The core question: kill-list (deny) vs short-lived tickets (allow)
|
||||
|
||||
The prior branch chose a **deny-list**: streams run by default; the async brain
|
||||
observes abuse, writes a revocation, and the edge refuses on an in-memory
|
||||
set-membership check. The rejected alternative was **short-lived tickets**: the
|
||||
credential itself expires quickly and must be continually re-minted, so an
|
||||
unauthorized stream dies when its ticket lapses.
|
||||
|
||||
### 2.1 Why the branch rejected tickets — and why that reasoning is incomplete
|
||||
|
||||
The plan rejected tickets because Silo's `master.m3u8` is **VOD, fetched once**, so
|
||||
a short ticket embedded in segment URLs would expire mid-movie and 401 the rest of
|
||||
the stream; and flipping the global token TTL is a break-all-playback change.
|
||||
|
||||
Both objections are about putting the short TTL **in the client's credential**.
|
||||
They do **not** apply if the short-lived state lives **server-side**. The client
|
||||
keeps its stable 24h token; the *server* holds a short-lived **lease** keyed by that
|
||||
token's `sid`. The edge hot path checks the local lease, not a client-presented
|
||||
ticket. This is a "ticket" in every property that matters, minted server-to-server,
|
||||
requiring zero client cooperation — which is exactly the gap the prior analysis
|
||||
missed.
|
||||
|
||||
### 2.2 Deny-list vs lease (server-side ticket) — honest trade
|
||||
|
||||
| Property | Deny-list (current branch) | Lease / server-side ticket |
|
||||
|---|---|---|
|
||||
| Default state | **allow** (runs unless revoked) | **deny** (runs only while leased) |
|
||||
| Hot-path cost | in-memory set lookup | in-memory `sid→expiry` lookup (identical class) |
|
||||
| To stop a stream | write a revocation, propagate | **stop renewing** (or expire) the lease |
|
||||
| Stop latency | detect → decide → propagate (~120s) | ≤ one lease TTL, **same path for every reason** |
|
||||
| Maintenance load | write **only on kill** (rare, cheap) | **renew every active stream every TTL** (continuous) |
|
||||
| Restart-resurrection | **must** bolt on a durable mirror (branch's GAP-4) | **free**: a reconstructed session has no lease until re-granted |
|
||||
| Failure posture | **fail-open** — brain/Redis down ⇒ new kills don't land, abuse continues, but playback survives | **fail-closed** — brain/Redis down ⇒ leases lapse, abuse stops, but legit playback also stops |
|
||||
| Policy checkpoint | scattered: each rule writes its own kill | **one** checkpoint: the renewal tick evaluates cap + ban + byte-budget + throughput at once |
|
||||
|
||||
The two are duals. Deny-list optimizes **availability** (fail-open, cheap,
|
||||
already-built) and is *reactive*. Lease optimizes **abuse control** (fail-closed,
|
||||
self-expiring, restart-safe, one uniform action) and is *preventive* — at the cost
|
||||
of continuous renewal load and an availability blast-radius if the brain stalls.
|
||||
|
||||
### 2.3 Decision: a **two-tier** enforcer (deny by default, lease for suspects)
|
||||
|
||||
Neither pure form is right for a media server that must not break legit playback
|
||||
*and* must reliably stop abuse. Use both, graduated by suspicion:
|
||||
|
||||
- **Tier 0 — normal streams: deny-list default (keep the branch as-is).** A stream
|
||||
runs unless explicitly revoked. Hot path = the existing `IsRevoked` set lookup.
|
||||
Fail-open. This preserves "playback never breaks because the brain hiccuped" for
|
||||
the 99% case, honoring *reliability first*.
|
||||
- **Tier 1 — flagged/suspect streams: promote to lease-required (fail-closed).**
|
||||
When the cold brain flags a session — over a *soft* cap, anomalous throughput /
|
||||
distinct-viewer fan-out (re-streaming), or over a rolling byte budget — it
|
||||
**demotes** that session to lease-required: the edge now additionally requires a
|
||||
fresh central lease for that `sid`, and the brain grants it only while policy
|
||||
permits. Stop = simply stop renewing. This gives suspects the lease's good
|
||||
properties (≤TTL cut, no restart-resurrection, and — crucially — **an outage can
|
||||
no longer be used to escape an existing flag**, because a lapsed lease fails
|
||||
closed) while the expensive renewal load is paid *only for the small suspect set*,
|
||||
not every stream (honoring *performance first*).
|
||||
- **Immediate admin kill stays deny-list + durable.** "Cut this now" must not wait
|
||||
≤TTL, so admin terminate/ban writes an immediate revocation and a **durable ban
|
||||
ledger** (Postgres) so a banned *user* is never re-leased or re-admitted across a
|
||||
restart. The durable store thus shrinks from "mirror every ephemeral kill" to
|
||||
"record standing bans + quota ledgers" — a more natural durability boundary.
|
||||
|
||||
This is the headline architecture. Everything below is the machinery to feed it and
|
||||
the adjacent pillars the matrix demands.
|
||||
|
||||
**What this reuses from `feat/sauron-async-enforcer`:** `streammonitor` (the
|
||||
authoritative cross-node snapshot) is the input to the renewal tick unchanged;
|
||||
`streamrevoke` becomes the Tier-0 + admin path unchanged; the pub/sub + Redis
|
||||
`silo:sessions:*` transport is reused to carry a central-stamped `lease_until` field
|
||||
(the lease is one extra field on records the brain already reads). The edge check
|
||||
flips from `if revoked → refuse` to `if revoked → refuse; else if leaseRequired[sid]
|
||||
&& now > leaseUntil[sid] → refuse`. Small, additive, off hot path.
|
||||
|
||||
---
|
||||
|
||||
## 3. The four capability pillars (mapped to the matrix)
|
||||
|
||||
The "kill-list vs ticket" decision only governs Pillar 1. The matrix demands three
|
||||
more, independent of that choice.
|
||||
|
||||
### Pillar 1 — Concurrency & policy decision engine (A1–A8, C15)
|
||||
|
||||
- **Fix the dormant-enforcer bug first (matrix correction #1).** The sauron
|
||||
enforcer reads the *raw* `users.max_streams` column, which migrations set to `0`
|
||||
for standard users (real cap lives in the Default Group). `limit<=0` ⇒ skip, so
|
||||
the async brain never fires for a normal user. Resolve the **group-merged
|
||||
effective policy** (`access.EffectivePolicyForUser` → `strictestPositive`) in the
|
||||
brain's `LimitFunc`, exactly as synchronous admission already does.
|
||||
- **Keep synchronous admission** (`ErrTooManyStreams`) as the immediate preventive
|
||||
gate (A1/A2). The cold brain is the cross-node backstop (A7/A8/C15) and now the
|
||||
lease authority for suspects.
|
||||
- **Multi-replica (A7, VERIFY-3):** have integrated replicas publish their local
|
||||
sessions into the shared `silo:sessions:*` picture (or elect a single brain
|
||||
leader) so the cross-node count is authoritative. The lease grant is the natural
|
||||
serialization point.
|
||||
|
||||
### Pillar 2 — Liveness & anti-evasion (A4, A5)
|
||||
|
||||
- **A4 buffer-ahead-then-pause is the one open monitoring hole.** Fix it
|
||||
model-independently: treat a **running transcode ffmpeg as liveness** (or a
|
||||
transcode-specific grace ≥ the client's max buffer) so a session that pre-buffers
|
||||
and pauses fetches >45s is neither reaped nor decounted, and cannot free a slot to
|
||||
admit another. This touches the sensitive session-reaping path (cf. #279) — do it
|
||||
behind a flag with tests.
|
||||
- **A5 progress-withholding is already solved** (existence is byte-observed). Keep;
|
||||
add a regression test so it stays that way.
|
||||
|
||||
### Pillar 3 — Volume/quota accounting (C16, D17–D19) — new
|
||||
|
||||
Concurrency caps count *streams*, never *bytes*, so every rip vector walks straight
|
||||
through them. Add a **rolling per-user byte budget** that spans **both** downloads
|
||||
and direct-play/stream egress:
|
||||
|
||||
- Meter served bytes per `uid` (the edge already meters egress; attribute it per
|
||||
session — the sauron branch added `BytesServed` to `SessionInfo`, reuse it) and
|
||||
per download (already tracked).
|
||||
- A rolling `bytes-per-period` ledger (durable, alongside the ban ledger). When a
|
||||
user exceeds it, the brain flags them → Tier-1 lease-required → new streams/
|
||||
downloads are refused and in-flight ones cut (`WatchAndCut`) ≤TTL. This is the
|
||||
only thing that stops **D19 sequential direct-play ripping** and **C16 token
|
||||
fan-out that manifests as sustained egress**.
|
||||
- **Close the compat-download hole (D18, widest gap):** route
|
||||
`/Items/{id}/Download` through the same `QuantityLimiter` + byte budget as native
|
||||
downloads (see §4 — today it has *no* quota at all). This is the single
|
||||
highest-value fix in the whole plan.
|
||||
|
||||
### Pillar 4 — Detection heuristics & perimeter (C14, E21–E29) — partly new, partly gap-closing
|
||||
|
||||
- **Re-streaming heuristic (C16 fan-out only — C14 is undetectable this way) — build it
|
||||
(it does not exist; matrix correction #2).** ⚠️ **Superseded by abuse-matrix correction
|
||||
#9 (2026-07-31): the fingerprints do NOT already flow in usable form.** `ClientIP` is a
|
||||
single value per session — stamped once at session creation in integrated mode,
|
||||
overwritten per request at the edge — so concurrent viewers collapse to one address and
|
||||
no consumer of the existing snapshot can see fan-out. It needs per-request observation
|
||||
at media-serve time. The rule sketched below is retained for its shape, not its
|
||||
feasibility claim:
|
||||
distinct client-IPs per `sid`, sustained throughput vs. media bitrate, distinct
|
||||
concurrent titles per `uid`. Output = a flag → Tier-1 demotion. Ship **disabled**;
|
||||
tune against real traffic before enabling (false-kill risk).
|
||||
- **CPU/transcode exhaustion (E29):** add a **per-node concurrent-transcode cap**
|
||||
at the node's `handleStart` admission (today `reconstructSem` guards only the
|
||||
restart path; fresh starts are unbounded). Wire `nodepool.MaxJobs` sensible
|
||||
defaults.
|
||||
- **API-flood perimeter (E25–E28):** extend the existing `internal/ratelimit` to
|
||||
the **jellycompat surface** and `/auth/refresh` (both currently unthrottled), and
|
||||
add a connection cap (`netutil.LimitListener` / per-IP). These live in existing
|
||||
subsystems — gap-closing, not new architecture.
|
||||
- **Library-list load (E21–E24):** out of scope for the stream enforcer, but the
|
||||
cold philosophy applies — add per-client browse-cost accounting + a cache/
|
||||
singleflight on the *general* paged browse (today only the Latest/hub rails are
|
||||
cached). Track separately; note here so it is not forgotten.
|
||||
|
||||
---
|
||||
|
||||
## 4. Download-quota verification (answering the explicit question)
|
||||
|
||||
**Claim under test:** the admin "max concurrent downloads per user" / "max downloads
|
||||
per period" / "period duration" settings let an operator cap e.g. **20 downloads per
|
||||
30 days**, and that cap actually holds.
|
||||
|
||||
**Result: it HOLDS for the native download routes, and is BYPASSED on the compat
|
||||
route.**
|
||||
|
||||
- Config keys exist and are operator-settable: `download.max_concurrent_per_user`
|
||||
(default 3), `download.max_per_period` (default **0 = unlimited**),
|
||||
`download.period_duration` (default 24h) — `internal/config/db_loader.go:545–553`.
|
||||
Setting `max_per_period=20`, `period_duration=720h` gives "20 per 30 days".
|
||||
- The period check is enforced **at download-record creation** at all four creation
|
||||
sites (single, direct, series batch, season batch) —
|
||||
`internal/downloads/service.go:356,427,597,689` call `limiter.Check(...)` *before*
|
||||
inserting rows.
|
||||
- **Crucially, the period count is not defeatable by completing downloads.**
|
||||
`CountByUserSince` counts every row with `created_at >= now-period AND status NOT
|
||||
IN ('cancelled','failed')` (`internal/downloads/repo.go:190–200`) — i.e.
|
||||
**completed downloads still consume the period quota**. Only the *concurrent*
|
||||
gate (`CountActiveByUser`, `status IN ('queued','downloading','preparing')`,
|
||||
`repo.go:176–186`) frees on completion. So a `create→complete→create` loop can
|
||||
keep 3-in-flight forever but **cannot** exceed 20-in-30-days. Cancelled/failed are
|
||||
excluded so transient failures don't burn quota — correct.
|
||||
|
||||
**Caveats the operator must know:**
|
||||
|
||||
1. **Compat `/Items/{id}/Download` bypasses all of it.** `HandleDownload`
|
||||
(`internal/jellycompat/streams.go`) creates no download row and never calls
|
||||
`QuantityLimiter` (grep confirms zero references). An Infuse-style client
|
||||
downloads unlimited full files regardless of the 20/30d setting. → Pillar 2/§3
|
||||
fix: route it through the same limiter + byte budget.
|
||||
2. **The period quota counts downloads, not bytes.** "20 downloads" of 20 different
|
||||
4K remuxes is ~1 TB; the quota bounds *count*, not *volume*. The Pillar-3 byte
|
||||
budget is what bounds volume.
|
||||
3. **Streaming-based ripping is unaffected.** Sequential direct-play (D19) creates
|
||||
stream sessions, not download rows, so the download quota never sees it.
|
||||
4. **Minor TOCTOU:** two concurrent `Check` calls can both pass before either
|
||||
inserts; the batch path mitigates via `batchSize`, but a burst of single
|
||||
creates could momentarily overshoot by the concurrency of the burst. Low
|
||||
severity; note for a follow-up (advisory lock or insert-then-verify).
|
||||
|
||||
**Bottom line for the operator:** yes, 20/30d works today for the native app's
|
||||
downloads and can't be gamed by finishing downloads — but it does **not** cover
|
||||
Infuse/compat downloads or streaming rips, and it counts files not bytes. The plan
|
||||
closes those three gaps in Pillars 2–3.
|
||||
|
||||
---
|
||||
|
||||
## 5. Matrix coverage after this plan
|
||||
|
||||
| Matrix rows | Today (sauron branch) | After this plan |
|
||||
|---|---|---|
|
||||
| A1–A3 over-cap | ✅ admission (once bug #1 fixed) | ✅ + reliable cross-node backstop |
|
||||
| A4 buffer-ahead | ❌ invisible window | ✅ ffmpeg-liveness (Pillar 2) |
|
||||
| A5 progress-withhold | ✅ | ✅ (+ regression test) |
|
||||
| A6–A8 churn / multi-replica / dormant enforcer | ⚠️ | ✅ bug-fix + replica publish + lease-for-suspects |
|
||||
| B9–B12 admin kill/ban/restart | ✅ | ✅ (durable ban ledger, unchanged) |
|
||||
| B13 Redis outage | ⚠️ fail-open (abuser escapes) | ✅ flagged suspects fail-closed; legit fail-open |
|
||||
| C14 under-cap re-stream | ❌ | ⚠️→✅ heuristic (Pillar 4, ship disabled) |
|
||||
| C15 over-cap re-stream | ✅ | ✅ + labeled |
|
||||
| C16 token hoard/fan-out | ❌ | ⚠️ caught via byte budget when it manifests as egress |
|
||||
| D17 native rip | ⚠️ count-only | ✅ byte budget (Pillar 3) |
|
||||
| D18 compat rip | ❌ no quota | ✅ route through quota + budget |
|
||||
| D19 sequential direct-play rip | ❌ | ✅ byte budget |
|
||||
| D20 admin stop rip | ⚠️ WatchAndCut | ✅ (unchanged) |
|
||||
| E21–E24 library load | ❌ | ⚠️ browse-cost accounting + cache (tracked separately) |
|
||||
| E25–E28 API floods | ⚠️ gaps | ✅ extend ratelimit to compat/`/auth/refresh` + conn cap |
|
||||
| E29 CPU exhaustion | ❌ | ✅ per-node transcode cap |
|
||||
|
||||
---
|
||||
|
||||
## 6. Phased plan
|
||||
|
||||
1. **P0 — correctness first (small, high value):** fix the dormant-enforcer limit
|
||||
source (matrix #1); close the compat-download quota hole (D18); add the per-node
|
||||
transcode cap (E29). No new architecture — all three are bounded fixes with
|
||||
outsized payoff.
|
||||
2. **P1 — byte budget (Pillar 3):** rolling per-user bytes-per-period ledger across
|
||||
downloads + stream egress; wire into a flag. Bounds all rip vectors.
|
||||
3. **P2 — two-tier lease (Pillar 1):** add the `lease_until` field + Tier-1
|
||||
demotion for flagged sessions; fail-closed for suspects, fail-open for normal.
|
||||
4. **P3 — liveness (Pillar 2):** ffmpeg-as-liveness for A4, behind a flag + tests.
|
||||
5. **P4 — detection (Pillar 4):** re-streaming heuristic (ship disabled) + perimeter
|
||||
ratelimit gap-closing + library-browse cost accounting.
|
||||
|
||||
Each phase is independently shippable and independently valuable; P0 alone closes
|
||||
the three worst holes.
|
||||
|
||||
## AI-use disclosure
|
||||
|
||||
Drafted with AI assistance (Claude) from a read-only audit of both branches and the
|
||||
four referenced docs, plus targeted verification of the download-quota enforcement
|
||||
path. No code behavior changed by writing this plan.
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
@@ -12,6 +13,7 @@ import (
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -20,7 +22,17 @@ const (
|
||||
)
|
||||
|
||||
type AdminPlaybackControlHandler struct {
|
||||
playback *PlaybackHandler
|
||||
playback *PlaybackHandler
|
||||
revocation *streamrevoke.Store
|
||||
}
|
||||
|
||||
// SetRevocationStore wires the kill switch so an admin terminate revokes the
|
||||
// stream credential (stops it at the edge and refuses reconnects), not just
|
||||
// dispatches a cooperative realtime command. Optional; nil keeps prior behavior.
|
||||
func (h *AdminPlaybackControlHandler) SetRevocationStore(store *streamrevoke.Store) {
|
||||
if h != nil {
|
||||
h.revocation = store
|
||||
}
|
||||
}
|
||||
|
||||
type playbackControlRequest struct {
|
||||
@@ -145,9 +157,38 @@ func (h *AdminPlaybackControlHandler) handleSessionCommand(w http.ResponseWriter
|
||||
return
|
||||
}
|
||||
|
||||
var req playbackControlRequest
|
||||
if err := decodeOptionalJSONBody(r, &req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
// A terminate must stick even if the client ignores the realtime command:
|
||||
// revoke the stream credential so the edge refuses further segments and any
|
||||
// reconnect within one propagation/poll interval. Stop stays cooperative.
|
||||
// The revocation is written BEFORE the local session lookup: the kill needs
|
||||
// only the id, and the stream may exist only as an edge Redis record — e.g.
|
||||
// after a central restart, or when a client that withholds progress let the
|
||||
// in-memory session be reaped — which is precisely the stream an operator
|
||||
// most needs to kill. Revoking an unknown id is harmless (idempotent, TTL'd).
|
||||
if name == playback.CommandTerminate && h.revocation != nil {
|
||||
if err := h.revocation.RevokeSession(r.Context(), sessionID, "admin_terminate"); err != nil {
|
||||
slog.Warn("admin terminate: revoke stream failed", "session_id", sessionID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
session, err := h.playback.sessionMgr.GetSession(sessionID)
|
||||
if err != nil {
|
||||
if errors.Is(err, playback.ErrSessionNotFound) {
|
||||
// The revocation above still cut the stream at every serve surface;
|
||||
// only the cooperative realtime command has no local session to go to.
|
||||
if name == playback.CommandTerminate && h.revocation != nil {
|
||||
writeJSON(w, http.StatusAccepted, playbackControlResponse{
|
||||
CommandID: uuid.NewString(),
|
||||
Status: "revoked",
|
||||
})
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusNotFound, "not_found", "Playback session not found")
|
||||
return
|
||||
}
|
||||
@@ -155,12 +196,6 @@ func (h *AdminPlaybackControlHandler) handleSessionCommand(w http.ResponseWriter
|
||||
return
|
||||
}
|
||||
|
||||
var req playbackControlRequest
|
||||
if err := decodeOptionalJSONBody(r, &req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
if requiresLivePlaybackControl(name) && (session == nil || !session.HasRealtimeConnection) {
|
||||
writeError(w, http.StatusConflict, "realtime_unavailable", "Realtime connection unavailable for playback session")
|
||||
return
|
||||
|
||||
@@ -0,0 +1,224 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
const (
|
||||
maxStreamRevocationBodyBytes = 16 << 10
|
||||
maxStreamRevocationIDLength = 256
|
||||
maxStreamRevocationReason = 512
|
||||
maxStreamRevocationTTL = 30 * 24 * 60 * 60
|
||||
)
|
||||
|
||||
// AdminStreamRevocationHandler exposes the operator-visible stream kill list.
|
||||
// Unrevoking an over-cap victim is legal, but the async enforcer may revoke it
|
||||
// again on its next pass while the owning user remains over cap.
|
||||
type AdminStreamRevocationHandler struct {
|
||||
store *streamrevoke.Store
|
||||
}
|
||||
|
||||
func NewAdminStreamRevocationHandler(store *streamrevoke.Store) *AdminStreamRevocationHandler {
|
||||
return &AdminStreamRevocationHandler{store: store}
|
||||
}
|
||||
|
||||
type streamRevocationRequest struct {
|
||||
Kind string `json:"kind"`
|
||||
ID string `json:"id"`
|
||||
Reason string `json:"reason"`
|
||||
TTLSeconds *int64 `json:"ttl_seconds"`
|
||||
}
|
||||
|
||||
type streamRevocationResponse struct {
|
||||
streamrevoke.Revocation
|
||||
Warnings []string `json:"warnings,omitempty"`
|
||||
}
|
||||
|
||||
// streamRevocationKindsByWire is the single source of truth for the closed
|
||||
// {kind} vocabulary the revocation routes accept. Both the wire parser
|
||||
// (revocationKeyFromWire) and the capability advertisement (streamRevocationKinds)
|
||||
// read it, so adding a kind here automatically advertises it and the two cannot
|
||||
// drift apart. An earlier version duplicated the list in a switch statement,
|
||||
// which meant a newly-accepted kind could go unadvertised — clients would then
|
||||
// feature-detect an incomplete vocabulary.
|
||||
var streamRevocationKindsByWire = map[string]streamrevoke.Kind{
|
||||
"session": streamrevoke.KindSession,
|
||||
string(streamrevoke.KindSession): streamrevoke.KindSession,
|
||||
string(streamrevoke.KindUser): streamrevoke.KindUser,
|
||||
}
|
||||
|
||||
// streamRevocationKinds returns the accepted {kind} values in sorted order.
|
||||
// Sorted because map iteration order is randomised and the advertisement is a
|
||||
// wire response that must be stable across calls.
|
||||
func streamRevocationKinds() []string {
|
||||
kinds := make([]string, 0, len(streamRevocationKindsByWire))
|
||||
for kind := range streamRevocationKindsByWire {
|
||||
kinds = append(kinds, kind)
|
||||
}
|
||||
sort.Strings(kinds)
|
||||
return kinds
|
||||
}
|
||||
|
||||
// streamRevocationCapabilitiesResponse advertises the operator kill-list
|
||||
// endpoints so clients and operator tooling can feature-detect them rather than
|
||||
// probing for a 404. These routes are mounted only when a revocation store is
|
||||
// wired, which is why this endpoint is mounted in the same block.
|
||||
type streamRevocationCapabilitiesResponse struct {
|
||||
// StreamRevocations reports that the admin kill-list endpoints exist.
|
||||
StreamRevocations bool `json:"stream_revocations"`
|
||||
// StreamRevocationKinds is the closed {kind} vocabulary accepted by
|
||||
// DELETE /admin/streams/revocations/{kind}/{id}.
|
||||
StreamRevocationKinds []string `json:"stream_revocation_kinds"`
|
||||
// StreamRevocationUnrevoke reports that revocations can be lifted, not just
|
||||
// listed and created.
|
||||
StreamRevocationUnrevoke bool `json:"stream_revocation_unrevoke"`
|
||||
}
|
||||
|
||||
// HandleGetCapabilities exposes additive feature support for the stream
|
||||
// kill list (GET /admin/streams/revocations/capabilities).
|
||||
func (h *AdminStreamRevocationHandler) HandleGetCapabilities(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, streamRevocationCapabilitiesResponse{
|
||||
StreamRevocations: true,
|
||||
StreamRevocationKinds: streamRevocationKinds(),
|
||||
StreamRevocationUnrevoke: true,
|
||||
})
|
||||
}
|
||||
|
||||
func (h *AdminStreamRevocationHandler) HandleList(w http.ResponseWriter, _ *http.Request) {
|
||||
revocations := h.store.List()
|
||||
sort.Slice(revocations, func(i, j int) bool {
|
||||
return revocations[i].RevokedAt.After(revocations[j].RevokedAt)
|
||||
})
|
||||
writeJSON(w, http.StatusOK, map[string]any{"revocations": revocations})
|
||||
}
|
||||
|
||||
func (h *AdminStreamRevocationHandler) HandleCreate(w http.ResponseWriter, r *http.Request) {
|
||||
var req streamRevocationRequest
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxStreamRevocationBodyBytes)
|
||||
decoder := json.NewDecoder(r.Body)
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
if err := ensureJSONEOF(decoder); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
||||
return
|
||||
}
|
||||
|
||||
key, err := revocationKeyFromWire(req.Kind, req.ID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
}
|
||||
reason := strings.TrimSpace(req.Reason)
|
||||
if len(reason) > maxStreamRevocationReason {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "Reason is too long")
|
||||
return
|
||||
}
|
||||
|
||||
var until time.Time
|
||||
if req.TTLSeconds != nil {
|
||||
if *req.TTLSeconds <= 0 || *req.TTLSeconds > maxStreamRevocationTTL {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", "ttl_seconds must be between 1 and 2592000")
|
||||
return
|
||||
}
|
||||
until = time.Now().Add(time.Duration(*req.TTLSeconds) * time.Second)
|
||||
} else {
|
||||
// A zero expiry asks the store to use its default via the public helper.
|
||||
var warnings []string
|
||||
switch key.Kind {
|
||||
case streamrevoke.KindSession:
|
||||
warnings, err = h.store.RevokeSessionWithWarnings(r.Context(), key.ID, reason)
|
||||
case streamrevoke.KindUser:
|
||||
userID, _ := strconv.Atoi(key.ID)
|
||||
warnings, err = h.store.RevokeUserWithWarnings(r.Context(), userID, reason)
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to create revocation")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, streamRevocationResponse{Revocation: findRevocation(h.store, key), Warnings: warnings})
|
||||
return
|
||||
}
|
||||
|
||||
warnings, err := h.store.RevokeWithWarnings(r.Context(), key, reason, until)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to create revocation")
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, streamRevocationResponse{Revocation: findRevocation(h.store, key), Warnings: warnings})
|
||||
}
|
||||
|
||||
func (h *AdminStreamRevocationHandler) HandleDelete(w http.ResponseWriter, r *http.Request) {
|
||||
key, err := revocationKeyFromWire(chi.URLParam(r, "kind"), chi.URLParam(r, "id"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "bad_request", err.Error())
|
||||
return
|
||||
}
|
||||
warnings, err := h.store.UnrevokeWithWarnings(r.Context(), key)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to remove revocation")
|
||||
return
|
||||
}
|
||||
if len(warnings) > 0 {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"warnings": warnings})
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func revocationKeyFromWire(kind, id string) (streamrevoke.Key, error) {
|
||||
internalKind, ok := streamRevocationKindsByWire[kind]
|
||||
if !ok {
|
||||
return streamrevoke.Key{}, fmt.Errorf("kind must be one of %s", strings.Join(streamRevocationKinds(), ", "))
|
||||
}
|
||||
|
||||
if internalKind == streamrevoke.KindSession {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
return streamrevoke.Key{}, errors.New("session id is required")
|
||||
}
|
||||
if len(id) > maxStreamRevocationIDLength {
|
||||
return streamrevoke.Key{}, errors.New("session id is too long")
|
||||
}
|
||||
return streamrevoke.Key{Kind: internalKind, ID: id}, nil
|
||||
}
|
||||
|
||||
userID, err := strconv.Atoi(id)
|
||||
if err != nil || userID <= 0 || strconv.Itoa(userID) != id {
|
||||
return streamrevoke.Key{}, errors.New("user id must be a canonical positive integer")
|
||||
}
|
||||
return streamrevoke.Key{Kind: internalKind, ID: id}, nil
|
||||
}
|
||||
|
||||
func ensureJSONEOF(decoder *json.Decoder) error {
|
||||
var extra any
|
||||
if err := decoder.Decode(&extra); !errors.Is(err, io.EOF) {
|
||||
if err == nil {
|
||||
return errors.New("multiple JSON values")
|
||||
}
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func findRevocation(store *streamrevoke.Store, key streamrevoke.Key) streamrevoke.Revocation {
|
||||
for _, revocation := range store.List() {
|
||||
if revocation.Kind == key.Kind && revocation.ID == key.ID {
|
||||
return revocation
|
||||
}
|
||||
}
|
||||
return streamrevoke.Revocation{Kind: key.Kind, ID: key.ID}
|
||||
}
|
||||
@@ -0,0 +1,203 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/go-chi/chi/v5"
|
||||
)
|
||||
|
||||
func streamRevocationTestRouter(store *streamrevoke.Store) http.Handler {
|
||||
h := NewAdminStreamRevocationHandler(store)
|
||||
r := chi.NewRouter()
|
||||
r.Get("/revocations", h.HandleList)
|
||||
r.Post("/revocations", h.HandleCreate)
|
||||
r.Delete("/revocations/{kind}/{id}", h.HandleDelete)
|
||||
return r
|
||||
}
|
||||
|
||||
func TestAdminStreamRevocationRequiresAdminMiddleware(t *testing.T) {
|
||||
const adminRole = "admin"
|
||||
protected := apimw.RequireActingAdmin(nil)(streamRevocationTestRouter(streamrevoke.New(streamrevoke.Options{})))
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
claims *auth.Claims
|
||||
status int
|
||||
}{
|
||||
{name: "unauthenticated", status: http.StatusUnauthorized},
|
||||
{name: "non admin", claims: &auth.Claims{UserID: 1, Role: string(scopeUser)}, status: http.StatusForbidden},
|
||||
{name: adminRole, claims: &auth.Claims{UserID: 1, Role: adminRole}, status: http.StatusOK},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/revocations", nil)
|
||||
if tc.claims != nil {
|
||||
req = req.WithContext(apimw.SetClaims(req.Context(), tc.claims))
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
protected.ServeHTTP(rec, req)
|
||||
if rec.Code != tc.status {
|
||||
t.Fatalf("status=%d, want %d", rec.Code, tc.status)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminStreamRevocationSessionMappingAndListDeleteRoundTrip(t *testing.T) {
|
||||
store := streamrevoke.New(streamrevoke.Options{})
|
||||
router := streamRevocationTestRouter(store)
|
||||
req := httptest.NewRequest(http.MethodPost, "/revocations", bytes.NewBufferString(`{"kind":"session","id":" s1 ","reason":" test "}`))
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusCreated {
|
||||
t.Fatalf("create status=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
var created map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &created); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if created["kind"] != "sess" || created["id"] != "s1" {
|
||||
t.Fatalf("created=%v, want internal sess kind and trimmed id", created)
|
||||
}
|
||||
|
||||
listReq := httptest.NewRequest(http.MethodGet, "/revocations", nil)
|
||||
listRec := httptest.NewRecorder()
|
||||
router.ServeHTTP(listRec, listReq)
|
||||
var listed struct {
|
||||
Revocations []streamrevoke.Revocation `json:"revocations"`
|
||||
}
|
||||
if err := json.Unmarshal(listRec.Body.Bytes(), &listed); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(listed.Revocations) != 1 {
|
||||
t.Fatalf("list=%+v", listed.Revocations)
|
||||
}
|
||||
deletePath := "/revocations/" + string(listed.Revocations[0].Kind) + "/" + listed.Revocations[0].ID
|
||||
deleteReq := httptest.NewRequest(http.MethodDelete, deletePath, nil)
|
||||
deleteRec := httptest.NewRecorder()
|
||||
router.ServeHTTP(deleteRec, deleteReq)
|
||||
if deleteRec.Code != http.StatusNoContent || len(store.List()) != 0 {
|
||||
t.Fatalf("delete status=%d remaining=%v", deleteRec.Code, store.List())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminStreamRevocationValidation(t *testing.T) {
|
||||
router := streamRevocationTestRouter(streamrevoke.New(streamrevoke.Options{}))
|
||||
tests := []struct {
|
||||
name string
|
||||
body string
|
||||
}{
|
||||
{name: "noncanonical user", body: `{"kind":"user","id":"01"}`},
|
||||
{name: "zero user", body: `{"kind":"user","id":"0"}`},
|
||||
{name: "unknown kind", body: `{"kind":"stream","id":"x"}`},
|
||||
{name: "missing session id", body: `{"kind":"session","id":" "}`},
|
||||
{name: "zero ttl", body: `{"kind":"sess","id":"x","ttl_seconds":0}`},
|
||||
{name: "negative ttl", body: `{"kind":"sess","id":"x","ttl_seconds":-1}`},
|
||||
{name: "ttl above max", body: `{"kind":"sess","id":"x","ttl_seconds":2592001}`},
|
||||
{name: "ttl integer overflow", body: `{"kind":"sess","id":"x","ttl_seconds":9223372036854775808}`},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodPost, "/revocations", bytes.NewBufferString(tt.body))
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminStreamRevocationDeleteAbsentIsIdempotent(t *testing.T) {
|
||||
router := streamRevocationTestRouter(streamrevoke.New(streamrevoke.Options{}))
|
||||
req := httptest.NewRequest(http.MethodDelete, "/revocations/sess/missing", nil)
|
||||
rec := httptest.NewRecorder()
|
||||
router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("status=%d body=%s", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamRevocationCapabilitiesAdvertisesKillListSurface(t *testing.T) {
|
||||
h := NewAdminStreamRevocationHandler(streamrevoke.New(streamrevoke.Options{}))
|
||||
rr := httptest.NewRecorder()
|
||||
h.HandleGetCapabilities(rr, httptest.NewRequest(http.MethodGet, "/revocations/capabilities", nil))
|
||||
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rr.Code)
|
||||
}
|
||||
var got streamRevocationCapabilitiesResponse
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode capabilities: %v", err)
|
||||
}
|
||||
if !got.StreamRevocations || !got.StreamRevocationUnrevoke {
|
||||
t.Fatalf("capabilities did not advertise the kill list: %+v", got)
|
||||
}
|
||||
if len(got.StreamRevocationKinds) == 0 {
|
||||
t.Fatal("no revocation kinds advertised")
|
||||
}
|
||||
}
|
||||
|
||||
// The advertised {kind} vocabulary must be exactly what the wire parser accepts,
|
||||
// in both directions. Sampling a handful of rejected strings is not enough: it
|
||||
// cannot catch a kind the parser starts accepting without advertising it. Since
|
||||
// both now read streamRevocationKindsByWire, the check is over the whole map.
|
||||
func TestAdvertisedRevocationKindsMatchWireParser(t *testing.T) {
|
||||
advertised := streamRevocationKinds()
|
||||
if len(advertised) != len(streamRevocationKindsByWire) {
|
||||
t.Fatalf("advertised %d kinds but the parser accepts %d; the two have drifted",
|
||||
len(advertised), len(streamRevocationKindsByWire))
|
||||
}
|
||||
for _, kind := range advertised {
|
||||
if _, ok := streamRevocationKindsByWire[kind]; !ok {
|
||||
t.Errorf("advertised kind %q is not in the parser vocabulary", kind)
|
||||
}
|
||||
if _, err := revocationKeyFromWire(kind, "1"); err != nil {
|
||||
t.Errorf("advertised kind %q rejected by revocationKeyFromWire: %v", kind, err)
|
||||
}
|
||||
}
|
||||
for kind := range streamRevocationKindsByWire {
|
||||
if !slices.Contains(advertised, kind) {
|
||||
t.Errorf("parser accepts kind %q but it is not advertised", kind)
|
||||
}
|
||||
}
|
||||
for _, kind := range []string{"", "users", "sessions", "profile", "USER", "device"} {
|
||||
if _, err := revocationKeyFromWire(kind, "1"); err == nil {
|
||||
t.Errorf("kind %q outside the vocabulary was accepted", kind)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdvertisedRevocationKindsAreSortedAndStable(t *testing.T) {
|
||||
first := streamRevocationKinds()
|
||||
if !slices.IsSorted(first) {
|
||||
t.Errorf("advertised kinds are not sorted: %v", first)
|
||||
}
|
||||
// Map iteration order is randomised, so an unsorted implementation would
|
||||
// return a different order across calls for the same wire contract.
|
||||
for i := 0; i < 20; i++ {
|
||||
if got := streamRevocationKinds(); !slices.Equal(got, first) {
|
||||
t.Fatalf("advertised kinds unstable across calls: %v then %v", first, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapabilitiesKindsAreNotAliasedToPackageState(t *testing.T) {
|
||||
h := NewAdminStreamRevocationHandler(streamrevoke.New(streamrevoke.Options{}))
|
||||
rr := httptest.NewRecorder()
|
||||
h.HandleGetCapabilities(rr, httptest.NewRequest(http.MethodGet, "/revocations/capabilities", nil))
|
||||
var got streamRevocationCapabilitiesResponse
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &got); err != nil {
|
||||
t.Fatalf("decode capabilities: %v", err)
|
||||
}
|
||||
got.StreamRevocationKinds[0] = "mutated"
|
||||
if slices.Contains(streamRevocationKinds(), "mutated") {
|
||||
t.Fatal("handler returned a slice aliasing package state; a caller can corrupt the vocabulary")
|
||||
}
|
||||
}
|
||||
@@ -7,14 +7,19 @@ import (
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/downloads"
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
// DownloadService is the interface that the download handler depends on. A
|
||||
@@ -31,8 +36,8 @@ type DownloadService interface {
|
||||
UpdateSubscription(ctx context.Context, userID int, profileID, deviceID, id string, patch downloads.SubscriptionPatch, filter catalog.AccessFilter) (*downloads.SubscriptionResult, error)
|
||||
DeleteSubscription(ctx context.Context, userID int, profileID, deviceID, id string) error
|
||||
SyncSubscriptions(ctx context.Context, userID int, profileID, deviceID string, filter catalog.AccessFilter) (int, error)
|
||||
ServeDirect(ctx context.Context, w http.ResponseWriter, r *http.Request, userID, fileID int, format string, filter catalog.AccessFilter) error
|
||||
ServeFile(ctx context.Context, w http.ResponseWriter, r *http.Request, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter) error
|
||||
ServeDirect(ctx context.Context, w http.ResponseWriter, r *http.Request, userID, fileID int, format string, filter catalog.AccessFilter, transfer transfers.Transfer) error
|
||||
ServeFile(ctx context.Context, w http.ResponseWriter, r *http.Request, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter, transfer transfers.Transfer) error
|
||||
List(ctx context.Context, userID int, profileID, deviceID string) ([]*downloads.Download, error)
|
||||
Delete(ctx context.Context, userID int, profileID, deviceID, downloadID string) error
|
||||
PatchStatus(ctx context.Context, userID int, profileID, deviceID, downloadID, status string) error
|
||||
@@ -44,7 +49,9 @@ type DownloadService interface {
|
||||
|
||||
// DownloadHandler handles download endpoints.
|
||||
type DownloadHandler struct {
|
||||
svc DownloadService
|
||||
svc DownloadService
|
||||
revocation *streamrevoke.Store
|
||||
transfers *transfers.Registry
|
||||
}
|
||||
|
||||
// NewDownloadHandler creates a new DownloadHandler.
|
||||
@@ -52,6 +59,35 @@ func NewDownloadHandler(svc DownloadService) *DownloadHandler {
|
||||
return &DownloadHandler{svc: svc}
|
||||
}
|
||||
|
||||
// SetRevocationStore wires the stream kill switch so a per-user revocation cuts
|
||||
// an IN-FLIGHT download pour. Downloads stay exempt from the live-stream cap
|
||||
// (creation-time download quotas do not gate serving), but a user whose sessions
|
||||
// were just revoked must not keep pulling a multi-GB file on a pre-revocation
|
||||
// connection. Optional; nil keeps prior behavior.
|
||||
func (h *DownloadHandler) SetRevocationStore(store *streamrevoke.Store) {
|
||||
if h != nil {
|
||||
h.revocation = store
|
||||
}
|
||||
}
|
||||
|
||||
// SetTransferRegistry wires process-local monitoring for active file pours.
|
||||
func (h *DownloadHandler) SetTransferRegistry(registry *transfers.Registry) {
|
||||
if h != nil {
|
||||
h.transfers = registry
|
||||
}
|
||||
}
|
||||
|
||||
// cutDownloadOnUserRevocation arms the shared in-flight cut for a download pour.
|
||||
// Sessionless (downloads have no stream session), so only user-kind kills apply;
|
||||
// the request entry time predates any future revocation, which is exactly the
|
||||
// user-kill cutoff contract. Returns a stop func the caller must defer.
|
||||
func (h *DownloadHandler) cutDownloadOnUserRevocation(ctx context.Context, w http.ResponseWriter, userID int, startedAt time.Time) func() {
|
||||
if h.revocation == nil {
|
||||
return func() {}
|
||||
}
|
||||
return h.revocation.WatchAndCutContext(ctx, w, "", userID, startedAt)
|
||||
}
|
||||
|
||||
// downloadRequest represents the JSON body for POST /downloads.
|
||||
type downloadRequest struct {
|
||||
ContentID string `json:"content_id"`
|
||||
@@ -116,14 +152,16 @@ type batchManifestsResponse struct {
|
||||
// downloadCapabilityResponse is the GET /downloads/capability payload clients
|
||||
// use for feature detection instead of introspecting admin settings.
|
||||
type downloadCapabilityResponse struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
DownloadAllowed bool `json:"download_allowed"`
|
||||
QualityPresets []string `json:"quality_presets"`
|
||||
TranscodeEnabled bool `json:"transcode_enabled"`
|
||||
TranscodeUserAllowed bool `json:"transcode_user_allowed"`
|
||||
SeasonDownload bool `json:"season_download"`
|
||||
SeriesMonitoring bool `json:"series_monitoring"`
|
||||
MonitoringModes []string `json:"monitoring_modes,omitempty"`
|
||||
Enabled bool `json:"enabled"`
|
||||
DownloadAllowed bool `json:"download_allowed"`
|
||||
QualityPresets []string `json:"quality_presets"`
|
||||
TranscodeEnabled bool `json:"transcode_enabled"`
|
||||
TranscodeUserAllowed bool `json:"transcode_user_allowed"`
|
||||
SeasonDownload bool `json:"season_download"`
|
||||
SeriesMonitoring bool `json:"series_monitoring"`
|
||||
MonitoringModes []string `json:"monitoring_modes,omitempty"`
|
||||
MaxUserConcurrentTransfers int `json:"max_user_concurrent_transfers"`
|
||||
TransferMonitoringFailClosed bool `json:"transfer_monitoring_fail_closed"`
|
||||
}
|
||||
|
||||
func toDownloadResponse(d *downloads.Download) downloadResponse {
|
||||
@@ -180,14 +218,16 @@ func (h *DownloadHandler) HandleCapability(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, downloadCapabilityResponse{
|
||||
Enabled: capInfo.Enabled,
|
||||
DownloadAllowed: capInfo.DownloadAllowed,
|
||||
QualityPresets: capInfo.QualityPresets,
|
||||
TranscodeEnabled: capInfo.TranscodeEnabled,
|
||||
TranscodeUserAllowed: capInfo.TranscodeUserAllowed,
|
||||
SeasonDownload: capInfo.SeasonDownload,
|
||||
SeriesMonitoring: capInfo.SeriesMonitoring,
|
||||
MonitoringModes: capInfo.MonitoringModes,
|
||||
Enabled: capInfo.Enabled,
|
||||
DownloadAllowed: capInfo.DownloadAllowed,
|
||||
QualityPresets: capInfo.QualityPresets,
|
||||
TranscodeEnabled: capInfo.TranscodeEnabled,
|
||||
TranscodeUserAllowed: capInfo.TranscodeUserAllowed,
|
||||
SeasonDownload: capInfo.SeasonDownload,
|
||||
SeriesMonitoring: capInfo.SeriesMonitoring,
|
||||
MonitoringModes: capInfo.MonitoringModes,
|
||||
MaxUserConcurrentTransfers: h.transfers.MaxPerUser(),
|
||||
TransferMonitoringFailClosed: true,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -381,6 +421,12 @@ func (h *DownloadHandler) HandlePatchDownload(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
// HandleDownloadFile handles GET /downloads/{id}/file.
|
||||
//
|
||||
// This is an offline download, NOT a live stream: it creates no SessionManager
|
||||
// session and no live-stream monitor record, so it is intentionally EXEMPT from the
|
||||
// concurrent-stream cap and never enters streammonitor. Active pours are exposed
|
||||
// separately through the in-memory transfer registry; creation-time download
|
||||
// quotas do not gate this serving request.
|
||||
func (h *DownloadHandler) HandleDownloadFile(w http.ResponseWriter, r *http.Request) {
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
if userID == 0 {
|
||||
@@ -397,12 +443,25 @@ func (h *DownloadHandler) HandleDownloadFile(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
profileID, deviceID, _, _ := managedIdentity(r)
|
||||
profileID, deviceID, deviceName, _ := managedIdentity(r)
|
||||
filter := requestAccessFilter(r)
|
||||
startedAt := time.Now()
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
// Full media downloads outlive the server's absolute WriteTimeout; roll
|
||||
// the write deadline with progress instead.
|
||||
sw := httpstream.NewRollingDeadlineWriter(w)
|
||||
if err := h.svc.ServeFile(r.Context(), sw, r, userID, profileID, deviceID, id, filter); err != nil {
|
||||
sw := httpstream.NewRollingDeadlineWriterCtx(r.Context(), w)
|
||||
transfer := h.newTransfer(r, userID, profileID, deviceName, "native_download")
|
||||
if !h.beginTransfer(w, r, transfer) {
|
||||
return
|
||||
}
|
||||
defer h.transfers.End(transfer.ID)
|
||||
metered := playback.NewSessionMeteredWriter(sw, h.transfers, transfer.ID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
// In-flight kill switch: a user revocation cuts this pour mid-transfer.
|
||||
stop := h.cutDownloadOnUserRevocation(r.Context(), metered, userID, startedAt)
|
||||
defer stop()
|
||||
|
||||
if err := h.svc.ServeFile(r.Context(), metered, r, userID, profileID, deviceID, id, filter, transfer); err != nil {
|
||||
if errors.Is(err, downloads.ErrNotFound) || errors.Is(err, catalog.ErrItemNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Download not found")
|
||||
return
|
||||
@@ -446,12 +505,63 @@ func (h *DownloadHandler) HandleDirectDownload(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
filter := requestAccessFilter(r)
|
||||
if err := h.svc.ServeDirect(r.Context(), w, r, userID, fileID, r.URL.Query().Get("format"), filter); err != nil {
|
||||
profileID, _, deviceName, _ := managedIdentity(r)
|
||||
startedAt := time.Now()
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
transfer := h.newTransfer(r, userID, profileID, deviceName, "native_direct")
|
||||
if !h.beginTransfer(w, r, transfer) {
|
||||
return
|
||||
}
|
||||
defer h.transfers.End(transfer.ID)
|
||||
metered := playback.NewSessionMeteredWriter(w, h.transfers, transfer.ID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
// In-flight kill switch: a user revocation cuts this pour mid-transfer.
|
||||
stop := h.cutDownloadOnUserRevocation(r.Context(), metered, userID, startedAt)
|
||||
defer stop()
|
||||
|
||||
if err := h.svc.ServeDirect(r.Context(), metered, r, userID, fileID, r.URL.Query().Get("format"), filter, transfer); err != nil {
|
||||
h.writeDownloadError(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (h *DownloadHandler) beginTransfer(w http.ResponseWriter, r *http.Request, transfer transfers.Transfer) bool {
|
||||
if err := h.transfers.Begin(transfer); err != nil {
|
||||
slog.DebugContext(r.Context(), "download transfer rejected",
|
||||
"component", "api",
|
||||
"transfer_id", transfer.ID,
|
||||
"error", err,
|
||||
)
|
||||
switch {
|
||||
case errors.Is(err, transfers.ErrUserTransferLimit):
|
||||
w.Header().Set("Retry-After", "5")
|
||||
writeError(w, http.StatusTooManyRequests, "transfer_limit_exceeded", "Concurrent transfer limit exceeded")
|
||||
case errors.Is(err, transfers.ErrRegistryFull):
|
||||
w.Header().Set("Retry-After", "5")
|
||||
writeError(w, http.StatusServiceUnavailable, "monitoring_unavailable", "Transfer monitoring unavailable")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to monitor transfer")
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *DownloadHandler) newTransfer(r *http.Request, userID int, profileID, clientName, route string) transfers.Transfer {
|
||||
if clientName == "" {
|
||||
clientName = r.UserAgent()
|
||||
}
|
||||
return transfers.Transfer{
|
||||
ID: uuid.NewString(),
|
||||
UserID: userID,
|
||||
ProfileID: profileID,
|
||||
Route: route,
|
||||
ClientIP: clientip.FromContext(r.Context()),
|
||||
ClientName: clientName,
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
}
|
||||
|
||||
// requireManaged validates a managed (device-scoped) request: authentication, a
|
||||
// configured service, and the device + profile identity (device_id from the
|
||||
// X-Silo-Device-Id header only, never the body). On failure it writes the error
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/downloads"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
// fakeDownloadService is a programmable DownloadService for handler tests. It
|
||||
@@ -54,6 +56,9 @@ type fakeDownloadService struct {
|
||||
gotSubtitleRef string
|
||||
gotDirectFormat string
|
||||
gotDirectFileID int
|
||||
gotTransfer transfers.Transfer
|
||||
transferRegistry *transfers.Registry
|
||||
beginTransfer bool
|
||||
|
||||
// Season download + series-monitoring (subscription) fakes.
|
||||
season []*downloads.Download
|
||||
@@ -154,9 +159,14 @@ func (f *fakeDownloadService) SyncSubscriptions(_ context.Context, userID int, p
|
||||
return f.syncRegistered, nil
|
||||
}
|
||||
|
||||
func (f *fakeDownloadService) ServeDirect(_ context.Context, w http.ResponseWriter, _ *http.Request, _, fileID int, format string, _ catalog.AccessFilter) error {
|
||||
func (f *fakeDownloadService) ServeDirect(_ context.Context, w http.ResponseWriter, _ *http.Request, _, fileID int, format string, _ catalog.AccessFilter, transfer transfers.Transfer) error {
|
||||
f.gotDirectFileID = fileID
|
||||
f.gotDirectFormat = format
|
||||
f.gotTransfer = transfer
|
||||
if f.beginTransfer && f.transferRegistry != nil {
|
||||
transfer.MediaFileID = fileID
|
||||
f.transferRegistry.Annotate(transfer.ID, "", fileID)
|
||||
}
|
||||
if f.directErr != nil {
|
||||
return f.directErr
|
||||
}
|
||||
@@ -164,8 +174,13 @@ func (f *fakeDownloadService) ServeDirect(_ context.Context, w http.ResponseWrit
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeDownloadService) ServeFile(_ context.Context, w http.ResponseWriter, _ *http.Request, userID int, profileID, deviceID, downloadID string, _ catalog.AccessFilter) error {
|
||||
func (f *fakeDownloadService) ServeFile(_ context.Context, w http.ResponseWriter, _ *http.Request, userID int, profileID, deviceID, downloadID string, _ catalog.AccessFilter, transfer transfers.Transfer) error {
|
||||
f.gotServe = identityCall{userID, profileID, deviceID, downloadID}
|
||||
f.gotTransfer = transfer
|
||||
if f.beginTransfer && f.transferRegistry != nil {
|
||||
transfer.DownloadID = downloadID
|
||||
f.transferRegistry.Annotate(transfer.ID, downloadID, 0)
|
||||
}
|
||||
if f.serveErr != nil {
|
||||
return f.serveErr
|
||||
}
|
||||
@@ -266,6 +281,7 @@ func TestHandleCapability(t *testing.T) {
|
||||
TranscodeUserAllowed: true,
|
||||
}}
|
||||
h := NewDownloadHandler(svc)
|
||||
h.SetTransferRegistry(transfers.New())
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleCapability(rec, downloadTestRequest(http.MethodGet, "/downloads/capability", nil, 7, "", ""))
|
||||
@@ -283,6 +299,9 @@ func TestHandleCapability(t *testing.T) {
|
||||
if len(resp.QualityPresets) != 1 || resp.QualityPresets[0] != downloads.QualityOriginal {
|
||||
t.Fatalf("quality presets = %v, want [original]", resp.QualityPresets)
|
||||
}
|
||||
if resp.MaxUserConcurrentTransfers != 24 || !resp.TransferMonitoringFailClosed {
|
||||
t.Fatalf("transfer capability = %+v", resp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleCapabilityUnauthorized(t *testing.T) {
|
||||
@@ -523,6 +542,119 @@ func TestHandleDirectDownloadMissingFileID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadTransferAdmissionFailsClosed(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
registry func(t *testing.T) *transfers.Registry
|
||||
wantStatus int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "per-user cap",
|
||||
registry: func(t *testing.T) *transfers.Registry {
|
||||
t.Helper()
|
||||
limit := 1
|
||||
r := transfers.NewWithOptions(transfers.Options{MaxPerUser: &limit})
|
||||
if err := r.Begin(transfers.Transfer{ID: "existing", UserID: 7}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
},
|
||||
wantStatus: http.StatusTooManyRequests,
|
||||
wantCode: "transfer_limit_exceeded",
|
||||
},
|
||||
{
|
||||
name: "global cap",
|
||||
registry: func(t *testing.T) *transfers.Registry {
|
||||
t.Helper()
|
||||
unlimited := 0
|
||||
r := transfers.NewWithOptions(transfers.Options{MaxEntries: 1, MaxPerUser: &unlimited})
|
||||
if err := r.Begin(transfers.Transfer{ID: "existing", UserID: 8}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
},
|
||||
wantStatus: http.StatusServiceUnavailable,
|
||||
wantCode: "monitoring_unavailable",
|
||||
},
|
||||
} {
|
||||
for _, route := range []string{"managed", "direct"} {
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead, "range"} {
|
||||
t.Run(tc.name+"/"+route+"/"+method, func(t *testing.T) {
|
||||
registry := tc.registry(t)
|
||||
svc := &fakeDownloadService{}
|
||||
h := NewDownloadHandler(svc)
|
||||
h.SetTransferRegistry(registry)
|
||||
requestMethod := method
|
||||
if method == "range" {
|
||||
requestMethod = http.MethodGet
|
||||
}
|
||||
var req *http.Request
|
||||
if route == "managed" {
|
||||
req = withChiID(downloadTestRequest(requestMethod, "/downloads/dl1/file", nil, 7, "p1", "dev1"), "dl1")
|
||||
} else {
|
||||
req = downloadTestRequest(requestMethod, "/direct-download?file_id=42", nil, 7, "p1", "")
|
||||
}
|
||||
if method == "range" {
|
||||
req.Header.Set("Range", "bytes=0-1")
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
if route == "managed" {
|
||||
h.HandleDownloadFile(rec, req)
|
||||
} else {
|
||||
h.HandleDirectDownload(rec, req)
|
||||
}
|
||||
if rec.Code != tc.wantStatus {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, tc.wantStatus, rec.Body.String())
|
||||
}
|
||||
if rec.Header().Get("Retry-After") != "5" {
|
||||
t.Fatalf("Retry-After = %q, want 5", rec.Header().Get("Retry-After"))
|
||||
}
|
||||
if !bytes.Contains(rec.Body.Bytes(), []byte(tc.wantCode)) || bytes.Contains(rec.Body.Bytes(), []byte("served")) {
|
||||
t.Fatalf("body = %q, want error code and no served bytes", rec.Body.String())
|
||||
}
|
||||
if svc.gotTransfer.ID != "" {
|
||||
t.Fatalf("service was called with transfer %+v", svc.gotTransfer)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadTransferResolutionFailureNeverRegisters(t *testing.T) {
|
||||
registry := transfers.New()
|
||||
svc := &fakeDownloadService{directErr: catalog.ErrItemNotFound}
|
||||
h := NewDownloadHandler(svc)
|
||||
h.SetTransferRegistry(registry)
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleDirectDownload(rec, downloadTestRequest(http.MethodGet, "/direct-download?file_id=42", nil, 7, "p1", ""))
|
||||
|
||||
if got := registry.Snapshot(); len(got) != 0 {
|
||||
t.Fatalf("failed resolution left transfers: %+v", got)
|
||||
}
|
||||
if svc.gotTransfer.ID == "" || svc.gotTransfer.Route != "native_direct" {
|
||||
t.Fatalf("transfer context = %+v", svc.gotTransfer)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadTransferServingErrorStillDeregisters(t *testing.T) {
|
||||
registry := transfers.New()
|
||||
svc := &fakeDownloadService{
|
||||
directErr: errors.New("write failed"),
|
||||
transferRegistry: registry,
|
||||
beginTransfer: true,
|
||||
}
|
||||
h := NewDownloadHandler(svc)
|
||||
h.SetTransferRegistry(registry)
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleDirectDownload(rec, downloadTestRequest(http.MethodGet, "/direct-download?file_id=42", nil, 7, "p1", ""))
|
||||
|
||||
if got := registry.Snapshot(); len(got) != 0 {
|
||||
t.Fatalf("serving error left transfers: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func withChiParams(r *http.Request, params map[string]string) *http.Request {
|
||||
rctx := chi.NewRouteContext()
|
||||
for k, v := range params {
|
||||
|
||||
@@ -157,7 +157,7 @@ func serveConvertedEpub(w http.ResponseWriter, r *http.Request, file *models.Med
|
||||
|
||||
name := strings.TrimSuffix(filepath.Base(file.FilePath), filepath.Ext(file.FilePath)) + ".epub"
|
||||
setConvertedEpubHeaders(w, file, key)
|
||||
http.ServeContent(httpstream.NewRollingDeadlineWriter(w), r, name, stat.ModTime(), f)
|
||||
http.ServeContent(httpstream.NewRollingDeadlineWriterCtx(r.Context(), w), r, name, stat.ModTime(), f)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"mime"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -20,8 +21,12 @@ import (
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
// EbookReaderProgress is the persisted reader position for one profile and ebook.
|
||||
@@ -100,6 +105,8 @@ type EbookReaderHandler struct {
|
||||
// Conversion, when set and enabled, transparently converts Kindle-family
|
||||
// files to EPUB on read. Nil means the feature is off.
|
||||
Conversion *EbookConversion
|
||||
Transfers *transfers.Registry
|
||||
Revocation *streamrevoke.Store
|
||||
}
|
||||
|
||||
func NewEbookReaderHandler(authorizer *MediaFileAuthorizer) *EbookReaderHandler {
|
||||
@@ -116,6 +123,9 @@ func (h *EbookReaderHandler) HandleReadFile(w http.ResponseWriter, r *http.Reque
|
||||
writeError(w, http.StatusServiceUnavailable, "unavailable", "Ebook reader is not configured")
|
||||
return
|
||||
}
|
||||
startedAt := time.Now()
|
||||
credentialIssuedAt := apimw.CredentialIssuedAt(r.Context())
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
|
||||
contentID := strings.TrimSpace(chi.URLParam(r, "content_id"))
|
||||
fileID, err := strconv.Atoi(chi.URLParam(r, "file_id"))
|
||||
@@ -134,7 +144,54 @@ func (h *EbookReaderHandler) HandleReadFile(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.serveEbook(w, r, file); err != nil {
|
||||
userID := apimw.GetUserID(r.Context())
|
||||
if h.Revocation != nil && h.Revocation.Refuse(w, "", userID, credentialIssuedAt) {
|
||||
return
|
||||
}
|
||||
|
||||
route := "ebook_read"
|
||||
if h.Conversion.active(r.Context()) && isKindleReaderFile(file) {
|
||||
route = "ebook_convert"
|
||||
}
|
||||
transfer := transfers.Transfer{
|
||||
ID: uuid.NewString(),
|
||||
UserID: userID,
|
||||
ProfileID: apimw.GetProfileID(r.Context()),
|
||||
MediaFileID: file.ID,
|
||||
Route: route,
|
||||
ClientIP: clientip.FromContext(r.Context()),
|
||||
ClientName: r.UserAgent(),
|
||||
StartedAt: startedAt,
|
||||
}
|
||||
if h.Transfers != nil {
|
||||
if err := h.Transfers.Begin(transfer); err != nil {
|
||||
slog.DebugContext(r.Context(), "ebook transfer rejected",
|
||||
"component", "ebooks",
|
||||
"transfer_id", transfer.ID,
|
||||
"error", err,
|
||||
)
|
||||
switch {
|
||||
case errors.Is(err, transfers.ErrUserTransferLimit):
|
||||
w.Header().Set("Retry-After", "5")
|
||||
writeError(w, http.StatusTooManyRequests, "transfer_limit_exceeded", "Concurrent transfer limit exceeded")
|
||||
case errors.Is(err, transfers.ErrRegistryFull):
|
||||
w.Header().Set("Retry-After", "5")
|
||||
writeError(w, http.StatusServiceUnavailable, "monitoring_unavailable", "Transfer monitoring unavailable")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to monitor transfer")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
defer h.Transfers.End(transfer.ID)
|
||||
metered := playback.NewSessionMeteredWriter(w, h.Transfers, transfer.ID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
if h.Revocation != nil {
|
||||
stop := h.Revocation.WatchAndCutContext(r.Context(), metered, "", userID, credentialIssuedAt)
|
||||
defer stop()
|
||||
}
|
||||
|
||||
if err := h.serveEbook(metered, r, file); err != nil {
|
||||
if errors.Is(err, catalog.ErrItemNotFound) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Ebook file not found")
|
||||
return
|
||||
@@ -623,7 +680,7 @@ func serveEbookInline(w http.ResponseWriter, r *http.Request, file *models.Media
|
||||
// Ebook files are served inline on the API origin (and reachable via the
|
||||
// ?token= query fallback), so browsers must never content-sniff them.
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
http.ServeContent(httpstream.NewRollingDeadlineWriter(w), r, stat.Name(), stat.ModTime(), f)
|
||||
http.ServeContent(httpstream.NewRollingDeadlineWriterCtx(r.Context(), w), r, stat.Name(), stat.ModTime(), f)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -4,20 +4,25 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"mime"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
func newEbookReaderAuthRequest(method, path string) *http.Request {
|
||||
@@ -26,6 +31,9 @@ func newEbookReaderAuthRequest(method, path string) *http.Request {
|
||||
UserID: 1,
|
||||
Role: "user",
|
||||
TokenType: auth.TokenTypeAccess,
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
IssuedAt: jwt.NewNumericDate(time.Now().Add(-time.Hour)),
|
||||
},
|
||||
})
|
||||
ctx = apimw.SetProfileID(ctx, "profile-1")
|
||||
return req.WithContext(ctx)
|
||||
@@ -202,6 +210,244 @@ func TestEbookReaderServesEbookInlineWithRangeSupport(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestEbookReaderTransferAdmissionFailsClosed(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
registry func(t *testing.T) *transfers.Registry
|
||||
wantStatus int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "per-user cap",
|
||||
registry: func(t *testing.T) *transfers.Registry {
|
||||
t.Helper()
|
||||
limit := 1
|
||||
r := transfers.NewWithOptions(transfers.Options{MaxPerUser: &limit})
|
||||
if err := r.Begin(transfers.Transfer{ID: "existing", UserID: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
},
|
||||
wantStatus: http.StatusTooManyRequests,
|
||||
wantCode: "transfer_limit_exceeded",
|
||||
},
|
||||
{
|
||||
name: "global cap",
|
||||
registry: func(t *testing.T) *transfers.Registry {
|
||||
t.Helper()
|
||||
unlimited := 0
|
||||
r := transfers.NewWithOptions(transfers.Options{MaxEntries: 1, MaxPerUser: &unlimited})
|
||||
if err := r.Begin(transfers.Transfer{ID: "existing", UserID: 2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
},
|
||||
wantStatus: http.StatusServiceUnavailable,
|
||||
wantCode: "monitoring_unavailable",
|
||||
},
|
||||
} {
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead, "range"} {
|
||||
t.Run(tc.name+"/"+method, func(t *testing.T) {
|
||||
registry := tc.registry(t)
|
||||
handler := NewEbookReaderHandler(&MediaFileAuthorizer{
|
||||
FileResolver: stubMediaFileResolver{file: &models.MediaFile{
|
||||
ID: 42, ContentID: "ebook-1", FilePath: "must-not-be-served.epub", Container: "epub", BaseType: "ebook",
|
||||
}},
|
||||
ItemAccess: stubItemAccessChecker{},
|
||||
})
|
||||
handler.Transfers = registry
|
||||
requestMethod := method
|
||||
if method == "range" {
|
||||
requestMethod = http.MethodGet
|
||||
}
|
||||
req := withEbookReaderRouteParams(
|
||||
newEbookReaderAuthRequest(requestMethod, "/ebooks/ebook-1/files/42/read"),
|
||||
"ebook-1",
|
||||
"42",
|
||||
)
|
||||
if method == "range" {
|
||||
req.Header.Set("Range", "bytes=0-1")
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
handler.HandleReadFile(rec, req)
|
||||
if rec.Code != tc.wantStatus {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, tc.wantStatus, rec.Body.String())
|
||||
}
|
||||
if rec.Header().Get("Retry-After") != "5" {
|
||||
t.Fatalf("Retry-After = %q, want 5", rec.Header().Get("Retry-After"))
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), tc.wantCode) || strings.Contains(rec.Body.String(), "must-not-be-served") {
|
||||
t.Fatalf("body = %q, want error code and no served bytes", rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type blockingEbookWriter struct {
|
||||
header http.Header
|
||||
reached chan struct{}
|
||||
release chan struct{}
|
||||
once sync.Once
|
||||
written int
|
||||
}
|
||||
|
||||
func (w *blockingEbookWriter) Header() http.Header { return w.header }
|
||||
func (w *blockingEbookWriter) WriteHeader(int) {}
|
||||
func (w *blockingEbookWriter) SetWriteDeadline(time.Time) error {
|
||||
return nil
|
||||
}
|
||||
func (w *blockingEbookWriter) Write(p []byte) (int, error) {
|
||||
w.written += len(p)
|
||||
if w.written >= (1<<20)+(64<<10) {
|
||||
w.once.Do(func() { close(w.reached) })
|
||||
<-w.release
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func TestEbookReaderTracksBytesWhilePourIsActiveAndFlushesBeforeRemoval(t *testing.T) {
|
||||
filePath := writePlaybackTestMediaFile(t, "large.epub")
|
||||
if err := os.WriteFile(filePath, make([]byte, 2<<20), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
registry := transfers.New()
|
||||
handler := NewEbookReaderHandler(&MediaFileAuthorizer{
|
||||
FileResolver: stubMediaFileResolver{file: &models.MediaFile{
|
||||
ID: 42, ContentID: "ebook-1", FilePath: filePath, Container: "epub", BaseType: "ebook",
|
||||
}},
|
||||
ItemAccess: stubItemAccessChecker{},
|
||||
})
|
||||
handler.Transfers = registry
|
||||
req := withEbookReaderRouteParams(newEbookReaderAuthRequest(http.MethodGet, "/ebooks/ebook-1/files/42/read"), "ebook-1", "42")
|
||||
writer := &blockingEbookWriter{
|
||||
header: make(http.Header),
|
||||
reached: make(chan struct{}),
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
defer close(done)
|
||||
handler.HandleReadFile(writer, req)
|
||||
}()
|
||||
|
||||
select {
|
||||
case <-writer.reached:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("ebook pour did not reach the blocking writer")
|
||||
}
|
||||
active := registry.Snapshot()
|
||||
if len(active) != 1 || active[0].Route != "ebook_read" || active[0].BytesServed == 0 {
|
||||
t.Fatalf("active transfer = %+v, want metered ebook_read pour", active)
|
||||
}
|
||||
close(writer.release)
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("ebook pour did not finish")
|
||||
}
|
||||
if got := registry.Snapshot(); len(got) != 0 {
|
||||
t.Fatalf("registry after completed pour = %+v, want empty", got)
|
||||
}
|
||||
}
|
||||
|
||||
type revokingEbookResolver struct {
|
||||
file *models.MediaFile
|
||||
revoke func()
|
||||
}
|
||||
|
||||
func (r revokingEbookResolver) GetByID(context.Context, int) (*models.MediaFile, error) {
|
||||
r.revoke()
|
||||
return r.file, nil
|
||||
}
|
||||
|
||||
func TestEbookReaderRefusesRevocationLandingBeforePour(t *testing.T) {
|
||||
filePath := writePlaybackTestMediaFile(t, "book.epub")
|
||||
store := streamrevoke.New(streamrevoke.Options{})
|
||||
handler := NewEbookReaderHandler(&MediaFileAuthorizer{
|
||||
FileResolver: revokingEbookResolver{
|
||||
file: &models.MediaFile{
|
||||
ID: 42, ContentID: "ebook-1", FilePath: filePath, Container: "epub", BaseType: "ebook",
|
||||
},
|
||||
revoke: func() {
|
||||
if err := store.RevokeUser(context.Background(), 1, "test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
},
|
||||
},
|
||||
ItemAccess: stubItemAccessChecker{},
|
||||
})
|
||||
handler.Revocation = store
|
||||
req := withEbookReaderRouteParams(newEbookReaderAuthRequest(http.MethodGet, "/ebooks/ebook-1/files/42/read"), "ebook-1", "42")
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
handler.HandleReadFile(rr, req)
|
||||
|
||||
if rr.Code != http.StatusForbidden {
|
||||
t.Fatalf("status = %d body = %q, want 403", rr.Code, rr.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestEbookReaderRevocationCutsInFlightRealSocket(t *testing.T) {
|
||||
filePath := writePlaybackTestMediaFile(t, "large-cut.epub")
|
||||
if err := os.Truncate(filePath, 128<<20); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store := streamrevoke.New(streamrevoke.Options{WatchInterval: 50 * time.Millisecond})
|
||||
handler := NewEbookReaderHandler(&MediaFileAuthorizer{
|
||||
FileResolver: stubMediaFileResolver{file: &models.MediaFile{
|
||||
ID: 42, ContentID: "ebook-1", FilePath: filePath, Container: "epub", BaseType: "ebook",
|
||||
}},
|
||||
ItemAccess: stubItemAccessChecker{},
|
||||
})
|
||||
handler.Revocation = store
|
||||
router := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
// A real access token always carries iat (auth.GenerateToken sets it), and
|
||||
// the cutoff must predate the revocation for a user kill to match it. With
|
||||
// no iat CredentialIssuedAt deliberately returns the zero time and the
|
||||
// cutoff fails open, so omitting it here would make this test vacuous.
|
||||
ctx := apimw.SetClaims(r.Context(), &auth.Claims{
|
||||
UserID: 1, Role: "user", TokenType: auth.TokenTypeAccess,
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
IssuedAt: jwt.NewNumericDate(time.Now().Add(-time.Hour)),
|
||||
},
|
||||
})
|
||||
ctx = apimw.SetProfileID(ctx, "profile-1")
|
||||
routeCtx := chi.NewRouteContext()
|
||||
routeCtx.URLParams.Add("content_id", "ebook-1")
|
||||
routeCtx.URLParams.Add("file_id", "42")
|
||||
ctx = context.WithValue(ctx, chi.RouteCtxKey, routeCtx)
|
||||
handler.HandleReadFile(w, r.WithContext(ctx))
|
||||
})
|
||||
server := httptest.NewServer(router)
|
||||
defer server.Close()
|
||||
|
||||
response, err := http.Get(server.URL + "/ebooks/ebook-1/files/42/read")
|
||||
if err != nil {
|
||||
if response != nil {
|
||||
_ = response.Body.Close()
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
if err := store.RevokeUser(context.Background(), 1, "test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, readErr := io.Copy(io.Discard, response.Body)
|
||||
readDone <- readErr
|
||||
}()
|
||||
select {
|
||||
case readErr := <-readDone:
|
||||
if readErr == nil {
|
||||
t.Fatal("client reached EOF; ebook revocation did not hang up the socket")
|
||||
}
|
||||
case <-time.After(8 * time.Second):
|
||||
t.Fatal("revocation did not cut the in-flight ebook pour")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEbookReaderServesContainerMimeForMismatchedExtension(t *testing.T) {
|
||||
filePath := writePlaybackTestMediaFile(t, "book.txt")
|
||||
if err := os.WriteFile(filePath, []byte("<html>not really</html>"), 0o644); err != nil {
|
||||
|
||||
+120
-36
@@ -15,6 +15,10 @@ import (
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/nodepool"
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/streammonitor"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
@@ -34,6 +38,12 @@ type NodeListEnabled interface {
|
||||
ListEnabled(ctx context.Context, nodeType string) ([]*nodepool.Node, error)
|
||||
}
|
||||
|
||||
// TransferSnapshotSource is the narrow process-local monitoring view needed by
|
||||
// the admin endpoint.
|
||||
type TransferSnapshotSource interface {
|
||||
Snapshot() []transfers.Transfer
|
||||
}
|
||||
|
||||
// NodeHandler handles CRUD operations and health checks for stream nodes.
|
||||
type NodeHandler struct {
|
||||
repo NodeRepository
|
||||
@@ -43,6 +53,25 @@ type NodeHandler struct {
|
||||
eventBus cache.EventBus
|
||||
redisClient *redis.Client // for reading session keys
|
||||
jwtSecret string // for bearer auth when calling force-reload on nodes
|
||||
|
||||
// sessionMgr + localNodeName let the session list include integrated
|
||||
// single-node streams (which never write Redis). Optional; nil in edge modes.
|
||||
sessionMgr *playback.SessionManager
|
||||
localNodeName string
|
||||
transfers TransferSnapshotSource
|
||||
}
|
||||
|
||||
// SetLocalSessionSource wires the in-process session manager so HandleListSessions
|
||||
// can union integrated streams with the Redis-backed edge records.
|
||||
func (h *NodeHandler) SetLocalSessionSource(sm *playback.SessionManager, nodeName string) {
|
||||
h.sessionMgr = sm
|
||||
h.localNodeName = nodeName
|
||||
}
|
||||
|
||||
// SetTransferSource wires process-local download monitoring into the unfiltered
|
||||
// admin response. Transfers do not belong to an edge node or live-stream source.
|
||||
func (h *NodeHandler) SetTransferSource(source TransferSnapshotSource) {
|
||||
h.transfers = source
|
||||
}
|
||||
|
||||
// NewNodeHandler creates a new NodeHandler.
|
||||
@@ -301,57 +330,112 @@ func (h *NodeHandler) HandleForceReloadNode(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
|
||||
// HandleListSessions handles GET /admin/nodes/sessions — lists active playback
|
||||
// sessions from Redis, optionally filtered by node_id query parameter.
|
||||
// sessions, unioning the Redis-backed edge records with the in-process
|
||||
// integrated sessions (which never write Redis) so a single-node deployment is
|
||||
// not blind. The union is deduped by session id (the same stream is tracked by
|
||||
// the central manager AND by the edge serving it — one row per stream, like the
|
||||
// enforcer's monitoring picture, so an operator never sees a single stream
|
||||
// counted twice against a cap). Optionally filtered by node_id: a filter
|
||||
// targets a specific edge node, so integrated sessions are only included in the
|
||||
// unfiltered listing.
|
||||
func (h *NodeHandler) HandleListSessions(w http.ResponseWriter, r *http.Request) {
|
||||
if h.redisClient == nil {
|
||||
writeError(w, http.StatusServiceUnavailable, "redis_unavailable", "Redis not configured")
|
||||
return
|
||||
}
|
||||
|
||||
ctx := r.Context()
|
||||
pattern := "silo:sessions:*"
|
||||
nodeFilter := r.URL.Query().Get("node_id")
|
||||
|
||||
if nodeIDStr := r.URL.Query().Get("node_id"); nodeIDStr != "" {
|
||||
nodeID, err := strconv.Atoi(nodeIDStr)
|
||||
if err == nil {
|
||||
node, err := h.repo.GetByID(ctx, nodeID)
|
||||
if err == nil {
|
||||
hashBytes := sha256.Sum256([]byte(node.URL))
|
||||
nodeHash := hex.EncodeToString(hashBytes[:4])
|
||||
pattern = "silo:sessions:" + nodeHash + ":*"
|
||||
var infos []nodesessions.SessionInfo
|
||||
|
||||
if h.redisClient != nil {
|
||||
pattern := nodesessions.KeyPrefix + "*"
|
||||
if nodeFilter != "" {
|
||||
if nodeID, err := strconv.Atoi(nodeFilter); err == nil {
|
||||
if node, err := h.repo.GetByID(ctx, nodeID); err == nil {
|
||||
hashBytes := sha256.Sum256([]byte(node.URL))
|
||||
nodeHash := hex.EncodeToString(hashBytes[:4])
|
||||
pattern = nodesessions.KeyPrefix + nodeHash + ":*"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var sessions []json.RawMessage
|
||||
var cursor uint64
|
||||
for {
|
||||
keys, next, err := h.redisClient.Scan(ctx, cursor, pattern, 100).Result()
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "redis_error", err.Error())
|
||||
return
|
||||
}
|
||||
for _, key := range keys {
|
||||
val, err := h.redisClient.Get(ctx, key).Result()
|
||||
var cursor uint64
|
||||
for {
|
||||
keys, next, err := h.redisClient.Scan(ctx, cursor, pattern, 100).Result()
|
||||
if err != nil {
|
||||
continue
|
||||
writeError(w, http.StatusInternalServerError, "redis_error", err.Error())
|
||||
return
|
||||
}
|
||||
for _, key := range keys {
|
||||
val, err := h.redisClient.Get(ctx, key).Result()
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var info nodesessions.SessionInfo
|
||||
if err := json.Unmarshal([]byte(val), &info); err != nil {
|
||||
slog.Debug("skip malformed session record", "key", key, "error", err)
|
||||
continue
|
||||
}
|
||||
infos = append(infos, info)
|
||||
}
|
||||
cursor = next
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
sessions = append(sessions, json.RawMessage(val))
|
||||
}
|
||||
cursor = next
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if sessions == nil {
|
||||
sessions = []json.RawMessage{}
|
||||
// Integrated single-node streams live only in the in-process session manager.
|
||||
// Include them in the unfiltered listing (a node_id filter targets an edge).
|
||||
if h.sessionMgr != nil && nodeFilter == "" {
|
||||
infos = append(infos, streammonitor.LiveLocalSessions(h.sessionMgr, h.localNodeName)...)
|
||||
}
|
||||
|
||||
sessions := []json.RawMessage{}
|
||||
for _, info := range streammonitor.DedupeSessionInfos(infos) {
|
||||
if data, err := json.Marshal(info); err == nil {
|
||||
sessions = append(sessions, json.RawMessage(data))
|
||||
}
|
||||
}
|
||||
|
||||
type sessionsResponse struct {
|
||||
Sessions []json.RawMessage `json:"sessions"`
|
||||
Sessions []json.RawMessage `json:"sessions"`
|
||||
Transfers []transfers.Transfer `json:"transfers"`
|
||||
}
|
||||
writeJSON(w, http.StatusOK, sessionsResponse{Sessions: sessions})
|
||||
activeTransfers := []transfers.Transfer{}
|
||||
if nodeFilter == "" && h.transfers != nil {
|
||||
activeTransfers = h.transfers.Snapshot()
|
||||
}
|
||||
writeJSON(w, http.StatusOK, sessionsResponse{Sessions: sessions, Transfers: activeTransfers})
|
||||
}
|
||||
|
||||
// nodeSessionsCapabilitiesResponse advertises the additive surface of
|
||||
// GET /admin/node-sessions so independently deployed clients (Android, Apple)
|
||||
// can feature-detect it instead of sniffing the server version.
|
||||
//
|
||||
// Schema support and runtime availability are deliberately separate booleans.
|
||||
// The transfers key is always present in the response shape once this endpoint
|
||||
// exists, but the process-local registry behind it is optional wiring — an edge
|
||||
// deployment can serve the field as an empty list forever. Collapsing the two
|
||||
// would advertise download monitoring that is not actually running.
|
||||
type nodeSessionsCapabilitiesResponse struct {
|
||||
// LogicalSessionID reports that session records may carry the stable logical
|
||||
// identity associated with a replaceable transport generation.
|
||||
LogicalSessionID bool `json:"logical_session_id"`
|
||||
// Transfers reports that the node-sessions payload carries a transfers array.
|
||||
Transfers bool `json:"transfers"`
|
||||
// TransfersActive reports that a transfer registry is wired on this server,
|
||||
// so the array reflects real in-flight download-class pours.
|
||||
TransfersActive bool `json:"transfers_active"`
|
||||
}
|
||||
|
||||
// HandleGetNodeSessionsCapabilities exposes additive feature support for the
|
||||
// live node-sessions payload (GET /admin/node-sessions/capabilities). It is
|
||||
// mounted alongside the endpoint it describes, so its presence tracks that
|
||||
// endpoint's availability rather than being advertised from an unrelated route.
|
||||
func (h *NodeHandler) HandleGetNodeSessionsCapabilities(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, nodeSessionsCapabilitiesResponse{
|
||||
LogicalSessionID: true,
|
||||
Transfers: true,
|
||||
TransfersActive: h.transfers != nil,
|
||||
})
|
||||
}
|
||||
|
||||
// reloadPools refreshes the in-memory proxy and transcode pools from the database.
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
const (
|
||||
testTransferID = "transfer-1"
|
||||
testProfileID = "profile-1"
|
||||
testLocalNode = "local"
|
||||
)
|
||||
|
||||
func TestHandleListSessionsAddsSiblingTransfersWithoutChangingSessionShape(t *testing.T) {
|
||||
sm := playback.NewSessionManager(0, 0)
|
||||
ctx := playback.WithClientInfo(context.Background(), playback.ClientInfo{Name: "native client"})
|
||||
session, err := sm.StartSessionWithContext(ctx, 42, testProfileID, 9, playback.PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSessionWithContext: %v", err)
|
||||
}
|
||||
registry := transfers.New()
|
||||
started := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
||||
if err := registry.Begin(transfers.Transfer{
|
||||
ID: testTransferID,
|
||||
UserID: 42,
|
||||
ProfileID: testProfileID,
|
||||
MediaFileID: 9,
|
||||
Route: "native_direct",
|
||||
StartedAt: started,
|
||||
}); err != nil {
|
||||
t.Fatalf("Begin: %v", err)
|
||||
}
|
||||
|
||||
h := NewNodeHandler(nil, nil, nil, nil, nil, nil, "")
|
||||
h.SetLocalSessionSource(sm, testLocalNode)
|
||||
h.SetTransferSource(registry)
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleListSessions(rec, httptest.NewRequest("GET", "/admin/nodes/sessions", nil))
|
||||
|
||||
var body struct {
|
||||
Sessions []json.RawMessage `json:"sessions"`
|
||||
Transfers []transfers.Transfer `json:"transfers"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if len(body.Sessions) != 1 {
|
||||
t.Fatalf("sessions = %s", body.Sessions)
|
||||
}
|
||||
// This session has never served a byte, so last_served_at is omitted
|
||||
// rather than reporting the client's last progress report as a serve time
|
||||
// (decision A5). The field is still emitted once real bytes are served —
|
||||
// asserted below.
|
||||
expectedSession, err := json.Marshal(nodesessions.SessionInfo{
|
||||
SessionID: session.ID,
|
||||
NodeName: testLocalNode,
|
||||
AuthUserID: session.UserID,
|
||||
ProfileID: session.ProfileID,
|
||||
Type: string(session.PlayMethod),
|
||||
Route: session.Origin(),
|
||||
MediaFileID: session.MediaFileID,
|
||||
ClientName: session.ClientName,
|
||||
StartedAt: session.StartedAt.UTC().Format(time.RFC3339),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal expected session: %v", err)
|
||||
}
|
||||
if !bytes.Equal(body.Sessions[0], expectedSession) {
|
||||
t.Fatalf("session JSON changed:\n got %s\nwant %s", body.Sessions[0], expectedSession)
|
||||
}
|
||||
if len(body.Transfers) != 1 || body.Transfers[0].ID != testTransferID {
|
||||
t.Fatalf("transfers = %+v", body.Transfers)
|
||||
}
|
||||
|
||||
// Once the server actually serves bytes, last_served_at reappears — the
|
||||
// field is omitted for want of a server-observed serve, not removed.
|
||||
if err := sm.BeginTransport(session.ID); err != nil {
|
||||
t.Fatalf("BeginTransport: %v", err)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
h.HandleListSessions(rec, httptest.NewRequest("GET", "/admin/nodes/sessions", nil))
|
||||
var served struct {
|
||||
Sessions []nodesessions.SessionInfo `json:"sessions"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &served); err != nil {
|
||||
t.Fatalf("decode served response: %v", err)
|
||||
}
|
||||
if len(served.Sessions) != 1 || served.Sessions[0].LastServedAt == "" {
|
||||
t.Fatalf("last_served_at missing after a served transport: %+v", served.Sessions)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleListSessionsNodeFilterExcludesLocalTransfers(t *testing.T) {
|
||||
registry := transfers.New()
|
||||
if err := registry.Begin(transfers.Transfer{ID: testTransferID}); err != nil {
|
||||
t.Fatalf("Begin: %v", err)
|
||||
}
|
||||
h := NewNodeHandler(nil, nil, nil, nil, nil, nil, "")
|
||||
h.SetTransferSource(registry)
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleListSessions(rec, httptest.NewRequest("GET", "/admin/nodes/sessions?node_id=12", nil))
|
||||
|
||||
var body struct {
|
||||
Transfers []transfers.Transfer `json:"transfers"`
|
||||
}
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if body.Transfers == nil || len(body.Transfers) != 0 {
|
||||
t.Fatalf("filtered transfers = %+v, want non-nil empty array", body.Transfers)
|
||||
}
|
||||
}
|
||||
|
||||
// Schema support and runtime availability are separate claims: the transfers key
|
||||
// is always in the payload once this endpoint exists, but the registry behind it
|
||||
// is optional wiring. Advertising them as one value would promise download
|
||||
// monitoring an edge deployment is not actually running.
|
||||
func TestNodeSessionsCapabilitiesSeparatesSchemaFromRuntimeWiring(t *testing.T) {
|
||||
t.Run("registry wired", func(t *testing.T) {
|
||||
h := NewNodeHandler(nil, nil, nil, nil, nil, nil, "")
|
||||
h.SetTransferSource(transfers.New())
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleGetNodeSessionsCapabilities(rec, httptest.NewRequest("GET", "/admin/node-sessions/capabilities", nil))
|
||||
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, want 200", rec.Code)
|
||||
}
|
||||
var body nodeSessionsCapabilitiesResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if !body.LogicalSessionID || !body.Transfers || !body.TransfersActive {
|
||||
t.Fatalf("capabilities = %+v, want schema flags and transfers_active true", body)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("no registry wired", func(t *testing.T) {
|
||||
h := NewNodeHandler(nil, nil, nil, nil, nil, nil, "")
|
||||
rec := httptest.NewRecorder()
|
||||
h.HandleGetNodeSessionsCapabilities(rec, httptest.NewRequest("GET", "/admin/node-sessions/capabilities", nil))
|
||||
|
||||
var body nodeSessionsCapabilitiesResponse
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode response: %v", err)
|
||||
}
|
||||
if !body.Transfers {
|
||||
t.Error("transfers = false; the response shape carries the key regardless of wiring")
|
||||
}
|
||||
if !body.LogicalSessionID {
|
||||
t.Error("logical_session_id = false; SessionInfo schema supports the field regardless of wiring")
|
||||
}
|
||||
if body.TransfersActive {
|
||||
t.Error("transfers_active = true with no registry wired; that advertises monitoring that is not running")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -612,7 +612,9 @@ func (h *PlaybackHandler) signSessionToken(card playback.RecipeCard) string {
|
||||
if h.JWTSecret == "" {
|
||||
return ""
|
||||
}
|
||||
token, err := streamtoken.Sign(card.ToClaims(), h.JWTSecret, playback.MaxTokenTTL)
|
||||
claims := card.ToClaims()
|
||||
claims.Origin = playback.OriginNative // native-only mint site
|
||||
token, err := streamtoken.Sign(claims, h.JWTSecret, playback.MaxTokenTTL)
|
||||
if err != nil {
|
||||
slog.Warn("sign stream token failed", "error", err, "session", card.SessionID, "playback_session_id", card.SessionID)
|
||||
return ""
|
||||
@@ -1952,6 +1954,8 @@ func (h *PlaybackHandler) handleStartPlaybackLegacy(w http.ResponseWriter, r *ht
|
||||
UserID: session.UserID,
|
||||
ProfileID: session.ProfileID,
|
||||
MediaFileID: session.MediaFileID,
|
||||
Origin: playback.OriginNative,
|
||||
ClientName: session.ClientName,
|
||||
}
|
||||
|
||||
// Resolve media path if possible.
|
||||
@@ -2585,6 +2589,7 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http.
|
||||
}
|
||||
nodeReq := transcodenode.TranscodeStartRequest{
|
||||
SessionID: restartTransportID,
|
||||
LogicalSessionID: updatedSession.ID,
|
||||
InputPath: file.FilePath,
|
||||
SourceVideoCodec: file.CodecVideo,
|
||||
SeekSeconds: restartSeekSeconds,
|
||||
@@ -2603,6 +2608,11 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http.
|
||||
SubtitleCodec: subtitleCodec,
|
||||
TotalDuration: float64(file.Duration),
|
||||
RequireReady: atomicLegacyReplacement,
|
||||
AuthUserID: updatedSession.UserID,
|
||||
ProfileID: updatedSession.ProfileID,
|
||||
MediaFileID: updatedSession.MediaFileID,
|
||||
Route: updatedSession.Origin(),
|
||||
ClientName: updatedSession.ClientName,
|
||||
}
|
||||
if strings.TrimSpace(nodeReq.HWAccel) == "" {
|
||||
nodeReq.HWAccel = h.playbackConfig().HWAccel
|
||||
@@ -2728,6 +2738,8 @@ func (h *PlaybackHandler) HandleChangeAudioTrack(w http.ResponseWriter, r *http.
|
||||
// mode; a re-minted token must not silently shed either.
|
||||
TranscodeTransportID: updatedSession.TranscodeTransportID,
|
||||
RemuxDVMode: string(updatedSession.RemuxDVMode),
|
||||
Origin: updatedSession.Origin(),
|
||||
ClientName: updatedSession.ClientName,
|
||||
}
|
||||
if plan.TranscodeNode != nil {
|
||||
tokenClaims.TranscodeNode = plan.TranscodeNode.URL
|
||||
@@ -3254,6 +3266,7 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
nodeReq := transcodenode.TranscodeStartRequest{
|
||||
SessionID: replacementTransportID,
|
||||
LogicalSessionID: session.ID,
|
||||
InputPath: file.FilePath,
|
||||
SourceVideoCodec: file.CodecVideo,
|
||||
VideoBitstreamFilter: videoBitstreamFilter,
|
||||
@@ -3273,6 +3286,11 @@ func (h *PlaybackHandler) HandleStartTranscode(w http.ResponseWriter, r *http.Re
|
||||
SubtitleCodec: subtitleCodec,
|
||||
TotalDuration: float64(file.Duration),
|
||||
RequireReady: atomicLegacyReplacement,
|
||||
AuthUserID: session.UserID,
|
||||
ProfileID: session.ProfileID,
|
||||
MediaFileID: session.MediaFileID,
|
||||
Route: session.Origin(),
|
||||
ClientName: session.ClientName,
|
||||
}
|
||||
|
||||
nodeResp, status, err := h.startRemotePlaybackTransport(context.Background(), tcNode.URL, nodeReq)
|
||||
@@ -3755,7 +3773,23 @@ func (h *PlaybackHandler) HandleGetTranscodeSegment(w http.ResponseWriter, r *ht
|
||||
|
||||
w.Header().Set("Cache-Control", "no-store, max-age=0")
|
||||
w.Header().Set("Pragma", "no-cache")
|
||||
http.ServeFile(w, r, segmentPath)
|
||||
// Hold a transport marker for the whole segment pour so a slow client draining
|
||||
// a single segment past the liveness grace cannot be reaped mid-serve and go
|
||||
// invisible to the monitor — matching the direct-play/remux handlers. This is
|
||||
// server-observed liveness: it keeps the session visible independent of any
|
||||
// client progress report (the hidden-stream defense).
|
||||
if err := h.sessionMgr.BeginTransport(sessionID); err == nil {
|
||||
defer func() { _ = h.sessionMgr.EndTransport(sessionID) }()
|
||||
} else {
|
||||
// The marker is what keeps a slow single-segment drain visible; the
|
||||
// segment is still served on failure, so log rather than fail — but log so
|
||||
// the hidden-stream window is observable.
|
||||
slog.Warn("begin transport marker failed", "session", sessionID, "segment", segmentPath, "error", err)
|
||||
}
|
||||
recorder, _ := h.sessionMgr.(playback.ServedBytesRecorder)
|
||||
metered := playback.NewSessionMeteredWriter(w, recorder, sessionID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
http.ServeFile(metered, r, segmentPath)
|
||||
}
|
||||
|
||||
// buildProxyManifestURL signs a stream token carrying the session's full
|
||||
@@ -3857,9 +3891,26 @@ func (h *PlaybackHandler) proxyToTranscodeNode(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
// Proxied transcode output can stream past the server's absolute
|
||||
// WriteTimeout; roll the write deadline with progress instead.
|
||||
sw := httpstream.NewRollingDeadlineWriter(w)
|
||||
dst := w
|
||||
var metered *playback.SessionMeteredWriter
|
||||
if !strings.HasSuffix(path, ".m3u8") {
|
||||
if err := h.sessionMgr.BeginTransport(sessionID); err == nil {
|
||||
defer func() { _ = h.sessionMgr.EndTransport(sessionID) }()
|
||||
}
|
||||
recorder, _ := h.sessionMgr.(playback.ServedBytesRecorder)
|
||||
metered = playback.NewSessionMeteredWriter(w, recorder, sessionID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
dst = metered
|
||||
}
|
||||
sw := httpstream.NewRollingDeadlineWriterCtx(r.Context(), dst)
|
||||
sw.WriteHeader(resp.StatusCode)
|
||||
io.Copy(sw, resp.Body)
|
||||
if _, copyErr := io.Copy(sw, resp.Body); copyErr != nil {
|
||||
slog.DebugContext(r.Context(), "copy transcode node response",
|
||||
"component", "api",
|
||||
"playback_session_id", sessionID,
|
||||
"error", copyErr,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// maybeStartThrottler reads throttle settings and starts the throttler if enabled.
|
||||
|
||||
@@ -1,17 +1,21 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -35,6 +39,32 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/watchsync"
|
||||
)
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
|
||||
return f(r)
|
||||
}
|
||||
|
||||
type recordingProxySessionManager struct {
|
||||
*playback.SessionManager
|
||||
events []string
|
||||
}
|
||||
|
||||
func (m *recordingProxySessionManager) BeginTransport(sessionID string) error {
|
||||
m.events = append(m.events, "begin")
|
||||
return m.SessionManager.BeginTransport(sessionID)
|
||||
}
|
||||
|
||||
func (m *recordingProxySessionManager) EndTransport(sessionID string) error {
|
||||
m.events = append(m.events, "end")
|
||||
return m.SessionManager.EndTransport(sessionID)
|
||||
}
|
||||
|
||||
func (m *recordingProxySessionManager) AddServedBytes(sessionID string, n int64) error {
|
||||
m.events = append(m.events, fmt.Sprintf("bytes:%d", n))
|
||||
return m.SessionManager.AddServedBytes(sessionID, n)
|
||||
}
|
||||
|
||||
type testUserStoreProvider struct {
|
||||
store userstore.UserStore
|
||||
}
|
||||
@@ -1981,6 +2011,96 @@ func TestHandleChangeAudioTrack_RemoteTranscodeNodeUnreachableSurfacesError(t *t
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyToTranscodeNodeMetersOnlyMediaBodiesAndFlushesBeforeTransportEnd(t *testing.T) {
|
||||
const body = "segment-bytes"
|
||||
previousTransport := http.DefaultClient.Transport
|
||||
http.DefaultClient.Transport = roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(bytes.NewBufferString(body)),
|
||||
}, nil
|
||||
})
|
||||
t.Cleanup(func() { http.DefaultClient.Transport = previousTransport })
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
path string
|
||||
wantBytes int64
|
||||
wantEvents []string
|
||||
}{
|
||||
{
|
||||
name: "segment",
|
||||
path: "/transcode/s/segment/part.m4s",
|
||||
wantBytes: int64(len(body)),
|
||||
wantEvents: []string{"begin", fmt.Sprintf("bytes:%d", len(body)), "end"},
|
||||
},
|
||||
{name: "playlist body", path: "/transcode/s/master.m3u8"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
manager := &recordingProxySessionManager{SessionManager: playback.NewSessionManager(0, 0)}
|
||||
session, err := manager.StartSession(1, "profile-1", 42, playback.PlayTranscode, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := NewPlaybackHandler(manager)
|
||||
handler.JWTSecret = "test-secret"
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/"+session.ID, nil)
|
||||
routeCtx := chi.NewRouteContext()
|
||||
routeCtx.URLParams.Add("session_id", session.ID)
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx))
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
handler.proxyToTranscodeNode(rec, req, "http://transcode.test", tc.path)
|
||||
|
||||
got, err := manager.GetSession(session.ID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.BytesServed != tc.wantBytes {
|
||||
t.Fatalf("bytes served = %d, want %d", got.BytesServed, tc.wantBytes)
|
||||
}
|
||||
if !slices.Equal(manager.events, tc.wantEvents) {
|
||||
t.Fatalf("events = %v, want %v", manager.events, tc.wantEvents)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type failingCopyBody struct{}
|
||||
|
||||
func (failingCopyBody) Read([]byte) (int, error) { return 0, errors.New("copy failed") }
|
||||
func (failingCopyBody) Close() error { return nil }
|
||||
|
||||
func TestProxyToTranscodeNodeLogsCopyFailure(t *testing.T) {
|
||||
previousTransport := http.DefaultClient.Transport
|
||||
http.DefaultClient.Transport = roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
return &http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Header: make(http.Header),
|
||||
Body: failingCopyBody{},
|
||||
}, nil
|
||||
})
|
||||
t.Cleanup(func() { http.DefaultClient.Transport = previousTransport })
|
||||
|
||||
var logs bytes.Buffer
|
||||
previousLogger := slog.Default()
|
||||
slog.SetDefault(slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelDebug})))
|
||||
t.Cleanup(func() { slog.SetDefault(previousLogger) })
|
||||
|
||||
handler := NewPlaybackHandler(failingSessionManager{})
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/session-1", nil)
|
||||
routeCtx := chi.NewRouteContext()
|
||||
routeCtx.URLParams.Add("session_id", "session-1")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, routeCtx))
|
||||
|
||||
handler.proxyToTranscodeNode(httptest.NewRecorder(), req, "http://transcode.test", "/transcode/session-1/segment/part.m4s")
|
||||
|
||||
if !strings.Contains(logs.String(), "copy transcode node response") || !strings.Contains(logs.String(), "copy failed") {
|
||||
t.Fatalf("copy failure log missing: %s", logs.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStartTranscode_LocalPathPropagatesSelectedAudioTrack(t *testing.T) {
|
||||
sessionMgr := playback.NewSessionManager(0, 0)
|
||||
filePath := writePlaybackTestMediaFile(t, "movie.mkv")
|
||||
|
||||
@@ -766,7 +766,33 @@ func (h *PlaybackHandler) prepareRemoteTransportV3(r *http.Request, session *pla
|
||||
videoCodec = "copy"
|
||||
}
|
||||
seekSeconds, startSegment := configureHLSTimelineV3(result.Plan, videoCodec, 2, float64(file.Duration))
|
||||
req := transcodenode.TranscodeStartRequest{SessionID: transportID, InputPath: file.FilePath, SourceVideoCodec: file.CodecVideo, VideoBitstreamFilter: videoBitstreamFilterForPlanV3(result.Plan), SeekSeconds: seekSeconds, StartSegmentNumber: startSegment, TargetResolution: result.TargetResolution, TargetCodecVideo: videoCodec, TargetCodecAudio: result.TargetAudioCodec, TargetAudioChannels: result.TargetAudioChannels, TargetBitrateKbps: result.TargetBitrateKbps, SegmentDuration: 2, HWAccel: h.playbackConfig().HWAccel, AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex), SubtitleTrackIndex: result.SubtitleTransportTrackIndex, SubtitleBurnIn: result.SubtitleBurnIn, SubtitleCodec: result.SubtitleCodec, TotalDuration: float64(file.Duration), RequireReady: true}
|
||||
req := transcodenode.TranscodeStartRequest{
|
||||
SessionID: transportID,
|
||||
LogicalSessionID: session.ID,
|
||||
InputPath: file.FilePath,
|
||||
SourceVideoCodec: file.CodecVideo,
|
||||
VideoBitstreamFilter: videoBitstreamFilterForPlanV3(result.Plan),
|
||||
SeekSeconds: seekSeconds,
|
||||
StartSegmentNumber: startSegment,
|
||||
TargetResolution: result.TargetResolution,
|
||||
TargetCodecVideo: videoCodec,
|
||||
TargetCodecAudio: result.TargetAudioCodec,
|
||||
TargetAudioChannels: result.TargetAudioChannels,
|
||||
TargetBitrateKbps: result.TargetBitrateKbps,
|
||||
SegmentDuration: 2,
|
||||
HWAccel: h.playbackConfig().HWAccel,
|
||||
AudioTrackIndex: plannedAudioTrackIndexV3(result, session.AudioTrackIndex),
|
||||
SubtitleTrackIndex: result.SubtitleTransportTrackIndex,
|
||||
SubtitleBurnIn: result.SubtitleBurnIn,
|
||||
SubtitleCodec: result.SubtitleCodec,
|
||||
TotalDuration: float64(file.Duration),
|
||||
RequireReady: true,
|
||||
AuthUserID: session.UserID,
|
||||
ProfileID: session.ProfileID,
|
||||
MediaFileID: file.ID,
|
||||
Route: session.Origin(),
|
||||
ClientName: session.ClientName,
|
||||
}
|
||||
nodeResp, status, err := h.startRemotePlaybackTransport(r.Context(), node.URL, req)
|
||||
if err != nil {
|
||||
// A timeout can fire after the node actually started the job; the
|
||||
|
||||
@@ -1156,6 +1156,14 @@ func TestPrepareTransportV3RequiresRemoteManifestReadiness(t *testing.T) {
|
||||
if !startRequest.RequireReady {
|
||||
t.Fatal("protocol-v3 remote start did not require manifest readiness")
|
||||
}
|
||||
file := v3HandlerFixtureFile(t)
|
||||
if startRequest.LogicalSessionID != "session-ready" ||
|
||||
startRequest.AuthUserID != 7 ||
|
||||
startRequest.ProfileID != "profile-1" ||
|
||||
startRequest.MediaFileID != file.ID ||
|
||||
startRequest.Route != playback.OriginNative {
|
||||
t.Fatalf("protocol-v3 remote monitoring attribution = %+v", startRequest)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleStartPlaybackUnknownProtocolUsesLegacyBranch(t *testing.T) {
|
||||
|
||||
@@ -143,7 +143,10 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) {
|
||||
_ = h.sessionMgr.EndTransport(sessionID)
|
||||
}()
|
||||
}
|
||||
if err := playback.ServeDirectPlay(w, r, file.FilePath); err != nil {
|
||||
recorder, _ := h.sessionMgr.(playback.ServedBytesRecorder)
|
||||
metered := playback.NewSessionMeteredWriter(w, recorder, sessionID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
if err := playback.ServeDirectPlay(metered, r, file.FilePath); err != nil {
|
||||
h.handleTransportStartFailure(r.Context(), session, file, err)
|
||||
}
|
||||
|
||||
@@ -153,13 +156,16 @@ func (h *StreamHandler) HandleStream(w http.ResponseWriter, r *http.Request) {
|
||||
_ = h.sessionMgr.EndTransport(sessionID)
|
||||
}()
|
||||
}
|
||||
recorder, _ := h.sessionMgr.(playback.ServedBytesRecorder)
|
||||
metered := playback.NewSessionMeteredWriter(w, recorder, sessionID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
seekSeconds := 0.0
|
||||
if seekStr := r.URL.Query().Get("seek"); seekStr != "" {
|
||||
if s, err := strconv.ParseFloat(seekStr, 64); err == nil && s >= 0 {
|
||||
seekSeconds = s
|
||||
}
|
||||
}
|
||||
if err := playback.ServeRemuxWithDVMode(w, r, file.FilePath, "mp4", seekSeconds, session.TranscodeAudio, session.AudioTrackIndex, file.PrimaryDVProfile(), session.RemuxDVMode, h.ffmpegPath()); err != nil {
|
||||
if err := playback.ServeRemuxWithDVMode(metered, r, file.FilePath, "mp4", seekSeconds, session.TranscodeAudio, session.AudioTrackIndex, file.PrimaryDVProfile(), session.RemuxDVMode, h.ffmpegPath()); err != nil {
|
||||
h.handleTransportStartFailure(r.Context(), session, file, err)
|
||||
}
|
||||
|
||||
@@ -218,6 +224,20 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusNotFound, "not_found", "Media file not found")
|
||||
return
|
||||
}
|
||||
beginTransport := func() (http.ResponseWriter, func()) {
|
||||
transportStarted := false
|
||||
if err := h.sessionMgr.BeginTransport(sessionID); err == nil {
|
||||
transportStarted = true
|
||||
}
|
||||
recorder, _ := h.sessionMgr.(playback.ServedBytesRecorder)
|
||||
metered := playback.NewSessionMeteredWriter(w, recorder, sessionID)
|
||||
return metered, func() {
|
||||
_ = metered.Close()
|
||||
if transportStarted {
|
||||
_ = h.sessionMgr.EndTransport(sessionID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
externalCount := len(file.ExternalSubtitles)
|
||||
if trackIndex < externalCount {
|
||||
@@ -231,7 +251,9 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
"Failed to load external subtitle")
|
||||
return
|
||||
}
|
||||
playback.ServeSubtitle(w, data, "ass")
|
||||
streamWriter, finish := beginTransport()
|
||||
defer finish()
|
||||
playback.ServeSubtitle(streamWriter, data, "ass")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -241,7 +263,9 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
"Failed to load external subtitle")
|
||||
return
|
||||
}
|
||||
playback.ServeSubtitle(w, vttData, "vtt")
|
||||
streamWriter, finish := beginTransport()
|
||||
defer finish()
|
||||
playback.ServeSubtitle(streamWriter, vttData, "vtt")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -264,7 +288,9 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
// demuxed, so the first byte lands within ~1s even on network
|
||||
// storage. Works identically for direct-play, remux, and
|
||||
// transcode because it doesn't depend on any other ffmpeg.
|
||||
h.streamEmbeddedSubtitle(w, r, file, embeddedIndex, session, requestedFormat)
|
||||
streamWriter, finish := beginTransport()
|
||||
defer finish()
|
||||
h.streamEmbeddedSubtitle(streamWriter, r, file, embeddedIndex, session, requestedFormat)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -296,13 +322,17 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Serve ASS/SSA downloaded subtitles as raw data.
|
||||
if playback.IsASS(string(dl.Format)) && requestedFormat != "vtt" {
|
||||
playback.ServeSubtitle(w, data, "ass")
|
||||
streamWriter, finish := beginTransport()
|
||||
defer finish()
|
||||
playback.ServeSubtitle(streamWriter, data, "ass")
|
||||
return
|
||||
}
|
||||
|
||||
// If the subtitle is already VTT, serve directly.
|
||||
if dl.Format == subtitles.FormatVTT {
|
||||
playback.ServeSubtitle(w, data, "vtt")
|
||||
streamWriter, finish := beginTransport()
|
||||
defer finish()
|
||||
playback.ServeSubtitle(streamWriter, data, "vtt")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -312,7 +342,9 @@ func (h *StreamHandler) HandleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusInternalServerError, "convert_error", "Failed to convert subtitle")
|
||||
return
|
||||
}
|
||||
playback.ServeSubtitle(w, vttData, "vtt")
|
||||
streamWriter, finish := beginTransport()
|
||||
defer finish()
|
||||
playback.ServeSubtitle(streamWriter, vttData, "vtt")
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -420,6 +452,14 @@ func (h *StreamHandler) HandleSubtitleFonts(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
if err := h.sessionMgr.BeginTransport(sessionID); err == nil {
|
||||
defer func() { _ = h.sessionMgr.EndTransport(sessionID) }()
|
||||
}
|
||||
recorder, _ := h.sessionMgr.(playback.ServedBytesRecorder)
|
||||
metered := playback.NewSessionMeteredWriter(w, recorder, sessionID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
w = metered
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
|
||||
@@ -5,8 +5,10 @@ package middleware
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/activitylog"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
@@ -258,6 +260,30 @@ func GetClaims(ctx context.Context) *auth.Claims {
|
||||
return claims
|
||||
}
|
||||
|
||||
// CredentialIssuedAt returns the authenticated access credential's iat. API
|
||||
// keys and legacy JWTs without iat deliberately return zero: streamrevoke's
|
||||
// documented fail-open contract means a user cutoff cannot cut pours owned by
|
||||
// those credentials. In particular, substituting time.Now would incorrectly
|
||||
// make every old API key appear newer than the cutoff.
|
||||
func CredentialIssuedAt(ctx context.Context) time.Time {
|
||||
claims := GetClaims(ctx)
|
||||
if claims == nil {
|
||||
slog.DebugContext(ctx, "stream credential has no claims; user revocation cutoff fails open")
|
||||
return time.Time{}
|
||||
}
|
||||
if claims.TokenType == auth.TokenTypeAPIKey {
|
||||
slog.DebugContext(ctx, "API-key stream credential has no issue time; user revocation cutoff fails open",
|
||||
"user_id", claims.UserID, "api_key_id", claims.APIKeyID)
|
||||
return time.Time{}
|
||||
}
|
||||
if claims.IssuedAt == nil {
|
||||
slog.DebugContext(ctx, "stream access credential has no iat; user revocation cutoff fails open",
|
||||
"user_id", claims.UserID, "session_id", claims.SessionID)
|
||||
return time.Time{}
|
||||
}
|
||||
return claims.IssuedAt.Time
|
||||
}
|
||||
|
||||
// IsAdmin reports whether the context's authenticated user account has the
|
||||
// admin role. Returns false when no claims are present. Note this is the
|
||||
// account-level role; it says nothing about which household profile is active.
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
package middleware
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
)
|
||||
|
||||
func TestCredentialIssuedAt(t *testing.T) {
|
||||
issuedAt := time.Now().Add(-time.Hour).Truncate(time.Second)
|
||||
access := &auth.Claims{
|
||||
UserID: 7, TokenType: auth.TokenTypeAccess,
|
||||
RegisteredClaims: jwt.RegisteredClaims{IssuedAt: jwt.NewNumericDate(issuedAt)},
|
||||
}
|
||||
if got := CredentialIssuedAt(SetClaims(context.Background(), access)); !got.Equal(issuedAt) {
|
||||
t.Fatalf("access credential time = %v, want %v", got, issuedAt)
|
||||
}
|
||||
|
||||
for name, claims := range map[string]*auth.Claims{
|
||||
"API key": {UserID: 7, TokenType: auth.TokenTypeAPIKey},
|
||||
"missing iat": {UserID: 7, TokenType: auth.TokenTypeAccess},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
if got := CredentialIssuedAt(SetClaims(context.Background(), claims)); !got.IsZero() {
|
||||
t.Fatalf("credential time = %v, want zero", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -83,8 +83,9 @@ func (w *statusWriter) Flush() {
|
||||
}
|
||||
}
|
||||
|
||||
// Unwrap lets http.ResponseController reach the underlying connection (e.g.
|
||||
// for the per-response write deadlines used by streaming handlers).
|
||||
// Unwrap exposes the wrapped ResponseWriter so http.ResponseController (used by
|
||||
// the stream kill switch's in-flight cut via SetWriteDeadline) can reach the
|
||||
// underlying socket instead of stopping at this wrapper and no-oping.
|
||||
func (w *statusWriter) Unwrap() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
|
||||
@@ -114,8 +114,9 @@ func (w *requestStatusWriter) Flush() {
|
||||
}
|
||||
}
|
||||
|
||||
// Unwrap lets http.ResponseController reach the underlying connection (e.g.
|
||||
// for the per-response write deadlines used by streaming handlers).
|
||||
// Unwrap exposes the wrapped ResponseWriter so http.ResponseController (used by
|
||||
// the stream kill switch's in-flight cut via SetWriteDeadline) can reach the
|
||||
// underlying socket instead of stopping at this wrapper and no-oping.
|
||||
func (w *requestStatusWriter) Unwrap() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
|
||||
+130
-6
@@ -36,6 +36,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/downloads"
|
||||
evt "github.com/Silo-Server/silo-server/internal/events"
|
||||
"github.com/Silo-Server/silo-server/internal/historyimport"
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
"github.com/Silo-Server/silo-server/internal/intromarkers"
|
||||
"github.com/Silo-Server/silo-server/internal/invitations"
|
||||
"github.com/Silo-Server/silo-server/internal/libraryingest"
|
||||
@@ -64,6 +65,8 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/scanqueue"
|
||||
"github.com/Silo-Server/silo-server/internal/secret"
|
||||
"github.com/Silo-Server/silo-server/internal/sections"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/Silo-Server/silo-server/internal/streamtoken"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
subtitleai "github.com/Silo-Server/silo-server/internal/subtitles/ai"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles/opensubtitles"
|
||||
@@ -71,6 +74,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles/subsource"
|
||||
"github.com/Silo-Server/silo-server/internal/taskmanager"
|
||||
"github.com/Silo-Server/silo-server/internal/taskmanager/repository"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
"github.com/Silo-Server/silo-server/internal/usercollections"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
"github.com/Silo-Server/silo-server/internal/watchstate"
|
||||
@@ -167,6 +171,10 @@ type Dependencies struct {
|
||||
ChapterThumbnailQueuer catalog.ChapterThumbnailQueuer
|
||||
PlaybackRealtimeHub *playback.RealtimeHub
|
||||
OnUserSessionsRevoked func(ctx context.Context, userID int)
|
||||
// RevocationStore is the stream kill switch (may be nil). Admin terminate
|
||||
// writes to it and integrated-mode serve paths consult it.
|
||||
RevocationStore *streamrevoke.Store
|
||||
TransferRegistry *transfers.Registry
|
||||
OnServerSettingUpdated func(ctx context.Context, key, value string)
|
||||
RequestServerRestart func(ctx context.Context) error
|
||||
ServerRestartStatus *handlers.ServerRestartStatusTracker
|
||||
@@ -680,6 +688,8 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
if conv := buildEbookConversion(deps, settingsRepo); conv != nil {
|
||||
ebookReaderHandler.Conversion = conv
|
||||
}
|
||||
ebookReaderHandler.Transfers = deps.TransferRegistry
|
||||
ebookReaderHandler.Revocation = deps.RevocationStore
|
||||
}
|
||||
catalogResourceHandler = handlers.NewCatalogResourceHandler(itemsHandler)
|
||||
catalogHandler = handlers.NewCatalogHandler(
|
||||
@@ -1002,6 +1012,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
playbackHandler.MarkerUpdateNotifier = playback.NewMarkerUpdateNotifier(deps.SessionMgr, realtimeHub)
|
||||
subtitleAINotifier = playback.NewSubtitleReadyNotifier(deps.SessionMgr, realtimeHub)
|
||||
adminPlaybackControlHandler = handlers.NewAdminPlaybackControlHandler(playbackHandler)
|
||||
adminPlaybackControlHandler.SetRevocationStore(deps.RevocationStore)
|
||||
|
||||
if deps.DB != nil && deps.FileRepo != nil && viewerResolver != nil && deps.Config != nil && detailSvc != nil {
|
||||
roomTokenService := watchtogether.NewRoomTokenService(deps.Config.Auth.JWTSecret, 24*time.Hour)
|
||||
@@ -1618,6 +1629,7 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
// devices sync on app open / background refresh; there is no server
|
||||
// background worker.
|
||||
downloadSvc.SetSubscriptions(downloads.NewSubscriptionRepository(deps.DB))
|
||||
downloadSvc.SetTransferRegistry(deps.TransferRegistry)
|
||||
downloadHandler = handlers.NewDownloadHandler(downloadSvc)
|
||||
if profileHandler != nil {
|
||||
// Profiles may live outside Postgres (sqlite userdb backend), so
|
||||
@@ -1628,6 +1640,12 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
} else {
|
||||
downloadHandler = handlers.NewDownloadHandler(nil)
|
||||
}
|
||||
// Kill switch for in-flight download pours: a per-user stream revocation
|
||||
// also hangs up a download mid-transfer (downloads stay exempt from the
|
||||
// live-stream cap; this only closes the "revoked user keeps pulling a
|
||||
// multi-GB file on a pre-revocation connection" hole).
|
||||
downloadHandler.SetRevocationStore(deps.RevocationStore)
|
||||
downloadHandler.SetTransferRegistry(deps.TransferRegistry)
|
||||
|
||||
var policyHandler *handlers.PolicyHandler
|
||||
if deps.PolicySystem != nil && deps.DB != nil {
|
||||
@@ -2480,8 +2498,8 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
// HLS transcode delivery — no profile auth needed;
|
||||
// session ID (UUID) serves as the access token, same
|
||||
// pattern as /stream/{session_id}.
|
||||
r.Get("/transcode/{session_id}/master.m3u8", playbackHandler.HandleGetTranscodeManifest)
|
||||
r.Get("/transcode/{session_id}/segment/{name}", playbackHandler.HandleGetTranscodeSegment)
|
||||
r.Get("/transcode/{session_id}/master.m3u8", guardRevocation(deps.RevocationStore, configJWTSecret(deps), playbackHandler.HandleGetTranscodeManifest))
|
||||
r.Get("/transcode/{session_id}/segment/{name}", guardRevocation(deps.RevocationStore, configJWTSecret(deps), playbackHandler.HandleGetTranscodeSegment))
|
||||
|
||||
// Playback realtime control socket — needs auth but not profile.
|
||||
r.Get("/sessions/{session_id}/control/ws", playbackHandler.HandleSessionWebSocket)
|
||||
@@ -2523,10 +2541,10 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
|
||||
// Stream routes.
|
||||
if streamHandler != nil {
|
||||
r.Get("/stream/{session_id}", streamHandler.HandleStream)
|
||||
r.Head("/stream/{session_id}", streamHandler.HandleStream)
|
||||
r.Get("/stream/{session_id}/subtitles/{track}", streamHandler.HandleSubtitle)
|
||||
r.Get("/stream/{session_id}/subtitles/{track}/fonts", streamHandler.HandleSubtitleFonts)
|
||||
r.Get("/stream/{session_id}", guardRevocationCut(deps.RevocationStore, configJWTSecret(deps), streamHandler.HandleStream))
|
||||
r.Head("/stream/{session_id}", guardRevocationCut(deps.RevocationStore, configJWTSecret(deps), streamHandler.HandleStream))
|
||||
r.Get("/stream/{session_id}/subtitles/{track}", guardRevocationCut(deps.RevocationStore, configJWTSecret(deps), streamHandler.HandleSubtitle))
|
||||
r.Get("/stream/{session_id}/subtitles/{track}/fonts", guardRevocationCut(deps.RevocationStore, configJWTSecret(deps), streamHandler.HandleSubtitleFonts))
|
||||
}
|
||||
|
||||
// Download routes.
|
||||
@@ -2928,6 +2946,8 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
jwtSecret = deps.Config.Auth.JWTSecret
|
||||
}
|
||||
nodeHandler := handlers.NewNodeHandler(deps.NodeRepo, deps.ProxyPool, deps.TranscodePool, deps.NodeRepo, deps.EventBus, deps.RedisClient, jwtSecret)
|
||||
nodeHandler.SetLocalSessionSource(deps.SessionMgr, deps.NodeID)
|
||||
nodeHandler.SetTransferSource(deps.TransferRegistry)
|
||||
r.Route("/nodes", func(r chi.Router) {
|
||||
r.Get("/", nodeHandler.HandleListNodes)
|
||||
r.Post("/", nodeHandler.HandleCreateNode)
|
||||
@@ -2940,6 +2960,9 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
// Live node sessions (reads from Redis)
|
||||
// Note: /admin/sessions is already used for playback sessions from PostgreSQL.
|
||||
r.Get("/node-sessions", nodeHandler.HandleListSessions)
|
||||
// Mounted beside the endpoint it describes so the advertisement
|
||||
// cannot outlive the route it advertises (both are gated on NodeRepo).
|
||||
r.Get("/node-sessions/capabilities", nodeHandler.HandleGetNodeSessionsCapabilities)
|
||||
}
|
||||
|
||||
// System inspection.
|
||||
@@ -3091,6 +3114,13 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
r.Post("/sessions/{session_id}/terminate", adminPlaybackControlHandler.HandleTerminateSession)
|
||||
r.Post("/sessions/{session_id}/message", adminPlaybackControlHandler.HandleMessageSession)
|
||||
}
|
||||
if deps.RevocationStore != nil {
|
||||
streamRevocationHandler := handlers.NewAdminStreamRevocationHandler(deps.RevocationStore)
|
||||
r.Get("/streams/revocations", streamRevocationHandler.HandleList)
|
||||
r.Get("/streams/revocations/capabilities", streamRevocationHandler.HandleGetCapabilities)
|
||||
r.Post("/streams/revocations", streamRevocationHandler.HandleCreate)
|
||||
r.Delete("/streams/revocations/{kind}/{id}", streamRevocationHandler.HandleDelete)
|
||||
}
|
||||
|
||||
if deps.TaskManager != nil {
|
||||
taskHistoryRepo := repository.NewPgExecutionRepository(deps.DB)
|
||||
@@ -3484,3 +3514,97 @@ func metadataAIConfigFromServer(cfg *config.Config) metadatatranslation.Config {
|
||||
OnView: cfg.MetadataAI.OnView,
|
||||
}
|
||||
}
|
||||
|
||||
// configJWTSecret returns the stream-token signing secret, or "" if unset.
|
||||
func configJWTSecret(deps Dependencies) string {
|
||||
if deps.Config != nil {
|
||||
return deps.Config.Auth.JWTSecret
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// guardRevocation wraps a native (integrated-mode) stream serve handler so the
|
||||
// server refuses a revoked session/user before serving. Session id comes from
|
||||
// the {session_id} URL param; user id (best-effort) from the ?st= stream token.
|
||||
// This is the integrated-mode analogue of the edge nodes' IsRevoked check —
|
||||
// without it a kill (admin terminate, over-cap, account revocation) would have
|
||||
// no teeth on a single-node deployment. It refuses new/next requests (stopping
|
||||
// HLS on its next segment and refusing reconnects); cutting an in-flight native
|
||||
// direct-play pour is a follow-up.
|
||||
var errStreamTokenSessionMismatch = errors.New("stream token session does not match route")
|
||||
|
||||
// streamRequestIdentity extracts the best-effort owner user id and the
|
||||
// credential-issue time for a native serve request. With a valid ?st= stream
|
||||
// token those are the token's uid and iat — the keys the user-kill cutoff in
|
||||
// streamrevoke compares (a token minted before a user revocation is killed; one
|
||||
// minted after re-auth plays). Without a token (legacy/bare URL or signing
|
||||
// disabled) these routes still run after API auth, so the authenticated user id
|
||||
// plus the access credential's iat apply. API keys and legacy JWTs without iat
|
||||
// use zero and therefore fail open for user cutoffs by explicit policy.
|
||||
func streamRequestIdentity(r *http.Request, secret, routeSessionID string) (int, time.Time, error) {
|
||||
if tok := r.URL.Query().Get("st"); tok != "" && secret != "" {
|
||||
if claims, err := streamtoken.Verify(tok, secret); err == nil {
|
||||
if claims.SessionID != routeSessionID {
|
||||
return 0, time.Time{}, errStreamTokenSessionMismatch
|
||||
}
|
||||
issuedAt := claims.IssuedTime()
|
||||
if issuedAt.IsZero() {
|
||||
slog.DebugContext(r.Context(), "stream token has no iat; user revocation cutoff fails open",
|
||||
"component", "api", "session_id", routeSessionID, "user_id", claims.UserID)
|
||||
}
|
||||
return claims.UserID, issuedAt, nil
|
||||
}
|
||||
}
|
||||
return apimw.GetUserID(r.Context()), apimw.CredentialIssuedAt(r.Context()), nil
|
||||
}
|
||||
|
||||
func rejectMismatchedStreamToken(w http.ResponseWriter, r *http.Request, err error) bool {
|
||||
if !errors.Is(err, errStreamTokenSessionMismatch) {
|
||||
return false
|
||||
}
|
||||
slog.WarnContext(r.Context(), "stream token session id does not match route",
|
||||
"component", "api",
|
||||
"route_session_id", chi.URLParam(r, "session_id"))
|
||||
http.Error(w, "stream token does not authorize this session", http.StatusForbidden)
|
||||
return true
|
||||
}
|
||||
|
||||
func guardRevocation(store *streamrevoke.Store, secret string, next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
sessionID := chi.URLParam(r, "session_id")
|
||||
userID, startedAt, err := streamRequestIdentity(r, secret, sessionID)
|
||||
if rejectMismatchedStreamToken(w, r, err) {
|
||||
return
|
||||
}
|
||||
if store != nil && store.Refuse(w, sessionID, userID, startedAt) {
|
||||
return
|
||||
}
|
||||
next(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
// guardRevocationCut is guardRevocation plus an in-flight connection cut, for the
|
||||
// long single-GET pours (native direct-play/remux on /stream). Transcode segment
|
||||
// routes use plain guardRevocation — per-segment refusal already stops HLS within
|
||||
// one segment, so a per-segment watcher goroutine would be wasteful.
|
||||
func guardRevocationCut(store *streamrevoke.Store, secret string, next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
sessionID := chi.URLParam(r, "session_id")
|
||||
userID, startedAt, err := streamRequestIdentity(r, secret, sessionID)
|
||||
if rejectMismatchedStreamToken(w, r, err) {
|
||||
return
|
||||
}
|
||||
if store == nil {
|
||||
next(w, r)
|
||||
return
|
||||
}
|
||||
latch := &httpstream.CutLatch{}
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), latch))
|
||||
if store.Refuse(w, sessionID, userID, startedAt) {
|
||||
return
|
||||
}
|
||||
stop := store.WatchAndCutContext(r.Context(), w, sessionID, userID, startedAt)
|
||||
defer stop()
|
||||
next(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
apimw "github.com/Silo-Server/silo-server/internal/api/middleware"
|
||||
"github.com/Silo-Server/silo-server/internal/auth"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/Silo-Server/silo-server/internal/streamtoken"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
)
|
||||
|
||||
func signStreamClaimsAt(
|
||||
t *testing.T,
|
||||
secret, sessionID string,
|
||||
userID int,
|
||||
issuedAt, expiresAt time.Time,
|
||||
) string {
|
||||
t.Helper()
|
||||
claims := streamtoken.Claims{
|
||||
SessionID: sessionID,
|
||||
UserID: userID,
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
ExpiresAt: jwt.NewNumericDate(expiresAt),
|
||||
},
|
||||
}
|
||||
if !issuedAt.IsZero() {
|
||||
claims.IssuedAt = jwt.NewNumericDate(issuedAt)
|
||||
}
|
||||
token := jwt.NewWithClaims(jwt.SigningMethodHS256, claims)
|
||||
signed, err := token.SignedString([]byte(secret))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return signed
|
||||
}
|
||||
|
||||
func mountedRevocationGuard(
|
||||
store *streamrevoke.Store,
|
||||
secret string,
|
||||
claims *auth.Claims,
|
||||
) http.Handler {
|
||||
router := chi.NewRouter()
|
||||
router.Use(func(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
next.ServeHTTP(w, r.WithContext(apimw.SetClaims(r.Context(), claims)))
|
||||
})
|
||||
})
|
||||
router.Get("/stream/{session_id}", guardRevocation(store, secret, func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
return router
|
||||
}
|
||||
|
||||
func TestMountedRevocationGuardUsesCredentialCutoff(t *testing.T) {
|
||||
const secret = "test-secret"
|
||||
const userID = 7
|
||||
store := streamrevoke.New(streamrevoke.Options{})
|
||||
beforeCutoff := time.Now().Add(-time.Hour)
|
||||
if err := store.RevokeUser(context.Background(), userID, "sessions_revoked"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterCutoff := time.Now().Add(time.Hour)
|
||||
handler := mountedRevocationGuard(store, secret, &auth.Claims{
|
||||
UserID: userID, TokenType: auth.TokenTypeAccess,
|
||||
RegisteredClaims: jwt.RegisteredClaims{IssuedAt: jwt.NewNumericDate(afterCutoff)},
|
||||
})
|
||||
|
||||
for name, tc := range map[string]struct {
|
||||
token string
|
||||
want int
|
||||
}{
|
||||
"credential before cutoff is refused": {
|
||||
token: signStreamClaimsAt(t, secret, "session-1", userID, beforeCutoff, time.Now().Add(time.Hour)),
|
||||
want: http.StatusForbidden,
|
||||
},
|
||||
"credential after cutoff plays": {
|
||||
token: signStreamClaimsAt(t, secret, "session-1", userID, afterCutoff, time.Now().Add(2*time.Hour)),
|
||||
want: http.StatusNoContent,
|
||||
},
|
||||
} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/session-1?st="+tc.token, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, req)
|
||||
if rec.Code != tc.want {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMountedRevocationGuardStreamTokenBinding(t *testing.T) {
|
||||
const secret = "test-secret"
|
||||
now := time.Now()
|
||||
accessClaims := &auth.Claims{
|
||||
UserID: 7, TokenType: auth.TokenTypeAccess,
|
||||
RegisteredClaims: jwt.RegisteredClaims{IssuedAt: jwt.NewNumericDate(now.Add(-time.Hour))},
|
||||
}
|
||||
handler := mountedRevocationGuard(streamrevoke.New(streamrevoke.Options{}), secret, accessClaims)
|
||||
matching := signStreamClaimsAt(t, secret, "session-1", 7, now, now.Add(time.Hour))
|
||||
mismatched := signStreamClaimsAt(t, secret, "other-session", 7, now, now.Add(time.Hour))
|
||||
badSignature := signStreamClaimsAt(t, "wrong-secret", "session-1", 7, now, now.Add(time.Hour))
|
||||
expired := signStreamClaimsAt(t, secret, "session-1", 7, now.Add(-2*time.Hour), now.Add(-time.Hour))
|
||||
missingIssuedAt := signStreamClaimsAt(t, secret, "session-1", 7, time.Time{}, now.Add(time.Hour))
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
query string
|
||||
wantStatus int
|
||||
}{
|
||||
{name: "matching", query: "?st=" + matching, wantStatus: http.StatusNoContent},
|
||||
{name: "absent", wantStatus: http.StatusNoContent},
|
||||
{name: "bad signature falls back to request auth", query: "?st=" + badSignature, wantStatus: http.StatusNoContent},
|
||||
{name: "expired falls back to request auth", query: "?st=" + expired, wantStatus: http.StatusNoContent},
|
||||
{name: "missing iat fails open", query: "?st=" + missingIssuedAt, wantStatus: http.StatusNoContent},
|
||||
{name: "mismatched session id", query: "?st=" + mismatched, wantStatus: http.StatusForbidden},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/session-1"+tc.query, nil)
|
||||
handler.ServeHTTP(rec, req)
|
||||
if rec.Code != tc.wantStatus {
|
||||
t.Fatalf("status = %d, want %d", rec.Code, tc.wantStatus)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMountedRevocationGuardAPIKeyFailsOpenForUserCutoff(t *testing.T) {
|
||||
store := streamrevoke.New(streamrevoke.Options{})
|
||||
if err := store.RevokeUser(context.Background(), 7, "sessions_revoked"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
handler := mountedRevocationGuard(store, "test-secret", &auth.Claims{
|
||||
UserID: 7, TokenType: auth.TokenTypeAPIKey, APIKeyID: 42,
|
||||
})
|
||||
rec := httptest.NewRecorder()
|
||||
handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/stream/session-1", nil))
|
||||
if rec.Code != http.StatusNoContent {
|
||||
t.Fatalf("API-key request status = %d, want %d (documented cutoff hole)", rec.Code, http.StatusNoContent)
|
||||
}
|
||||
}
|
||||
@@ -87,6 +87,14 @@ type statusRecorder struct {
|
||||
bytes int
|
||||
}
|
||||
|
||||
// Unwrap exposes the wrapped ResponseWriter to http.NewResponseController so
|
||||
// write-deadline control — the revocation in-flight cut — reaches the real
|
||||
// socket. Without this, SetWriteDeadline returns ErrNotSupported and a
|
||||
// RevokeUser cannot hang up an in-flight multi-GB audiobook pour.
|
||||
func (s *statusRecorder) Unwrap() http.ResponseWriter {
|
||||
return s.ResponseWriter
|
||||
}
|
||||
|
||||
func (s *statusRecorder) WriteHeader(code int) {
|
||||
s.status = code
|
||||
s.ResponseWriter.WriteHeader(code)
|
||||
|
||||
@@ -3,6 +3,9 @@ package abs
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
@@ -10,8 +13,11 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
const publicTrackSessionTTL = 24 * time.Hour
|
||||
@@ -52,12 +58,26 @@ func trackInoFor(contentID string, fileIdx int) string {
|
||||
// - Serve the bytes directly with Range-request support via playback.ServeDirectPlay.
|
||||
// - Set Content-Disposition: attachment on /download paths to encourage
|
||||
// browser save-to-disk / mobile offline-save behaviour.
|
||||
//
|
||||
// Both forms serve real playback bytes, but per A4 they are deliberately
|
||||
// exempt from the live-stream cap and absent from streammonitor. Each active
|
||||
// pour is still metered, registered in the process-local transfer registry,
|
||||
// and revocable before and during transfer.
|
||||
func (h *Handler) handleFileStream(w http.ResponseWriter, r *http.Request) {
|
||||
a, ok := absAuthFrom(r)
|
||||
if !ok || a.UserID == "" {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
userID, err := strconv.Atoi(a.UserID)
|
||||
if err != nil || userID <= 0 {
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
if h.deps.Revocation != nil && h.deps.Revocation.Refuse(w, "", userID, a.IssuedAt) {
|
||||
return
|
||||
}
|
||||
|
||||
contentID := chi.URLParam(r, "libraryItemId")
|
||||
inoStr := chi.URLParam(r, "ino")
|
||||
@@ -101,26 +121,74 @@ func (h *Handler) handleFileStream(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
mediaFile := files[fileIdx]
|
||||
|
||||
// /download variant: hint the client to save rather than stream.
|
||||
route := "abs_file_stream"
|
||||
if strings.HasSuffix(r.URL.Path, "/download") {
|
||||
route = "abs_file_download"
|
||||
}
|
||||
transfer := transfers.Transfer{
|
||||
ID: uuid.NewString(),
|
||||
UserID: userID,
|
||||
ProfileID: a.ProfileID,
|
||||
MediaFileID: mediaFile.ID,
|
||||
Route: route,
|
||||
ClientIP: requestClientIP(r),
|
||||
ClientName: r.UserAgent(),
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
if h.deps.Transfers != nil {
|
||||
if err := h.deps.Transfers.Begin(transfer); err != nil {
|
||||
slog.DebugContext(r.Context(), "ABS transfer rejected", "component", "audiobooks", "transfer_id", transfer.ID, "error", err)
|
||||
writeTransferBeginError(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
defer h.deps.Transfers.End(transfer.ID)
|
||||
|
||||
// Success-only response headers must follow transfer admission so a
|
||||
// rejection is not mislabeled as audio or an attachment.
|
||||
if strings.HasSuffix(r.URL.Path, "/download") {
|
||||
filename := filepath.Base(mediaFile.FilePath)
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="`+filename+`"`)
|
||||
}
|
||||
|
||||
// Set Content-Type for audio files. ServeDirectPlay uses MimeFromExtension
|
||||
// which covers video containers; we override with audio-specific MIME
|
||||
// types because ABS clients pattern-match on Content-Type.
|
||||
ext := strings.ToLower(filepath.Ext(mediaFile.FilePath))
|
||||
if ct := audioContentType(ext); ct != "" {
|
||||
w.Header().Set("Content-Type", ct)
|
||||
}
|
||||
|
||||
if err := playback.ServeDirectPlay(w, r, mediaFile.FilePath); err != nil {
|
||||
metered := playback.NewSessionMeteredWriter(w, h.deps.Transfers, transfer.ID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
|
||||
if h.deps.Revocation != nil {
|
||||
stop := h.deps.Revocation.WatchAndCutContext(r.Context(), metered, "", userID, a.IssuedAt)
|
||||
defer stop()
|
||||
}
|
||||
if err := playback.ServeDirectPlay(metered, r, mediaFile.FilePath); err != nil {
|
||||
// ServeDirectPlay has already written an error response; just log.
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func writeTransferBeginError(w http.ResponseWriter, err error) {
|
||||
status := http.StatusInternalServerError
|
||||
code := "internal_error"
|
||||
message := "Failed to monitor transfer"
|
||||
switch {
|
||||
case errors.Is(err, transfers.ErrUserTransferLimit):
|
||||
status = http.StatusTooManyRequests
|
||||
code = "transfer_limit_exceeded"
|
||||
message = "Concurrent transfer limit exceeded"
|
||||
w.Header().Set("Retry-After", "5")
|
||||
case errors.Is(err, transfers.ErrRegistryFull):
|
||||
status = http.StatusServiceUnavailable
|
||||
code = "monitoring_unavailable"
|
||||
message = "Transfer monitoring unavailable"
|
||||
w.Header().Set("Retry-After", "5")
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(status)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"code": code, "message": message})
|
||||
}
|
||||
|
||||
// handlePublicTrack serves audio bytes for ONE track of a playback session.
|
||||
//
|
||||
// Real ABS Android client (v2.22.0+) builds the streaming URL as
|
||||
@@ -172,6 +240,15 @@ func (h *Handler) handlePublicTrack(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "session expired", http.StatusGone)
|
||||
return
|
||||
}
|
||||
userID, err := strconv.Atoi(sess.UserID)
|
||||
if err != nil || userID <= 0 {
|
||||
http.Error(w, "session owner invalid", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
if h.deps.Revocation != nil && h.deps.Revocation.Refuse(w, sid, userID, sess.StartedAt) {
|
||||
return
|
||||
}
|
||||
if h.beginNativePlaybackTransport(sid) {
|
||||
defer h.endNativePlaybackTransport(sid)
|
||||
}
|
||||
@@ -196,7 +273,12 @@ func (h *Handler) handlePublicTrack(w http.ResponseWriter, r *http.Request) {
|
||||
if ct := audioContentType(ext); ct != "" {
|
||||
w.Header().Set("Content-Type", ct)
|
||||
}
|
||||
_ = playback.ServeDirectPlay(w, r, mediaFile.FilePath)
|
||||
recorder, _ := h.deps.NativeSessions.(playback.ServedBytesRecorder)
|
||||
metered := playback.NewSessionMeteredWriter(w, recorder, sid)
|
||||
defer func() { _ = metered.Close() }()
|
||||
stop := h.deps.Revocation.WatchAndCutContext(r.Context(), metered, sid, userID, sess.StartedAt)
|
||||
defer stop()
|
||||
_ = playback.ServeDirectPlay(metered, r, mediaFile.FilePath)
|
||||
}
|
||||
|
||||
func publicTrackSessionExpired(sess ABSPlaybackSession, now time.Time) bool {
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -13,6 +14,7 @@ import (
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
// fakePlaybackSessionStore is an in-memory ABSPlaybackSessionStore for the
|
||||
@@ -88,7 +90,7 @@ func newPublicTrackHandler(t *testing.T, sid, contentID string, closed bool) (*H
|
||||
t.Helper()
|
||||
audioPath := makeTempAudio(t)
|
||||
sessStore := &fakePlaybackSessionStore{}
|
||||
sess := ABSPlaybackSession{ID: sid, UserID: "u1", ContentID: contentID}
|
||||
sess := ABSPlaybackSession{ID: sid, UserID: "1", ContentID: contentID, StartedAt: time.Now()}
|
||||
if closed {
|
||||
now := time.Now()
|
||||
sess.ClosedAt = &now
|
||||
@@ -134,6 +136,81 @@ func TestHandlePublicTrack_ServesBytesForValidSession(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileTransferAdmissionFailsClosedBeforeSuccessHeaders(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
registry func(t *testing.T) *transfers.Registry
|
||||
wantStatus int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "per-user cap",
|
||||
registry: func(t *testing.T) *transfers.Registry {
|
||||
t.Helper()
|
||||
limit := 1
|
||||
r := transfers.NewWithOptions(transfers.Options{MaxPerUser: &limit})
|
||||
if err := r.Begin(transfers.Transfer{ID: "existing", UserID: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
},
|
||||
wantStatus: http.StatusTooManyRequests,
|
||||
wantCode: "transfer_limit_exceeded",
|
||||
},
|
||||
{
|
||||
name: "global cap",
|
||||
registry: func(t *testing.T) *transfers.Registry {
|
||||
t.Helper()
|
||||
unlimited := 0
|
||||
r := transfers.NewWithOptions(transfers.Options{MaxEntries: 1, MaxPerUser: &unlimited})
|
||||
if err := r.Begin(transfers.Transfer{ID: "existing", UserID: 2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
},
|
||||
wantStatus: http.StatusServiceUnavailable,
|
||||
wantCode: "monitoring_unavailable",
|
||||
},
|
||||
} {
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead, "range"} {
|
||||
t.Run(tc.name+"/"+method, func(t *testing.T) {
|
||||
h, _ := newPublicTrackHandler(t, "unused", "book-1", false)
|
||||
h.deps.Transfers = tc.registry(t)
|
||||
requestMethod := method
|
||||
if method == "range" {
|
||||
requestMethod = http.MethodGet
|
||||
}
|
||||
req := httptest.NewRequest(requestMethod, "/api/items/book-1/file/0/download", nil)
|
||||
if method == "range" {
|
||||
req.Header.Set("Range", "bytes=0-1")
|
||||
}
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("libraryItemId", "book-1")
|
||||
rctx.URLParams.Add("ino", "0")
|
||||
ctx := context.WithValue(req.Context(), chi.RouteCtxKey, rctx)
|
||||
ctx = context.WithValue(ctx, ctxKey{}, ctxAuth{UserID: "1", ProfileID: "profile-1"})
|
||||
rec := httptest.NewRecorder()
|
||||
h.handleFileStream(rec, req.WithContext(ctx))
|
||||
if rec.Code != tc.wantStatus {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, tc.wantStatus, rec.Body.String())
|
||||
}
|
||||
if rec.Header().Get("Retry-After") != "5" {
|
||||
t.Fatalf("Retry-After = %q, want 5", rec.Header().Get("Retry-After"))
|
||||
}
|
||||
if rec.Header().Get("Content-Disposition") != "" {
|
||||
t.Fatalf("rejection has Content-Disposition %q", rec.Header().Get("Content-Disposition"))
|
||||
}
|
||||
if rec.Header().Get("Content-Type") != "application/json" {
|
||||
t.Fatalf("Content-Type = %q, want application/json", rec.Header().Get("Content-Type"))
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), tc.wantCode) || strings.Contains(rec.Body.String(), "audio-bytes") {
|
||||
t.Fatalf("body = %q, want error code and no audio bytes", rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestHandlePublicTrack_HeadProbe covers the iOS/Android HEAD pre-flight
|
||||
// some players issue before the GET. http.ServeContent returns headers
|
||||
// without a body for HEAD; the handler must not 404.
|
||||
|
||||
@@ -21,6 +21,8 @@ import (
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/catalog"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -277,6 +279,11 @@ type Dependencies struct {
|
||||
// see Audiobookshelf-compatible clients. May be nil; ABS playback still
|
||||
// functions, but admin live-session visibility is unavailable.
|
||||
NativeSessions PlaybackSessionManager
|
||||
// Revocation makes ABS byte-serving routes obey the shared stream kill
|
||||
// switch. Nil preserves compatibility for isolated tests.
|
||||
Revocation *streamrevoke.Store
|
||||
// Transfers tracks file pours separately from native playback sessions.
|
||||
Transfers *transfers.Registry
|
||||
// NativeSessionSyncer flushes native session-manager state into the shared
|
||||
// admin live-session table after ABS play/sync/close events.
|
||||
NativeSessionSyncer PlaybackSessionSyncer
|
||||
@@ -569,6 +576,7 @@ type ctxAuth struct {
|
||||
ProfileID string
|
||||
JTI string
|
||||
Token string // raw bearer token
|
||||
IssuedAt time.Time
|
||||
}
|
||||
|
||||
// absAuthFrom extracts ABS auth from the request context. Returns (zero, false)
|
||||
@@ -682,11 +690,16 @@ func (h *Handler) bearerAuth(next http.Handler) http.Handler {
|
||||
return
|
||||
}
|
||||
_ = h.deps.TokenStore.TouchToken(r.Context(), claims.JTI)
|
||||
var issuedAt time.Time
|
||||
if claims.IssuedAt != nil {
|
||||
issuedAt = claims.IssuedAt.Time
|
||||
}
|
||||
ctx := context.WithValue(r.Context(), ctxKey{}, ctxAuth{
|
||||
UserID: claims.UserID,
|
||||
ProfileID: claims.ProfileID,
|
||||
JTI: claims.JTI,
|
||||
Token: raw,
|
||||
IssuedAt: issuedAt,
|
||||
})
|
||||
next.ServeHTTP(w, r.WithContext(ctx))
|
||||
})
|
||||
|
||||
@@ -0,0 +1,278 @@
|
||||
package abs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
)
|
||||
|
||||
var _ interface{ Unwrap() http.ResponseWriter } = (*statusRecorder)(nil)
|
||||
|
||||
func TestStatusRecorderLetsResponseControllerReachSocketDeadline(t *testing.T) {
|
||||
base := &deadlineResponseRecorder{ResponseRecorder: httptest.NewRecorder()}
|
||||
wrapped := &statusRecorder{ResponseWriter: base}
|
||||
if err := http.NewResponseController(wrapped).SetWriteDeadline(time.Now()); err != nil {
|
||||
t.Fatalf("SetWriteDeadline through statusRecorder: %v", err)
|
||||
}
|
||||
if base.deadlines != 1 {
|
||||
t.Fatalf("underlying deadline calls = %d, want 1", base.deadlines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMountedAccessLogMiddlewareLetsRevocationCutRealSocket(t *testing.T) {
|
||||
store := streamrevoke.New(streamrevoke.Options{WatchInterval: 50 * time.Millisecond})
|
||||
startedAt := time.Now()
|
||||
started := make(chan struct{})
|
||||
var startedOnce sync.Once
|
||||
h := &Handler{}
|
||||
router := chi.NewRouter()
|
||||
router.Use(h.accessLog)
|
||||
router.Get("/pour", func(w http.ResponseWriter, r *http.Request) {
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
stop := store.WatchAndCutContext(r.Context(), w, "abs-mounted", 1, startedAt)
|
||||
defer stop()
|
||||
sw := httpstream.NewRollingDeadlineWriterCtx(r.Context(), w)
|
||||
sw.WriteHeader(http.StatusOK)
|
||||
chunk := make([]byte, 32<<10)
|
||||
for {
|
||||
if _, err := sw.Write(chunk); err != nil {
|
||||
return
|
||||
}
|
||||
sw.Flush()
|
||||
startedOnce.Do(func() { close(started) })
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
})
|
||||
server := httptest.NewServer(router)
|
||||
defer server.Close()
|
||||
|
||||
response, err := http.Get(server.URL + "/pour")
|
||||
if err != nil {
|
||||
if response != nil {
|
||||
_ = response.Body.Close()
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer func() { _ = response.Body.Close() }()
|
||||
select {
|
||||
case <-started:
|
||||
case <-time.After(3 * time.Second):
|
||||
t.Fatal("mounted ABS pour did not start")
|
||||
}
|
||||
if err := store.RevokeSession(context.Background(), "abs-mounted", "test"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
readDone := make(chan error, 1)
|
||||
go func() {
|
||||
_, readErr := io.Copy(io.Discard, response.Body)
|
||||
readDone <- readErr
|
||||
}()
|
||||
select {
|
||||
case readErr := <-readDone:
|
||||
if readErr == nil {
|
||||
t.Fatal("client reached EOF; revocation did not hang up the socket")
|
||||
}
|
||||
case <-time.After(8 * time.Second):
|
||||
t.Fatal("revocation did not cut the mounted ABS pour")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBearerAuthCarriesJWTTimestampForUserCutoff(t *testing.T) {
|
||||
const accessTokenType = "access"
|
||||
secret := []byte("test-secret-32-bytes-aaaaaaaaaaaaa")
|
||||
tokens := newMemTokenStore()
|
||||
revocations := streamrevoke.New(streamrevoke.Options{})
|
||||
if err := revocations.RevokeUser(context.Background(), 7, "cutoff"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutoff := revocations.List()[0].RevokedAt
|
||||
h := New(Dependencies{
|
||||
MediaStore: noopMediaStore{},
|
||||
TokenStore: tokens,
|
||||
Config: &staticConfig{secret: secret},
|
||||
Revocation: revocations,
|
||||
})
|
||||
|
||||
issue := func(jti string, issuedAt time.Time) string {
|
||||
t.Helper()
|
||||
raw, err := issueJWT(secret, Claims{
|
||||
Type: accessTokenType,
|
||||
UserID: "7",
|
||||
JTI: jti,
|
||||
RegisteredClaims: jwt.RegisteredClaims{
|
||||
IssuedAt: jwt.NewNumericDate(issuedAt),
|
||||
ExpiresAt: jwt.NewNumericDate(time.Now().Add(time.Hour)),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := tokens.InsertToken(context.Background(), ABSToken{
|
||||
UserID: "7", Type: accessTokenType, JTI: jti, ExpiresAt: time.Now().Add(time.Hour),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return raw
|
||||
}
|
||||
|
||||
next := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
auth, ok := absAuthFrom(r)
|
||||
if !ok {
|
||||
t.Fatal("missing ctxAuth")
|
||||
}
|
||||
if revocations.Refuse(w, "", 7, auth.IssuedAt) {
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
})
|
||||
protected := h.bearerAuth(next)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
token string
|
||||
status int
|
||||
}{
|
||||
{name: "before cutoff refused", token: issue("old", cutoff.Add(-time.Hour)), status: http.StatusForbidden},
|
||||
{name: "after cutoff admitted", token: issue("new", cutoff.Add(time.Hour)), status: http.StatusNoContent},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+tc.token)
|
||||
rec := httptest.NewRecorder()
|
||||
protected.ServeHTTP(rec, req)
|
||||
if rec.Code != tc.status {
|
||||
t.Fatalf("status=%d body=%s, want %d", rec.Code, rec.Body.String(), tc.status)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicTrackUsesSessionStartedAtForCutoff(t *testing.T) {
|
||||
h, _ := newPublicTrackHandler(t, "sid-cutoff", "book-1", false)
|
||||
store := streamrevoke.New(streamrevoke.Options{})
|
||||
h.deps.Revocation = store
|
||||
if err := store.RevokeUser(context.Background(), 1, "cutoff"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rec := dispatchTrack(h, http.MethodGet, "sid-cutoff", "1")
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("status=%d body=%s, want 403", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
sessions, ok := h.deps.PlaybackSessionStore.(*fakePlaybackSessionStore)
|
||||
if !ok {
|
||||
t.Fatal("unexpected playback session store type")
|
||||
}
|
||||
session := sessions.sessions["sid-cutoff"]
|
||||
session.StartedAt = time.Now().Add(time.Hour)
|
||||
sessions.sessions["sid-cutoff"] = session
|
||||
rec = dispatchTrack(h, http.MethodGet, "sid-cutoff", "1")
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("post-cutoff session status=%d body=%s, want 200", rec.Code, rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
type revocationFeedFileMediaStore struct {
|
||||
noopMediaStore
|
||||
file *models.MediaFile
|
||||
onGet func()
|
||||
}
|
||||
|
||||
func (s *revocationFeedFileMediaStore) GetMediaFileByID(_ context.Context, id int) (*models.MediaFile, error) {
|
||||
if s.onGet != nil {
|
||||
s.onGet()
|
||||
}
|
||||
if s.file != nil && s.file.ID == id {
|
||||
return s.file, nil
|
||||
}
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
|
||||
type deadlineResponseRecorder struct {
|
||||
*httptest.ResponseRecorder
|
||||
deadlines int
|
||||
}
|
||||
|
||||
func (w *deadlineResponseRecorder) SetWriteDeadline(time.Time) error {
|
||||
w.deadlines++
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestPublicFeedFileUsesFeedCreatedAtForCutoff(t *testing.T) {
|
||||
const (
|
||||
feedSlug = "cutoff-feed"
|
||||
ino = 9
|
||||
)
|
||||
ctx := context.Background()
|
||||
feeds := newMemRSSFeedStore()
|
||||
feed := RSSFeed{
|
||||
ID: "feed-1",
|
||||
UserID: "1",
|
||||
LibraryItemID: "book-1",
|
||||
Slug: feedSlug,
|
||||
CreatedAt: time.Now().Add(-time.Hour),
|
||||
}
|
||||
feeds.rows[feed.ID] = feed
|
||||
revocations := streamrevoke.New(streamrevoke.Options{})
|
||||
if err := revocations.RevokeUser(ctx, 1, "cutoff"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cutoff := revocations.List()[0].RevokedAt
|
||||
mediaStore := &revocationFeedFileMediaStore{file: &models.MediaFile{
|
||||
ID: ino, ContentID: feed.LibraryItemID, FilePath: makeTempAudio(t),
|
||||
}}
|
||||
h := New(Dependencies{
|
||||
MediaStore: mediaStore,
|
||||
RSSFeedStore: feeds,
|
||||
Revocation: revocations,
|
||||
})
|
||||
|
||||
rec := dispatchABSWithParams(http.MethodGet, "/feed/"+feedSlug+"/file/9",
|
||||
map[string]string{"slug": feedSlug, "ino": "9"}, nil, "", "", h.handlePublicFeedFile)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Fatalf("pre-cutoff feed status=%d body=%s, want 403", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
feed.CreatedAt = cutoff.Add(time.Nanosecond)
|
||||
feeds.rows[feed.ID] = feed
|
||||
for !time.Now().After(feed.CreatedAt) {
|
||||
time.Sleep(time.Nanosecond)
|
||||
}
|
||||
rec = dispatchABSWithParams(http.MethodGet, "/feed/"+feedSlug+"/file/9",
|
||||
map[string]string{"slug": feedSlug, "ino": "9"}, nil, "", "", h.handlePublicFeedFile)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("post-cutoff feed status=%d body=%s, want 200", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
// Revoke again between the handler's initial Refuse check and WatchAndCut.
|
||||
// The deadline recorder makes the watcher observable without a real socket.
|
||||
mediaStore.onGet = func() {
|
||||
mediaStore.onGet = nil
|
||||
if err := revocations.RevokeUser(ctx, 1, "cut during pour"); err != nil {
|
||||
t.Fatalf("RevokeUser during pour: %v", err)
|
||||
}
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/feed/"+feedSlug+"/file/9", nil)
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("slug", feedSlug)
|
||||
rctx.URLParams.Add("ino", "9")
|
||||
req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
|
||||
deadlineRec := &deadlineResponseRecorder{ResponseRecorder: httptest.NewRecorder()}
|
||||
h.handlePublicFeedFile(deadlineRec, req)
|
||||
if deadlineRec.Code != http.StatusOK {
|
||||
t.Fatalf("watched feed status=%d body=%s, want 200", deadlineRec.Code, deadlineRec.Body.String())
|
||||
}
|
||||
if deadlineRec.deadlines != 1 {
|
||||
t.Fatalf("write deadlines=%d, want 1 from WatchAndCut", deadlineRec.deadlines)
|
||||
}
|
||||
}
|
||||
@@ -13,9 +13,13 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/google/uuid"
|
||||
"github.com/oklog/ulid/v2"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
var slugRe = regexp.MustCompile(`^[a-z0-9-]{4,64}$`)
|
||||
@@ -224,6 +228,15 @@ func (h *Handler) handlePublicFeedFile(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "feed get failed", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
userID, parseUserErr := strconv.Atoi(f.UserID)
|
||||
if parseUserErr != nil || userID <= 0 {
|
||||
http.Error(w, "feed not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
if h.deps.Revocation != nil && h.deps.Revocation.Refuse(w, "", userID, f.CreatedAt) {
|
||||
return
|
||||
}
|
||||
inoStr := chi.URLParam(r, "ino")
|
||||
ino, parseErr := strconv.Atoi(inoStr)
|
||||
if parseErr != nil {
|
||||
@@ -235,5 +248,30 @@ func (h *Handler) handlePublicFeedFile(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "file not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
http.ServeFile(w, r, mf.FilePath)
|
||||
transfer := transfers.Transfer{
|
||||
ID: uuid.NewString(),
|
||||
UserID: userID,
|
||||
ProfileID: f.ProfileID,
|
||||
MediaFileID: mf.ID,
|
||||
Route: "abs_feed",
|
||||
ClientIP: requestClientIP(r),
|
||||
ClientName: r.UserAgent(),
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
if h.deps.Transfers != nil {
|
||||
if err := h.deps.Transfers.Begin(transfer); err != nil {
|
||||
slog.DebugContext(r.Context(), "ABS feed transfer rejected", "component", "audiobooks", "transfer_id", transfer.ID, "error", err)
|
||||
writeTransferBeginError(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
defer h.deps.Transfers.End(transfer.ID)
|
||||
metered := playback.NewSessionMeteredWriter(w, h.deps.Transfers, transfer.ID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
|
||||
if h.deps.Revocation != nil {
|
||||
stop := h.deps.Revocation.WatchAndCutContext(r.Context(), metered, "", userID, f.CreatedAt)
|
||||
defer stop()
|
||||
}
|
||||
http.ServeFile(metered, r, mf.FilePath)
|
||||
}
|
||||
|
||||
@@ -5,13 +5,17 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
type memRSSFeedStore struct {
|
||||
@@ -219,3 +223,79 @@ func TestPublicFeed_HappyPath_XML(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFeedFileTransferAdmissionFailsClosed(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
registry func(t *testing.T) *transfers.Registry
|
||||
wantStatus int
|
||||
wantCode string
|
||||
}{
|
||||
{
|
||||
name: "per-user cap",
|
||||
registry: func(t *testing.T) *transfers.Registry {
|
||||
t.Helper()
|
||||
limit := 1
|
||||
r := transfers.NewWithOptions(transfers.Options{MaxPerUser: &limit})
|
||||
if err := r.Begin(transfers.Transfer{ID: "existing", UserID: 1}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
},
|
||||
wantStatus: http.StatusTooManyRequests,
|
||||
wantCode: "transfer_limit_exceeded",
|
||||
},
|
||||
{
|
||||
name: "global cap",
|
||||
registry: func(t *testing.T) *transfers.Registry {
|
||||
t.Helper()
|
||||
unlimited := 0
|
||||
r := transfers.NewWithOptions(transfers.Options{MaxEntries: 1, MaxPerUser: &unlimited})
|
||||
if err := r.Begin(transfers.Transfer{ID: "existing", UserID: 2}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return r
|
||||
},
|
||||
wantStatus: http.StatusServiceUnavailable,
|
||||
wantCode: "monitoring_unavailable",
|
||||
},
|
||||
} {
|
||||
for _, method := range []string{http.MethodGet, http.MethodHead, "range"} {
|
||||
t.Run(tc.name+"/"+method, func(t *testing.T) {
|
||||
store := newMemRSSFeedStore()
|
||||
store.rows["feed-id"] = RSSFeed{
|
||||
ID: "feed-id", UserID: "1", ProfileID: "profile-1", LibraryItemID: "book-1", Slug: "feed", CreatedAt: time.Now(),
|
||||
}
|
||||
h := New(Dependencies{
|
||||
RSSFeedStore: store,
|
||||
MediaStore: &revocationFeedFileMediaStore{file: &models.MediaFile{
|
||||
ID: 9, ContentID: "book-1", FilePath: "must-not-be-served.mp3",
|
||||
}},
|
||||
Transfers: tc.registry(t),
|
||||
})
|
||||
requestMethod := method
|
||||
if method == "range" {
|
||||
requestMethod = http.MethodGet
|
||||
}
|
||||
req := httptest.NewRequest(requestMethod, "/feed/feed/file/9", nil)
|
||||
if method == "range" {
|
||||
req.Header.Set("Range", "bytes=0-1")
|
||||
}
|
||||
rctx := chi.NewRouteContext()
|
||||
rctx.URLParams.Add("slug", "feed")
|
||||
rctx.URLParams.Add("ino", "9")
|
||||
rec := httptest.NewRecorder()
|
||||
h.handlePublicFeedFile(rec, req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx)))
|
||||
if rec.Code != tc.wantStatus {
|
||||
t.Fatalf("status = %d, want %d; body=%s", rec.Code, tc.wantStatus, rec.Body.String())
|
||||
}
|
||||
if rec.Header().Get("Retry-After") != "5" {
|
||||
t.Fatalf("Retry-After = %q, want 5", rec.Header().Get("Retry-After"))
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), tc.wantCode) || strings.Contains(rec.Body.String(), "must-not-be-served") {
|
||||
t.Fatalf("body = %q, want error code and no file bytes", rec.Body.String())
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -12,6 +12,8 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/recommendations"
|
||||
"github.com/Silo-Server/silo-server/internal/scanner"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
@@ -60,6 +62,8 @@ type ABSHandlerDeps struct {
|
||||
Detail *catalog.DetailService
|
||||
SessionMgr *playback.SessionManager
|
||||
SessionSyncer abs.PlaybackSessionSyncer
|
||||
Revocation *streamrevoke.Store
|
||||
Transfers *transfers.Registry
|
||||
}
|
||||
|
||||
// absAuthAdapter is the narrow slice of internal/auth that BuildABSHandler
|
||||
@@ -174,6 +178,8 @@ func (s *Service) BuildABSHandler(deps ABSHandlerDeps) *abs.Handler {
|
||||
SocketIO: socketServer,
|
||||
NativeSessions: deps.SessionMgr,
|
||||
NativeSessionSyncer: deps.SessionSyncer,
|
||||
Revocation: deps.Revocation,
|
||||
Transfers: deps.Transfers,
|
||||
CoverResolver: func(ctx context.Context, path, variant string) string {
|
||||
if deps.Detail == nil {
|
||||
return ""
|
||||
|
||||
@@ -8,7 +8,9 @@ import (
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/streamtoken"
|
||||
redisv9 "github.com/redis/go-redis/v9"
|
||||
"github.com/robfig/cron/v3"
|
||||
)
|
||||
@@ -57,6 +59,8 @@ var adminSettingDefaults = map[string]string{
|
||||
"playback.chapter_thumbnail_execution": "local",
|
||||
"playback.chapter_thumbnail_node_capacity": "1",
|
||||
"playback.chapter_thumbnail_hdr_policy": "best_effort",
|
||||
"playback.over_cap_revocation_ttl": streamtoken.MaxTTL.String(),
|
||||
"playback.max_user_concurrent_transfers": "24",
|
||||
"playback.watched_threshold": "90",
|
||||
"playback.min_resume_threshold": "5",
|
||||
"allow_4k_transcode": "false",
|
||||
@@ -358,6 +362,10 @@ func NormalizeAdminSetting(key, raw string) (string, error) {
|
||||
"download.period_duration", "jellyfin_compat.session_ttl",
|
||||
"jellyfin_compat.playback_session_ttl":
|
||||
return normalizeAdminDuration(key, value)
|
||||
case "playback.over_cap_revocation_ttl":
|
||||
return normalizeAdminDurationRange(key, value, 5*time.Minute, streamtoken.MaxTTL)
|
||||
case "playback.max_user_concurrent_transfers":
|
||||
return normalizeAdminInt(key, value, 0, 10_000)
|
||||
|
||||
case "server.log_level":
|
||||
return normalizeAdminEnum(key, value, "debug", "info", "warn", "error")
|
||||
@@ -583,6 +591,14 @@ func normalizeAdminDuration(key, value string) (string, error) {
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func normalizeAdminDurationRange(key, value string, minValue, maxValue time.Duration) (string, error) {
|
||||
parsed, err := parseDuration(value)
|
||||
if err != nil || parsed < minValue || parsed > maxValue {
|
||||
return "", fmt.Errorf("%s must be a duration between %s and %s", key, minValue, maxValue)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func normalizeAdminEnum(key, value string, allowed ...string) (string, error) {
|
||||
normalized := strings.ToLower(value)
|
||||
for _, candidate := range allowed {
|
||||
|
||||
@@ -210,6 +210,10 @@ func TestNormalizeAdminSettingRejectsInvalidValues(t *testing.T) {
|
||||
{key: "database.max_connections", value: "0"},
|
||||
{key: "metadata.cache_images", value: "maybe"},
|
||||
{key: "auth.access_token_expiry", value: "forever"},
|
||||
{key: "playback.over_cap_revocation_ttl", value: "4m59s"},
|
||||
{key: "playback.over_cap_revocation_ttl", value: "24h1s"},
|
||||
{key: "playback.max_user_concurrent_transfers", value: "-1"},
|
||||
{key: "playback.max_user_concurrent_transfers", value: "10001"},
|
||||
{key: "recommendations.embeddings_cron", value: "not a cron"},
|
||||
{key: "notifications.server_channels.batch_seconds", value: "119"},
|
||||
{key: "catalog.search.meilisearch.semantic_ratio", value: "1.2"},
|
||||
@@ -227,6 +231,24 @@ func TestNormalizeAdminSettingRejectsInvalidValues(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverCapRevocationTTLDefaultAndValidation(t *testing.T) {
|
||||
effective := EffectiveAdminSettings(nil)
|
||||
if got := effective["playback.over_cap_revocation_ttl"]; got != "24h0m0s" {
|
||||
t.Fatalf("default over-cap revocation TTL = %q, want 24h0m0s", got)
|
||||
}
|
||||
if got, err := NormalizeAdminSetting("playback.over_cap_revocation_ttl", "6h"); err != nil || got != "6h" {
|
||||
t.Fatalf("valid over-cap revocation TTL = %q, %v", got, err)
|
||||
}
|
||||
if got := effective["playback.max_user_concurrent_transfers"]; got != "24" {
|
||||
t.Fatalf("default max user concurrent transfers = %q, want 24", got)
|
||||
}
|
||||
for _, value := range []string{"0", "48", "10000"} {
|
||||
if got, err := NormalizeAdminSetting("playback.max_user_concurrent_transfers", value); err != nil || got != value {
|
||||
t.Fatalf("max user concurrent transfers %q = %q, %v", value, got, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeAdminSettingAcceptsApprovedThemeCatalogURL(t *testing.T) {
|
||||
got, err := NormalizeAdminSetting(
|
||||
"theme.catalog_url",
|
||||
|
||||
@@ -36,10 +36,12 @@ var restartRequiredKeys = map[string]bool{
|
||||
// thumbnails, audiobook enricher) — keep restart-required until those
|
||||
// convert. transcode_dir is fully live (only the playback handler reads
|
||||
// it). The chapter-thumbnail worker pool is sized at construction.
|
||||
"playback.ffmpeg_path": true,
|
||||
"playback.hw_accel": true,
|
||||
"playback.hw_device": true,
|
||||
"playback.chapter_thumbnail_workers": true,
|
||||
"playback.ffmpeg_path": true,
|
||||
"playback.hw_accel": true,
|
||||
"playback.hw_device": true,
|
||||
"playback.chapter_thumbnail_workers": true,
|
||||
"playback.over_cap_revocation_ttl": true,
|
||||
"playback.max_user_concurrent_transfers": true,
|
||||
|
||||
// Scanner / matcher toggles captured at construction. Worker counts,
|
||||
// batch size, metadata.cache_images, and mdblist.api_key hot-reload via
|
||||
|
||||
@@ -11,6 +11,8 @@ func TestRestartRequired(t *testing.T) {
|
||||
{"server.listen", true},
|
||||
{"auth.jwt_secret", true},
|
||||
{"ratelimit.backend", true},
|
||||
{"playback.over_cap_revocation_ttl", true},
|
||||
{"playback.max_user_concurrent_transfers", true},
|
||||
// Prefix-covered namespaces.
|
||||
{"database.max_connections", true},
|
||||
{"userdb.backend", true},
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/idgen"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
)
|
||||
|
||||
// FileResolver looks up media files by various keys.
|
||||
@@ -90,6 +91,7 @@ type Service struct {
|
||||
groupProvider access.GroupPolicyProvider
|
||||
itemAccess ItemAccessChecker
|
||||
settings SettingsReader
|
||||
transfers *transfers.Registry
|
||||
|
||||
// Offline-manifest dependencies (Phase 2); nil until SetOfflineDeps wires them.
|
||||
manifest *ManifestBuilder
|
||||
@@ -142,6 +144,11 @@ func (s *Service) SetSubscriptions(subRepo *SubscriptionRepository) {
|
||||
s.subRepo = subRepo
|
||||
}
|
||||
|
||||
// SetTransferRegistry wires process-local monitoring for active file pours.
|
||||
func (s *Service) SetTransferRegistry(registry *transfers.Registry) {
|
||||
s.transfers = registry
|
||||
}
|
||||
|
||||
// SetGroupPolicyProvider wires access-group policy composition into download
|
||||
// checks. Nil keeps legacy user-row behavior.
|
||||
func (s *Service) SetGroupPolicyProvider(provider access.GroupPolicyProvider) {
|
||||
@@ -846,7 +853,7 @@ func (s *Service) List(ctx context.Context, userID int, profileID, deviceID stri
|
||||
|
||||
// ServeDirect validates permissions and serves a file directly for browser
|
||||
// download. No persistent download record is created.
|
||||
func (s *Service) ServeDirect(ctx context.Context, w http.ResponseWriter, r *http.Request, userID, fileID int, format string, filter catalog.AccessFilter) error {
|
||||
func (s *Service) ServeDirect(ctx context.Context, w http.ResponseWriter, r *http.Request, userID, fileID int, format string, filter catalog.AccessFilter, transfer transfers.Transfer) error {
|
||||
if _, _, err := s.downloadConfigForUser(ctx, userID, ""); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -866,6 +873,8 @@ func (s *Service) ServeDirect(ctx context.Context, w http.ResponseWriter, r *htt
|
||||
if !catalog.FileAllowedByAccess(file, filter) {
|
||||
return catalog.ErrItemNotFound
|
||||
}
|
||||
transfer.MediaFileID = file.ID
|
||||
s.annotateTransfer(transfer)
|
||||
return s.serveLocalFile(ctx, w, r, file.FilePath, userID)
|
||||
}
|
||||
|
||||
@@ -873,14 +882,14 @@ func (s *Service) ServeDirect(ctx context.Context, w http.ResponseWriter, r *htt
|
||||
// authorize on (user, profile, device) and re-check per-profile content access
|
||||
// before serving; ephemeral rows keep today's queued→downloading→completed
|
||||
// behavior. The ephemeral path never serves a managed row, and vice versa.
|
||||
func (s *Service) ServeFile(ctx context.Context, w http.ResponseWriter, r *http.Request, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter) error {
|
||||
func (s *Service) ServeFile(ctx context.Context, w http.ResponseWriter, r *http.Request, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter, transfer transfers.Transfer) error {
|
||||
// Re-check policy — admin may have disabled downloads or revoked permission.
|
||||
if _, _, err := s.downloadConfigForUser(ctx, userID, deviceID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if deviceID != "" {
|
||||
return s.serveManaged(ctx, w, r, userID, profileID, deviceID, downloadID, filter)
|
||||
return s.serveManaged(ctx, w, r, userID, profileID, deviceID, downloadID, filter, transfer)
|
||||
}
|
||||
|
||||
dl, err := s.repo.GetByID(ctx, downloadID)
|
||||
@@ -909,7 +918,7 @@ func (s *Service) ServeFile(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
}
|
||||
}
|
||||
|
||||
if err := s.serveDownloadBytes(ctx, w, r, dl, userID, filter); err != nil {
|
||||
if err := s.serveDownloadBytes(ctx, w, r, dl, userID, filter, transfer); err != nil {
|
||||
if dl.Format == FormatOriginal {
|
||||
if updateErr := s.repo.UpdateStatus(ctx, dl.ID, StatusFailed, 0, nil); updateErr != nil {
|
||||
slog.ErrorContext(ctx, "failed to mark download as failed", "component", "downloads", "download_id", dl.ID, "error", updateErr)
|
||||
@@ -927,7 +936,7 @@ func (s *Service) ServeFile(ctx context.Context, w http.ResponseWriter, r *http.
|
||||
|
||||
// serveManaged authorizes a managed entry on (user, profile, device), re-checks
|
||||
// per-profile content access (invariant 2), and streams the original source.
|
||||
func (s *Service) serveManaged(ctx context.Context, w http.ResponseWriter, r *http.Request, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter) error {
|
||||
func (s *Service) serveManaged(ctx context.Context, w http.ResponseWriter, r *http.Request, userID int, profileID, deviceID, downloadID string, filter catalog.AccessFilter, transfer transfers.Transfer) error {
|
||||
if profileID == "" {
|
||||
return ErrProfileRequired
|
||||
}
|
||||
@@ -943,7 +952,7 @@ func (s *Service) serveManaged(ctx context.Context, w http.ResponseWriter, r *ht
|
||||
if err := s.itemAccess.EnsureAccessible(ctx, dl.ContentID, filter); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.serveDownloadBytes(ctx, w, r, dl, userID, filter)
|
||||
return s.serveDownloadBytes(ctx, w, r, dl, userID, filter, transfer)
|
||||
}
|
||||
|
||||
// PatchStatus lets a client confirm a managed entry's local state
|
||||
@@ -1044,7 +1053,7 @@ func (s *Service) resolveFile(ctx context.Context, req CreateRequest) (*models.M
|
||||
// original rows. Both paths mirror playback's per-file authorization
|
||||
// (catalog.FileAllowedByAccess): library scope and the profile's max playback
|
||||
// quality can change after a row was registered.
|
||||
func (s *Service) serveDownloadBytes(ctx context.Context, w http.ResponseWriter, r *http.Request, dl *Download, userID int, filter catalog.AccessFilter) error {
|
||||
func (s *Service) serveDownloadBytes(ctx context.Context, w http.ResponseWriter, r *http.Request, dl *Download, userID int, filter catalog.AccessFilter, transfer transfers.Transfer) error {
|
||||
file, err := s.fileRepo.GetByID(ctx, dl.MediaFileID)
|
||||
if err != nil {
|
||||
return fmt.Errorf("loading media file: %w", err)
|
||||
@@ -1070,6 +1079,9 @@ func (s *Service) serveDownloadBytes(ctx context.Context, w http.ResponseWriter,
|
||||
if !catalog.FileAllowedByAccess(&served, filter) {
|
||||
return catalog.ErrItemNotFound
|
||||
}
|
||||
transfer.DownloadID = dl.ID
|
||||
transfer.MediaFileID = dl.MediaFileID
|
||||
s.annotateTransfer(transfer)
|
||||
return s.serveLocalFile(ctx, w, r, artifact.OutputPath, userID)
|
||||
}
|
||||
if file.MissingSince != nil {
|
||||
@@ -1078,9 +1090,19 @@ func (s *Service) serveDownloadBytes(ctx context.Context, w http.ResponseWriter,
|
||||
if !catalog.FileAllowedByAccess(file, filter) {
|
||||
return catalog.ErrItemNotFound
|
||||
}
|
||||
transfer.DownloadID = dl.ID
|
||||
transfer.MediaFileID = dl.MediaFileID
|
||||
s.annotateTransfer(transfer)
|
||||
return s.serveLocalFile(ctx, w, r, file.FilePath, userID)
|
||||
}
|
||||
|
||||
func (s *Service) annotateTransfer(transfer transfers.Transfer) {
|
||||
if s.transfers == nil {
|
||||
return
|
||||
}
|
||||
s.transfers.Annotate(transfer.ID, transfer.DownloadID, transfer.MediaFileID)
|
||||
}
|
||||
|
||||
func (s *Service) serveLocalFile(ctx context.Context, w http.ResponseWriter, r *http.Request, path string, userID int) error {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -70,30 +71,76 @@ type RollingDeadlineWriter struct {
|
||||
step time.Duration
|
||||
lastBump time.Time
|
||||
disabled bool
|
||||
latch *CutLatch
|
||||
|
||||
statusCode int
|
||||
bytesWritten int64
|
||||
firstWriteErr error
|
||||
}
|
||||
|
||||
// CutLatch records that a stream has been terminally cut. Once latched, a
|
||||
// RollingDeadlineWriter must never push the write deadline back out again: the
|
||||
// cut is a deliberate hang-up, not a stall.
|
||||
type CutLatch struct {
|
||||
cut atomic.Bool
|
||||
}
|
||||
|
||||
func (l *CutLatch) Cut() {
|
||||
if l != nil {
|
||||
l.cut.Store(true)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *CutLatch) IsCut() bool {
|
||||
return l != nil && l.cut.Load()
|
||||
}
|
||||
|
||||
type cutLatchContextKey struct{}
|
||||
|
||||
// WithCutLatch carries l on ctx so rolling writers constructed inside serving
|
||||
// helpers can observe a cut made by a watcher around an inner writer.
|
||||
func WithCutLatch(ctx context.Context, l *CutLatch) context.Context {
|
||||
return context.WithValue(ctx, cutLatchContextKey{}, l)
|
||||
}
|
||||
|
||||
// CutLatchFrom returns the stream cut latch carried by ctx, if any.
|
||||
func CutLatchFrom(ctx context.Context) *CutLatch {
|
||||
if ctx == nil {
|
||||
return nil
|
||||
}
|
||||
l, _ := ctx.Value(cutLatchContextKey{}).(*CutLatch)
|
||||
return l
|
||||
}
|
||||
|
||||
// NewRollingDeadlineWriter wraps w with the configured stall window.
|
||||
func NewRollingDeadlineWriter(w http.ResponseWriter) *RollingDeadlineWriter {
|
||||
return newRollingDeadlineWriter(w, StallWindow(), bumpStep)
|
||||
return newRollingDeadlineWriterWithLatch(w, StallWindow(), bumpStep, nil)
|
||||
}
|
||||
|
||||
// NewRollingDeadlineWriterCtx wraps w and observes a terminal cut latch carried
|
||||
// by ctx. Callers without a revocable request can use NewRollingDeadlineWriter.
|
||||
func NewRollingDeadlineWriterCtx(ctx context.Context, w http.ResponseWriter) *RollingDeadlineWriter {
|
||||
return newRollingDeadlineWriterWithLatch(w, StallWindow(), bumpStep, CutLatchFrom(ctx))
|
||||
}
|
||||
|
||||
func newRollingDeadlineWriter(w http.ResponseWriter, window, step time.Duration) *RollingDeadlineWriter {
|
||||
return newRollingDeadlineWriterWithLatch(w, window, step, nil)
|
||||
}
|
||||
|
||||
func newRollingDeadlineWriterWithLatch(w http.ResponseWriter, window, step time.Duration, latch *CutLatch) *RollingDeadlineWriter {
|
||||
s := &RollingDeadlineWriter{
|
||||
w: w,
|
||||
rc: http.NewResponseController(w),
|
||||
window: window,
|
||||
step: step,
|
||||
latch: latch,
|
||||
}
|
||||
s.bump()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *RollingDeadlineWriter) bump() {
|
||||
if s.disabled {
|
||||
if s.disabled || s.latch.IsCut() {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
@@ -104,6 +151,13 @@ func (s *RollingDeadlineWriter) bump() {
|
||||
s.disabled = true
|
||||
return
|
||||
}
|
||||
// Close the check/set race with a concurrent cut: if the watcher latched
|
||||
// after the first check but before the future deadline landed, immediately
|
||||
// restore the terminal deadline instead of leaving the socket re-armed.
|
||||
if s.latch.IsCut() {
|
||||
_ = s.rc.SetWriteDeadline(time.Now())
|
||||
return
|
||||
}
|
||||
s.lastBump = now
|
||||
}
|
||||
|
||||
|
||||
@@ -211,6 +211,49 @@ func TestWriteOutcomeCompletedAndCounted(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type recordingDeadlineWriter struct {
|
||||
*httptest.ResponseRecorder
|
||||
deadlines []time.Time
|
||||
}
|
||||
|
||||
func (w *recordingDeadlineWriter) SetWriteDeadline(deadline time.Time) error {
|
||||
w.deadlines = append(w.deadlines, deadline)
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestRollingDeadlineWriterNeverRearmsAfterCut(t *testing.T) {
|
||||
base := &recordingDeadlineWriter{ResponseRecorder: httptest.NewRecorder()}
|
||||
latch := &CutLatch{}
|
||||
sw := newRollingDeadlineWriterWithLatch(base, time.Minute, 0, latch)
|
||||
if got := len(base.deadlines); got != 1 {
|
||||
t.Fatalf("constructor deadlines = %d, want 1", got)
|
||||
}
|
||||
|
||||
latch.Cut()
|
||||
sw.bump()
|
||||
if _, err := sw.Write([]byte("post-cut")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(base.deadlines); got != 1 {
|
||||
t.Fatalf("deadlines after cut = %d, want constructor deadline only", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRollingDeadlineWriterStartsLatchedWithoutBump(t *testing.T) {
|
||||
base := &recordingDeadlineWriter{ResponseRecorder: httptest.NewRecorder()}
|
||||
latch := &CutLatch{}
|
||||
latch.Cut()
|
||||
ctx := WithCutLatch(context.Background(), latch)
|
||||
|
||||
sw := newRollingDeadlineWriterWithLatch(base, time.Minute, 0, CutLatchFrom(ctx))
|
||||
if _, err := sw.Write([]byte("still no bump")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := len(base.deadlines); got != 0 {
|
||||
t.Fatalf("deadlines = %d, want 0 for pre-latched writer", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServeContentReadFromOutcomeCompletedAndCounted(t *testing.T) {
|
||||
const totalSize = 2 << 20
|
||||
filePath := filepath.Join(t.TempDir(), "source.bin")
|
||||
|
||||
@@ -25,9 +25,11 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/nodepool"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/Silo-Server/silo-server/internal/streamtoken"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
"github.com/Silo-Server/silo-server/internal/transcodenode"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
"github.com/Silo-Server/silo-server/internal/watchsync"
|
||||
)
|
||||
@@ -206,6 +208,13 @@ type PlaybackHandler struct {
|
||||
// server-authoritative store instead (see internal/noderecipe). Optional
|
||||
// (nil disables it — integrated/no-node deployments need no handoff).
|
||||
RecipeNodeStore recipeNodePutter
|
||||
// Revocation is the shared stream kill switch. jellycompat local serving (the
|
||||
// integrated path, and multi-node local-transcode fallback) must consult it so
|
||||
// an admin terminate / over-cap / account revocation actually stops the bytes —
|
||||
// the multi-node redirect path is already guarded by the proxy. Optional.
|
||||
Revocation *streamrevoke.Store
|
||||
// Transfers tracks download-class pours without entering live-stream limits.
|
||||
Transfers *transfers.Registry
|
||||
}
|
||||
|
||||
// recipeNodePutter persists and removes a remote transcode's reconstruction
|
||||
@@ -372,6 +381,20 @@ func (h *PlaybackHandler) buildProxyRedirectURL(
|
||||
AudioTrackIndex: audioTrackIndex,
|
||||
TranscodeNode: transcodeNodeURL,
|
||||
DVProfile: file.PrimaryDVProfile(),
|
||||
Origin: playback.OriginJellyfin,
|
||||
}
|
||||
// Carry the owner (uid/pid/mfid) so the edge tracker attributes this stream to
|
||||
// the real user — without it jellycompat streams land under user 0 in the
|
||||
// monitoring picture, breaking per-user over-cap enforcement and user-key kills
|
||||
// at the edge. Native tokens already carry the owner; keep parity. Also carry
|
||||
// the client name so the edge record identifies the viewer/app.
|
||||
if h.sessionMgr != nil {
|
||||
if s, err := h.sessionMgr.GetSession(upstreamSessionID); err == nil && s != nil {
|
||||
claims.UserID = s.UserID
|
||||
claims.ProfileID = s.ProfileID
|
||||
claims.MediaFileID = s.MediaFileID
|
||||
claims.ClientName = s.ClientName
|
||||
}
|
||||
}
|
||||
token, err := streamtoken.Sign(claims, h.JWTSecret, 24*time.Hour)
|
||||
if err != nil {
|
||||
@@ -441,6 +464,7 @@ func (h *PlaybackHandler) startRemoteTranscode(
|
||||
|
||||
reqBody := transcodenode.TranscodeStartRequest{
|
||||
SessionID: upstreamSessionID,
|
||||
LogicalSessionID: playSessionID,
|
||||
InputPath: file.FilePath,
|
||||
SeekSeconds: initialSeekSeconds,
|
||||
StartSegmentNumber: startSegmentNumber,
|
||||
@@ -454,6 +478,18 @@ func (h *PlaybackHandler) startRemoteTranscode(
|
||||
if source.TranscodeAudio {
|
||||
reqBody.TargetCodecVideo = "copy"
|
||||
}
|
||||
// Attribute the node's live-session record to the real owner + route + client
|
||||
// so it is not an ownerless (user 0), route-less record when no proxy record
|
||||
// fronts it. Mirrors the redirect token's owner-carry.
|
||||
reqBody.Route = playback.OriginJellyfin
|
||||
if h.sessionMgr != nil {
|
||||
if s, err := h.sessionMgr.GetSession(upstreamSessionID); err == nil && s != nil {
|
||||
reqBody.AuthUserID = s.UserID
|
||||
reqBody.ProfileID = s.ProfileID
|
||||
reqBody.MediaFileID = s.MediaFileID
|
||||
reqBody.ClientName = s.ClientName
|
||||
}
|
||||
}
|
||||
|
||||
body, err := json.Marshal(reqBody)
|
||||
if err != nil {
|
||||
|
||||
@@ -96,6 +96,14 @@ func (w *compatImageProxyTagResponseWriter) finish() {
|
||||
}
|
||||
}
|
||||
|
||||
// Unwrap exposes the wrapped ResponseWriter so http.ResponseController (used by
|
||||
// the stream kill switch's in-flight cut via SetWriteDeadline) can reach the
|
||||
// underlying socket instead of stopping at this wrapper and no-oping. Video
|
||||
// stream responses are non-JSON, so they pass through this writer untouched.
|
||||
func (w *compatImageProxyTagResponseWriter) Unwrap() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
|
||||
func isJSONResponse(contentType string) bool {
|
||||
contentType = strings.ToLower(strings.TrimSpace(contentType))
|
||||
return contentType == "application/json" || strings.HasPrefix(contentType, "application/json;")
|
||||
|
||||
@@ -50,7 +50,9 @@ func (w *loggingResponseWriter) Flush() {
|
||||
}
|
||||
}
|
||||
|
||||
// Unwrap returns the underlying ResponseWriter for http.ResponseController.
|
||||
// Unwrap exposes the wrapped ResponseWriter so http.ResponseController (used by
|
||||
// the stream kill switch's in-flight cut via SetWriteDeadline) can reach the
|
||||
// underlying socket instead of stopping at this wrapper and no-oping.
|
||||
func (w *loggingResponseWriter) Unwrap() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
@@ -156,7 +158,9 @@ func (w *debugResponseWriter) Flush() {
|
||||
}
|
||||
}
|
||||
|
||||
// Unwrap returns the underlying ResponseWriter for http.ResponseController.
|
||||
// Unwrap exposes the wrapped ResponseWriter so http.ResponseController (used by
|
||||
// the stream kill switch's in-flight cut via SetWriteDeadline) can reach the
|
||||
// underlying socket instead of stopping at this wrapper and no-oping.
|
||||
func (w *debugResponseWriter) Unwrap() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
|
||||
@@ -108,6 +108,8 @@ func NewRouter(deps Dependencies) chi.Router {
|
||||
}
|
||||
playbackHandler.NodePlanner = deps.NodePlanner
|
||||
playbackHandler.JWTSecret = deps.JWTSecret
|
||||
playbackHandler.Revocation = deps.RevocationStore
|
||||
playbackHandler.Transfers = deps.TransferRegistry
|
||||
// Compat transcode reconstruct is driven by the recipe carried in the durable
|
||||
// compat playback store (jellycompat_playback_sessions); no separate native
|
||||
// recipe table is needed.
|
||||
|
||||
@@ -17,7 +17,9 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/recommendations"
|
||||
"github.com/Silo-Server/silo-server/internal/scantrigger"
|
||||
"github.com/Silo-Server/silo-server/internal/secret"
|
||||
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
"github.com/Silo-Server/silo-server/internal/userstore"
|
||||
"github.com/Silo-Server/silo-server/internal/watchstate"
|
||||
"github.com/Silo-Server/silo-server/internal/watchsync"
|
||||
@@ -30,6 +32,11 @@ type Dependencies struct {
|
||||
// periodic orphan-transcode sweep so it stops on shutdown; nil (tests) makes
|
||||
// the sweep a single boot-time run instead of a long-lived ticker.
|
||||
AppContext context.Context
|
||||
|
||||
// RevocationStore is the shared stream kill switch consulted by local serving.
|
||||
RevocationStore *streamrevoke.Store
|
||||
// TransferRegistry exposes download-class pours separately from live streams.
|
||||
TransferRegistry *transfers.Registry
|
||||
// LiveConfig returns the current hot-reloaded config. May be nil (tests,
|
||||
// worker modes); read through CurrentConfig(), which falls back to Config.
|
||||
LiveConfig func() *config.Config
|
||||
|
||||
@@ -21,10 +21,13 @@ import (
|
||||
|
||||
"github.com/go-chi/chi/v5"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
"github.com/Silo-Server/silo-server/internal/models"
|
||||
"github.com/Silo-Server/silo-server/internal/nodepool"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/subtitles"
|
||||
"github.com/Silo-Server/silo-server/internal/transfers"
|
||||
"github.com/Silo-Server/silo-server/internal/watchsync"
|
||||
)
|
||||
|
||||
@@ -33,6 +36,24 @@ import (
|
||||
// the near-head follow-up segments arrive quickly enough for browser playback.
|
||||
const compatSegmentDuration = 2
|
||||
|
||||
// compatCredentialIssuedAt returns the issue time of the credential the client
|
||||
// actually presented. A normal compat login keeps its CreatedAt across bridged
|
||||
// Silo access-token refreshes, so refresh cannot escape a user cutoff.
|
||||
// Synthetic API-key sessions have no credential issue time and deliberately
|
||||
// return zero: by streamrevoke's fail-open contract, user cutoffs cannot cut an
|
||||
// API-key-owned pour.
|
||||
func compatCredentialIssuedAt(ctx context.Context, session *Session) time.Time {
|
||||
if session == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
if strings.HasPrefix(session.Token, "sa_") {
|
||||
slog.DebugContext(ctx, "jellycompat API-key stream has no issue time; user revocation cutoff fails open",
|
||||
"component", "jellycompat", "user_id", session.StreamAppUserID)
|
||||
return time.Time{}
|
||||
}
|
||||
return session.CreatedAt
|
||||
}
|
||||
|
||||
// errUpstreamReplaced signals that a concurrent request attached a different
|
||||
// upstream session to the play session while this one was being created.
|
||||
var errUpstreamReplaced = errors.New("upstream session replaced concurrently")
|
||||
@@ -54,6 +75,7 @@ func (h *PlaybackHandler) HandleVideoStream(w http.ResponseWriter, r *http.Reque
|
||||
writeError(w, http.StatusUnauthorized, "Unauthorized", "Missing authentication token")
|
||||
return
|
||||
}
|
||||
startedAt := compatCredentialIssuedAt(r.Context(), session)
|
||||
|
||||
routeID := chiURLParam(r, "id")
|
||||
mediaSourceID := firstNonEmpty(r.URL.Query().Get("mediaSourceId"), r.URL.Query().Get("MediaSourceId"))
|
||||
@@ -87,12 +109,35 @@ func (h *PlaybackHandler) HandleVideoStream(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
|
||||
// Kill switch, BEFORE ensureUpstreamPlayback: a killed upstream session must
|
||||
// be refused while it is still identifiable. ensureUpstreamPlayback revives a
|
||||
// lost session — or, when the revoked one is no longer reconstructable, mints
|
||||
// a REPLACEMENT with a fresh id that the post-ensure check below would wave
|
||||
// through, letting a client dodge a session kill by simply re-hitting the same
|
||||
// stream URL. (User-kind kills don't need this: they match by user id either
|
||||
// side of the ensure.)
|
||||
if h.Revocation != nil && playSession.UpstreamSessionID != "" &&
|
||||
h.Revocation.Refuse(w, playSession.UpstreamSessionID, session.StreamAppUserID, startedAt) {
|
||||
return
|
||||
}
|
||||
|
||||
playSession, err = h.ensureUpstreamPlayback(r.Context(), session, playSession.ID, *source, method)
|
||||
if err != nil {
|
||||
writeCompatUpstreamError(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Kill switch (local serving): refuse a revoked session/user before serving.
|
||||
// The multi-node redirect path below is guarded by the proxy; this covers the
|
||||
// integrated / local-fallback path that the proxy never sees. The request
|
||||
// entry time is the credential time for the user-kill cutoff: a compat
|
||||
// request uses the compat login's creation time, which remains stable when
|
||||
// its bridged Silo access token refreshes.
|
||||
if h.Revocation != nil && playSession.UpstreamSessionID != "" &&
|
||||
h.Revocation.Refuse(w, playSession.UpstreamSessionID, session.StreamAppUserID, startedAt) {
|
||||
return
|
||||
}
|
||||
|
||||
if h.fileResolver == nil {
|
||||
writeError(w, http.StatusInternalServerError, "ServerError", "File resolver not available")
|
||||
return
|
||||
@@ -127,15 +172,27 @@ func (h *PlaybackHandler) HandleVideoStream(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
}
|
||||
|
||||
// In-flight cut: hang up a long local direct-play/remux pour the moment it is
|
||||
// revoked. The Refuse check above only covers new/reconnect requests; a single
|
||||
// long GET needs the connection cut to stop mid-stream.
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
if h.Revocation != nil && playSession.UpstreamSessionID != "" {
|
||||
stop := h.Revocation.WatchAndCutContext(r.Context(), w, playSession.UpstreamSessionID, session.StreamAppUserID, startedAt)
|
||||
defer stop()
|
||||
}
|
||||
recorder, _ := h.sessionMgr.(playback.ServedBytesRecorder)
|
||||
metered := playback.NewSessionMeteredWriter(w, recorder, playSession.UpstreamSessionID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
|
||||
switch method {
|
||||
case "remux":
|
||||
audioTrackIndex := -1
|
||||
if resolvedAudioTrackIndex, ok := compatAudioTrackIndex(*source); ok {
|
||||
audioTrackIndex = resolvedAudioTrackIndex
|
||||
}
|
||||
_ = playback.ServeRemux(w, r, file.FilePath, "mp4", seekSeconds, source.TranscodeAudio, audioTrackIndex, file.PrimaryDVProfile())
|
||||
_ = playback.ServeRemux(metered, r, file.FilePath, "mp4", seekSeconds, source.TranscodeAudio, audioTrackIndex, file.PrimaryDVProfile())
|
||||
default:
|
||||
_ = playback.ServeDirectPlay(w, r, file.FilePath)
|
||||
_ = playback.ServeDirectPlay(metered, r, file.FilePath)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,6 +201,20 @@ func (h *PlaybackHandler) HandleVideoStream(w http.ResponseWriter, r *http.Reque
|
||||
// load-bearing for Infuse: it refuses Direct Play (Static=true streaming)
|
||||
// for items it believes it cannot download, so the flag must stay true and
|
||||
// this route must exist.
|
||||
// HandleDownload serves a full file for offline/download clients (e.g. Infuse
|
||||
// with CanDownload). By design it is NOT a live stream: it creates no
|
||||
// SessionManager session and never enters streammonitor, so it is EXEMPT from the
|
||||
// concurrent-stream cap. Its active pour is exposed through the separate,
|
||||
// process-local transfer registry. NOTE: unlike the native
|
||||
// /downloads/{id}/file route, this route is NOT covered by the download
|
||||
// concurrency/period quota either — it requires no download row, only compat
|
||||
// auth. Access control is compat authentication plus the library access filter;
|
||||
// a per-user stream revocation cuts an in-flight pour (below) and, because the
|
||||
// same hook deletes every compat login, refuses reconnects until re-auth.
|
||||
// Bringing this route under a real download quota (Infuse-compatible: plain
|
||||
// GETs, Range requests, no download rows) is an open follow-up in the coverage
|
||||
// matrix; if downloads should ever count against the live-stream cap instead,
|
||||
// register a tracked session here — but that conflates two distinct limits.
|
||||
func (h *PlaybackHandler) HandleDownload(w http.ResponseWriter, r *http.Request) {
|
||||
session := SessionFromContext(r.Context())
|
||||
if session == nil {
|
||||
@@ -179,13 +250,56 @@ func (h *PlaybackHandler) HandleDownload(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
file, err := h.fileResolver.GetByID(r.Context(), version.FileID)
|
||||
if err != nil {
|
||||
if err != nil || file == nil {
|
||||
writeError(w, http.StatusNotFound, "NotFound", "Media file not found")
|
||||
return
|
||||
}
|
||||
|
||||
client := playback.ClientInfoFromContext(r.Context())
|
||||
transfer := transfers.Transfer{
|
||||
ID: uuidNewString(),
|
||||
UserID: session.StreamAppUserID,
|
||||
ProfileID: session.ProfileID,
|
||||
MediaFileID: file.ID,
|
||||
Route: "jellycompat_download",
|
||||
ClientIP: clientip.FromContext(r.Context()),
|
||||
ClientName: firstNonEmpty(client.Name, client.UserAgent),
|
||||
StartedAt: time.Now(),
|
||||
}
|
||||
if h.Transfers != nil {
|
||||
if err := h.Transfers.Begin(transfer); err != nil {
|
||||
slog.DebugContext(r.Context(), "compat transfer rejected", "component", "jellycompat", "transfer_id", transfer.ID, "error", err)
|
||||
switch {
|
||||
case errors.Is(err, transfers.ErrUserTransferLimit):
|
||||
w.Header().Set("Retry-After", "5")
|
||||
writeError(w, http.StatusTooManyRequests, "transfer_limit_exceeded", "Concurrent transfer limit exceeded")
|
||||
case errors.Is(err, transfers.ErrRegistryFull):
|
||||
w.Header().Set("Retry-After", "5")
|
||||
writeError(w, http.StatusServiceUnavailable, "monitoring_unavailable", "Transfer monitoring unavailable")
|
||||
default:
|
||||
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to monitor transfer")
|
||||
}
|
||||
return
|
||||
}
|
||||
}
|
||||
defer h.Transfers.End(transfer.ID)
|
||||
metered := playback.NewSessionMeteredWriter(w, h.Transfers, transfer.ID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
|
||||
// In-flight kill switch: sessionless pour, so only user-kind kills apply;
|
||||
// the entry time predates any future revocation (the user-kill cutoff), so a
|
||||
// revocation issued mid-transfer hangs this connection up.
|
||||
if h.Revocation != nil {
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
stop := h.Revocation.WatchAndCutContext(
|
||||
r.Context(), metered, "", session.StreamAppUserID,
|
||||
compatCredentialIssuedAt(r.Context(), session),
|
||||
)
|
||||
defer stop()
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Disposition", "attachment; filename*=UTF-8''"+url.PathEscape(filepath.Base(file.FilePath)))
|
||||
_ = playback.ServeDirectPlay(w, r, file.FilePath)
|
||||
_ = playback.ServeDirectPlay(metered, r, file.FilePath)
|
||||
}
|
||||
|
||||
// HandleMasterManifest serves the compat-owned HLS manifest route.
|
||||
@@ -210,6 +324,19 @@ func (h *PlaybackHandler) HandleMasterManifest(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
|
||||
// Kill switch: refuse a revoked session before the ensure/start machinery
|
||||
// below can revive its transcode. Without this a killed session's player
|
||||
// polling the manifest keeps ffmpeg alive (and re-spawns it after a restart)
|
||||
// even though every segment request is refused — the same resurrection hole
|
||||
// the transcode node's reconstruct guard closes on the edge.
|
||||
if h.Revocation != nil && playSession.UpstreamSessionID != "" &&
|
||||
h.Revocation.Refuse(
|
||||
w, playSession.UpstreamSessionID, session.StreamAppUserID,
|
||||
compatCredentialIssuedAt(r.Context(), session),
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
source := findMediaSource(playSession, firstNonEmpty(r.URL.Query().Get("MediaSourceId"), r.URL.Query().Get("mediaSourceId")))
|
||||
if source == nil {
|
||||
writeError(w, http.StatusBadRequest, "BadRequest", "Media source is required")
|
||||
@@ -319,6 +446,15 @@ func (h *PlaybackHandler) HandleHLSManifest(w http.ResponseWriter, r *http.Reque
|
||||
writeError(w, http.StatusNotFound, "NotFound", "Playback session not found")
|
||||
return
|
||||
}
|
||||
// Kill switch: refuse before ensureTranscodeManifest can keep/restart the
|
||||
// killed session's ffmpeg (see HandleMasterManifest).
|
||||
if h.Revocation != nil && playSession.UpstreamSessionID != "" &&
|
||||
h.Revocation.Refuse(
|
||||
w, playSession.UpstreamSessionID, session.StreamAppUserID,
|
||||
compatCredentialIssuedAt(r.Context(), session),
|
||||
) {
|
||||
return
|
||||
}
|
||||
source := firstMediaSource(playSession)
|
||||
if mediaSourceID := firstNonEmpty(r.URL.Query().Get("MediaSourceId"), r.URL.Query().Get("mediaSourceId")); mediaSourceID != "" {
|
||||
source = findMediaSource(playSession, mediaSourceID)
|
||||
@@ -372,6 +508,15 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
// Kill switch (local serving): refuse a revoked session/user on every segment.
|
||||
// Request entry time = credential time (compat auth is live, see HandleVideoStream).
|
||||
if h.Revocation != nil && h.Revocation.Refuse(
|
||||
w, playSession.UpstreamSessionID, session.StreamAppUserID,
|
||||
compatCredentialIssuedAt(r.Context(), session),
|
||||
) {
|
||||
return
|
||||
}
|
||||
|
||||
name := chiURLParam(r, "segmentId")
|
||||
ext := chiURLParam(r, "segmentContainer")
|
||||
|
||||
@@ -531,7 +676,22 @@ func (h *PlaybackHandler) HandleHLSSegment(w http.ResponseWriter, r *http.Reques
|
||||
transcodeSession.ReportSegmentDownloaded(segNum)
|
||||
}
|
||||
|
||||
http.ServeFile(w, r, segmentPath)
|
||||
// Hold a transport marker for the whole segment pour so a compat HLS transcode
|
||||
// stays visible via server-observed liveness rather than the client's progress
|
||||
// POST: a client that keeps pulling segments but withholds progress reports
|
||||
// must still be counted (the hidden-stream defense). Mirrors the direct/remux
|
||||
// path above and the native transcode segment handler.
|
||||
if h.sessionMgr != nil && playSession.UpstreamSessionID != "" {
|
||||
if err := h.sessionMgr.BeginTransport(playSession.UpstreamSessionID); err == nil {
|
||||
upstreamSessionID := playSession.UpstreamSessionID
|
||||
defer func() { _ = h.sessionMgr.EndTransport(upstreamSessionID) }()
|
||||
}
|
||||
}
|
||||
|
||||
recorder, _ := h.sessionMgr.(playback.ServedBytesRecorder)
|
||||
metered := playback.NewSessionMeteredWriter(w, recorder, playSession.UpstreamSessionID)
|
||||
defer func() { _ = metered.Close() }()
|
||||
http.ServeFile(metered, r, segmentPath)
|
||||
}
|
||||
|
||||
// hlsSegmentErrorResponse maps a segment-retrieval error to a Jellyfin-faithful
|
||||
@@ -559,12 +719,26 @@ func (h *PlaybackHandler) HandleSubtitleStream(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
|
||||
_, source, err := h.resolvePlaybackRoute(r, session, chiURLParam(r, "routeMediaSourceId"), chiURLParam(r, "routeMediaSourceId"))
|
||||
playSession, source, err := h.resolvePlaybackRoute(r, session, chiURLParam(r, "routeMediaSourceId"), chiURLParam(r, "routeMediaSourceId"))
|
||||
if err != nil || source == nil {
|
||||
writeError(w, http.StatusNotFound, "NotFound", "Playback session not found")
|
||||
return
|
||||
}
|
||||
|
||||
startedAt := compatCredentialIssuedAt(r.Context(), session)
|
||||
// Kill switch: a revoked session must not keep pulling subtitle bytes (or
|
||||
// triggering server-side ffmpeg subtitle extraction below). Compat
|
||||
// extraction is buffered, so a socket cut stops response delivery but does
|
||||
// not interrupt extraction already in progress.
|
||||
if h.Revocation != nil && playSession != nil && playSession.UpstreamSessionID != "" &&
|
||||
h.Revocation.Refuse(w, playSession.UpstreamSessionID, session.StreamAppUserID, startedAt) {
|
||||
return
|
||||
}
|
||||
if h.Revocation != nil && playSession != nil && playSession.UpstreamSessionID != "" {
|
||||
stop := h.Revocation.WatchAndCutContext(r.Context(), w, playSession.UpstreamSessionID, session.StreamAppUserID, startedAt)
|
||||
defer stop()
|
||||
}
|
||||
|
||||
if h.fileResolver == nil {
|
||||
writeError(w, http.StatusInternalServerError, "ServerError", "File resolver not available")
|
||||
return
|
||||
|
||||
@@ -14,6 +14,18 @@ import (
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
)
|
||||
|
||||
func TestCompatCredentialIssuedAtUsesLoginCreationAndAPIKeyZero(t *testing.T) {
|
||||
createdAt := time.Now().Add(-time.Hour)
|
||||
session := &Session{Token: "compat-token", StreamAppUserID: 7, CreatedAt: createdAt}
|
||||
if got := compatCredentialIssuedAt(context.Background(), session); !got.Equal(createdAt) {
|
||||
t.Fatalf("normal compat credential time = %v, want %v", got, createdAt)
|
||||
}
|
||||
session.Token = "sa_test_key"
|
||||
if got := compatCredentialIssuedAt(context.Background(), session); !got.IsZero() {
|
||||
t.Fatalf("API-key compat credential time = %v, want zero", got)
|
||||
}
|
||||
}
|
||||
|
||||
// withCompatSession attaches a compat session carrying tok to req, so the
|
||||
// ActiveEncodings ownership guard (CompatToken == session.Token) is satisfied.
|
||||
func withCompatSession(req *http.Request, tok string) *http.Request {
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package nodesessions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
// Publisher mirrors the integrated API process's live sessions into the
|
||||
// shared node-session keyspace.
|
||||
type Publisher struct {
|
||||
rdb *redis.Client
|
||||
nodeName string
|
||||
nodeHash string
|
||||
ttl time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
ids map[string]struct{}
|
||||
}
|
||||
|
||||
// NewPublisher creates a publisher whose key namespace is derived from the
|
||||
// process-unique instanceID. nodeName remains display metadata only.
|
||||
func NewPublisher(rdb *redis.Client, instanceID, nodeName string) *Publisher {
|
||||
return &Publisher{
|
||||
rdb: rdb,
|
||||
nodeName: nodeName,
|
||||
nodeHash: namespaceHash(instanceID),
|
||||
ttl: sessionTTL,
|
||||
ids: make(map[string]struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// Publish refreshes every currently-live record and immediately removes only
|
||||
// records which disappeared since the preceding successful snapshot.
|
||||
func (p *Publisher) Publish(ctx context.Context, infos []SessionInfo) {
|
||||
if p == nil || p.rdb == nil {
|
||||
return
|
||||
}
|
||||
|
||||
type record struct {
|
||||
id string
|
||||
data []byte
|
||||
}
|
||||
records := make([]record, 0, len(infos))
|
||||
current := make(map[string]struct{}, len(infos))
|
||||
for _, info := range infos {
|
||||
if info.SessionID == "" {
|
||||
continue
|
||||
}
|
||||
info.NodeName = p.nodeName
|
||||
data, err := json.Marshal(info)
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "integrated session marshal failed",
|
||||
"component", "nodesessions", "session", info.SessionID, "error", err)
|
||||
continue
|
||||
}
|
||||
records = append(records, record{id: info.SessionID, data: data})
|
||||
current[info.SessionID] = struct{}{}
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
pipe := p.rdb.Pipeline()
|
||||
for _, rec := range records {
|
||||
pipe.Set(ctx, sessionRedisKey(p.nodeHash, rec.id), rec.data, p.ttl)
|
||||
}
|
||||
for id := range p.ids {
|
||||
if _, live := current[id]; !live {
|
||||
pipe.Del(ctx, sessionRedisKey(p.nodeHash, id))
|
||||
}
|
||||
}
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
slog.DebugContext(ctx, "integrated session publish failed",
|
||||
"component", "nodesessions", "error", err)
|
||||
return
|
||||
}
|
||||
p.ids = current
|
||||
}
|
||||
|
||||
// Start publishes snapshots at interval until ctx is canceled.
|
||||
func (p *Publisher) Start(ctx context.Context, interval time.Duration, snapshot func() []SessionInfo) {
|
||||
if p == nil || p.rdb == nil || interval <= 0 || snapshot == nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
p.Publish(ctx, snapshot())
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Cleanup removes all records owned by this process.
|
||||
func (p *Publisher) Cleanup(ctx context.Context) {
|
||||
if p == nil || p.rdb == nil {
|
||||
return
|
||||
}
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
if len(p.ids) == 0 {
|
||||
return
|
||||
}
|
||||
pipe := p.rdb.Pipeline()
|
||||
for id := range p.ids {
|
||||
pipe.Del(ctx, sessionRedisKey(p.nodeHash, id))
|
||||
}
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
slog.DebugContext(ctx, "integrated session cleanup failed",
|
||||
"component", "nodesessions", "error", err)
|
||||
return
|
||||
}
|
||||
p.ids = make(map[string]struct{})
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
package nodesessions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type expiringRedisHook struct {
|
||||
mu sync.Mutex
|
||||
expires map[string]time.Time
|
||||
}
|
||||
|
||||
func (h *expiringRedisHook) DialHook(next redis.DialHook) redis.DialHook {
|
||||
return next
|
||||
}
|
||||
|
||||
func (h *expiringRedisHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
|
||||
return func(ctx context.Context, cmd redis.Cmder) error {
|
||||
return h.process(ctx, cmd)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *expiringRedisHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
|
||||
return func(ctx context.Context, cmds []redis.Cmder) error {
|
||||
for _, cmd := range cmds {
|
||||
if err := h.process(ctx, cmd); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func (h *expiringRedisHook) process(_ context.Context, cmd redis.Cmder) error {
|
||||
args := cmd.Args()
|
||||
if len(args) < 2 {
|
||||
return nil
|
||||
}
|
||||
key, _ := args[1].(string)
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
switch strings.ToLower(cmd.Name()) {
|
||||
case "set":
|
||||
expiry := time.Time{}
|
||||
for i := 3; i+1 < len(args); i++ {
|
||||
if strings.EqualFold(strings.TrimSpace(toString(args[i])), "px") {
|
||||
ms, _ := strconv.ParseInt(toString(args[i+1]), 10, 64)
|
||||
expiry = time.Now().Add(time.Duration(ms) * time.Millisecond)
|
||||
break
|
||||
}
|
||||
}
|
||||
h.expires[key] = expiry
|
||||
case "del":
|
||||
delete(h.expires, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func toString(value any) string {
|
||||
switch value := value.(type) {
|
||||
case string:
|
||||
return value
|
||||
case []byte:
|
||||
return string(value)
|
||||
case int:
|
||||
return strconv.Itoa(value)
|
||||
case int64:
|
||||
return strconv.FormatInt(value, 10)
|
||||
case time.Duration:
|
||||
return strconv.FormatInt(value.Milliseconds(), 10)
|
||||
default:
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
func (h *expiringRedisHook) has(key string) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
expiry, ok := h.expires[key]
|
||||
if ok && !expiry.IsZero() && !time.Now().Before(expiry) {
|
||||
delete(h.expires, key)
|
||||
return false
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
func testPublisherRedis(t *testing.T) (*redis.Client, *expiringRedisHook) {
|
||||
t.Helper()
|
||||
hook := &expiringRedisHook{expires: make(map[string]time.Time)}
|
||||
rdb := redis.NewClient(&redis.Options{Addr: "unused.invalid:6379"})
|
||||
rdb.AddHook(hook)
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
return rdb, hook
|
||||
}
|
||||
|
||||
func TestPublishersWithSharedNodeNameDoNotClobberEachOther(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
rdb, redisState := testPublisherRedis(t)
|
||||
suffix := time.Now().UTC().Format("20060102150405.000000000")
|
||||
first := NewPublisher(rdb, "first:"+suffix, "shared-name")
|
||||
second := NewPublisher(rdb, "second:"+suffix, "shared-name")
|
||||
t.Cleanup(func() {
|
||||
first.Cleanup(ctx)
|
||||
second.Cleanup(ctx)
|
||||
})
|
||||
|
||||
info := SessionInfo{SessionID: "same-session"}
|
||||
first.Publish(ctx, []SessionInfo{info})
|
||||
second.Publish(ctx, []SessionInfo{info})
|
||||
first.Publish(ctx, nil)
|
||||
|
||||
firstKey := sessionRedisKey(first.nodeHash, info.SessionID)
|
||||
secondKey := sessionRedisKey(second.nodeHash, info.SessionID)
|
||||
if redisState.has(firstKey) {
|
||||
t.Fatal("departed first key is still present")
|
||||
}
|
||||
if !redisState.has(secondKey) {
|
||||
t.Fatal("live second key is absent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublisherRefreshesStableRecordPastTTL(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
rdb, redisState := testPublisherRedis(t)
|
||||
suffix := time.Now().UTC().Format("20060102150405.000000000")
|
||||
publisher := NewPublisher(rdb, "refresh:"+suffix, "node")
|
||||
publisher.ttl = 80 * time.Millisecond
|
||||
t.Cleanup(func() { publisher.Cleanup(context.Background()) })
|
||||
|
||||
publisher.Start(ctx, 20*time.Millisecond, func() []SessionInfo {
|
||||
return []SessionInfo{{SessionID: "stable"}}
|
||||
})
|
||||
time.Sleep(220 * time.Millisecond)
|
||||
|
||||
key := sessionRedisKey(publisher.nodeHash, "stable")
|
||||
if !redisState.has(key) {
|
||||
t.Fatal("stable key is absent after > TTL")
|
||||
}
|
||||
}
|
||||
@@ -13,25 +13,44 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
keyPrefix = "silo:sessions:"
|
||||
sessionTTL = 60 * time.Second
|
||||
refreshInt = 30 * time.Second
|
||||
// KeyPrefix is the Redis key namespace for node session records
|
||||
// (silo:sessions:{nodeHash}:{sessionID}). Exported so readers of the
|
||||
// monitoring picture (streammonitor, the admin session list) SCAN the same
|
||||
// namespace this tracker writes instead of duplicating the literal.
|
||||
KeyPrefix = "silo:sessions:"
|
||||
|
||||
sessionTTL = 60 * time.Second
|
||||
refreshInt = 30 * time.Second
|
||||
transcodeIdleTTL = 180 * time.Second
|
||||
sessionTypeTranscode = "transcode"
|
||||
)
|
||||
|
||||
// SessionInfo represents an active streaming session stored in Redis.
|
||||
type SessionInfo struct {
|
||||
SessionID string `json:"session_id"`
|
||||
NodeURL string `json:"node_url"`
|
||||
NodeName string `json:"node_name"`
|
||||
UserID string `json:"user_id,omitempty"`
|
||||
MediaItemID string `json:"media_item_id,omitempty"`
|
||||
MediaTitle string `json:"media_title,omitempty"`
|
||||
Type string `json:"type"` // "direct_play", "remux", "transcode"
|
||||
CodecVideo string `json:"codec_video,omitempty"`
|
||||
CodecAudio string `json:"codec_audio,omitempty"`
|
||||
Resolution string `json:"resolution,omitempty"`
|
||||
HWAccel string `json:"hw_accel,omitempty"`
|
||||
StartedAt string `json:"started_at"`
|
||||
SessionID string `json:"session_id"`
|
||||
// LogicalSessionID is the stable playback-session identity when SessionID is
|
||||
// a replaceable transport generation. It is opaque monitoring metadata;
|
||||
// SessionID remains the node-addressing key.
|
||||
LogicalSessionID string `json:"logical_session_id,omitempty"`
|
||||
NodeURL string `json:"node_url"`
|
||||
NodeName string `json:"node_name"`
|
||||
UserID string `json:"user_id,omitempty"`
|
||||
MediaItemID string `json:"media_item_id,omitempty"`
|
||||
MediaTitle string `json:"media_title,omitempty"`
|
||||
Type string `json:"type"` // "direct_play", "remux", "transcode"
|
||||
CodecVideo string `json:"codec_video,omitempty"`
|
||||
CodecAudio string `json:"codec_audio,omitempty"`
|
||||
Resolution string `json:"resolution,omitempty"`
|
||||
HWAccel string `json:"hw_accel,omitempty"`
|
||||
StartedAt string `json:"started_at"`
|
||||
|
||||
// LastServedAt / BytesServed are the authoritative, server-observed liveness
|
||||
// signals: they are refreshed only when the node actually serves bytes for this
|
||||
// session (a segment written, a direct-play/remux pour advancing), never from a
|
||||
// client progress report. A "quiet" stream that keeps pulling stays fresh here;
|
||||
// one that stops pulling ages out. This is the anti-abuse monitoring surface.
|
||||
LastServedAt string `json:"last_served_at,omitempty"`
|
||||
BytesServed int64 `json:"bytes_served,omitempty"`
|
||||
|
||||
// AuthUserID / ProfileID / MediaFileID are the numeric ownership keys the
|
||||
// node copies from the verified stream token. They enrich the live admin
|
||||
@@ -41,6 +60,31 @@ type SessionInfo struct {
|
||||
AuthUserID int `json:"auth_user_id,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
MediaFileID int `json:"media_file_id,omitempty"`
|
||||
|
||||
// Route is the origin protocol ("native" | "jellycompat"). Type is the play
|
||||
// method (direct_play/remux/transcode); Route is orthogonal — a jellycompat
|
||||
// stream and a native one share a Type but differ in Route. Client* are
|
||||
// best-effort viewer identity for oversight and abuse heuristics (e.g. one
|
||||
// re-streaming client fanning out to many viewers). Position is the last
|
||||
// known playback position in seconds (secondary timing).
|
||||
Route string `json:"route,omitempty"`
|
||||
ClientIP string `json:"client_ip,omitempty"`
|
||||
ClientName string `json:"client_name,omitempty"`
|
||||
Position float64 `json:"position,omitempty"`
|
||||
}
|
||||
|
||||
// Lease identifies one request-scoped reference to a tracked session. Its
|
||||
// fields are deliberately private: only the Tracker that issued it can release
|
||||
// it, and stale leases from an invalidated generation are harmless.
|
||||
type Lease struct {
|
||||
sessionID string
|
||||
epoch uint64
|
||||
id uint64
|
||||
}
|
||||
|
||||
type sessionRefs struct {
|
||||
epoch uint64
|
||||
leases map[uint64]struct{}
|
||||
}
|
||||
|
||||
// Tracker manages session lifecycle in Redis for a single node.
|
||||
@@ -51,29 +95,45 @@ type Tracker struct {
|
||||
nodeType string
|
||||
nodeHash string // first 8 chars of SHA-256 of nodeURL
|
||||
|
||||
mu sync.Mutex
|
||||
sessions map[string]struct{} // set of active session IDs
|
||||
touched map[string]time.Time // ephemeral sessions by last-activity time
|
||||
mu sync.Mutex
|
||||
sessions map[string]sessionRefs // explicitly tracked sessions and request leases
|
||||
ephemeral map[string]time.Time // ephemeral sessions by last-observed request time
|
||||
touched map[string]time.Time // last successful serve time
|
||||
records map[string]SessionInfo // last-written record per session, for enriched refresh
|
||||
bytes map[string]int64 // cumulative bytes served per session (monitoring only)
|
||||
nextEpoch uint64
|
||||
nextLease uint64
|
||||
}
|
||||
|
||||
// NewTracker creates a session tracker for the given node.
|
||||
// rdb may be nil, in which case all operations are no-ops.
|
||||
func NewTracker(rdb *redis.Client, nodeURL, nodeName, nodeType string) *Tracker {
|
||||
h := sha256.Sum256([]byte(nodeURL))
|
||||
return &Tracker{
|
||||
rdb: rdb,
|
||||
nodeURL: nodeURL,
|
||||
nodeName: nodeName,
|
||||
nodeType: nodeType,
|
||||
nodeHash: hex.EncodeToString(h[:4]), // 8 hex chars
|
||||
sessions: make(map[string]struct{}),
|
||||
touched: make(map[string]time.Time),
|
||||
rdb: rdb,
|
||||
nodeURL: nodeURL,
|
||||
nodeName: nodeName,
|
||||
nodeType: nodeType,
|
||||
nodeHash: namespaceHash(nodeURL),
|
||||
sessions: make(map[string]sessionRefs),
|
||||
ephemeral: make(map[string]time.Time),
|
||||
touched: make(map[string]time.Time),
|
||||
records: make(map[string]SessionInfo),
|
||||
bytes: make(map[string]int64),
|
||||
}
|
||||
}
|
||||
|
||||
// redisKey returns the full Redis key for a session.
|
||||
func (tr *Tracker) redisKey(sessionID string) string {
|
||||
return keyPrefix + tr.nodeHash + ":" + sessionID
|
||||
return sessionRedisKey(tr.nodeHash, sessionID)
|
||||
}
|
||||
|
||||
func namespaceHash(namespace string) string {
|
||||
h := sha256.Sum256([]byte(namespace))
|
||||
return hex.EncodeToString(h[:4])
|
||||
}
|
||||
|
||||
func sessionRedisKey(nodeHash, sessionID string) string {
|
||||
return KeyPrefix + nodeHash + ":" + sessionID
|
||||
}
|
||||
|
||||
// NodeHash returns the node's hash prefix used in Redis keys.
|
||||
@@ -98,72 +158,247 @@ func (tr *Tracker) ActiveCount() int {
|
||||
defer tr.mu.Unlock()
|
||||
now := time.Now()
|
||||
count := len(tr.sessions)
|
||||
for id, last := range tr.touched {
|
||||
for id, last := range tr.ephemeral {
|
||||
if _, dup := tr.sessions[id]; dup {
|
||||
continue
|
||||
}
|
||||
if now.Sub(last) <= sessionTTL {
|
||||
if now.Sub(last) <= idleTTLFor(tr.records[id]) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// Track registers an active session in Redis with a TTL.
|
||||
func (tr *Tracker) Track(ctx context.Context, info SessionInfo) {
|
||||
if tr.rdb == nil {
|
||||
return
|
||||
// Snapshot returns a copy of the currently-known session records on this node
|
||||
// (including ephemeral ones still within their TTL), stamped with the latest
|
||||
// LastServedAt/BytesServed. Used for the node /status view; central aggregates
|
||||
// the equivalent picture by reading Redis.
|
||||
func (tr *Tracker) Snapshot() []SessionInfo {
|
||||
tr.mu.Lock()
|
||||
defer tr.mu.Unlock()
|
||||
now := time.Now()
|
||||
out := make([]SessionInfo, 0, len(tr.records))
|
||||
for id, rec := range tr.records {
|
||||
// Session-backed entries are always live until Remove; only ephemeral
|
||||
// (non-session) entries age out by idle timeout.
|
||||
if refs, isSession := tr.sessions[id]; !isSession || len(refs.leases) == 0 {
|
||||
if last, ok := tr.ephemeral[id]; !ok || now.Sub(last) > idleTTLFor(rec) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, tr.enrichLocked(id, rec))
|
||||
}
|
||||
data, err := json.Marshal(info)
|
||||
return out
|
||||
}
|
||||
|
||||
// enrichLocked stamps a record with the latest observed liveness/bytes. Caller
|
||||
// holds tr.mu.
|
||||
func (tr *Tracker) enrichLocked(id string, rec SessionInfo) SessionInfo {
|
||||
if last, ok := tr.touched[id]; ok {
|
||||
rec.LastServedAt = last.UTC().Format(time.RFC3339)
|
||||
}
|
||||
if b, ok := tr.bytes[id]; ok {
|
||||
rec.BytesServed = b
|
||||
}
|
||||
return rec
|
||||
}
|
||||
|
||||
// Track registers an active session in Redis with a TTL and returns a lease.
|
||||
// Request-scoped callers must release it exactly once. Lifecycle-owned callers
|
||||
// may instead use Remove for unconditional teardown.
|
||||
func (tr *Tracker) Track(ctx context.Context, info SessionInfo) Lease {
|
||||
if tr.rdb == nil {
|
||||
return Lease{}
|
||||
}
|
||||
now := time.Now()
|
||||
if info.LastServedAt == "" {
|
||||
info.LastServedAt = now.UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// Explicitly-tracked sessions (direct play / remux) live in tr.sessions for
|
||||
// the whole connection. touched holds their last-served time so LastServedAt
|
||||
// reflects real activity; refreshAll only idle-prunes ephemeral (non-session)
|
||||
// touched entries, so a long quiet pour is never expired while its connection
|
||||
// is open.
|
||||
tr.mu.Lock()
|
||||
tr.preserveStartedAtLocked(&info)
|
||||
refs, exists := tr.sessions[info.SessionID]
|
||||
if !exists {
|
||||
tr.nextEpoch++
|
||||
refs = sessionRefs{epoch: tr.nextEpoch, leases: make(map[uint64]struct{})}
|
||||
}
|
||||
tr.nextLease++
|
||||
lease := Lease{sessionID: info.SessionID, epoch: refs.epoch, id: tr.nextLease}
|
||||
refs.leases[lease.id] = struct{}{}
|
||||
tr.sessions[info.SessionID] = refs
|
||||
delete(tr.ephemeral, info.SessionID)
|
||||
tr.records[info.SessionID] = info
|
||||
tr.touched[info.SessionID] = now
|
||||
enriched := tr.enrichLocked(info.SessionID, info)
|
||||
tr.mu.Unlock()
|
||||
if exists {
|
||||
return lease
|
||||
}
|
||||
|
||||
data, err := json.Marshal(enriched)
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "session track marshal failed", "component", "nodesessions", "error", err)
|
||||
return
|
||||
return lease
|
||||
}
|
||||
key := tr.redisKey(info.SessionID)
|
||||
if err := tr.rdb.Set(ctx, key, data, sessionTTL).Err(); err != nil {
|
||||
slog.DebugContext(ctx, "session track set failed", "component", "nodesessions", "error", err, "session", info.SessionID)
|
||||
return
|
||||
}
|
||||
|
||||
tr.mu.Lock()
|
||||
tr.sessions[info.SessionID] = struct{}{}
|
||||
tr.mu.Unlock()
|
||||
return lease
|
||||
}
|
||||
|
||||
// Touch registers or refreshes an ephemeral session that has no explicit end,
|
||||
// such as HLS manifest/segment fetches flowing through a proxy. The session is
|
||||
// written to Redis on first touch and drops out of the active count after
|
||||
// sessionTTL without further touches (pruned by the refresh loop).
|
||||
func (tr *Tracker) Touch(ctx context.Context, info SessionInfo) {
|
||||
// EnsureEphemeral makes a short-lived session visible without claiming that any
|
||||
// bytes were served. The first write is synchronous; later liveness/byte updates
|
||||
// are flushed by the refresh loop.
|
||||
func (tr *Tracker) EnsureEphemeral(ctx context.Context, info SessionInfo) {
|
||||
if tr.rdb == nil {
|
||||
return
|
||||
}
|
||||
now := time.Now()
|
||||
tr.mu.Lock()
|
||||
_, known := tr.touched[info.SessionID]
|
||||
tr.touched[info.SessionID] = time.Now()
|
||||
tr.preserveStartedAtLocked(&info)
|
||||
_, known := tr.records[info.SessionID]
|
||||
tr.ephemeral[info.SessionID] = now
|
||||
tr.records[info.SessionID] = info
|
||||
enriched := tr.enrichLocked(info.SessionID, info)
|
||||
tr.mu.Unlock()
|
||||
if known {
|
||||
return
|
||||
}
|
||||
|
||||
data, err := json.Marshal(info)
|
||||
data, err := json.Marshal(enriched)
|
||||
if err != nil {
|
||||
slog.DebugContext(ctx, "session touch marshal failed", "component", "nodesessions", "error", err)
|
||||
slog.DebugContext(ctx, "session ensure marshal failed", "component", "nodesessions", "error", err)
|
||||
return
|
||||
}
|
||||
if err := tr.rdb.Set(ctx, tr.redisKey(info.SessionID), data, sessionTTL).Err(); err != nil {
|
||||
slog.DebugContext(ctx, "session touch set failed", "component", "nodesessions", "error", err, "session", info.SessionID)
|
||||
slog.DebugContext(ctx, "session ensure set failed", "component", "nodesessions", "error", err, "session", info.SessionID)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove deletes a session from Redis and the in-memory set.
|
||||
// Touch registers or refreshes an ephemeral session that has no explicit end,
|
||||
// such as HLS manifest/segment fetches flowing through a proxy. The session is
|
||||
// written to Redis on first touch and drops out of the active count after
|
||||
// sessionTTL without further touches (pruned by the refresh loop). Subsequent
|
||||
// touches update LastServedAt in memory; the body is re-flushed on the refresh
|
||||
// tick so the monitoring record reflects real serve activity.
|
||||
func (tr *Tracker) Touch(ctx context.Context, info SessionInfo) {
|
||||
if tr.rdb == nil {
|
||||
return
|
||||
}
|
||||
tr.EnsureEphemeral(ctx, info)
|
||||
tr.mu.Lock()
|
||||
if _, known := tr.records[info.SessionID]; known {
|
||||
tr.touched[info.SessionID] = time.Now()
|
||||
}
|
||||
tr.mu.Unlock()
|
||||
}
|
||||
|
||||
// preserveStartedAtLocked keeps the first-seen StartedAt when a record is
|
||||
// re-written for a session we already track. Track re-fires on every range
|
||||
// reconnect and Touch stamps a fresh sessionInfo per segment fetch; without
|
||||
// this, StartedAt degrades to "time of last request", corrupting the admin
|
||||
// view and the enforcer's StartedAt tie-break in victim selection. Caller
|
||||
// holds tr.mu.
|
||||
func (tr *Tracker) preserveStartedAtLocked(info *SessionInfo) {
|
||||
if prev, ok := tr.records[info.SessionID]; ok && prev.StartedAt != "" {
|
||||
info.StartedAt = prev.StartedAt
|
||||
}
|
||||
}
|
||||
|
||||
// AddBytes records that n bytes were served for a session and marks it as active
|
||||
// now. Cheap and lock-guarded; the accumulated value is flushed to Redis on the
|
||||
// refresh tick. Bytes attribution is best-effort monitoring and never gates the
|
||||
// hot path. Bytes for a session with no live record are dropped: a late tally
|
||||
// arriving after the record was idle-pruned (or removed) must not recreate a
|
||||
// bytes entry nothing will ever clean up — that was a slow permanent leak on
|
||||
// busy edges.
|
||||
func (tr *Tracker) AddBytes(sessionID string, n int64) {
|
||||
if tr.rdb == nil || n <= 0 {
|
||||
return
|
||||
}
|
||||
tr.mu.Lock()
|
||||
if _, known := tr.records[sessionID]; known {
|
||||
tr.bytes[sessionID] += n
|
||||
// Mark real serve activity so LastServedAt advances for every session
|
||||
// type, including long direct-play/remux pours.
|
||||
tr.touched[sessionID] = time.Now()
|
||||
}
|
||||
tr.mu.Unlock()
|
||||
}
|
||||
|
||||
// MarkServed records real serve activity for a session without byte
|
||||
// attribution, advancing LastServedAt (flushed on the refresh tick). Used by
|
||||
// the transcode node's own serve paths, where wrapping ServeFile for byte
|
||||
// counting would cost the sendfile fast path for a signal the fronting proxy
|
||||
// already measures — without it the node record's LastServedAt stays frozen at
|
||||
// start time, which starves the enforcer's freshness ordering and the admin
|
||||
// view of the node-side truth.
|
||||
func (tr *Tracker) MarkServed(sessionID string) {
|
||||
if tr == nil || tr.rdb == nil {
|
||||
return
|
||||
}
|
||||
tr.mu.Lock()
|
||||
if _, known := tr.records[sessionID]; known {
|
||||
tr.touched[sessionID] = time.Now()
|
||||
}
|
||||
tr.mu.Unlock()
|
||||
}
|
||||
|
||||
// Release drops one request-scoped lease. A stale, duplicate, or already
|
||||
// invalidated release is a no-op. The final live lease tears the record down.
|
||||
func (tr *Tracker) Release(ctx context.Context, lease Lease) {
|
||||
if tr.rdb == nil || lease.sessionID == "" {
|
||||
return
|
||||
}
|
||||
tr.mu.Lock()
|
||||
refs, ok := tr.sessions[lease.sessionID]
|
||||
if !ok || refs.epoch != lease.epoch {
|
||||
tr.mu.Unlock()
|
||||
slog.DebugContext(ctx, "stale session lease release ignored", "component", "nodesessions", "session", lease.sessionID)
|
||||
return
|
||||
}
|
||||
if _, ok := refs.leases[lease.id]; !ok {
|
||||
tr.mu.Unlock()
|
||||
slog.DebugContext(ctx, "duplicate session lease release ignored", "component", "nodesessions", "session", lease.sessionID)
|
||||
return
|
||||
}
|
||||
delete(refs.leases, lease.id)
|
||||
if len(refs.leases) > 0 {
|
||||
tr.sessions[lease.sessionID] = refs
|
||||
tr.mu.Unlock()
|
||||
return
|
||||
}
|
||||
delete(tr.sessions, lease.sessionID)
|
||||
delete(tr.ephemeral, lease.sessionID)
|
||||
delete(tr.touched, lease.sessionID)
|
||||
delete(tr.records, lease.sessionID)
|
||||
delete(tr.bytes, lease.sessionID)
|
||||
tr.mu.Unlock()
|
||||
|
||||
if err := tr.rdb.Del(ctx, tr.redisKey(lease.sessionID)).Err(); err != nil {
|
||||
slog.DebugContext(ctx, "session release failed", "component", "nodesessions", "error", err, "session", lease.sessionID)
|
||||
}
|
||||
}
|
||||
|
||||
// Remove deletes a session unconditionally and invalidates every outstanding
|
||||
// lease for its generation.
|
||||
func (tr *Tracker) Remove(ctx context.Context, sessionID string) {
|
||||
if tr.rdb == nil {
|
||||
return
|
||||
}
|
||||
tr.mu.Lock()
|
||||
tr.nextEpoch++
|
||||
delete(tr.sessions, sessionID)
|
||||
delete(tr.ephemeral, sessionID)
|
||||
delete(tr.touched, sessionID)
|
||||
delete(tr.records, sessionID)
|
||||
delete(tr.bytes, sessionID)
|
||||
tr.mu.Unlock()
|
||||
|
||||
if err := tr.rdb.Del(ctx, tr.redisKey(sessionID)).Err(); err != nil {
|
||||
@@ -177,17 +412,21 @@ func (tr *Tracker) Cleanup(ctx context.Context) {
|
||||
return
|
||||
}
|
||||
tr.mu.Lock()
|
||||
ids := make([]string, 0, len(tr.sessions)+len(tr.touched))
|
||||
ids := make([]string, 0, len(tr.sessions)+len(tr.ephemeral))
|
||||
for id := range tr.sessions {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
for id := range tr.touched {
|
||||
for id := range tr.ephemeral {
|
||||
if _, dup := tr.sessions[id]; !dup {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
tr.sessions = make(map[string]struct{})
|
||||
tr.nextEpoch++
|
||||
tr.sessions = make(map[string]sessionRefs)
|
||||
tr.ephemeral = make(map[string]time.Time)
|
||||
tr.touched = make(map[string]time.Time)
|
||||
tr.records = make(map[string]SessionInfo)
|
||||
tr.bytes = make(map[string]int64)
|
||||
tr.mu.Unlock()
|
||||
|
||||
if len(ids) == 0 {
|
||||
@@ -225,33 +464,63 @@ func (tr *Tracker) StartRefresh(ctx context.Context) {
|
||||
|
||||
func (tr *Tracker) refreshAll(ctx context.Context) {
|
||||
now := time.Now()
|
||||
type flush struct {
|
||||
id string
|
||||
data []byte
|
||||
}
|
||||
tr.mu.Lock()
|
||||
ids := make([]string, 0, len(tr.sessions)+len(tr.touched))
|
||||
ids := make([]string, 0, len(tr.sessions))
|
||||
for id := range tr.sessions {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
for id, last := range tr.touched {
|
||||
if now.Sub(last) > sessionTTL {
|
||||
// Idle ephemeral session: stop refreshing and let the Redis
|
||||
// key expire on its own.
|
||||
for id, last := range tr.ephemeral {
|
||||
_, isSession := tr.sessions[id]
|
||||
if !isSession && now.Sub(last) > idleTTLFor(tr.records[id]) {
|
||||
// Idle ephemeral session: stop refreshing and let the Redis key
|
||||
// expire on its own. Session-backed entries (direct/remux) are pruned
|
||||
// on Remove, never by idle timeout, so a quiet-but-open pour stays live.
|
||||
delete(tr.ephemeral, id)
|
||||
delete(tr.touched, id)
|
||||
delete(tr.records, id)
|
||||
delete(tr.bytes, id)
|
||||
continue
|
||||
}
|
||||
if _, dup := tr.sessions[id]; !dup {
|
||||
if !isSession {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
}
|
||||
flushes := make([]flush, 0, len(ids))
|
||||
for _, id := range ids {
|
||||
rec, ok := tr.records[id]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
data, err := json.Marshal(tr.enrichLocked(id, rec))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
flushes = append(flushes, flush{id: id, data: data})
|
||||
}
|
||||
tr.mu.Unlock()
|
||||
|
||||
if len(ids) == 0 {
|
||||
if len(flushes) == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// Re-SET (not just EXPIRE) so LastServedAt/BytesServed stay current in the
|
||||
// monitoring record. 30s cadence keeps this cheap.
|
||||
pipe := tr.rdb.Pipeline()
|
||||
for _, id := range ids {
|
||||
pipe.Expire(ctx, tr.redisKey(id), sessionTTL)
|
||||
for _, f := range flushes {
|
||||
pipe.Set(ctx, tr.redisKey(f.id), f.data, sessionTTL)
|
||||
}
|
||||
if _, err := pipe.Exec(ctx); err != nil {
|
||||
slog.DebugContext(ctx, "session refresh pipeline failed", "component", "nodesessions", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func idleTTLFor(rec SessionInfo) time.Duration {
|
||||
if rec.Type == sessionTypeTranscode {
|
||||
return transcodeIdleTTL
|
||||
}
|
||||
return sessionTTL
|
||||
}
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package nodesessions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func TestTranscodeIdleTTLAgreesAcrossCountSnapshotAndRefresh(t *testing.T) {
|
||||
rdb := redis.NewClient(&redis.Options{Addr: "127.0.0.1:1", DialTimeout: 10 * time.Millisecond})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
tr := NewTracker(rdb, "http://node", "node", "proxy")
|
||||
idle := time.Now().Add(-90 * time.Second)
|
||||
tr.ephemeral[sessionTypeTranscode] = idle
|
||||
tr.records[sessionTypeTranscode] = SessionInfo{SessionID: sessionTypeTranscode, Type: sessionTypeTranscode}
|
||||
tr.ephemeral["direct"] = idle
|
||||
tr.records["direct"] = SessionInfo{SessionID: "direct", Type: "direct_play"}
|
||||
|
||||
if got := tr.ActiveCount(); got != 1 {
|
||||
t.Fatalf("ActiveCount=%d, want only transcode", got)
|
||||
}
|
||||
snapshot := tr.Snapshot()
|
||||
if len(snapshot) != 1 || snapshot[0].SessionID != sessionTypeTranscode {
|
||||
t.Fatalf("Snapshot=%+v, want only transcode", snapshot)
|
||||
}
|
||||
|
||||
tr.refreshAll(context.Background())
|
||||
tr.mu.Lock()
|
||||
_, transcodeRetained := tr.records[sessionTypeTranscode]
|
||||
_, directRetained := tr.records["direct"]
|
||||
tr.mu.Unlock()
|
||||
if !transcodeRetained || directRetained {
|
||||
t.Fatalf("refresh records: transcode=%v direct=%v, want true/false", transcodeRetained, directRetained)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package nodesessions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type memoryRedisHook struct {
|
||||
mu sync.Mutex
|
||||
values map[string]string
|
||||
}
|
||||
|
||||
func (h *memoryRedisHook) DialHook(next redis.DialHook) redis.DialHook { return next }
|
||||
|
||||
func (h *memoryRedisHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
|
||||
return func(ctx context.Context, cmd redis.Cmder) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
switch c := cmd.(type) {
|
||||
case *redis.StatusCmd:
|
||||
if c.Name() == "set" {
|
||||
args := c.Args()
|
||||
var value string
|
||||
switch v := args[2].(type) {
|
||||
case string:
|
||||
value = v
|
||||
case []byte:
|
||||
value = string(v)
|
||||
}
|
||||
h.values[args[1].(string)] = value
|
||||
c.SetVal("OK")
|
||||
return nil
|
||||
}
|
||||
case *redis.IntCmd:
|
||||
if c.Name() == "del" {
|
||||
for _, arg := range c.Args()[1:] {
|
||||
delete(h.values, arg.(string))
|
||||
}
|
||||
c.SetVal(1)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return next(ctx, cmd)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *memoryRedisHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
|
||||
return func(ctx context.Context, cmds []redis.Cmder) error {
|
||||
for _, cmd := range cmds {
|
||||
if err := h.ProcessHook(func(context.Context, redis.Cmder) error { return nil })(ctx, cmd); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func newMemoryTracker(t *testing.T) (*Tracker, *memoryRedisHook) {
|
||||
t.Helper()
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Dialer: func(context.Context, string, string) (net.Conn, error) {
|
||||
t.Fatal("unexpected Redis dial")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
hook := &memoryRedisHook{values: make(map[string]string)}
|
||||
rdb.AddHook(hook)
|
||||
return NewTracker(rdb, "http://node", "node", "proxy"), hook
|
||||
}
|
||||
|
||||
func (h *memoryRedisHook) has(key string) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
_, ok := h.values[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
func TestStaleLeaseCannotReleaseReplacementAfterRemove(t *testing.T) {
|
||||
tr, _ := newMemoryTracker(t)
|
||||
ctx := context.Background()
|
||||
old := tr.Track(ctx, SessionInfo{SessionID: "s"})
|
||||
tr.Remove(ctx, "s")
|
||||
current := tr.Track(ctx, SessionInfo{SessionID: "s"})
|
||||
|
||||
tr.Release(ctx, old)
|
||||
if got := tr.Snapshot(); len(got) != 1 || got[0].SessionID != "s" {
|
||||
t.Fatalf("stale release removed replacement: %+v", got)
|
||||
}
|
||||
tr.Release(ctx, current)
|
||||
}
|
||||
|
||||
func TestStaleLeaseCannotReleaseReplacementAfterCleanup(t *testing.T) {
|
||||
tr, _ := newMemoryTracker(t)
|
||||
ctx := context.Background()
|
||||
old := tr.Track(ctx, SessionInfo{SessionID: "s"})
|
||||
tr.Cleanup(ctx)
|
||||
current := tr.Track(ctx, SessionInfo{SessionID: "s"})
|
||||
|
||||
tr.Release(ctx, old)
|
||||
if got := tr.Snapshot(); len(got) != 1 || got[0].SessionID != "s" {
|
||||
t.Fatalf("stale release after cleanup removed replacement: %+v", got)
|
||||
}
|
||||
tr.Release(ctx, current)
|
||||
}
|
||||
|
||||
func TestDoubleReleaseDoesNotConsumeAnotherLease(t *testing.T) {
|
||||
tr, _ := newMemoryTracker(t)
|
||||
ctx := context.Background()
|
||||
first := tr.Track(ctx, SessionInfo{SessionID: "s"})
|
||||
second := tr.Track(ctx, SessionInfo{SessionID: "s"})
|
||||
|
||||
tr.Release(ctx, first)
|
||||
tr.Release(ctx, first)
|
||||
if got := tr.Snapshot(); len(got) != 1 {
|
||||
t.Fatalf("double release consumed live lease: %+v", got)
|
||||
}
|
||||
tr.Release(ctx, second)
|
||||
}
|
||||
|
||||
func TestOverlappingLeasesKeepRecordKeyAndBytesAlive(t *testing.T) {
|
||||
tr, redisState := newMemoryTracker(t)
|
||||
ctx := context.Background()
|
||||
first := tr.Track(ctx, SessionInfo{SessionID: "s"})
|
||||
second := tr.Track(ctx, SessionInfo{SessionID: "s"})
|
||||
|
||||
tr.Release(ctx, first)
|
||||
tr.AddBytes("s", 123)
|
||||
got := tr.Snapshot()
|
||||
if len(got) != 1 || got[0].BytesServed != 123 {
|
||||
t.Fatalf("live overlap record = %+v, want 123 bytes", got)
|
||||
}
|
||||
if !redisState.has(tr.redisKey("s")) {
|
||||
t.Fatal("first release deleted Redis key while second lease was live")
|
||||
}
|
||||
|
||||
tr.Release(ctx, second)
|
||||
if redisState.has(tr.redisKey("s")) {
|
||||
t.Fatal("final release left Redis key behind")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnsureEphemeralSeparatesVisibilityFromServedLiveness(t *testing.T) {
|
||||
tr, redisState := newMemoryTracker(t)
|
||||
ctx := context.Background()
|
||||
tr.EnsureEphemeral(ctx, SessionInfo{SessionID: "s", Type: "transcode"})
|
||||
got := tr.Snapshot()
|
||||
if len(got) != 1 || got[0].LastServedAt != "" || got[0].BytesServed != 0 {
|
||||
t.Fatalf("ensured record claims served liveness: %+v", got)
|
||||
}
|
||||
if !redisState.has(tr.redisKey("s")) {
|
||||
t.Fatal("ensured record was not projected to Redis")
|
||||
}
|
||||
|
||||
tr.AddBytes("s", 7)
|
||||
got = tr.Snapshot()
|
||||
if len(got) != 1 || got[0].LastServedAt == "" || got[0].BytesServed != 7 {
|
||||
t.Fatalf("served record = %+v, want liveness and bytes", got)
|
||||
}
|
||||
|
||||
redisState.mu.Lock()
|
||||
var projected SessionInfo
|
||||
err := json.Unmarshal([]byte(redisState.values[tr.redisKey("s")]), &projected)
|
||||
redisState.mu.Unlock()
|
||||
if err != nil {
|
||||
t.Fatalf("decode projected record: %v", err)
|
||||
}
|
||||
if projected.LastServedAt != "" {
|
||||
t.Fatalf("first projection LastServedAt = %q, want empty", projected.LastServedAt)
|
||||
}
|
||||
}
|
||||
@@ -57,7 +57,7 @@ func MimeFromExtension(name string) string {
|
||||
func ServeDirectPlay(w http.ResponseWriter, r *http.Request, filePath string) error {
|
||||
// Media bodies routinely take longer than the server's absolute
|
||||
// WriteTimeout; roll the write deadline with progress instead.
|
||||
streamWriter := httpstream.NewRollingDeadlineWriter(w)
|
||||
streamWriter := httpstream.NewRollingDeadlineWriterCtx(r.Context(), w)
|
||||
w = streamWriter
|
||||
f, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package playback
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
const meteredWriterFlushBytes = 1 << 20
|
||||
|
||||
// ServedBytesRecorder receives server-observed byte counts.
|
||||
type ServedBytesRecorder interface {
|
||||
AddServedBytes(sessionID string, n int64) error
|
||||
}
|
||||
|
||||
// SessionMeteredWriter preserves optional ResponseWriter capabilities while
|
||||
// attributing bytes to a playback session in coarse chunks.
|
||||
type SessionMeteredWriter struct {
|
||||
http.ResponseWriter
|
||||
recorder ServedBytesRecorder
|
||||
sessionID string
|
||||
pending int64
|
||||
}
|
||||
|
||||
func NewSessionMeteredWriter(w http.ResponseWriter, recorder ServedBytesRecorder, sessionID string) *SessionMeteredWriter {
|
||||
return &SessionMeteredWriter{ResponseWriter: w, recorder: recorder, sessionID: sessionID}
|
||||
}
|
||||
|
||||
func (w *SessionMeteredWriter) Write(p []byte) (int, error) {
|
||||
n, err := w.ResponseWriter.Write(p)
|
||||
w.account(int64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
func (w *SessionMeteredWriter) ReadFrom(src io.Reader) (int64, error) {
|
||||
if rf, ok := w.ResponseWriter.(io.ReaderFrom); ok {
|
||||
n, err := rf.ReadFrom(src)
|
||||
w.account(n)
|
||||
return n, err
|
||||
}
|
||||
return io.Copy(meteredWriteOnly{w}, src)
|
||||
}
|
||||
|
||||
type meteredWriteOnly struct{ io.Writer }
|
||||
|
||||
func (w *SessionMeteredWriter) account(n int64) {
|
||||
if n <= 0 {
|
||||
return
|
||||
}
|
||||
w.pending += n
|
||||
if w.pending >= meteredWriterFlushBytes {
|
||||
w.flush()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *SessionMeteredWriter) flush() {
|
||||
if w.pending <= 0 {
|
||||
return
|
||||
}
|
||||
if w.recorder != nil {
|
||||
_ = w.recorder.AddServedBytes(w.sessionID, w.pending)
|
||||
}
|
||||
w.pending = 0
|
||||
}
|
||||
|
||||
// Close flushes the final partial chunk. Callers must defer it.
|
||||
func (w *SessionMeteredWriter) Close() error {
|
||||
w.flush()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *SessionMeteredWriter) Flush() {
|
||||
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// Unwrap lets http.ResponseController reach the underlying connection.
|
||||
func (w *SessionMeteredWriter) Unwrap() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package playback
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
)
|
||||
|
||||
type byteRecorder struct{ total int64 }
|
||||
|
||||
func (r *byteRecorder) AddServedBytes(_ string, n int64) error {
|
||||
r.total += n
|
||||
return nil
|
||||
}
|
||||
|
||||
type readerFromResponseWriter struct {
|
||||
header http.Header
|
||||
body bytes.Buffer
|
||||
readFromCalls int
|
||||
deadlineSet bool
|
||||
}
|
||||
|
||||
func (w *readerFromResponseWriter) Header() http.Header { return w.header }
|
||||
func (w *readerFromResponseWriter) WriteHeader(int) {}
|
||||
func (w *readerFromResponseWriter) Write(p []byte) (int, error) { return w.body.Write(p) }
|
||||
func (w *readerFromResponseWriter) SetWriteDeadline(time.Time) error {
|
||||
w.deadlineSet = true
|
||||
return nil
|
||||
}
|
||||
func (w *readerFromResponseWriter) ReadFrom(src io.Reader) (int64, error) {
|
||||
w.readFromCalls++
|
||||
return w.body.ReadFrom(src)
|
||||
}
|
||||
|
||||
type readOnly struct{ io.Reader }
|
||||
|
||||
func TestSessionMeteredWriterPreservesReadFromAndUnwrapChain(t *testing.T) {
|
||||
base := &readerFromResponseWriter{header: make(http.Header)}
|
||||
production := httpstream.NewRollingDeadlineWriter(base)
|
||||
recorder := &byteRecorder{}
|
||||
metered := NewSessionMeteredWriter(production, recorder, "s1")
|
||||
|
||||
payload := bytes.Repeat([]byte("x"), 2<<20)
|
||||
n, err := metered.ReadFrom(readOnly{bytes.NewReader(payload)})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFrom: %v", err)
|
||||
}
|
||||
if n != int64(len(payload)) || base.readFromCalls == 0 {
|
||||
t.Fatalf("ReadFrom n=%d calls=%d, want %d and fast-path calls", n, base.readFromCalls, len(payload))
|
||||
}
|
||||
if err := metered.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
if recorder.total != int64(len(payload)) {
|
||||
t.Fatalf("recorded bytes=%d, want %d", recorder.total, len(payload))
|
||||
}
|
||||
if metered.Unwrap() != production {
|
||||
t.Fatal("Unwrap did not expose the next production writer")
|
||||
}
|
||||
if err := http.NewResponseController(metered).SetWriteDeadline(time.Now()); err != nil {
|
||||
t.Fatalf("SetWriteDeadline through chain: %v", err)
|
||||
}
|
||||
if !base.deadlineSet {
|
||||
t.Fatal("response controller did not reach base writer")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionMeteredWriterFallbackDoesNotRecurseAndFlushesTail(t *testing.T) {
|
||||
base := httptest.NewRecorder()
|
||||
recorder := &byteRecorder{}
|
||||
metered := NewSessionMeteredWriter(base, recorder, "s1")
|
||||
payload := []byte("final partial chunk")
|
||||
|
||||
n, err := metered.ReadFrom(readOnly{bytes.NewReader(payload)})
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFrom fallback: %v", err)
|
||||
}
|
||||
if n != int64(len(payload)) {
|
||||
t.Fatalf("ReadFrom n=%d, want %d", n, len(payload))
|
||||
}
|
||||
if recorder.total != 0 {
|
||||
t.Fatalf("tail flushed before Close: %d", recorder.total)
|
||||
}
|
||||
defer func() { _ = metered.Close() }()
|
||||
if err := metered.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
if recorder.total != int64(len(payload)) {
|
||||
t.Fatalf("tail bytes=%d, want %d", recorder.total, len(payload))
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,6 @@
|
||||
package playback
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/streamtoken"
|
||||
)
|
||||
import "github.com/Silo-Server/silo-server/internal/streamtoken"
|
||||
|
||||
// RecipeCard is the small, durable "recipe" needed to reconstruct a transcode
|
||||
// session after the server forgets its in-memory state (e.g. a restart). It
|
||||
@@ -183,7 +179,7 @@ func (c RecipeCard) TranscodeOpts(outputDir, ffmpegPath string, logSink FFmpegLo
|
||||
// cleanup spares a dir that is not live in memory only until this age elapses:
|
||||
// past it, no surviving token could still reconstruct the session, so the dir is
|
||||
// safe to reap. It must comfortably outlast any realistic restart outage.
|
||||
const MaxTokenTTL = 24 * time.Hour
|
||||
const MaxTokenTTL = streamtoken.MaxTTL
|
||||
|
||||
// ToClaims projects the reconstruction recipe into stream-token claims so the
|
||||
// card can travel with the client instead of a shared per-session store. The
|
||||
|
||||
@@ -335,7 +335,7 @@ func ServeRemux(w http.ResponseWriter, r *http.Request, filePath, outputFormat s
|
||||
func ServeRemuxWithDVMode(w http.ResponseWriter, r *http.Request, filePath, outputFormat string, seekSeconds float64, transcodeAudio bool, audioTrackIndex int, dvProfile int, mode RemuxDVMode, ffmpegPath string) error {
|
||||
// Remux output streams for the length of the title; roll the write
|
||||
// deadline with progress instead of the server's absolute WriteTimeout.
|
||||
w = httpstream.NewRollingDeadlineWriter(w)
|
||||
w = httpstream.NewRollingDeadlineWriterCtx(r.Context(), w)
|
||||
// Check file exists before starting ffmpeg to return a proper 404.
|
||||
// Headers must be written before streaming begins, so we can't detect
|
||||
// ffmpeg errors after WriteHeader(200) has been sent.
|
||||
|
||||
+116
-21
@@ -56,6 +56,8 @@ type Session struct {
|
||||
StartedAt time.Time
|
||||
UpdatedAt time.Time
|
||||
LastActivityAt time.Time
|
||||
BytesServed int64
|
||||
LastServedAt time.Time
|
||||
activeTransportCount int
|
||||
replacementPlayMethod PlayMethod
|
||||
streamRevision uint64
|
||||
@@ -135,6 +137,14 @@ var ErrSessionReplacementSuperseded = errors.New("session replacement was supers
|
||||
|
||||
type clientInfoContextKey struct{}
|
||||
|
||||
// Origin identifies the protocol a playback session was started through, for
|
||||
// first-class monitoring attribution (a jellycompat session shares this manager
|
||||
// with native ones and is otherwise indistinguishable in the live snapshot).
|
||||
const (
|
||||
OriginNative = "native"
|
||||
OriginJellyfin = "jellycompat"
|
||||
)
|
||||
|
||||
// ClientInfo carries best-effort client metadata from request handling into
|
||||
// the playback session manager.
|
||||
type ClientInfo struct {
|
||||
@@ -171,6 +181,8 @@ type SessionManager struct {
|
||||
admissionDecider AdmissionDecider
|
||||
activeGrace time.Duration
|
||||
pausedGrace time.Duration
|
||||
transcodeGrace time.Duration
|
||||
unservedGrace time.Duration
|
||||
expireHook func(*Session)
|
||||
}
|
||||
|
||||
@@ -233,6 +245,14 @@ const (
|
||||
// pressing Play after a long pause freeze the client (issue #243).
|
||||
// Keep in sync with pausedSessionGrace in internal/worker/cleanup.go.
|
||||
DefaultPausedSessionGrace = 30 * time.Minute
|
||||
|
||||
// DefaultTranscodeSessionGrace covers buffer-ahead gaps between segment
|
||||
// requests without depending on local ffmpeg process state.
|
||||
DefaultTranscodeSessionGrace = 10 * time.Minute
|
||||
|
||||
// DefaultUnservedSessionGrace gives a newly-created session time to begin
|
||||
// serving media before admission and cleanup treat it as inactive.
|
||||
DefaultUnservedSessionGrace = 2 * time.Minute
|
||||
)
|
||||
|
||||
// NewSessionManager creates a SessionManager with the given concurrency limits.
|
||||
@@ -240,11 +260,13 @@ const (
|
||||
// maxTranscodes limits concurrent transcode streams per user.
|
||||
func NewSessionManager(maxStreams, maxTranscodes int) *SessionManager {
|
||||
return &SessionManager{
|
||||
sessions: make(map[string]*Session),
|
||||
maxStreams: maxStreams,
|
||||
maxTranscodes: maxTranscodes,
|
||||
activeGrace: DefaultActiveSessionGrace,
|
||||
pausedGrace: DefaultPausedSessionGrace,
|
||||
sessions: make(map[string]*Session),
|
||||
maxStreams: maxStreams,
|
||||
maxTranscodes: maxTranscodes,
|
||||
activeGrace: DefaultActiveSessionGrace,
|
||||
pausedGrace: DefaultPausedSessionGrace,
|
||||
transcodeGrace: DefaultTranscodeSessionGrace,
|
||||
unservedGrace: DefaultUnservedSessionGrace,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,8 +286,9 @@ func (m *SessionManager) SetAdmissionDecider(decider AdmissionDecider) {
|
||||
m.admissionDecider = decider
|
||||
}
|
||||
|
||||
// SetLivenessGracePeriods overrides the grace periods used by admission
|
||||
// control and stale-session cleanup.
|
||||
// SetLivenessGracePeriods overrides the active and paused grace periods used by
|
||||
// admission control and stale-session cleanup. The never-served grace is
|
||||
// configured independently with SetUnservedSessionGrace.
|
||||
func (m *SessionManager) SetLivenessGracePeriods(active, paused time.Duration) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
@@ -278,6 +301,23 @@ func (m *SessionManager) SetLivenessGracePeriods(active, paused time.Duration) {
|
||||
}
|
||||
}
|
||||
|
||||
// SetTranscodeLivenessGrace overrides the unpaused transcode idle window.
|
||||
func (m *SessionManager) SetTranscodeLivenessGrace(grace time.Duration) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if grace > 0 {
|
||||
m.transcodeGrace = grace
|
||||
}
|
||||
}
|
||||
|
||||
// SetUnservedSessionGrace overrides the idle window for sessions that have not
|
||||
// served media. A non-positive duration disables this additional grace.
|
||||
func (m *SessionManager) SetUnservedSessionGrace(grace time.Duration) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.unservedGrace = grace
|
||||
}
|
||||
|
||||
// SetExpirationHook registers a callback that runs after a session is removed
|
||||
// by stale cleanup. The hook executes outside the manager lock.
|
||||
func (m *SessionManager) SetExpirationHook(fn func(*Session)) {
|
||||
@@ -294,6 +334,17 @@ func normalizeClientMetadataValue(value string, maxLen int) string {
|
||||
return value
|
||||
}
|
||||
|
||||
// Origin reports the protocol this session was started through, as the label
|
||||
// monitoring and stream tokens attribute by. It derives from the stored
|
||||
// IsJellyfinCompat identity so the two can never disagree; native is the
|
||||
// historical default for anything not minted through the compat layer.
|
||||
func (s *Session) Origin() string {
|
||||
if s == nil || !s.IsJellyfinCompat {
|
||||
return OriginNative
|
||||
}
|
||||
return OriginJellyfin
|
||||
}
|
||||
|
||||
// StartSession creates a new playback session using the same file as both the
|
||||
// requested and effective source.
|
||||
func (m *SessionManager) StartSession(userID int, profileID string, fileID int, method PlayMethod, transcodeAudio bool) (*Session, error) {
|
||||
@@ -550,6 +601,19 @@ func (m *SessionManager) RegisterReconstructedWithLimits(ctx context.Context, s
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// EffectiveLimits resolves a user's admission limits through the installed
|
||||
// limit provider, falling back to the manager defaults when none is set.
|
||||
//
|
||||
// Out-of-band consumers — notably the async stream enforcer — must resolve caps
|
||||
// through here rather than reading users.max_streams directly: that column is 0
|
||||
// ("inherit from group") for standard accounts, which an enforcer treating
|
||||
// limit <= 0 as unlimited would read as "no cap". Going through the same
|
||||
// provider synchronous admission uses means the two can never disagree, and the
|
||||
// lookup is late-bound so it sees whatever provider the API wiring installed.
|
||||
func (m *SessionManager) EffectiveLimits(ctx context.Context, userID int) (SessionLimits, error) {
|
||||
return m.limitsForUser(ctx, userID)
|
||||
}
|
||||
|
||||
func (m *SessionManager) limitsForUser(ctx context.Context, userID int) (SessionLimits, error) {
|
||||
m.mu.RLock()
|
||||
provider := m.limitProvider
|
||||
@@ -1080,6 +1144,7 @@ func (m *SessionManager) BeginTransport(sessionID string) error {
|
||||
}
|
||||
|
||||
s.activeTransportCount++
|
||||
s.LastServedAt = time.Now()
|
||||
m.touchSessionLocked(s)
|
||||
return nil
|
||||
}
|
||||
@@ -1098,6 +1163,25 @@ func (m *SessionManager) EndTransport(sessionID string) error {
|
||||
if s.activeTransportCount > 0 {
|
||||
s.activeTransportCount--
|
||||
}
|
||||
s.LastServedAt = time.Now()
|
||||
m.touchSessionLocked(s)
|
||||
return nil
|
||||
}
|
||||
|
||||
// AddServedBytes records bytes observed by a server-side media pour. It never
|
||||
// recreates an unknown or already-reaped session.
|
||||
func (m *SessionManager) AddServedBytes(sessionID string, n int64) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
s, ok := m.sessions[sessionID]
|
||||
if !ok {
|
||||
return ErrSessionNotFound
|
||||
}
|
||||
if n <= 0 {
|
||||
return nil
|
||||
}
|
||||
s.BytesServed += n
|
||||
s.LastServedAt = time.Now()
|
||||
m.touchSessionLocked(s)
|
||||
return nil
|
||||
}
|
||||
@@ -1236,8 +1320,9 @@ func (m *SessionManager) AllSessions() []*Session {
|
||||
return result
|
||||
}
|
||||
|
||||
// CleanExpired removes sessions whose last playback activity exceeds maxIdle.
|
||||
// Paused sessions receive a 3x grace period for backwards compatibility.
|
||||
// CleanExpired removes sessions whose last server-observed liveness exceeds
|
||||
// maxIdle. Paused sessions receive a 3x grace period for backwards
|
||||
// compatibility.
|
||||
func (m *SessionManager) CleanExpired(maxIdle time.Duration) []*Session {
|
||||
return m.CleanInactive(maxIdle, maxIdle*3)
|
||||
}
|
||||
@@ -1252,9 +1337,9 @@ func (m *SessionManager) CleanStale() []*Session {
|
||||
return m.CleanInactive(active, paused)
|
||||
}
|
||||
|
||||
// CleanInactive removes sessions whose last playback activity exceeds the
|
||||
// provided grace period. Sessions with an active media transport request are
|
||||
// preserved even if they have not emitted a recent heartbeat yet.
|
||||
// CleanInactive removes sessions whose last server-observed liveness exceeds
|
||||
// the provided grace period. Sessions with an active media transport request
|
||||
// are preserved even if they have not emitted a recent heartbeat yet.
|
||||
func (m *SessionManager) CleanInactive(activeIdle, pausedIdle time.Duration) []*Session {
|
||||
m.mu.Lock()
|
||||
|
||||
@@ -1302,26 +1387,36 @@ func (m *SessionManager) sessionIsInactiveLocked(s *Session, now time.Time, acti
|
||||
return true
|
||||
}
|
||||
|
||||
lastActivity := s.LastActivityAt
|
||||
if lastActivity.IsZero() {
|
||||
lastActivity = s.UpdatedAt
|
||||
if s.IsPaused && (s.HasRealtimeConnection || s.HasWebSocket) {
|
||||
return false
|
||||
}
|
||||
if lastActivity.IsZero() {
|
||||
lastActivity = s.StartedAt
|
||||
}
|
||||
if lastActivity.IsZero() {
|
||||
|
||||
livenessAt := s.serverObservedLivenessAt()
|
||||
if livenessAt.IsZero() {
|
||||
return false
|
||||
}
|
||||
|
||||
grace := activeIdle
|
||||
if s.IsPaused {
|
||||
grace = pausedIdle
|
||||
} else if s.PlayMethod == PlayTranscode && m.transcodeGrace > grace {
|
||||
grace = m.transcodeGrace
|
||||
}
|
||||
if s.LastServedAt.IsZero() && s.activeTransportCount == 0 && m.unservedGrace > grace {
|
||||
grace = m.unservedGrace
|
||||
}
|
||||
if grace <= 0 {
|
||||
return !lastActivity.After(now)
|
||||
return !livenessAt.After(now)
|
||||
}
|
||||
|
||||
return !lastActivity.Add(grace).After(now)
|
||||
return !livenessAt.Add(grace).After(now)
|
||||
}
|
||||
|
||||
func (s *Session) serverObservedLivenessAt() time.Time {
|
||||
if !s.LastServedAt.IsZero() {
|
||||
return s.LastServedAt
|
||||
}
|
||||
return s.StartedAt
|
||||
}
|
||||
|
||||
// String returns a human-readable summary of a session.
|
||||
|
||||
@@ -24,23 +24,34 @@ func TestPausedSessionSurvivesIntentionalPause(t *testing.T) {
|
||||
t.Fatalf("UpdateProgress(paused): %v", err)
|
||||
}
|
||||
|
||||
setLastActivity := func(age time.Duration) {
|
||||
setLastServed := func(age time.Duration) {
|
||||
m.mu.Lock()
|
||||
s := m.sessions[session.ID]
|
||||
s.LastActivityAt = time.Now().Add(-age)
|
||||
s.UpdatedAt = s.LastActivityAt
|
||||
s.LastServedAt = time.Now().Add(-age)
|
||||
m.mu.Unlock()
|
||||
}
|
||||
|
||||
if err := m.SetRealtimeConnection(session.ID, true); err != nil {
|
||||
t.Fatalf("SetRealtimeConnection(true): %v", err)
|
||||
}
|
||||
setLastServed(DefaultPausedSessionGrace + time.Minute)
|
||||
m.CleanStale()
|
||||
if _, err := m.GetSession(session.ID); err != nil {
|
||||
t.Fatalf("realtime-connected paused session was reaped: %v", err)
|
||||
}
|
||||
if err := m.SetRealtimeConnection(session.ID, false); err != nil {
|
||||
t.Fatalf("SetRealtimeConnection(false): %v", err)
|
||||
}
|
||||
|
||||
// Paused for 10 minutes: must survive.
|
||||
setLastActivity(10 * time.Minute)
|
||||
setLastServed(10 * time.Minute)
|
||||
m.CleanStale()
|
||||
if _, err := m.GetSession(session.ID); err != nil {
|
||||
t.Fatalf("session reaped after 10-minute pause; paused grace must allow intentional pauses (err: %v)", err)
|
||||
}
|
||||
|
||||
// Abandoned well past the paused grace: must still be reaped.
|
||||
setLastActivity(DefaultPausedSessionGrace + time.Minute)
|
||||
setLastServed(DefaultPausedSessionGrace + time.Minute)
|
||||
m.CleanStale()
|
||||
if _, err := m.GetSession(session.ID); err == nil {
|
||||
t.Fatal("session survived past the paused grace; abandoned sessions must still be reaped")
|
||||
@@ -54,12 +65,11 @@ func TestSequentialRangedTransportsSurviveIdleAndPausedGrace(t *testing.T) {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
|
||||
setLastActivity := func(age time.Duration) {
|
||||
setLastServed := func(age time.Duration) {
|
||||
t.Helper()
|
||||
m.mu.Lock()
|
||||
s := m.sessions[session.ID]
|
||||
s.LastActivityAt = time.Now().Add(-age)
|
||||
s.UpdatedAt = s.LastActivityAt
|
||||
s.LastServedAt = time.Now().Add(-age)
|
||||
m.mu.Unlock()
|
||||
}
|
||||
assertPresent := func(stage string) {
|
||||
@@ -71,14 +81,14 @@ func TestSequentialRangedTransportsSurviveIdleAndPausedGrace(t *testing.T) {
|
||||
|
||||
const activeGrace = 2 * time.Minute
|
||||
for cycle := 1; cycle <= 3; cycle++ {
|
||||
setLastActivity(activeGrace / 2)
|
||||
setLastServed(activeGrace / 2)
|
||||
m.CleanInactive(activeGrace, DefaultPausedSessionGrace)
|
||||
assertPresent(fmt.Sprintf("idle gap before transport %d", cycle))
|
||||
|
||||
if err := m.BeginTransport(session.ID); err != nil {
|
||||
t.Fatalf("BeginTransport(%d): %v", cycle, err)
|
||||
}
|
||||
setLastActivity(activeGrace + time.Minute)
|
||||
setLastServed(activeGrace + time.Minute)
|
||||
m.CleanInactive(activeGrace, DefaultPausedSessionGrace)
|
||||
assertPresent(fmt.Sprintf("active transport %d", cycle))
|
||||
if err := m.EndTransport(session.ID); err != nil {
|
||||
@@ -89,14 +99,14 @@ func TestSequentialRangedTransportsSurviveIdleAndPausedGrace(t *testing.T) {
|
||||
if err := m.UpdateProgress(session.ID, 42, true); err != nil {
|
||||
t.Fatalf("UpdateProgress(paused): %v", err)
|
||||
}
|
||||
setLastActivity(DefaultPausedSessionGrace - time.Minute)
|
||||
setLastServed(DefaultPausedSessionGrace - time.Minute)
|
||||
m.CleanInactive(activeGrace, DefaultPausedSessionGrace)
|
||||
assertPresent("late paused idle gap")
|
||||
|
||||
if err := m.BeginTransport(session.ID); err != nil {
|
||||
t.Fatalf("BeginTransport(late range): %v", err)
|
||||
}
|
||||
setLastActivity(DefaultPausedSessionGrace + time.Minute)
|
||||
setLastServed(DefaultPausedSessionGrace + time.Minute)
|
||||
m.CleanInactive(activeGrace, DefaultPausedSessionGrace)
|
||||
assertPresent("late ranged transport active")
|
||||
if err := m.EndTransport(session.ID); err != nil {
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
package playback
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAddServedBytesUnknownDoesNotCreateSession(t *testing.T) {
|
||||
m := NewSessionManager(0, 0)
|
||||
if err := m.AddServedBytes("missing", 100); !errors.Is(err, ErrSessionNotFound) {
|
||||
t.Fatalf("AddServedBytes error=%v, want ErrSessionNotFound", err)
|
||||
}
|
||||
if len(m.AllSessions()) != 0 {
|
||||
t.Fatal("AddServedBytes recreated an unknown session")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLastServedAtOnlyAdvancesForServerObservedEvents(t *testing.T) {
|
||||
m := NewSessionManager(0, 0)
|
||||
s, err := m.StartSession(1, "p", 1, PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := m.UpdateProgress(s.ID, 10, false); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterProgress, _ := m.GetSession(s.ID)
|
||||
if !afterProgress.LastServedAt.IsZero() {
|
||||
t.Fatalf("UpdateProgress advanced LastServedAt: %v", afterProgress.LastServedAt)
|
||||
}
|
||||
if err := m.AddServedBytes(s.ID, 7); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterBytes, _ := m.GetSession(s.ID)
|
||||
if afterBytes.LastServedAt.IsZero() || afterBytes.BytesServed != 7 {
|
||||
t.Fatalf("AddServedBytes did not update served state: %+v", afterBytes)
|
||||
}
|
||||
first := afterBytes.LastServedAt
|
||||
time.Sleep(time.Millisecond)
|
||||
if err := m.BeginTransport(s.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
afterBegin, _ := m.GetSession(s.ID)
|
||||
if !afterBegin.LastServedAt.After(first) {
|
||||
t.Fatal("BeginTransport did not advance LastServedAt")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTranscodeLivenessGraceAndPausedPrecedence(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
method PlayMethod
|
||||
paused bool
|
||||
idle time.Duration
|
||||
wantReaped bool
|
||||
}{
|
||||
{name: "transcode five minutes retained", method: PlayTranscode, idle: 5 * time.Minute},
|
||||
{name: "transcode fifteen minutes reaped", method: PlayTranscode, idle: 15 * time.Minute, wantReaped: true},
|
||||
{name: "paused transcode keeps thirty minutes", method: PlayTranscode, paused: true, idle: 15 * time.Minute},
|
||||
{name: "direct keeps active grace", method: PlayDirect, idle: time.Minute, wantReaped: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
m := NewSessionManager(0, 0)
|
||||
s, err := m.StartSession(1, "p", 1, tt.method, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.sessions[s.ID].IsPaused = tt.paused
|
||||
m.sessions[s.ID].LastServedAt = time.Now().Add(-tt.idle)
|
||||
m.mu.Unlock()
|
||||
reaped := m.CleanStale()
|
||||
if (len(reaped) > 0) != tt.wantReaped {
|
||||
t.Fatalf("reaped=%d, wantReaped=%v", len(reaped), tt.wantReaped)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
package playback
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNeverServedProgressPingDoesNotPreventReaping(t *testing.T) {
|
||||
m := NewSessionManager(0, 0)
|
||||
m.SetLivenessGracePeriods(time.Minute, time.Hour)
|
||||
m.SetUnservedSessionGrace(2 * time.Minute)
|
||||
|
||||
session, err := m.StartSession(1, "profile", 100, PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.sessions[session.ID].StartedAt = time.Now().Add(-90 * time.Second)
|
||||
m.mu.Unlock()
|
||||
if err := m.UpdateProgress(session.ID, 42, false); err != nil {
|
||||
t.Fatalf("UpdateProgress: %v", err)
|
||||
}
|
||||
if expired := m.CleanStale(); len(expired) != 0 {
|
||||
t.Fatalf("expired within configured unserved grace = %+v, want none", expired)
|
||||
}
|
||||
|
||||
m.mu.Lock()
|
||||
m.sessions[session.ID].StartedAt = time.Now().Add(-3 * time.Minute)
|
||||
m.mu.Unlock()
|
||||
if err := m.UpdateProgress(session.ID, 43, false); err != nil {
|
||||
t.Fatalf("UpdateProgress after grace: %v", err)
|
||||
}
|
||||
expired := m.CleanStale()
|
||||
if len(expired) != 1 || expired[0].ID != session.ID {
|
||||
t.Fatalf("expired = %+v, want progress-pinged never-served session %q", expired, session.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActiveTransportPreventsServerObservedReaping(t *testing.T) {
|
||||
m := NewSessionManager(0, 0)
|
||||
m.SetUnservedSessionGrace(0)
|
||||
|
||||
session, err := m.StartSession(1, "profile", 100, PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
if err := m.BeginTransport(session.ID); err != nil {
|
||||
t.Fatalf("BeginTransport: %v", err)
|
||||
}
|
||||
|
||||
if expired := m.CleanInactive(0, 0); len(expired) != 0 {
|
||||
t.Fatalf("expired active transport = %+v, want none", expired)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFreshLastServedSurvivesStaleClientActivity(t *testing.T) {
|
||||
m := NewSessionManager(0, 0)
|
||||
m.SetUnservedSessionGrace(0)
|
||||
|
||||
session, err := m.StartSession(1, "profile", 100, PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
m.mu.Lock()
|
||||
s := m.sessions[session.ID]
|
||||
s.LastServedAt = time.Now()
|
||||
s.LastActivityAt = time.Now().Add(-time.Hour)
|
||||
s.UpdatedAt = s.LastActivityAt
|
||||
m.mu.Unlock()
|
||||
|
||||
if expired := m.CleanInactive(time.Minute, time.Hour); len(expired) != 0 {
|
||||
t.Fatalf("expired freshly-served session = %+v, want none", expired)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPausedWebSocketConnectionPreventsServerObservedReaping(t *testing.T) {
|
||||
m := NewSessionManager(0, 0)
|
||||
m.SetUnservedSessionGrace(0)
|
||||
|
||||
session, err := m.StartSession(1, "profile", 100, PlayTranscode, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
if err := m.UpdateProgress(session.ID, 42, true); err != nil {
|
||||
t.Fatalf("UpdateProgress(paused): %v", err)
|
||||
}
|
||||
if err := m.SetWebSocket(session.ID, true); err != nil {
|
||||
t.Fatalf("SetWebSocket: %v", err)
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.sessions[session.ID].LastServedAt = time.Now().Add(-time.Hour)
|
||||
m.mu.Unlock()
|
||||
|
||||
if expired := m.CleanInactive(time.Minute, 30*time.Minute); len(expired) != 0 {
|
||||
t.Fatalf("expired WebSocket-connected paused session = %+v, want none", expired)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPausedWithoutRealtimeConnectionReapsFromLastServed(t *testing.T) {
|
||||
m := NewSessionManager(0, 0)
|
||||
m.SetUnservedSessionGrace(0)
|
||||
|
||||
session, err := m.StartSession(1, "profile", 100, PlayTranscode, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
m.mu.Lock()
|
||||
m.sessions[session.ID].LastServedAt = time.Now().Add(-time.Hour)
|
||||
m.mu.Unlock()
|
||||
if err := m.UpdateProgress(session.ID, 42, true); err != nil {
|
||||
t.Fatalf("UpdateProgress(paused): %v", err)
|
||||
}
|
||||
|
||||
expired := m.CleanInactive(time.Minute, 30*time.Minute)
|
||||
if len(expired) != 1 || expired[0].ID != session.ID {
|
||||
t.Fatalf("expired = %+v, want paused disconnected session %q", expired, session.ID)
|
||||
}
|
||||
}
|
||||
@@ -864,6 +864,35 @@ func TestStartSessionWithFiles(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionOriginFromContext(t *testing.T) {
|
||||
mgr := playback.NewSessionManager(0, 0)
|
||||
|
||||
// A jellycompat-originated session records its origin for monitor attribution.
|
||||
ctx := playback.WithClientInfo(context.Background(), playback.ClientInfo{
|
||||
Name: "Infuse",
|
||||
IsCompat: true,
|
||||
})
|
||||
s, err := mgr.StartSessionWithContext(ctx, 1, "p1", 100, playback.PlayTranscode, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSessionWithContext: %v", err)
|
||||
}
|
||||
if s.Origin() != playback.OriginJellyfin {
|
||||
t.Errorf("Origin = %q, want %q", s.Origin(), playback.OriginJellyfin)
|
||||
}
|
||||
if s.ClientName != "Infuse" {
|
||||
t.Errorf("ClientName = %q, want Infuse", s.ClientName)
|
||||
}
|
||||
|
||||
// An unset origin defaults to native (historical behavior), never empty.
|
||||
s2, err := mgr.StartSession(2, "p2", 101, playback.PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
if s2.Origin() != playback.OriginNative {
|
||||
t.Errorf("default Origin = %q, want %q", s2.Origin(), playback.OriginNative)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpdateStreamState(t *testing.T) {
|
||||
mgr := playback.NewSessionManager(0, 0)
|
||||
session, err := mgr.StartSession(1, "profile-1", 100, playback.PlayRemux, false)
|
||||
@@ -1063,6 +1092,8 @@ func TestSetRealtimeConnection(t *testing.T) {
|
||||
func TestSessionManager_LimitCountsIgnoreStaleSessions(t *testing.T) {
|
||||
sm := playback.NewSessionManager(5, 2)
|
||||
sm.SetLivenessGracePeriods(20*time.Millisecond, 40*time.Millisecond)
|
||||
sm.SetTranscodeLivenessGrace(20 * time.Millisecond)
|
||||
sm.SetUnservedSessionGrace(20 * time.Millisecond)
|
||||
|
||||
if _, err := sm.StartSession(1, "profile-1", 100, playback.PlayDirect, false); err != nil {
|
||||
t.Fatalf("StartSession direct: %v", err)
|
||||
@@ -1081,6 +1112,22 @@ func TestSessionManager_LimitCountsIgnoreStaleSessions(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionManager_LivenessGraceDoesNotOverwriteTranscodeGrace(t *testing.T) {
|
||||
sm := playback.NewSessionManager(5, 2)
|
||||
sm.SetTranscodeLivenessGrace(time.Hour)
|
||||
sm.SetLivenessGracePeriods(20*time.Millisecond, 40*time.Millisecond)
|
||||
|
||||
if _, err := sm.StartSession(1, "profile-1", 101, playback.PlayTranscode, false); err != nil {
|
||||
t.Fatalf("StartSession transcode: %v", err)
|
||||
}
|
||||
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
|
||||
if got := sm.TranscodeCount(1); got != 1 {
|
||||
t.Fatalf("TranscodeCount after active grace = %d, want 1", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionManager_ActiveTransportKeepsSessionLive(t *testing.T) {
|
||||
sm := playback.NewSessionManager(5, 2)
|
||||
sm.SetLivenessGracePeriods(20*time.Millisecond, 40*time.Millisecond)
|
||||
@@ -1202,6 +1249,7 @@ func TestSessionManager_ZeroLimits(t *testing.T) {
|
||||
|
||||
func TestSessionManager_CleanExpired(t *testing.T) {
|
||||
sm := playback.NewSessionManager(0, 0)
|
||||
sm.SetUnservedSessionGrace(0)
|
||||
|
||||
// Create three sessions.
|
||||
idle, _ := sm.StartSession(1, "prof", 100, playback.PlayDirect, false)
|
||||
@@ -1211,21 +1259,20 @@ func TestSessionManager_CleanExpired(t *testing.T) {
|
||||
// Mark the third session as actively transporting media.
|
||||
_ = sm.BeginTransport(transportActive.ID)
|
||||
|
||||
// Send a recent progress update on the "active" session so it stays fresh.
|
||||
// Send a recent progress update on the "active" session. Client-reported
|
||||
// progress does not confer server-observed liveness.
|
||||
_ = sm.UpdateProgress(active.ID, 42.0, false)
|
||||
|
||||
// The "idle" session has no update since StartSession.
|
||||
// The "transportActive" session has an open media transport, so it should
|
||||
// be exempt.
|
||||
|
||||
// CleanExpired with 0 duration expires everything without a WebSocket
|
||||
// that hasn't been updated "recently". Since all sessions were just
|
||||
// created, use a very short maxIdle to ensure only truly idle ones are
|
||||
// caught. We simulate staleness by using a large duration that none can
|
||||
// exceed.
|
||||
// CleanExpired with 0 duration expires everything without an active
|
||||
// transport. Disabling the never-served grace above preserves exact control
|
||||
// for this explicit zero-window cleanup.
|
||||
expired := sm.CleanExpired(0) // 0 means "expire anything older than now"
|
||||
|
||||
// Both inactive sessions should be expired (UpdatedAt <= time.Now()).
|
||||
// Both sessions without a server-observed transport should be expired.
|
||||
// The transport-active session should survive regardless of staleness.
|
||||
if len(expired) != 2 {
|
||||
t.Fatalf("CleanExpired(0) removed %d sessions, want 2", len(expired))
|
||||
@@ -1273,6 +1320,7 @@ func TestSessionManager_CleanExpired_RespectsMaxIdle(t *testing.T) {
|
||||
|
||||
func TestSessionManager_CleanInactive_TriggersExpirationHook(t *testing.T) {
|
||||
sm := playback.NewSessionManager(0, 0)
|
||||
sm.SetUnservedSessionGrace(0)
|
||||
|
||||
session, err := sm.StartSession(1, "prof", 100, playback.PlayDirect, false)
|
||||
if err != nil {
|
||||
@@ -1301,6 +1349,7 @@ func TestSessionManager_CleanInactive_TriggersExpirationHook(t *testing.T) {
|
||||
|
||||
func TestSessionManager_CleanExpired_PausedGracePeriod(t *testing.T) {
|
||||
sm := playback.NewSessionManager(0, 0)
|
||||
sm.SetUnservedSessionGrace(0)
|
||||
|
||||
// Create two sessions and mark one as paused.
|
||||
playing, _ := sm.StartSession(1, "prof", 100, playback.PlayDirect, false)
|
||||
@@ -1337,6 +1386,8 @@ func TestSessionManager_CleanExpired_PausedGracePeriod(t *testing.T) {
|
||||
func TestCheckReplacementAllowedExcludesOnlyTheReplacedSession(t *testing.T) {
|
||||
sm := playback.NewSessionManager(10, 2)
|
||||
sm.SetLivenessGracePeriods(25*time.Millisecond, time.Hour)
|
||||
sm.SetTranscodeLivenessGrace(25 * time.Millisecond)
|
||||
sm.SetUnservedSessionGrace(25 * time.Millisecond)
|
||||
|
||||
failed, err := sm.StartSession(1, "profile-1", 100, playback.PlayTranscode, false)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
)
|
||||
|
||||
func mustCIDRs(t *testing.T, raw ...string) []*net.IPNet {
|
||||
t.Helper()
|
||||
out := make([]*net.IPNet, 0, len(raw))
|
||||
for _, entry := range raw {
|
||||
_, network, err := net.ParseCIDR(entry)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseCIDR(%q): %v", entry, err)
|
||||
}
|
||||
out = append(out, network)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// The edge attributes monitoring records to a viewer address. Behind an ingress
|
||||
// or load balancer every viewer shares the connecting peer address, so without
|
||||
// forwarded-header resolution the admin session view — and any per-viewer
|
||||
// analysis built on it — collapses to one indistinguishable client.
|
||||
func TestEdgeClientIPResolvesViewerBehindTrustedIngress(t *testing.T) {
|
||||
srv := &Server{}
|
||||
srv.SetClientIPResolver(clientip.NewResolver(mustCIDRs(t, "10.0.0.0/8")))
|
||||
|
||||
first := httptest.NewRequest(http.MethodGet, "/stream/direct/tok", nil)
|
||||
first.RemoteAddr = "10.10.10.100:54321"
|
||||
first.Header.Set("X-Forwarded-For", "203.0.113.7")
|
||||
|
||||
second := httptest.NewRequest(http.MethodGet, "/stream/direct/tok", nil)
|
||||
second.RemoteAddr = "10.10.10.100:54322"
|
||||
second.Header.Set("X-Forwarded-For", "198.51.100.42")
|
||||
|
||||
firstIP := srv.edgeClientIP(first)
|
||||
secondIP := srv.edgeClientIP(second)
|
||||
|
||||
if firstIP != "203.0.113.7" {
|
||||
t.Errorf("first viewer = %q, want 203.0.113.7", firstIP)
|
||||
}
|
||||
if secondIP != "198.51.100.42" {
|
||||
t.Errorf("second viewer = %q, want 198.51.100.42", secondIP)
|
||||
}
|
||||
if firstIP == secondIP {
|
||||
t.Fatal("two viewers behind one ingress collapsed to a single address")
|
||||
}
|
||||
}
|
||||
|
||||
// A client that is not itself a trusted proxy must not be able to choose its
|
||||
// own recorded address: with auto-enforcement built on these records, a
|
||||
// believed forged header would be a spoofable primitive.
|
||||
func TestEdgeClientIPIgnoresForwardedHeaderFromUntrustedPeer(t *testing.T) {
|
||||
srv := &Server{}
|
||||
srv.SetClientIPResolver(clientip.NewResolver(mustCIDRs(t, "10.0.0.0/8")))
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/direct/tok", nil)
|
||||
req.RemoteAddr = "203.0.113.9:44444"
|
||||
req.Header.Set("X-Forwarded-For", "198.51.100.1")
|
||||
req.Header.Set("X-Real-IP", "198.51.100.2")
|
||||
|
||||
if got := srv.edgeClientIP(req); got != "203.0.113.9" {
|
||||
t.Fatalf("edgeClientIP = %q, want the connecting peer 203.0.113.9", got)
|
||||
}
|
||||
}
|
||||
|
||||
// With no trusted proxies configured, forwarding headers are never consulted —
|
||||
// the fail-closed direction for the trust boundary.
|
||||
func TestEdgeClientIPWithEmptyTrustListIgnoresForwardedHeader(t *testing.T) {
|
||||
srv := &Server{}
|
||||
srv.SetClientIPResolver(clientip.NewResolver(nil))
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/direct/tok", nil)
|
||||
req.RemoteAddr = "10.10.10.100:54321"
|
||||
req.Header.Set("X-Forwarded-For", "203.0.113.7")
|
||||
|
||||
if got := srv.edgeClientIP(req); got != "10.10.10.100" {
|
||||
t.Fatalf("edgeClientIP = %q, want the connecting peer 10.10.10.100", got)
|
||||
}
|
||||
}
|
||||
|
||||
// A directly-exposed edge with no resolver wired keeps the previous behavior.
|
||||
func TestEdgeClientIPWithoutResolverUsesRemoteAddr(t *testing.T) {
|
||||
srv := &Server{}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/direct/tok", nil)
|
||||
req.RemoteAddr = "203.0.113.5:12345"
|
||||
req.Header.Set("X-Forwarded-For", "198.51.100.1")
|
||||
|
||||
if got := srv.edgeClientIP(req); got != "203.0.113.5" {
|
||||
t.Fatalf("edgeClientIP = %q, want 203.0.113.5", got)
|
||||
}
|
||||
}
|
||||
|
||||
// IPv6 peers carry bracketed host:port in RemoteAddr; the fallback must not
|
||||
// mangle them.
|
||||
func TestEdgeClientIPHandlesIPv6RemoteAddr(t *testing.T) {
|
||||
srv := &Server{}
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/direct/tok", nil)
|
||||
req.RemoteAddr = "[2001:db8::1]:9999"
|
||||
|
||||
if got := srv.edgeClientIP(req); got != "2001:db8::1" {
|
||||
t.Fatalf("edgeClientIP = %q, want 2001:db8::1", got)
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -57,9 +58,11 @@ func (m *egressMeter) RateKbps() int {
|
||||
return int(total * 8 / 1000 / meterWindowSeconds)
|
||||
}
|
||||
|
||||
// meteredResponseWriter counts every byte written to the client.
|
||||
// Embedding the interface intentionally hides optimizations like
|
||||
// io.ReaderFrom so all writes flow through Write.
|
||||
// meteredResponseWriter counts every byte written to the client. Embedding the
|
||||
// interface hides optimizations like io.ReaderFrom, so ReadFrom is forwarded
|
||||
// explicitly below — every stream handler runs inside meterEgress, so without
|
||||
// that forwarding NO proxied pour could ever reach the kernel sendfile path,
|
||||
// regardless of what the inner writers (sessionByteWriter) forward.
|
||||
type meteredResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
meter *egressMeter
|
||||
@@ -71,12 +74,32 @@ func (w *meteredResponseWriter) Write(b []byte) (int, error) {
|
||||
return n, err
|
||||
}
|
||||
|
||||
// ReadFrom preserves the underlying ResponseWriter's sendfile fast path
|
||||
// (disk→socket inside the kernel) while still metering the bytes: the count
|
||||
// only needs the total, which sendfile reports on return. Without a ReaderFrom
|
||||
// underneath it falls back to a plain copy through Write (which meters), via
|
||||
// writeOnly so io.Copy cannot re-enter this method and recurse.
|
||||
func (w *meteredResponseWriter) ReadFrom(src io.Reader) (int64, error) {
|
||||
if rf, ok := w.ResponseWriter.(io.ReaderFrom); ok {
|
||||
n, err := rf.ReadFrom(src)
|
||||
w.meter.Add(n)
|
||||
return n, err
|
||||
}
|
||||
return io.Copy(writeOnly{w}, src)
|
||||
}
|
||||
|
||||
func (w *meteredResponseWriter) Flush() {
|
||||
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
// Unwrap lets http.NewResponseController reach the underlying ResponseWriter so
|
||||
// SetWriteDeadline (used by the revocation connection-cut) can find the socket.
|
||||
func (w *meteredResponseWriter) Unwrap() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
|
||||
// meterEgress wraps stream handlers so their responses count toward the
|
||||
// node's measured egress bandwidth.
|
||||
func (s *Server) meterEgress(next http.Handler) http.Handler {
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type deleteCaptureHook struct {
|
||||
mu sync.Mutex
|
||||
called bool
|
||||
contextErr error
|
||||
values map[string]string
|
||||
}
|
||||
|
||||
func (h *deleteCaptureHook) DialHook(next redis.DialHook) redis.DialHook {
|
||||
return next
|
||||
}
|
||||
|
||||
func (h *deleteCaptureHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
|
||||
return func(ctx context.Context, cmd redis.Cmder) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
switch cmd.Name() {
|
||||
case "set":
|
||||
args := cmd.Args()
|
||||
h.values[args[1].(string)] = "set"
|
||||
if status, ok := cmd.(*redis.StatusCmd); ok {
|
||||
status.SetVal("OK")
|
||||
}
|
||||
return nil
|
||||
case "del":
|
||||
h.called = true
|
||||
h.contextErr = ctx.Err()
|
||||
for _, arg := range cmd.Args()[1:] {
|
||||
delete(h.values, arg.(string))
|
||||
}
|
||||
if count, ok := cmd.(*redis.IntCmd); ok {
|
||||
count.SetVal(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return next(ctx, cmd)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *deleteCaptureHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
|
||||
return next
|
||||
}
|
||||
|
||||
func newProxyTestTracker(t *testing.T) (*nodesessions.Tracker, *deleteCaptureHook) {
|
||||
t.Helper()
|
||||
rdb := redis.NewClient(&redis.Options{
|
||||
Dialer: func(context.Context, string, string) (net.Conn, error) {
|
||||
t.Fatal("Redis command should be intercepted before dialing")
|
||||
return nil, nil
|
||||
},
|
||||
})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
hook := &deleteCaptureHook{values: make(map[string]string)}
|
||||
rdb.AddHook(hook)
|
||||
return nodesessions.NewTracker(rdb, "http://node", "node", "proxy"), hook
|
||||
}
|
||||
|
||||
func (h *deleteCaptureHook) has(key string) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
_, ok := h.values[key]
|
||||
return ok
|
||||
}
|
||||
|
||||
func TestReleaseTrackedUsesLiveContextAfterRequestCancellation(t *testing.T) {
|
||||
tracker, hook := newProxyTestTracker(t)
|
||||
server := &Server{tracker: tracker}
|
||||
lease := tracker.Track(context.Background(), nodesessions.SessionInfo{SessionID: "session-1"})
|
||||
|
||||
requestCtx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
if requestCtx.Err() == nil {
|
||||
t.Fatal("request context was not canceled")
|
||||
}
|
||||
func() {
|
||||
defer server.releaseTracked(lease)
|
||||
}()
|
||||
|
||||
hook.mu.Lock()
|
||||
defer hook.mu.Unlock()
|
||||
if !hook.called {
|
||||
t.Fatal("Redis DEL was not attempted")
|
||||
}
|
||||
if hook.contextErr != nil {
|
||||
t.Fatalf("DEL context error = %v, want live bounded cleanup context", hook.contextErr)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/config"
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
"github.com/Silo-Server/silo-server/internal/nodeconfig"
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
"github.com/Silo-Server/silo-server/internal/streamtoken"
|
||||
)
|
||||
|
||||
type latchCapturingRevocationStore struct {
|
||||
latch *httpstream.CutLatch
|
||||
}
|
||||
|
||||
func (s *latchCapturingRevocationStore) IsRevoked(string, int, time.Time) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *latchCapturingRevocationStore) Refuse(http.ResponseWriter, string, int, time.Time) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *latchCapturingRevocationStore) WatchAndCutContext(ctx context.Context, _ http.ResponseWriter, _ string, _ int, _ time.Time) func() {
|
||||
s.latch = httpstream.CutLatchFrom(ctx)
|
||||
return func() {}
|
||||
}
|
||||
|
||||
func TestHandleRemuxCarriesCutLatchToRevocationWatcher(t *testing.T) {
|
||||
const secret = "proxy-revocation-test-secret"
|
||||
watcher := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{})
|
||||
cfg := &config.Config{}
|
||||
cfg.Auth.JWTSecret = secret
|
||||
watcher.SetConfigForTest(cfg)
|
||||
|
||||
server := NewServer(watcher, nodesessions.NewTracker(nil, "http://proxy", "proxy", "proxy"))
|
||||
revocation := &latchCapturingRevocationStore{}
|
||||
server.SetRevocationStore(revocation)
|
||||
|
||||
token, err := streamtoken.Sign(streamtoken.Claims{
|
||||
SessionID: "remux-latch",
|
||||
MediaPath: t.TempDir() + "/missing.mkv",
|
||||
}, secret, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/remux/"+token, nil)
|
||||
rr := httptest.NewRecorder()
|
||||
|
||||
server.Handler().ServeHTTP(rr, req)
|
||||
|
||||
if revocation.latch == nil {
|
||||
t.Fatal("remux revocation watcher request context has no cut latch")
|
||||
}
|
||||
}
|
||||
+228
-18
@@ -1,10 +1,12 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -13,12 +15,23 @@ import (
|
||||
"github.com/go-chi/chi/v5"
|
||||
"github.com/go-chi/cors"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/clientip"
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
"github.com/Silo-Server/silo-server/internal/nodeconfig"
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/streamtoken"
|
||||
)
|
||||
|
||||
// revocationStore is the subset of *streamrevoke.Store the edge consults to
|
||||
// enforce kills. IsRevoked is a pure in-memory lookup (no I/O) so it is safe on
|
||||
// the per-request hot path. Nil disables enforcement.
|
||||
type revocationStore interface {
|
||||
IsRevoked(sessionID string, userID int, startedAt time.Time) bool
|
||||
Refuse(w http.ResponseWriter, sessionID string, userID int, startedAt time.Time) bool
|
||||
WatchAndCutContext(ctx context.Context, w http.ResponseWriter, sessionID string, userID int, startedAt time.Time) func()
|
||||
}
|
||||
|
||||
// Server is the HTTP handler for proxy mode.
|
||||
type Server struct {
|
||||
watcher *nodeconfig.Watcher
|
||||
@@ -28,6 +41,26 @@ type Server struct {
|
||||
// subCache stores full-track PGS (.sup) extracts under the transcode dir
|
||||
// so repeat selections skip the whole-file ffmpeg demux.
|
||||
subCache *playback.SubtitleCache
|
||||
|
||||
revocation revocationStore
|
||||
ipResolver *clientip.Resolver
|
||||
}
|
||||
|
||||
// SetRevocationStore wires the kill-switch the edge consults per request. The
|
||||
// store's cache is kept current out-of-band (pub/sub + poll), so the hot-path
|
||||
// check is a local map read.
|
||||
func (s *Server) SetRevocationStore(store revocationStore) {
|
||||
s.revocation = store
|
||||
}
|
||||
|
||||
// SetClientIPResolver wires the trusted-proxy-aware client address resolver so
|
||||
// edge monitoring records the real viewer rather than the ingress that fronts
|
||||
// this node. Without one the edge falls back to the connecting address, which
|
||||
// behind a reverse proxy or load balancer is the same value for every viewer.
|
||||
// The resolver's trusted list is updated in place on setting changes, so this
|
||||
// is wired once at startup.
|
||||
func (s *Server) SetClientIPResolver(resolver *clientip.Resolver) {
|
||||
s.ipResolver = resolver
|
||||
}
|
||||
|
||||
// NewServer creates a new proxy server backed by a config watcher and session
|
||||
@@ -137,31 +170,148 @@ func (s *Server) verifyToken(w http.ResponseWriter, r *http.Request) *streamtoke
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return nil
|
||||
}
|
||||
// Kill switch: a revoked session/user is refused here, on every request. For
|
||||
// chunked (HLS) playback this stops the stream on its next segment fetch and
|
||||
// refuses reconnects; long direct-play/remux pours are additionally cut mid-
|
||||
// stream by cutOnRevocation.
|
||||
// The token's iat is the credential-issue time the user-kill cutoff compares
|
||||
// against: edge requests carry no fresh auth, only this token, so a user
|
||||
// revocation kills exactly the tokens minted before it.
|
||||
if s.revocation != nil && s.revocation.Refuse(w, claims.SessionID, claims.UserID, claims.IssuedTime()) {
|
||||
return nil
|
||||
}
|
||||
return claims
|
||||
}
|
||||
|
||||
// cutOnRevocation watches a long-lived pour (direct play / remux) and hangs up
|
||||
// the socket the moment the session/user is revoked, via the shared store helper.
|
||||
// Returns a stop func to cancel the watcher when the request finishes normally.
|
||||
func (s *Server) cutOnRevocation(ctx context.Context, w http.ResponseWriter, claims *streamtoken.Claims) func() {
|
||||
if s.revocation == nil {
|
||||
return func() {}
|
||||
}
|
||||
return s.revocation.WatchAndCutContext(ctx, w, claims.SessionID, claims.UserID, claims.IssuedTime())
|
||||
}
|
||||
|
||||
// releaseTracked releases one request lease with a bounded background context. The
|
||||
// request context is already canceled by the time this defer runs on a client
|
||||
// disconnect, which would skip the Redis DEL and leave a phantom session until
|
||||
// TTL — a false over-cap window.
|
||||
func (s *Server) releaseTracked(lease nodesessions.Lease) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
s.tracker.Release(ctx, lease)
|
||||
}
|
||||
|
||||
func (s *Server) handleDirectPlay(w http.ResponseWriter, r *http.Request) {
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
claims := s.verifyToken(w, r)
|
||||
if claims == nil {
|
||||
return
|
||||
}
|
||||
|
||||
info := sessionInfo(s.tracker, claims, "direct_play")
|
||||
s.tracker.Track(r.Context(), info)
|
||||
defer s.tracker.Remove(r.Context(), claims.SessionID)
|
||||
info := sessionInfo(s.tracker, claims, "direct_play", s.edgeClientIP(r))
|
||||
lease := s.tracker.Track(r.Context(), info)
|
||||
defer s.releaseTracked(lease)
|
||||
|
||||
http.ServeFile(w, r, claims.MediaPath)
|
||||
sw := &sessionByteWriter{ResponseWriter: w, tracker: s.tracker, sessionID: claims.SessionID}
|
||||
defer sw.flush()
|
||||
|
||||
stop := s.cutOnRevocation(r.Context(), sw, claims)
|
||||
defer stop()
|
||||
|
||||
http.ServeFile(sw, r, claims.MediaPath)
|
||||
}
|
||||
|
||||
// sessionByteWriter attributes served bytes to a session so LastServedAt advances
|
||||
// during long direct-play/remux pours (authoritative liveness). Bytes are
|
||||
// flushed to the tracker in coarse chunks to avoid per-write lock churn. Unwrap
|
||||
// lets the revocation connection-cut reach the socket through this layer.
|
||||
type sessionByteWriter struct {
|
||||
http.ResponseWriter
|
||||
tracker *nodesessions.Tracker
|
||||
sessionID string
|
||||
acc int64
|
||||
}
|
||||
|
||||
func (w *sessionByteWriter) Write(b []byte) (int, error) {
|
||||
n, err := w.ResponseWriter.Write(b)
|
||||
w.account(int64(n))
|
||||
return n, err
|
||||
}
|
||||
|
||||
// account tallies served bytes, flushing to the tracker in coarse ~1 MiB chunks
|
||||
// to avoid per-write lock churn. Shared by Write and the ReadFrom fast path.
|
||||
func (w *sessionByteWriter) account(n int64) {
|
||||
if n <= 0 {
|
||||
return
|
||||
}
|
||||
w.acc += n
|
||||
if w.acc >= 1<<20 { // flush every ~1 MiB
|
||||
w.tracker.AddBytes(w.sessionID, w.acc)
|
||||
w.acc = 0
|
||||
}
|
||||
}
|
||||
|
||||
// ReadFrom preserves the underlying writer's sendfile fast path while still
|
||||
// attributing served bytes. http.ServeFile pours an *os.File through the
|
||||
// ResponseWriter's io.ReaderFrom (sendfile: disk->socket inside the kernel, no
|
||||
// userspace copy) — but only if it can see that method. Wrapping the writer for
|
||||
// byte counting would hide it and force every direct-play/remux byte through
|
||||
// userspace; forwarding here keeps zero-copy AND the count. Liveness does not
|
||||
// depend on this coarse count: the session stays live from Track to Remove
|
||||
// (tracker re-SETs it, never idle-prunes an open pour), so a single sendfile
|
||||
// call that only tallies on return still stays visible for the whole pour.
|
||||
func (w *sessionByteWriter) ReadFrom(src io.Reader) (int64, error) {
|
||||
if rf, ok := w.ResponseWriter.(io.ReaderFrom); ok {
|
||||
n, err := rf.ReadFrom(src)
|
||||
w.account(n)
|
||||
return n, err
|
||||
}
|
||||
// No sendfile fast path underneath. Copy manually — but NOT via io.Copy(w,
|
||||
// src), which would re-detect this very ReadFrom and recurse forever.
|
||||
// writeOnly exposes only Write, so io.Copy falls back to the Write loop
|
||||
// (which accounts bytes).
|
||||
return io.Copy(writeOnly{w}, src)
|
||||
}
|
||||
|
||||
// writeOnly wraps an io.Writer to expose ONLY Write, hiding any ReadFrom so
|
||||
// io.Copy cannot re-enter sessionByteWriter.ReadFrom.
|
||||
type writeOnly struct{ io.Writer }
|
||||
|
||||
func (w *sessionByteWriter) flush() {
|
||||
if w.acc > 0 {
|
||||
w.tracker.AddBytes(w.sessionID, w.acc)
|
||||
w.acc = 0
|
||||
}
|
||||
}
|
||||
|
||||
func (w *sessionByteWriter) Flush() {
|
||||
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
|
||||
func (w *sessionByteWriter) Unwrap() http.ResponseWriter {
|
||||
return w.ResponseWriter
|
||||
}
|
||||
|
||||
func (s *Server) handleRemux(w http.ResponseWriter, r *http.Request) {
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
claims := s.verifyToken(w, r)
|
||||
if claims == nil {
|
||||
return
|
||||
}
|
||||
|
||||
info := sessionInfo(s.tracker, claims, "remux")
|
||||
s.tracker.Track(r.Context(), info)
|
||||
defer s.tracker.Remove(r.Context(), claims.SessionID)
|
||||
info := sessionInfo(s.tracker, claims, "remux", s.edgeClientIP(r))
|
||||
lease := s.tracker.Track(r.Context(), info)
|
||||
defer s.releaseTracked(lease)
|
||||
|
||||
sw := &sessionByteWriter{ResponseWriter: w, tracker: s.tracker, sessionID: claims.SessionID}
|
||||
defer sw.flush()
|
||||
|
||||
stop := s.cutOnRevocation(r.Context(), sw, claims)
|
||||
defer stop()
|
||||
|
||||
seekSeconds := 0.0
|
||||
if seekStr := r.URL.Query().Get("seek"); seekStr != "" {
|
||||
@@ -171,8 +321,9 @@ func (s *Server) handleRemux(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
// Honor the Dolby Vision mode frozen in the token (empty decodes as the
|
||||
// legacy auto behavior for old tokens), mirroring how the integrated
|
||||
// server's stream handler serves the same claims.
|
||||
_ = playback.ServeRemuxWithDVMode(w, r, claims.MediaPath, "mp4", seekSeconds, claims.TranscodeAudio, claims.AudioTrackIndex, claims.DVProfile, playback.RemuxDVMode(claims.RemuxDVMode), s.watcher.Config().Playback.FFmpegPath)
|
||||
// server's stream handler serves the same claims. Serve through sw so the
|
||||
// bytes are metered and the revocation cut can reach this response.
|
||||
_ = playback.ServeRemuxWithDVMode(sw, r, claims.MediaPath, "mp4", seekSeconds, claims.TranscodeAudio, claims.AudioTrackIndex, claims.DVProfile, playback.RemuxDVMode(claims.RemuxDVMode), s.watcher.Config().Playback.FFmpegPath)
|
||||
}
|
||||
|
||||
func (s *Server) handleTranscodeManifest(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -180,7 +331,7 @@ func (s *Server) handleTranscodeManifest(w http.ResponseWriter, r *http.Request)
|
||||
if claims == nil {
|
||||
return
|
||||
}
|
||||
s.touchTranscodeSession(r, claims)
|
||||
s.ensureTranscodeSession(r, claims)
|
||||
s.proxyToTranscodeNode(w, r, claims, "/transcode/"+transcodeTransportIDFromClaims(claims)+"/master.m3u8")
|
||||
}
|
||||
|
||||
@@ -189,7 +340,7 @@ func (s *Server) handleTranscodeSegment(w http.ResponseWriter, r *http.Request)
|
||||
if claims == nil {
|
||||
return
|
||||
}
|
||||
s.touchTranscodeSession(r, claims)
|
||||
s.ensureTranscodeSession(r, claims)
|
||||
name := chi.URLParam(r, "name")
|
||||
s.proxyToTranscodeNode(w, r, claims, "/transcode/"+transcodeTransportIDFromClaims(claims)+"/segment/"+name)
|
||||
}
|
||||
@@ -204,22 +355,30 @@ func transcodeTransportIDFromClaims(claims *streamtoken.Claims) string {
|
||||
return claims.SessionID
|
||||
}
|
||||
|
||||
// touchTranscodeSession keeps HLS sessions visible in the active stream count.
|
||||
// ensureTranscodeSession keeps HLS sessions visible in the active stream count.
|
||||
// Unlike direct play and remux, transcode playback reaches the proxy as many
|
||||
// short manifest/segment requests, so the session is tracked by recent
|
||||
// activity instead of request lifetime.
|
||||
func (s *Server) touchTranscodeSession(r *http.Request, claims *streamtoken.Claims) {
|
||||
s.tracker.Touch(r.Context(), sessionInfo(s.tracker, claims, "transcode"))
|
||||
// requests instead of request lifetime. Visibility is separate from served
|
||||
// liveness: only a successful upstream media response advances LastServedAt.
|
||||
func (s *Server) ensureTranscodeSession(r *http.Request, claims *streamtoken.Claims) {
|
||||
s.tracker.EnsureEphemeral(r.Context(), sessionInfo(s.tracker, claims, "transcode", s.edgeClientIP(r)))
|
||||
}
|
||||
|
||||
// sessionInfo builds the node-session tracker record for a verified token,
|
||||
// copying the numeric ownership keys the node-session tracker needs.
|
||||
func sessionInfo(tr *nodesessions.Tracker, claims *streamtoken.Claims, kind string) nodesessions.SessionInfo {
|
||||
// copying the numeric ownership keys plus the monitoring attribution (route +
|
||||
// client identity) the first-class monitor view needs. clientIP is the connecting
|
||||
// address observed at the edge (best-effort; the client reaches the proxy
|
||||
// directly). Route/ClientName come from the token so the edge — which never sees
|
||||
// the originating API path — can still tag native vs jellycompat.
|
||||
func sessionInfo(tr *nodesessions.Tracker, claims *streamtoken.Claims, kind, clientIP string) nodesessions.SessionInfo {
|
||||
return nodesessions.SessionInfo{
|
||||
SessionID: claims.SessionID,
|
||||
NodeURL: tr.NodeURL(),
|
||||
NodeName: tr.NodeName(),
|
||||
Type: kind,
|
||||
Route: claims.Origin,
|
||||
ClientIP: clientIP,
|
||||
ClientName: claims.ClientName,
|
||||
StartedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
AuthUserID: claims.UserID,
|
||||
ProfileID: claims.ProfileID,
|
||||
@@ -227,11 +386,47 @@ func sessionInfo(tr *nodesessions.Tracker, claims *streamtoken.Claims, kind stri
|
||||
}
|
||||
}
|
||||
|
||||
// edgeClientIP resolves the viewer's address for monitoring attribution.
|
||||
//
|
||||
// When a trusted-proxy resolver is wired it walks the forwarding headers the
|
||||
// same way the native API surface does, honouring the operator's
|
||||
// `clientip.trusted_proxies` boundary — headers are consulted only when the
|
||||
// connecting peer is itself trusted, so an untrusted client cannot forge its
|
||||
// own address. Without a resolver (or when the peer is untrusted) this is the
|
||||
// connecting address, which is correct for a directly-exposed edge but
|
||||
// collapses every viewer to one value behind an ingress or load balancer.
|
||||
func (s *Server) edgeClientIP(r *http.Request) string {
|
||||
if s != nil && s.ipResolver != nil {
|
||||
return s.ipResolver.ClientIP(r)
|
||||
}
|
||||
return remoteAddrIP(r)
|
||||
}
|
||||
|
||||
// remoteAddrIP extracts the connecting peer's IP, best-effort.
|
||||
func remoteAddrIP(r *http.Request) string {
|
||||
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
return r.RemoteAddr
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
func (s *Server) handleSubtitle(w http.ResponseWriter, r *http.Request) {
|
||||
r = r.WithContext(httpstream.WithCutLatch(r.Context(), &httpstream.CutLatch{}))
|
||||
claims := s.verifyToken(w, r)
|
||||
if claims == nil {
|
||||
return
|
||||
}
|
||||
// Attribute subtitle bytes only when the session already has a tracker
|
||||
// record. Do not take lifecycle ownership here: Track/Remove per subtitle
|
||||
// request can delete a concurrently active media request until Batch 2 adds
|
||||
// reference-counted edge tracking.
|
||||
metered := &sessionByteWriter{ResponseWriter: w, tracker: s.tracker, sessionID: claims.SessionID}
|
||||
defer metered.flush()
|
||||
stop := s.cutOnRevocation(r.Context(), metered, claims)
|
||||
defer stop()
|
||||
w = metered
|
||||
|
||||
cfg := s.watcher.Config()
|
||||
trackParam := chi.URLParam(r, "track")
|
||||
trackIndex, requestedFormat, err := playback.ParseSubtitleTrackParam(trackParam)
|
||||
@@ -370,7 +565,22 @@ func (s *Server) proxyToTranscodeNode(w http.ResponseWriter, r *http.Request, cl
|
||||
}
|
||||
}
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
io.Copy(w, resp.Body)
|
||||
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent {
|
||||
_, _ = io.Copy(w, resp.Body)
|
||||
return
|
||||
}
|
||||
// Attribute served bytes to the session for authoritative monitoring —
|
||||
// incrementally (~1 MiB granularity), not in one post-copy tally: a slow
|
||||
// segment drain that outlives the 60s record TTL would otherwise go
|
||||
// invisible mid-pour and then post bytes to a record that no longer exists.
|
||||
// Manifest bytes are tiny; segment bytes are the real signal. Best-effort,
|
||||
// never gates. writeOnly forces the per-chunk Write path: sw.ReadFrom would
|
||||
// tally only once on return, which is the single-post-copy behavior this
|
||||
// replaces (there is no file to sendfile here — the source is the node's
|
||||
// HTTP response body).
|
||||
sw := &sessionByteWriter{ResponseWriter: w, tracker: s.tracker, sessionID: claims.SessionID}
|
||||
defer sw.flush()
|
||||
_, _ = io.Copy(writeOnly{sw}, resp.Body)
|
||||
}
|
||||
|
||||
func (s *Server) handleForceReload(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
)
|
||||
|
||||
// rfResponseWriter is the sendfile-capable writer shape: an http.ResponseWriter
|
||||
// that also implements io.ReaderFrom. It records whether ReadFrom was taken.
|
||||
type rfResponseWriter struct {
|
||||
buf bytes.Buffer
|
||||
hdr http.Header
|
||||
usedRF bool
|
||||
}
|
||||
|
||||
func (w *rfResponseWriter) Header() http.Header {
|
||||
if w.hdr == nil {
|
||||
w.hdr = http.Header{}
|
||||
}
|
||||
return w.hdr
|
||||
}
|
||||
func (w *rfResponseWriter) Write(b []byte) (int, error) { return w.buf.Write(b) }
|
||||
func (w *rfResponseWriter) WriteHeader(int) {}
|
||||
func (w *rfResponseWriter) ReadFrom(src io.Reader) (int64, error) {
|
||||
w.usedRF = true
|
||||
return w.buf.ReadFrom(src)
|
||||
}
|
||||
|
||||
// plainResponseWriter implements ONLY http.ResponseWriter (no ReadFrom), forcing
|
||||
// sessionByteWriter.ReadFrom down its manual-copy fallback.
|
||||
type plainResponseWriter struct {
|
||||
buf bytes.Buffer
|
||||
hdr http.Header
|
||||
}
|
||||
|
||||
func (w *plainResponseWriter) Header() http.Header {
|
||||
if w.hdr == nil {
|
||||
w.hdr = http.Header{}
|
||||
}
|
||||
return w.hdr
|
||||
}
|
||||
func (w *plainResponseWriter) Write(b []byte) (int, error) { return w.buf.Write(b) }
|
||||
func (w *plainResponseWriter) WriteHeader(int) {}
|
||||
|
||||
// TestSessionByteWriterReadFromFastPath verifies that when the underlying writer
|
||||
// supports sendfile (io.ReaderFrom), ReadFrom forwards to it (preserving
|
||||
// zero-copy) and still attributes the served bytes.
|
||||
func TestSessionByteWriterReadFromFastPath(t *testing.T) {
|
||||
under := &rfResponseWriter{}
|
||||
sw := &sessionByteWriter{ResponseWriter: under, sessionID: "s"}
|
||||
|
||||
const payload = "the quick brown fox jumps"
|
||||
n, err := sw.ReadFrom(strings.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFrom: %v", err)
|
||||
}
|
||||
if !under.usedRF {
|
||||
t.Fatal("expected the underlying io.ReaderFrom (sendfile) fast path to be used")
|
||||
}
|
||||
if n != int64(len(payload)) {
|
||||
t.Fatalf("bytes forwarded = %d, want %d", n, len(payload))
|
||||
}
|
||||
if sw.acc != int64(len(payload)) {
|
||||
t.Fatalf("accounted bytes = %d, want %d", sw.acc, len(payload))
|
||||
}
|
||||
if got := under.buf.String(); got != payload {
|
||||
t.Fatalf("served body = %q, want %q", got, payload)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMeteredWriterChainPreservesSendfile proves the PRODUCTION writer chain —
|
||||
// sessionByteWriter over meteredResponseWriter (every stream route runs inside
|
||||
// meterEgress) — still reaches the underlying sendfile fast path AND meters the
|
||||
// bytes. Regression guard: meteredResponseWriter used to hide io.ReaderFrom,
|
||||
// which silently forced every proxied pour through a userspace copy no matter
|
||||
// what the inner writer forwarded, making sessionByteWriter's fast path dead
|
||||
// code on real requests.
|
||||
func TestMeteredWriterChainPreservesSendfile(t *testing.T) {
|
||||
under := &rfResponseWriter{}
|
||||
meter := newEgressMeter()
|
||||
mw := &meteredResponseWriter{ResponseWriter: under, meter: meter}
|
||||
sw := &sessionByteWriter{ResponseWriter: mw, sessionID: "s"}
|
||||
|
||||
const payload = "kernel to socket, no userspace detours"
|
||||
n, err := sw.ReadFrom(strings.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFrom: %v", err)
|
||||
}
|
||||
if !under.usedRF {
|
||||
t.Fatal("expected sendfile fast path through the full metered chain")
|
||||
}
|
||||
if n != int64(len(payload)) {
|
||||
t.Fatalf("bytes forwarded = %d, want %d", n, len(payload))
|
||||
}
|
||||
if sw.acc != int64(len(payload)) {
|
||||
t.Fatalf("session-accounted bytes = %d, want %d", sw.acc, len(payload))
|
||||
}
|
||||
var metered int64
|
||||
meter.mu.Lock()
|
||||
for _, b := range meter.buckets {
|
||||
metered += b
|
||||
}
|
||||
meter.mu.Unlock()
|
||||
if metered != int64(len(payload)) {
|
||||
t.Fatalf("metered bytes = %d, want %d", metered, len(payload))
|
||||
}
|
||||
if got := under.buf.String(); got != payload {
|
||||
t.Fatalf("served body = %q, want %q", got, payload)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSessionByteWriterAccountFlushBranch exercises account()'s coarse-flush
|
||||
// branch (w.acc >= 1<<20) that the short-payload tests never reach: a >=1MiB
|
||||
// pour must flush the accumulator to the tracker and reset acc to 0. It also
|
||||
// pins that the flush is safe against a real *nodesessions.Tracker — the other
|
||||
// tests leave tracker nil, which only stays safe while the branch is untaken. A
|
||||
// Redis-less tracker makes AddBytes a no-op, so the branch runs without Redis.
|
||||
func TestSessionByteWriterAccountFlushBranch(t *testing.T) {
|
||||
tracker := nodesessions.NewTracker(nil, "http://node", "node", "proxy")
|
||||
under := &rfResponseWriter{}
|
||||
sw := &sessionByteWriter{ResponseWriter: under, tracker: tracker, sessionID: "s"}
|
||||
|
||||
const oneMiB = 1 << 20
|
||||
payload := strings.Repeat("x", oneMiB)
|
||||
n, err := sw.ReadFrom(strings.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFrom: %v", err)
|
||||
}
|
||||
if !under.usedRF {
|
||||
t.Fatal("expected the sendfile fast path even for a large payload")
|
||||
}
|
||||
if n != int64(oneMiB) {
|
||||
t.Fatalf("bytes forwarded = %d, want %d", n, oneMiB)
|
||||
}
|
||||
// The >=1MiB pour must have tripped the flush branch, resetting the accumulator.
|
||||
if sw.acc != 0 {
|
||||
t.Fatalf("accumulator = %d, want 0 (coarse-flush branch not taken)", sw.acc)
|
||||
}
|
||||
if got := under.buf.Len(); got != oneMiB {
|
||||
t.Fatalf("served bytes = %d, want %d", got, oneMiB)
|
||||
}
|
||||
}
|
||||
|
||||
// TestSessionByteWriterReadFromFallbackNoRecursion verifies the fallback path
|
||||
// (underlying writer has no sendfile support) copies via Write without
|
||||
// re-entering ReadFrom. A naive io.Copy(sw, src) fallback would re-detect this
|
||||
// ReadFrom and recurse until the stack overflows, so simply returning here — and
|
||||
// counting the bytes — proves the guard works.
|
||||
func TestSessionByteWriterReadFromFallbackNoRecursion(t *testing.T) {
|
||||
under := &plainResponseWriter{}
|
||||
sw := &sessionByteWriter{ResponseWriter: under, sessionID: "s"}
|
||||
|
||||
const payload = "fallback path must not recurse"
|
||||
n, err := sw.ReadFrom(strings.NewReader(payload))
|
||||
if err != nil {
|
||||
t.Fatalf("ReadFrom: %v", err)
|
||||
}
|
||||
if n != int64(len(payload)) {
|
||||
t.Fatalf("bytes copied = %d, want %d", n, len(payload))
|
||||
}
|
||||
if sw.acc != int64(len(payload)) {
|
||||
t.Fatalf("accounted bytes = %d, want %d", sw.acc, len(payload))
|
||||
}
|
||||
if got := under.buf.String(); got != payload {
|
||||
t.Fatalf("served body = %q, want %q", got, payload)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/config"
|
||||
"github.com/Silo-Server/silo-server/internal/nodeconfig"
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
"github.com/Silo-Server/silo-server/internal/streamtoken"
|
||||
)
|
||||
|
||||
const proxyTrackerTestSecret = "proxy-tracker-test-secret"
|
||||
|
||||
func newMountedProxyTestServer(t *testing.T, mediaPath string) (*Server, *deleteCaptureHook, string) {
|
||||
t.Helper()
|
||||
watcher := nodeconfig.NewWatcher(nil, nil, nil, nodeconfig.BootstrapOverrides{})
|
||||
cfg := &config.Config{}
|
||||
cfg.Auth.JWTSecret = proxyTrackerTestSecret
|
||||
watcher.SetConfigForTest(cfg)
|
||||
tracker, redisState := newProxyTestTracker(t)
|
||||
server := NewServer(watcher, tracker)
|
||||
token, err := streamtoken.Sign(streamtoken.Claims{
|
||||
SessionID: "session-1",
|
||||
MediaPath: mediaPath,
|
||||
TranscodeNode: "http://transcode",
|
||||
UserID: 7,
|
||||
ProfileID: "profile-1",
|
||||
MediaFileID: 42,
|
||||
}, proxyTrackerTestSecret, time.Minute)
|
||||
if err != nil {
|
||||
t.Fatalf("sign stream token: %v", err)
|
||||
}
|
||||
return server, redisState, token
|
||||
}
|
||||
|
||||
type blockingResponseWriter struct {
|
||||
header http.Header
|
||||
entered chan struct{}
|
||||
release chan struct{}
|
||||
once sync.Once
|
||||
}
|
||||
|
||||
func (w *blockingResponseWriter) Header() http.Header { return w.header }
|
||||
func (w *blockingResponseWriter) WriteHeader(int) {}
|
||||
func (w *blockingResponseWriter) Write(p []byte) (int, error) {
|
||||
w.once.Do(func() { close(w.entered) })
|
||||
<-w.release
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func TestMountedDirectOverlappingRangesKeepSessionUntilFinalRequest(t *testing.T) {
|
||||
path := t.TempDir() + "/movie.bin"
|
||||
if err := os.WriteFile(path, bytes.Repeat([]byte("x"), 64<<10), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server, redisState, token := newMountedProxyTestServer(t, path)
|
||||
firstWriter := &blockingResponseWriter{
|
||||
header: make(http.Header),
|
||||
entered: make(chan struct{}),
|
||||
release: make(chan struct{}),
|
||||
}
|
||||
firstDone := make(chan struct{})
|
||||
go func() {
|
||||
defer close(firstDone)
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/direct/"+token, nil)
|
||||
req.Header.Set("Range", "bytes=0-32767")
|
||||
server.Handler().ServeHTTP(firstWriter, req)
|
||||
}()
|
||||
select {
|
||||
case <-firstWriter.entered:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("first mounted range request did not begin its pour")
|
||||
}
|
||||
|
||||
second := httptest.NewRequest(http.MethodGet, "/stream/direct/"+token, nil)
|
||||
second.Header.Set("Range", "bytes=32768-65535")
|
||||
secondRec := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(secondRec, second)
|
||||
if secondRec.Code != http.StatusPartialContent {
|
||||
t.Fatalf("second range status = %d, want 206", secondRec.Code)
|
||||
}
|
||||
snapshot := server.tracker.Snapshot()
|
||||
if len(snapshot) != 1 || snapshot[0].BytesServed == 0 {
|
||||
t.Fatalf("first request lost tracking after second completed: %+v", snapshot)
|
||||
}
|
||||
key := nodesessions.KeyPrefix + server.tracker.NodeHash() + ":session-1"
|
||||
if !redisState.has(key) {
|
||||
t.Fatal("Redis key was deleted while the first range was still pouring")
|
||||
}
|
||||
|
||||
close(firstWriter.release)
|
||||
select {
|
||||
case <-firstDone:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("first range request did not complete")
|
||||
}
|
||||
if got := server.tracker.Snapshot(); len(got) != 0 {
|
||||
t.Fatalf("final request release left session behind: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMountedDirectReleasesLeaseOnEveryCompletionPath(t *testing.T) {
|
||||
path := t.TempDir() + "/movie.bin"
|
||||
if err := os.WriteFile(path, []byte("payload"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
server, _, token := newMountedProxyTestServer(t, path)
|
||||
|
||||
t.Run("head", func(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodHead, "/stream/direct/"+token, nil))
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d", rec.Code)
|
||||
}
|
||||
if got := server.tracker.Snapshot(); len(got) != 0 {
|
||||
t.Fatalf("HEAD leaked lease: %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing file", func(t *testing.T) {
|
||||
missingServer, _, missingToken := newMountedProxyTestServer(t, path+".missing")
|
||||
rec := httptest.NewRecorder()
|
||||
missingServer.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/stream/direct/"+missingToken, nil))
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Fatalf("status = %d", rec.Code)
|
||||
}
|
||||
if got := missingServer.tracker.Snapshot(); len(got) != 0 {
|
||||
t.Fatalf("missing-file completion leaked lease: %+v", got)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("client cancellation", func(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
req := httptest.NewRequest(http.MethodGet, "/stream/direct/"+token, nil).WithContext(ctx)
|
||||
cancel()
|
||||
rec := httptest.NewRecorder()
|
||||
server.Handler().ServeHTTP(rec, req)
|
||||
if got := server.tracker.Snapshot(); len(got) != 0 {
|
||||
t.Fatalf("canceled request leaked lease: %+v", got)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
type roundTripFunc func(*http.Request) (*http.Response, error)
|
||||
|
||||
func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) }
|
||||
|
||||
type limitedFailureWriter struct {
|
||||
header http.Header
|
||||
remaining int
|
||||
}
|
||||
|
||||
func (w *limitedFailureWriter) Header() http.Header { return w.header }
|
||||
func (w *limitedFailureWriter) WriteHeader(int) {}
|
||||
func (w *limitedFailureWriter) Write(p []byte) (int, error) {
|
||||
if w.remaining <= 0 {
|
||||
return 0, errors.New("client disconnected")
|
||||
}
|
||||
n := min(len(p), w.remaining)
|
||||
w.remaining -= n
|
||||
return n, errors.New("client disconnected")
|
||||
}
|
||||
|
||||
func TestMountedTranscodeLivenessRequiresSuccessfulServedBytes(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
method string
|
||||
status int
|
||||
body string
|
||||
transportErr error
|
||||
writer http.ResponseWriter
|
||||
wantBytes int64
|
||||
wantServed bool
|
||||
}{
|
||||
{name: "200 bytes", method: http.MethodGet, status: http.StatusOK, body: "manifest", wantBytes: 8, wantServed: true},
|
||||
{name: "206 bytes", method: http.MethodGet, status: http.StatusPartialContent, body: "segment", wantBytes: 7, wantServed: true},
|
||||
{name: "head zero bytes", method: http.MethodHead, status: http.StatusOK},
|
||||
{name: "404 error body", method: http.MethodGet, status: http.StatusNotFound, body: "not found"},
|
||||
{name: "upstream failure", method: http.MethodGet, transportErr: errors.New("dial failed")},
|
||||
{name: "client write failure", method: http.MethodGet, status: http.StatusOK, body: "abcdefgh", writer: &limitedFailureWriter{header: make(http.Header), remaining: 3}, wantBytes: 3, wantServed: true},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
server, _, token := newMountedProxyTestServer(t, "")
|
||||
server.httpClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||
if tc.transportErr != nil {
|
||||
return nil, tc.transportErr
|
||||
}
|
||||
return &http.Response{
|
||||
StatusCode: tc.status,
|
||||
Header: make(http.Header),
|
||||
Body: io.NopCloser(bytes.NewBufferString(tc.body)),
|
||||
}, nil
|
||||
})}
|
||||
writer := tc.writer
|
||||
if writer == nil {
|
||||
writer = httptest.NewRecorder()
|
||||
}
|
||||
server.Handler().ServeHTTP(writer, httptest.NewRequest(tc.method, "/stream/transcode/"+token+"/master.m3u8", nil))
|
||||
|
||||
got := server.tracker.Snapshot()
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("snapshot = %+v, want visible ephemeral record", got)
|
||||
}
|
||||
if got[0].BytesServed != tc.wantBytes {
|
||||
t.Fatalf("BytesServed = %d, want %d", got[0].BytesServed, tc.wantBytes)
|
||||
}
|
||||
if (got[0].LastServedAt != "") != tc.wantServed {
|
||||
t.Fatalf("LastServedAt = %q, want served=%v", got[0].LastServedAt, tc.wantServed)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package streamenforcer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const coordinatorKey = "silo:enforcer:leader"
|
||||
|
||||
var acquireLeaseScript = redis.NewScript(`
|
||||
local v = redis.call('GET', KEYS[1])
|
||||
if v == false or v == ARGV[1] then
|
||||
redis.call('SET', KEYS[1], ARGV[1], 'PX', ARGV[2])
|
||||
return 1
|
||||
end
|
||||
return 0
|
||||
`)
|
||||
|
||||
// Coordinator elects one enforcer replica for an evaluation pass.
|
||||
type Coordinator interface {
|
||||
Acquire(ctx context.Context) (bool, error)
|
||||
}
|
||||
|
||||
// RedisCoordinator uses a renewable lease to select one evaluator.
|
||||
type RedisCoordinator struct {
|
||||
rdb *redis.Client
|
||||
instanceID string
|
||||
leaseTTL time.Duration
|
||||
}
|
||||
|
||||
// NewRedisCoordinator creates a coordinator whose lease lasts twice the
|
||||
// evaluation interval.
|
||||
func NewRedisCoordinator(rdb *redis.Client, instanceID string, interval time.Duration) *RedisCoordinator {
|
||||
if interval <= 0 {
|
||||
interval = DefaultInterval
|
||||
}
|
||||
return &RedisCoordinator{rdb: rdb, instanceID: instanceID, leaseTTL: 2 * interval}
|
||||
}
|
||||
|
||||
// Acquire claims or renews this instance's lease.
|
||||
func (c *RedisCoordinator) Acquire(ctx context.Context) (bool, error) {
|
||||
if c == nil || c.rdb == nil {
|
||||
return true, nil
|
||||
}
|
||||
result, err := acquireLeaseScript.Run(
|
||||
ctx,
|
||||
c.rdb,
|
||||
[]string{coordinatorKey},
|
||||
c.instanceID,
|
||||
c.leaseTTL.Milliseconds(),
|
||||
).Int()
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
return result == 1, nil
|
||||
}
|
||||
@@ -0,0 +1,226 @@
|
||||
// Package streamenforcer is the async decision loop ("the brain") of the
|
||||
// monitor-and-kill design. It runs on central only, off the hot path: it reads
|
||||
// the authoritative live-streams snapshot, compares each user's live count to
|
||||
// that user's limit, and issues revocations for the over-cap sessions. The
|
||||
// revocation kill switch (internal/streamrevoke) then stops them at the edge
|
||||
// within one propagation/poll interval.
|
||||
//
|
||||
// Every enforcement reason (over-cap here, admin terminate and account
|
||||
// revocation elsewhere) collapses to the same action: write a revocation. This
|
||||
// package owns only the over-cap rule; other reasons call the revoker directly.
|
||||
package streamenforcer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/streammonitor"
|
||||
)
|
||||
|
||||
// DefaultInterval is how often the enforcer evaluates the live picture. The
|
||||
// ~120s enforcement budget = this interval + the revocation propagation/poll.
|
||||
const DefaultInterval = 30 * time.Second
|
||||
|
||||
const (
|
||||
// RevocationTTLSetting controls the lifetime of future over-cap
|
||||
// revocations. Changing it cannot shorten an existing monotonic kill; an
|
||||
// explicit unrevoke is required to clear one.
|
||||
RevocationTTLSetting = "playback.over_cap_revocation_ttl"
|
||||
// MinRevocationTTL prevents a setting typo from making automated kills
|
||||
// churn faster than the evaluation and propagation cadence.
|
||||
MinRevocationTTL = 5 * time.Minute
|
||||
// DefaultRevocationTTL matches the full reconstructable stream-token life.
|
||||
// Derive it from the token contract instead of duplicating 24h here.
|
||||
DefaultRevocationTTL = playback.MaxTokenTTL
|
||||
MaxRevocationTTL = playback.MaxTokenTTL
|
||||
)
|
||||
|
||||
// Revoker is the subset of *streamrevoke.Store the enforcer needs. It uses the
|
||||
// if-absent variant so recurring evaluation never slides its own kill expiry.
|
||||
type Revoker interface {
|
||||
RevokeSessionForIfAbsent(ctx context.Context, sessionID, reason string, ttl time.Duration) (bool, error)
|
||||
}
|
||||
|
||||
// LimitFunc returns the maximum concurrent streams allowed for a user. A return
|
||||
// of <= 0 means "unlimited" (no enforcement for that user), matching the
|
||||
// SessionManager's convention. An error means the limit is currently unknown;
|
||||
// the enforcer fails OPEN (does not kill) so a limit-lookup blip never
|
||||
// terminates legitimate playback.
|
||||
type LimitFunc func(ctx context.Context, userID int) (maxStreams int, err error)
|
||||
|
||||
// Enforcer periodically trims over-cap streams.
|
||||
type Enforcer struct {
|
||||
source streammonitor.Source
|
||||
limits LimitFunc
|
||||
revoker Revoker
|
||||
interval time.Duration
|
||||
revocationTTL time.Duration
|
||||
now func() time.Time
|
||||
coordinator Coordinator
|
||||
}
|
||||
|
||||
// SetCoordinator installs the optional cross-replica evaluator coordinator.
|
||||
// Without one, every tick evaluates as before.
|
||||
func (e *Enforcer) SetCoordinator(c Coordinator) {
|
||||
if e != nil {
|
||||
e.coordinator = c
|
||||
}
|
||||
}
|
||||
|
||||
// New builds an enforcer. interval <= 0 uses DefaultInterval.
|
||||
func New(source streammonitor.Source, limits LimitFunc, revoker Revoker, interval time.Duration) *Enforcer {
|
||||
return NewWithRevocationTTL(source, limits, revoker, interval, DefaultRevocationTTL)
|
||||
}
|
||||
|
||||
// NewWithRevocationTTL builds an enforcer with the validated lifetime used for
|
||||
// future automated revocations.
|
||||
func NewWithRevocationTTL(
|
||||
source streammonitor.Source,
|
||||
limits LimitFunc,
|
||||
revoker Revoker,
|
||||
interval, revocationTTL time.Duration,
|
||||
) *Enforcer {
|
||||
if interval <= 0 {
|
||||
interval = DefaultInterval
|
||||
}
|
||||
if revocationTTL < MinRevocationTTL || revocationTTL > MaxRevocationTTL {
|
||||
revocationTTL = DefaultRevocationTTL
|
||||
}
|
||||
return &Enforcer{
|
||||
source: source,
|
||||
limits: limits,
|
||||
revoker: revoker,
|
||||
interval: interval,
|
||||
revocationTTL: revocationTTL,
|
||||
now: time.Now,
|
||||
}
|
||||
}
|
||||
|
||||
// ParseRevocationTTL validates the server setting. Empty uses the long default.
|
||||
func ParseRevocationTTL(raw string) (time.Duration, error) {
|
||||
if raw == "" {
|
||||
return DefaultRevocationTTL, nil
|
||||
}
|
||||
ttl, err := time.ParseDuration(raw)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("%s must be a duration: %w", RevocationTTLSetting, err)
|
||||
}
|
||||
if ttl < MinRevocationTTL || ttl > MaxRevocationTTL {
|
||||
return 0, fmt.Errorf(
|
||||
"%s must be between %s and %s",
|
||||
RevocationTTLSetting, MinRevocationTTL, MaxRevocationTTL,
|
||||
)
|
||||
}
|
||||
return ttl, nil
|
||||
}
|
||||
|
||||
// Start runs the evaluation loop until ctx is canceled. Non-blocking.
|
||||
func (e *Enforcer) Start(ctx context.Context) {
|
||||
if e == nil || e.source == nil || e.limits == nil || e.revoker == nil {
|
||||
return
|
||||
}
|
||||
go func() {
|
||||
ticker := time.NewTicker(e.interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
e.evaluate(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// evaluate runs one pass: snapshot → per-user over-cap check → revoke victims.
|
||||
// Exported behavior is covered by EvaluateOnce for tests.
|
||||
func (e *Enforcer) evaluate(ctx context.Context) {
|
||||
if e.coordinator != nil {
|
||||
acquired, err := e.coordinator.Acquire(ctx)
|
||||
if err != nil {
|
||||
// Coordination failure must not disable enforcement.
|
||||
slog.DebugContext(ctx, "stream enforcer coordination failed; evaluating anyway", "error", err)
|
||||
} else if !acquired {
|
||||
return
|
||||
}
|
||||
}
|
||||
passCtx, cancel := context.WithTimeout(ctx, e.interval)
|
||||
defer cancel()
|
||||
if err := e.EvaluateOnce(passCtx); err != nil {
|
||||
slog.DebugContext(ctx, "stream enforcer evaluate failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// EvaluateOnce performs a single enforcement pass and returns the number of
|
||||
// sessions revoked. Deterministic and side-effect-scoped for testing.
|
||||
func (e *Enforcer) EvaluateOnce(ctx context.Context) error {
|
||||
snap, err := e.source.Snapshot(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for userID, streams := range snap.ByUser() {
|
||||
if userID <= 0 || len(streams) == 0 {
|
||||
// user 0 == records with no resolved owner; never enforce against it.
|
||||
continue
|
||||
}
|
||||
limit, err := e.limits(ctx, userID)
|
||||
if err != nil {
|
||||
// Fail open: a limit-lookup error must never kill legitimate streams.
|
||||
slog.DebugContext(ctx, "stream enforcer: limit lookup failed; skipping user",
|
||||
"user_id", userID, "error", err)
|
||||
continue
|
||||
}
|
||||
if limit <= 0 || len(streams) <= limit {
|
||||
continue
|
||||
}
|
||||
for _, victim := range e.selectVictims(streams, limit) {
|
||||
created, err := e.revoker.RevokeSessionForIfAbsent(
|
||||
ctx, victim.SessionID, "over_concurrent_stream_limit", e.revocationTTL,
|
||||
)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "stream enforcer: revoke failed",
|
||||
"user_id", userID, "session_id", victim.SessionID, "error", err)
|
||||
continue
|
||||
}
|
||||
if !created {
|
||||
continue
|
||||
}
|
||||
slog.InfoContext(ctx, "stream enforcer: revoked over-cap session",
|
||||
"user_id", userID, "session_id", victim.SessionID,
|
||||
"limit", limit, "live", len(streams))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// selectVictims returns the streams beyond the limit, keeping the `limit`
|
||||
// MOST-RECENTLY-SERVED sessions and trimming the rest. Ordering by real serve
|
||||
// activity (LastServedAt), not StartedAt, matters: after a network blip a client
|
||||
// reconnects with a new session while the old ghost lingers in the monitor for
|
||||
// up to its TTL. Keeping the freshly-served sessions means the live reconnect
|
||||
// survives and the stale ghost is the one trimmed (and it would have aged out
|
||||
// anyway). Falls back to StartedAt, then session id, for deterministic ties.
|
||||
func (e *Enforcer) selectVictims(streams []streammonitor.LiveStream, limit int) []streammonitor.LiveStream {
|
||||
ordered := make([]streammonitor.LiveStream, len(streams))
|
||||
copy(ordered, streams)
|
||||
// Most-recently-served first.
|
||||
sort.SliceStable(ordered, func(i, j int) bool {
|
||||
if !ordered[i].LastServedAt.Equal(ordered[j].LastServedAt) {
|
||||
return ordered[i].LastServedAt.After(ordered[j].LastServedAt)
|
||||
}
|
||||
if !ordered[i].StartedAt.Equal(ordered[j].StartedAt) {
|
||||
return ordered[i].StartedAt.After(ordered[j].StartedAt)
|
||||
}
|
||||
return ordered[i].SessionID < ordered[j].SessionID
|
||||
})
|
||||
if limit >= len(ordered) {
|
||||
return nil
|
||||
}
|
||||
// Keep the first `limit` (freshest); revoke the rest (stalest).
|
||||
return ordered[limit:]
|
||||
}
|
||||
@@ -0,0 +1,406 @@
|
||||
package streamenforcer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"sort"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
"github.com/Silo-Server/silo-server/internal/streammonitor"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
type fakeSource struct {
|
||||
snap streammonitor.Snapshot
|
||||
err error
|
||||
}
|
||||
|
||||
func (f fakeSource) Snapshot(context.Context) (streammonitor.Snapshot, error) {
|
||||
return f.snap, f.err
|
||||
}
|
||||
|
||||
type fakeRevoker struct {
|
||||
revoked []string
|
||||
err error
|
||||
seen map[string]struct{}
|
||||
ttls []time.Duration
|
||||
}
|
||||
|
||||
func (f *fakeRevoker) RevokeSessionForIfAbsent(
|
||||
_ context.Context,
|
||||
sessionID, _ string,
|
||||
ttl time.Duration,
|
||||
) (bool, error) {
|
||||
if f.err != nil {
|
||||
return false, f.err
|
||||
}
|
||||
if f.seen == nil {
|
||||
f.seen = make(map[string]struct{})
|
||||
}
|
||||
if _, ok := f.seen[sessionID]; ok {
|
||||
return false, nil
|
||||
}
|
||||
f.seen[sessionID] = struct{}{}
|
||||
f.revoked = append(f.revoked, sessionID)
|
||||
f.ttls = append(f.ttls, ttl)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// stream builds a LiveStream whose StartedAt and LastServedAt are both ts.
|
||||
func stream(id string, user int, ts time.Time) streammonitor.LiveStream {
|
||||
return streammonitor.LiveStream{SessionID: id, UserID: user, StartedAt: ts, LastServedAt: ts}
|
||||
}
|
||||
|
||||
// served builds a LiveStream started at `started` but last served at `lastServed`
|
||||
// (to model a ghost: old activity) — StartedAt independent from real liveness.
|
||||
func served(id string, user int, started, lastServed time.Time) streammonitor.LiveStream {
|
||||
return streammonitor.LiveStream{SessionID: id, UserID: user, StartedAt: started, LastServedAt: lastServed}
|
||||
}
|
||||
|
||||
func TestEvaluateOnce(t *testing.T) {
|
||||
base := time.Date(2026, 7, 4, 12, 0, 0, 0, time.UTC)
|
||||
limit3 := func(context.Context, int) (int, error) { return 3, nil }
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
streams []streammonitor.LiveStream
|
||||
limits LimitFunc
|
||||
wantRevoke []string
|
||||
}{
|
||||
{
|
||||
name: "under cap: no revokes",
|
||||
streams: []streammonitor.LiveStream{stream("a", 1, base), stream("b", 1, base.Add(time.Minute))},
|
||||
limits: limit3,
|
||||
wantRevoke: nil,
|
||||
},
|
||||
{
|
||||
name: "over cap: least-recently-served trimmed, freshest kept",
|
||||
streams: []streammonitor.LiveStream{
|
||||
stream("stale", 1, base),
|
||||
stream("old", 1, base.Add(1*time.Minute)),
|
||||
stream("keep1", 1, base.Add(2*time.Minute)),
|
||||
stream("keep2", 1, base.Add(3*time.Minute)),
|
||||
stream("keep3", 1, base.Add(4*time.Minute)),
|
||||
},
|
||||
limits: limit3,
|
||||
wantRevoke: []string{"stale", "old"},
|
||||
},
|
||||
{
|
||||
name: "ghost trimmed, fresh reconnect survives",
|
||||
// Two fresh (served just now) and two ghosts (served long ago), limit 2.
|
||||
streams: []streammonitor.LiveStream{
|
||||
served("ghost1", 1, base, base),
|
||||
served("ghost2", 1, base.Add(time.Second), base.Add(time.Second)),
|
||||
served("fresh1", 1, base.Add(10*time.Minute), base.Add(10*time.Minute)),
|
||||
served("fresh2", 1, base.Add(10*time.Minute), base.Add(11*time.Minute)),
|
||||
},
|
||||
limits: func(context.Context, int) (int, error) { return 2, nil },
|
||||
wantRevoke: []string{"ghost1", "ghost2"},
|
||||
},
|
||||
{
|
||||
name: "limit lookup error: fail open",
|
||||
streams: []streammonitor.LiveStream{stream("a", 1, base), stream("b", 1, base), stream("c", 1, base), stream("d", 1, base)},
|
||||
limits: func(context.Context, int) (int, error) { return 0, errors.New("db down") },
|
||||
wantRevoke: nil,
|
||||
},
|
||||
{
|
||||
name: "unlimited (limit<=0): no revokes",
|
||||
streams: []streammonitor.LiveStream{stream("a", 1, base), stream("b", 1, base), stream("c", 1, base)},
|
||||
limits: func(context.Context, int) (int, error) { return 0, nil },
|
||||
wantRevoke: nil,
|
||||
},
|
||||
{
|
||||
name: "user 0 never enforced",
|
||||
streams: []streammonitor.LiveStream{stream("a", 0, base), stream("b", 0, base), stream("c", 0, base), stream("d", 0, base)},
|
||||
limits: limit3,
|
||||
wantRevoke: nil,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rev := &fakeRevoker{}
|
||||
e := New(fakeSource{snap: streammonitor.Snapshot{Streams: tc.streams}}, tc.limits, rev, 0)
|
||||
if err := e.EvaluateOnce(context.Background()); err != nil {
|
||||
t.Fatalf("EvaluateOnce: %v", err)
|
||||
}
|
||||
got := append([]string{}, rev.revoked...)
|
||||
sort.Strings(got)
|
||||
want := append([]string{}, tc.wantRevoke...)
|
||||
sort.Strings(want)
|
||||
if len(got) != len(want) {
|
||||
t.Fatalf("revoked = %v, want %v", got, want)
|
||||
}
|
||||
for i := range got {
|
||||
if got[i] != want[i] {
|
||||
t.Fatalf("revoked = %v, want %v", got, want)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateOnceProgressOnlySessionIsVictimBeforeServingStreams(t *testing.T) {
|
||||
sm := playback.NewSessionManager(0, 0)
|
||||
phantom, err := sm.StartSession(7, "profile", 1, playback.PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession(phantom): %v", err)
|
||||
}
|
||||
|
||||
for i := 0; i < 2; i++ {
|
||||
session, startErr := sm.StartSession(7, "profile", 10+i, playback.PlayDirect, false)
|
||||
if startErr != nil {
|
||||
t.Fatalf("StartSession(serving %d): %v", i, startErr)
|
||||
}
|
||||
if beginErr := sm.BeginTransport(session.ID); beginErr != nil {
|
||||
t.Fatalf("BeginTransport(serving %d): %v", i, beginErr)
|
||||
}
|
||||
if bytesErr := sm.AddServedBytes(session.ID, 1024); bytesErr != nil {
|
||||
t.Fatalf("AddServedBytes(serving %d): %v", i, bytesErr)
|
||||
}
|
||||
if endErr := sm.EndTransport(session.ID); endErr != nil {
|
||||
t.Fatalf("EndTransport(serving %d): %v", i, endErr)
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(time.Until(time.Now().Truncate(time.Second).Add(time.Second)) + 10*time.Millisecond)
|
||||
if err := sm.UpdateProgress(phantom.ID, 30, false); err != nil {
|
||||
t.Fatalf("UpdateProgress(phantom): %v", err)
|
||||
}
|
||||
|
||||
source := streammonitor.NewFuncSource(func(context.Context) ([]nodesessions.SessionInfo, error) {
|
||||
return streammonitor.LiveLocalSessions(sm, "local"), nil
|
||||
})
|
||||
rev := &fakeRevoker{}
|
||||
e := New(source, func(context.Context, int) (int, error) { return 2, nil }, rev, 0)
|
||||
if err := e.EvaluateOnce(context.Background()); err != nil {
|
||||
t.Fatalf("EvaluateOnce: %v", err)
|
||||
}
|
||||
if len(rev.revoked) != 1 || rev.revoked[0] != phantom.ID {
|
||||
t.Fatalf("revoked = %v, want progress-only phantom %q", rev.revoked, phantom.ID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateOnceSourceError(t *testing.T) {
|
||||
rev := &fakeRevoker{}
|
||||
e := New(fakeSource{err: errors.New("scan failed")}, func(context.Context, int) (int, error) { return 1, nil }, rev, 0)
|
||||
if err := e.EvaluateOnce(context.Background()); err == nil {
|
||||
t.Fatal("expected error from source")
|
||||
}
|
||||
if len(rev.revoked) != 0 {
|
||||
t.Fatalf("expected no revokes on source error, got %v", rev.revoked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateOnceRevokesCanonicalLogicalSessionID(t *testing.T) {
|
||||
source := streammonitor.NewFuncSource(func(context.Context) ([]nodesessions.SessionInfo, error) {
|
||||
return []nodesessions.SessionInfo{
|
||||
{SessionID: "transport-a", LogicalSessionID: "logical-a", AuthUserID: 7, LastServedAt: "2026-07-30T01:00:00Z"},
|
||||
{SessionID: "transport-b", LogicalSessionID: "logical-b", AuthUserID: 7, LastServedAt: "2026-07-30T02:00:00Z"},
|
||||
}, nil
|
||||
})
|
||||
rev := &fakeRevoker{}
|
||||
e := New(source, func(context.Context, int) (int, error) { return 1, nil }, rev, 0)
|
||||
if err := e.EvaluateOnce(context.Background()); err != nil {
|
||||
t.Fatalf("EvaluateOnce: %v", err)
|
||||
}
|
||||
if len(rev.revoked) != 1 || rev.revoked[0] != "logical-a" {
|
||||
t.Fatalf("revoked = %v, want logical-a", rev.revoked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEvaluateOnceDoesNotExtendExistingAutomatedRevocation(t *testing.T) {
|
||||
base := time.Now()
|
||||
source := fakeSource{snap: streammonitor.Snapshot{Streams: []streammonitor.LiveStream{
|
||||
stream("keep", 7, base.Add(time.Minute)),
|
||||
stream("victim", 7, base),
|
||||
}}}
|
||||
rev := &fakeRevoker{}
|
||||
e := New(source, func(context.Context, int) (int, error) { return 1, nil }, rev, 0)
|
||||
if err := e.EvaluateOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := e.EvaluateOnce(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(rev.revoked) != 1 {
|
||||
t.Fatalf("revocation writes = %d, want 1", len(rev.revoked))
|
||||
}
|
||||
if len(rev.ttls) != 1 || rev.ttls[0] != playback.MaxTokenTTL {
|
||||
t.Fatalf("revocation TTLs = %v, want [%v]", rev.ttls, playback.MaxTokenTTL)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRevocationTTL(t *testing.T) {
|
||||
if got, err := ParseRevocationTTL(""); err != nil || got != playback.MaxTokenTTL {
|
||||
t.Fatalf("default = %v, %v; want %v", got, err, playback.MaxTokenTTL)
|
||||
}
|
||||
if got, err := ParseRevocationTTL("6h"); err != nil || got != 6*time.Hour {
|
||||
t.Fatalf("valid = %v, %v; want 6h", got, err)
|
||||
}
|
||||
for _, raw := range []string{"bad", "4m59s", "24h1s"} {
|
||||
if _, err := ParseRevocationTTL(raw); err == nil {
|
||||
t.Fatalf("ParseRevocationTTL(%q) accepted invalid value", raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type firstCoordinator struct {
|
||||
mu sync.Mutex
|
||||
acquired bool
|
||||
}
|
||||
|
||||
func (c *firstCoordinator) Acquire(context.Context) (bool, error) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if c.acquired {
|
||||
return false, nil
|
||||
}
|
||||
c.acquired = true
|
||||
return true, nil
|
||||
}
|
||||
|
||||
type errorCoordinator struct{}
|
||||
|
||||
func (errorCoordinator) Acquire(context.Context) (bool, error) {
|
||||
return false, errors.New("redis unavailable")
|
||||
}
|
||||
|
||||
type countingSource struct {
|
||||
mu sync.Mutex
|
||||
snap streammonitor.Snapshot
|
||||
calls int
|
||||
}
|
||||
|
||||
func (s *countingSource) Snapshot(context.Context) (streammonitor.Snapshot, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.calls++
|
||||
return s.snap, nil
|
||||
}
|
||||
|
||||
func TestSharedCoordinatorAllowsExactlyOneEnforcerEvaluation(t *testing.T) {
|
||||
base := time.Now()
|
||||
source := &countingSource{snap: streammonitor.Snapshot{Streams: []streammonitor.LiveStream{
|
||||
stream("keep", 7, base.Add(time.Minute)),
|
||||
stream("victim", 7, base),
|
||||
}}}
|
||||
revoker := &fakeRevoker{}
|
||||
coordinator := &firstCoordinator{}
|
||||
enforcers := []*Enforcer{
|
||||
New(source, func(context.Context, int) (int, error) { return 1, nil }, revoker, time.Second),
|
||||
New(source, func(context.Context, int) (int, error) { return 1, nil }, revoker, time.Second),
|
||||
}
|
||||
for _, enforcer := range enforcers {
|
||||
enforcer.SetCoordinator(coordinator)
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for _, enforcer := range enforcers {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
enforcer.evaluate(context.Background())
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
source.mu.Lock()
|
||||
snapshotCalls := source.calls
|
||||
source.mu.Unlock()
|
||||
if snapshotCalls != 1 {
|
||||
t.Fatalf("snapshot calls = %d, want exactly 1", snapshotCalls)
|
||||
}
|
||||
if len(revoker.revoked) != 1 || revoker.revoked[0] != "victim" {
|
||||
t.Fatalf("revoked = %v, want exactly [victim]", revoker.revoked)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCoordinatorErrorStillEvaluates(t *testing.T) {
|
||||
base := time.Now()
|
||||
revoker := &fakeRevoker{}
|
||||
enforcer := New(fakeSource{snap: streammonitor.Snapshot{Streams: []streammonitor.LiveStream{
|
||||
stream("keep", 7, base.Add(time.Minute)),
|
||||
stream("victim", 7, base),
|
||||
}}}, func(context.Context, int) (int, error) { return 1, nil }, revoker, time.Second)
|
||||
enforcer.SetCoordinator(errorCoordinator{})
|
||||
enforcer.evaluate(context.Background())
|
||||
if len(revoker.revoked) != 1 || revoker.revoked[0] != "victim" {
|
||||
t.Fatalf("revoked = %v, want [victim]", revoker.revoked)
|
||||
}
|
||||
}
|
||||
|
||||
type blockingSource struct {
|
||||
deadlineObserved chan struct{}
|
||||
}
|
||||
|
||||
func (s blockingSource) Snapshot(ctx context.Context) (streammonitor.Snapshot, error) {
|
||||
if _, ok := ctx.Deadline(); !ok {
|
||||
close(s.deadlineObserved)
|
||||
return streammonitor.Snapshot{}, errors.New("evaluation context has no deadline")
|
||||
}
|
||||
<-ctx.Done()
|
||||
close(s.deadlineObserved)
|
||||
return streammonitor.Snapshot{}, ctx.Err()
|
||||
}
|
||||
|
||||
func TestEvaluationPassIsBoundedByInterval(t *testing.T) {
|
||||
observed := make(chan struct{})
|
||||
enforcer := New(
|
||||
blockingSource{deadlineObserved: observed},
|
||||
func(context.Context, int) (int, error) { return 1, nil },
|
||||
&fakeRevoker{},
|
||||
25*time.Millisecond,
|
||||
)
|
||||
start := time.Now()
|
||||
enforcer.evaluate(context.Background())
|
||||
elapsed := time.Since(start)
|
||||
select {
|
||||
case <-observed:
|
||||
default:
|
||||
t.Fatal("source did not observe the evaluation deadline")
|
||||
}
|
||||
if elapsed < 20*time.Millisecond || elapsed > 250*time.Millisecond {
|
||||
t.Fatalf("evaluation elapsed %v, want bounded near 25ms", elapsed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisCoordinatorElectsOneHolderAndRenews(t *testing.T) {
|
||||
addr := os.Getenv("SILO_TEST_REDIS_ADDR")
|
||||
if addr == "" {
|
||||
t.Skip("SILO_TEST_REDIS_ADDR is not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
rdb := redis.NewClient(&redis.Options{Addr: addr})
|
||||
t.Cleanup(func() {
|
||||
_ = rdb.Del(ctx, coordinatorKey).Err()
|
||||
_ = rdb.Close()
|
||||
})
|
||||
if err := rdb.Ping(ctx).Err(); err != nil {
|
||||
t.Fatalf("ping test Redis: %v", err)
|
||||
}
|
||||
if err := rdb.Del(ctx, coordinatorKey).Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
const interval = 80 * time.Millisecond
|
||||
first := NewRedisCoordinator(rdb, "first", interval)
|
||||
second := NewRedisCoordinator(rdb, "second", interval)
|
||||
if acquired, err := first.Acquire(ctx); err != nil || !acquired {
|
||||
t.Fatalf("first acquire=%v err=%v, want true", acquired, err)
|
||||
}
|
||||
if acquired, err := second.Acquire(ctx); err != nil || acquired {
|
||||
t.Fatalf("second acquire=%v err=%v, want false", acquired, err)
|
||||
}
|
||||
time.Sleep(90 * time.Millisecond)
|
||||
if acquired, err := first.Acquire(ctx); err != nil || !acquired {
|
||||
t.Fatalf("renew acquire=%v err=%v, want true", acquired, err)
|
||||
}
|
||||
if ttl, err := rdb.PTTL(ctx, coordinatorKey).Result(); err != nil || ttl < 120*time.Millisecond {
|
||||
t.Fatalf("renewed lease TTL=%v err=%v, want >=120ms", ttl, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,438 @@
|
||||
// Package streammonitor produces a normalized "live streams" snapshot grouped by
|
||||
// user from the authoritative server-side monitoring records written by
|
||||
// internal/nodesessions. It backs an async enforcement loop and admin views.
|
||||
//
|
||||
// EXISTENCE is server-observed on every path and never gated by a client report,
|
||||
// so a "hidden stream" (a disguised client that pulls bytes but withholds
|
||||
// progress) is always counted:
|
||||
// - Edge (multi-node): a record exists in Redis for the whole connection
|
||||
// (direct/remux Track..Remove) or while segments are pulled (transcode Touch),
|
||||
// and BytesServed/LastServedAt advance only on real bytes.
|
||||
// - Integrated: the FuncSource reflects SessionManager.AllSessions(), and a
|
||||
// session is unreapable while it holds an in-flight transport marker
|
||||
// (BeginTransport/EndTransport around every byte pour) — no client progress
|
||||
// required to stay visible.
|
||||
//
|
||||
// TIMING is server-observed on every path. LastServedAt advances only when the
|
||||
// server begins, ends, or writes a media transport. A session that has not
|
||||
// served media projects an empty LastServedAt and sorts as the stalest over-cap
|
||||
// victim. See internal/nodesessions for record production.
|
||||
package streammonitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
)
|
||||
|
||||
// sessionKeyPrefix is the Redis key prefix under which nodesessions.Tracker
|
||||
// stores its SessionInfo records — the tracker's own exported constant, so the
|
||||
// write and read sides can never drift.
|
||||
const sessionKeyPrefix = nodesessions.KeyPrefix
|
||||
|
||||
// scanCount is the COUNT hint passed to SCAN. It bounds the amount of work per
|
||||
// round trip while keeping the number of round trips reasonable.
|
||||
const scanCount = 256
|
||||
|
||||
// LiveStream is a normalized view of a single active streaming session.
|
||||
type LiveStream struct {
|
||||
SessionID string
|
||||
LogicalSessionID string
|
||||
UserID int // from SessionInfo.AuthUserID
|
||||
ProfileID string
|
||||
NodeName string
|
||||
NodeURL string
|
||||
Type string // play method: direct_play | remux | transcode
|
||||
Route string // origin protocol: native | jellycompat
|
||||
MediaFileID int
|
||||
ClientIP string
|
||||
ClientName string
|
||||
Position float64 // last known playback position (seconds); secondary timing
|
||||
HWAccel string
|
||||
LastServedAt time.Time // parsed from SessionInfo.LastServedAt (zero if absent)
|
||||
BytesServed int64
|
||||
StartedAt time.Time // parsed from SessionInfo.StartedAt (zero if unparseable)
|
||||
}
|
||||
|
||||
// Snapshot is a point-in-time picture of the live streams.
|
||||
type Snapshot struct {
|
||||
Streams []LiveStream
|
||||
}
|
||||
|
||||
// CountByUser returns the number of live streams owned by userID.
|
||||
func (s Snapshot) CountByUser(userID int) int {
|
||||
n := 0
|
||||
for _, st := range s.Streams {
|
||||
if st.UserID == userID {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// StreamsForUser returns the live streams owned by userID.
|
||||
func (s Snapshot) StreamsForUser(userID int) []LiveStream {
|
||||
out := make([]LiveStream, 0)
|
||||
for _, st := range s.Streams {
|
||||
if st.UserID == userID {
|
||||
out = append(out, st)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ByUser groups the live streams by owning user id.
|
||||
func (s Snapshot) ByUser() map[int][]LiveStream {
|
||||
out := make(map[int][]LiveStream)
|
||||
for _, st := range s.Streams {
|
||||
out[st.UserID] = append(out[st.UserID], st)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Source yields the current live picture.
|
||||
type Source interface {
|
||||
Snapshot(ctx context.Context) (Snapshot, error)
|
||||
}
|
||||
|
||||
// MultiSource unions several sources into one snapshot, de-duplicating sessions
|
||||
// that appear in more than one backend (e.g. a session held both in the central
|
||||
// session manager and mirrored by an edge's Redis record) by keeping the most-
|
||||
// recently-served copy. A source that errors is skipped (logged), so one
|
||||
// unavailable backend never blinds the enforcer to the others. This is what lets
|
||||
// the enforcer see BOTH locally-served (integrated) and edge-served (multi-node)
|
||||
// streams regardless of whether Redis is configured.
|
||||
type MultiSource struct {
|
||||
sources []Source
|
||||
}
|
||||
|
||||
// NewMultiSource builds a union source over the given sources (nil entries are
|
||||
// ignored).
|
||||
func NewMultiSource(sources ...Source) *MultiSource {
|
||||
return &MultiSource{sources: sources}
|
||||
}
|
||||
|
||||
// Snapshot merges every sub-source's snapshot, de-duplicated by canonical
|
||||
// logical session identity when available.
|
||||
func (m *MultiSource) Snapshot(ctx context.Context) (Snapshot, error) {
|
||||
var all []LiveStream
|
||||
for _, src := range m.sources {
|
||||
if src == nil {
|
||||
continue
|
||||
}
|
||||
snap, err := src.Snapshot(ctx)
|
||||
if err != nil {
|
||||
slog.Warn("streammonitor: source snapshot failed; skipping", "error", err)
|
||||
continue
|
||||
}
|
||||
all = append(all, snap.Streams...)
|
||||
}
|
||||
return Snapshot{Streams: mergeStreams(all)}, nil
|
||||
}
|
||||
|
||||
// toLiveStream converts a nodesessions.SessionInfo into a LiveStream, parsing
|
||||
// the RFC3339 timestamps and tolerating empty/unparseable values (which map to
|
||||
// the zero time).
|
||||
func toLiveStream(info nodesessions.SessionInfo) LiveStream {
|
||||
return LiveStream{
|
||||
SessionID: info.SessionID,
|
||||
LogicalSessionID: info.LogicalSessionID,
|
||||
UserID: info.AuthUserID,
|
||||
ProfileID: info.ProfileID,
|
||||
NodeName: info.NodeName,
|
||||
NodeURL: info.NodeURL,
|
||||
Type: info.Type,
|
||||
Route: info.Route,
|
||||
MediaFileID: info.MediaFileID,
|
||||
ClientIP: info.ClientIP,
|
||||
ClientName: info.ClientName,
|
||||
Position: info.Position,
|
||||
HWAccel: info.HWAccel,
|
||||
LastServedAt: parseTime(info.LastServedAt),
|
||||
BytesServed: info.BytesServed,
|
||||
StartedAt: parseTime(info.StartedAt),
|
||||
}
|
||||
}
|
||||
|
||||
// sessionIdentity returns the stable identity used to merge and enforce a
|
||||
// playback stream. SessionID remains the transport-addressing key on raw node
|
||||
// records; a logical id, when present, takes precedence only in monitoring.
|
||||
func sessionIdentity(sessionID, logicalSessionID string) string {
|
||||
if logicalSessionID != "" {
|
||||
return logicalSessionID
|
||||
}
|
||||
return sessionID
|
||||
}
|
||||
|
||||
// parseTime parses an RFC3339 timestamp, returning the zero time for empty or
|
||||
// unparseable input.
|
||||
func parseTime(s string) time.Time {
|
||||
if s == "" {
|
||||
return time.Time{}
|
||||
}
|
||||
t, err := time.Parse(time.RFC3339, s)
|
||||
if err != nil {
|
||||
slog.Debug("streammonitor: unparseable timestamp", "value", s, "error", err)
|
||||
return time.Time{}
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// mergeStreams collapses records that share a canonical identity (the logical
|
||||
// session id when present, otherwise the transport SessionID). The same stream
|
||||
// can be tracked by more than one node or transport generation. The freshest
|
||||
// record wins; genuinely distinct logical sessions are retained. The relative
|
||||
// order of kept records is not guaranteed.
|
||||
//
|
||||
// Ownership and attribution are carried forward independently of the freshness
|
||||
// pick: the transcode node's own start record has no resolved owner (UserID 0)
|
||||
// and thinner attribution, while the proxy record fronting it does. If the
|
||||
// ownerless node record happened to be the freshest copy, taking it wholesale
|
||||
// would bucket the session under user 0 (which the enforcer skips — silently
|
||||
// exempting it from the concurrency cap) and drop route/client detail from the
|
||||
// monitor view. So a merged record adopts a resolved owner from either candidate
|
||||
// and backfills any empty attribution/display field from the other copy.
|
||||
func mergeStreams(streams []LiveStream) []LiveStream {
|
||||
bySession := make(map[string]LiveStream, len(streams))
|
||||
for _, st := range streams {
|
||||
key := sessionIdentity(st.SessionID, st.LogicalSessionID)
|
||||
existing, ok := bySession[key]
|
||||
if !ok {
|
||||
st.SessionID = key
|
||||
bySession[key] = st
|
||||
continue
|
||||
}
|
||||
winner := existing
|
||||
other := st
|
||||
if st.LastServedAt.After(existing.LastServedAt) {
|
||||
winner, other = st, existing
|
||||
}
|
||||
// If the freshest copy is ownerless, adopt a resolved owner from either
|
||||
// candidate so the session is still attributed (and enforced) correctly.
|
||||
if winner.UserID <= 0 {
|
||||
for _, cand := range []LiveStream{existing, st} {
|
||||
if cand.UserID > 0 {
|
||||
winner.UserID = cand.UserID
|
||||
winner.ProfileID = cand.ProfileID
|
||||
winner.MediaFileID = cand.MediaFileID
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// Backfill display/attribution fields the freshest copy lacks so the
|
||||
// merged record is as complete as possible for the monitor view.
|
||||
if winner.Route == "" {
|
||||
winner.Route = other.Route
|
||||
}
|
||||
if winner.ClientIP == "" {
|
||||
winner.ClientIP = other.ClientIP
|
||||
}
|
||||
if winner.ClientName == "" {
|
||||
winner.ClientName = other.ClientName
|
||||
}
|
||||
if winner.HWAccel == "" {
|
||||
winner.HWAccel = other.HWAccel
|
||||
}
|
||||
if winner.Position == 0 {
|
||||
winner.Position = other.Position
|
||||
}
|
||||
// These are observers of one pour, not independent byte sources.
|
||||
winner.BytesServed = max(winner.BytesServed, other.BytesServed)
|
||||
winner.SessionID = key
|
||||
bySession[key] = winner
|
||||
}
|
||||
out := make([]LiveStream, 0, len(bySession))
|
||||
for _, st := range bySession {
|
||||
out = append(out, st)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// DedupeSessionInfos collapses raw monitoring records that share a canonical
|
||||
// identity, applying the same rules as mergeStreams — keep the most-recently-
|
||||
// served copy, carry a resolved owner forward, backfill missing attribution —
|
||||
// but preserving the SessionInfo shape and transport SessionID for surfaces
|
||||
// whose wire format IS the raw record. Kept as a sibling of mergeStreams rather
|
||||
// than a shared generic because mergeStreams operates on parsed LiveStream.
|
||||
func DedupeSessionInfos(infos []nodesessions.SessionInfo) []nodesessions.SessionInfo {
|
||||
bySession := make(map[string]nodesessions.SessionInfo, len(infos))
|
||||
order := make([]string, 0, len(infos))
|
||||
for _, in := range infos {
|
||||
key := sessionIdentity(in.SessionID, in.LogicalSessionID)
|
||||
existing, ok := bySession[key]
|
||||
if !ok {
|
||||
bySession[key] = in
|
||||
order = append(order, key)
|
||||
continue
|
||||
}
|
||||
winner, other := existing, in
|
||||
if parseTime(in.LastServedAt).After(parseTime(existing.LastServedAt)) {
|
||||
winner, other = in, existing
|
||||
}
|
||||
if winner.AuthUserID <= 0 && other.AuthUserID > 0 {
|
||||
winner.AuthUserID = other.AuthUserID
|
||||
winner.ProfileID = other.ProfileID
|
||||
winner.MediaFileID = other.MediaFileID
|
||||
}
|
||||
if winner.Route == "" {
|
||||
winner.Route = other.Route
|
||||
}
|
||||
if winner.ClientIP == "" {
|
||||
winner.ClientIP = other.ClientIP
|
||||
}
|
||||
if winner.ClientName == "" {
|
||||
winner.ClientName = other.ClientName
|
||||
}
|
||||
if winner.HWAccel == "" {
|
||||
winner.HWAccel = other.HWAccel
|
||||
}
|
||||
if winner.Position == 0 {
|
||||
winner.Position = other.Position
|
||||
}
|
||||
winner.BytesServed = max(winner.BytesServed, other.BytesServed)
|
||||
bySession[key] = winner
|
||||
}
|
||||
out := make([]nodesessions.SessionInfo, 0, len(bySession))
|
||||
for _, id := range order {
|
||||
out = append(out, bySession[id])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// LiveLocalSessions maps in-process playback sessions into monitoring records
|
||||
// so integrated streams are visible to both the enforcer and admin view.
|
||||
func LiveLocalSessions(sm *playback.SessionManager, nodeName string) []nodesessions.SessionInfo {
|
||||
if sm == nil {
|
||||
return nil
|
||||
}
|
||||
live := sm.AllSessions()
|
||||
out := make([]nodesessions.SessionInfo, 0, len(live))
|
||||
for _, s := range live {
|
||||
lastServedAt := ""
|
||||
if !s.LastServedAt.IsZero() {
|
||||
lastServedAt = s.LastServedAt.UTC().Format(time.RFC3339)
|
||||
}
|
||||
out = append(out, nodesessions.SessionInfo{
|
||||
SessionID: s.ID,
|
||||
NodeName: nodeName,
|
||||
AuthUserID: s.UserID,
|
||||
ProfileID: s.ProfileID,
|
||||
Type: string(s.PlayMethod),
|
||||
Route: s.Origin(),
|
||||
MediaFileID: s.MediaFileID,
|
||||
ClientIP: s.ClientIP,
|
||||
ClientName: s.ClientName,
|
||||
Position: s.Position,
|
||||
Resolution: s.TargetResolution,
|
||||
HWAccel: s.TranscodeHWAccel,
|
||||
StartedAt: s.StartedAt.UTC().Format(time.RFC3339),
|
||||
LastServedAt: lastServedAt,
|
||||
BytesServed: s.BytesServed,
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// RedisSource reads silo:sessions:* records, producing the multi-node
|
||||
// authoritative picture.
|
||||
type RedisSource struct {
|
||||
rdb *redis.Client
|
||||
}
|
||||
|
||||
// NewRedisSource creates a RedisSource backed by rdb.
|
||||
func NewRedisSource(rdb *redis.Client) *RedisSource {
|
||||
return &RedisSource{rdb: rdb}
|
||||
}
|
||||
|
||||
// Snapshot SCANs every silo:sessions:* record, decodes it, and returns the
|
||||
// deduped live picture. The same logical session appearing on multiple nodes or
|
||||
// transport generations is collapsed to the record with the newest liveness.
|
||||
func (r *RedisSource) Snapshot(ctx context.Context) (Snapshot, error) {
|
||||
if r.rdb == nil {
|
||||
return Snapshot{Streams: []LiveStream{}}, nil
|
||||
}
|
||||
|
||||
// Fully iterate the cursor to collect matching keys. SCAN (not KEYS) keeps
|
||||
// this non-blocking against a large keyspace.
|
||||
var keys []string
|
||||
var cursor uint64
|
||||
for {
|
||||
batch, next, err := r.rdb.Scan(ctx, cursor, sessionKeyPrefix+"*", scanCount).Result()
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
keys = append(keys, batch...)
|
||||
cursor = next
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if len(keys) == 0 {
|
||||
return Snapshot{Streams: []LiveStream{}}, nil
|
||||
}
|
||||
|
||||
vals, err := r.rdb.MGet(ctx, keys...).Result()
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
|
||||
streams := make([]LiveStream, 0, len(vals))
|
||||
for i, v := range vals {
|
||||
if v == nil {
|
||||
// Key expired between SCAN and MGET; skip.
|
||||
continue
|
||||
}
|
||||
var raw string
|
||||
switch val := v.(type) {
|
||||
case string:
|
||||
raw = val
|
||||
case []byte:
|
||||
raw = string(val)
|
||||
default:
|
||||
slog.Debug("streammonitor: unexpected redis value type", "key", keys[i])
|
||||
continue
|
||||
}
|
||||
var info nodesessions.SessionInfo
|
||||
if err := json.Unmarshal([]byte(raw), &info); err != nil {
|
||||
slog.Debug("streammonitor: unmarshal session record failed", "key", keys[i], "error", err)
|
||||
continue
|
||||
}
|
||||
streams = append(streams, toLiveStream(info))
|
||||
}
|
||||
|
||||
return Snapshot{Streams: mergeStreams(streams)}, nil
|
||||
}
|
||||
|
||||
// FuncSource adapts an in-process provider (integrated single-node) that returns
|
||||
// the local tracker records.
|
||||
type FuncSource struct {
|
||||
fn func(ctx context.Context) ([]nodesessions.SessionInfo, error)
|
||||
}
|
||||
|
||||
// NewFuncSource creates a FuncSource backed by fn.
|
||||
func NewFuncSource(fn func(ctx context.Context) ([]nodesessions.SessionInfo, error)) *FuncSource {
|
||||
return &FuncSource{fn: fn}
|
||||
}
|
||||
|
||||
// Snapshot invokes the wrapped provider and returns the deduped live picture.
|
||||
func (f *FuncSource) Snapshot(ctx context.Context) (Snapshot, error) {
|
||||
if f.fn == nil {
|
||||
return Snapshot{Streams: []LiveStream{}}, nil
|
||||
}
|
||||
infos, err := f.fn(ctx)
|
||||
if err != nil {
|
||||
return Snapshot{}, err
|
||||
}
|
||||
streams := make([]LiveStream, 0, len(infos))
|
||||
for _, info := range infos {
|
||||
streams = append(streams, toLiveStream(info))
|
||||
}
|
||||
return Snapshot{Streams: mergeStreams(streams)}, nil
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
package streammonitor
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
)
|
||||
|
||||
func fakeFn(infos []nodesessions.SessionInfo) func(ctx context.Context) ([]nodesessions.SessionInfo, error) {
|
||||
return func(ctx context.Context) ([]nodesessions.SessionInfo, error) {
|
||||
return infos, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestFuncSourceGrouping(t *testing.T) {
|
||||
infos := []nodesessions.SessionInfo{
|
||||
{SessionID: "s1", AuthUserID: 1, Type: "direct_play", StartedAt: "2026-07-04T10:00:00Z", LastServedAt: "2026-07-04T10:05:00Z"},
|
||||
{SessionID: "s2", AuthUserID: 1, Type: "transcode", StartedAt: "2026-07-04T10:01:00Z", LastServedAt: "2026-07-04T10:06:00Z"},
|
||||
{SessionID: "s3", AuthUserID: 2, Type: "remux", StartedAt: "2026-07-04T10:02:00Z", LastServedAt: "2026-07-04T10:07:00Z"},
|
||||
{SessionID: "s4", AuthUserID: 0, Type: "direct_play"}, // no user, no timestamps
|
||||
}
|
||||
|
||||
snap, err := NewFuncSource(fakeFn(infos)).Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Snapshot: %v", err)
|
||||
}
|
||||
|
||||
if got := len(snap.Streams); got != 4 {
|
||||
t.Fatalf("Streams len = %d, want 4", got)
|
||||
}
|
||||
|
||||
if got := snap.CountByUser(1); got != 2 {
|
||||
t.Errorf("CountByUser(1) = %d, want 2", got)
|
||||
}
|
||||
if got := snap.CountByUser(2); got != 1 {
|
||||
t.Errorf("CountByUser(2) = %d, want 1", got)
|
||||
}
|
||||
if got := snap.CountByUser(0); got != 1 {
|
||||
t.Errorf("CountByUser(0) = %d, want 1", got)
|
||||
}
|
||||
if got := snap.CountByUser(99); got != 0 {
|
||||
t.Errorf("CountByUser(99) = %d, want 0", got)
|
||||
}
|
||||
|
||||
u1 := snap.StreamsForUser(1)
|
||||
if len(u1) != 2 {
|
||||
t.Fatalf("StreamsForUser(1) len = %d, want 2", len(u1))
|
||||
}
|
||||
for _, st := range u1 {
|
||||
if st.UserID != 1 {
|
||||
t.Errorf("StreamsForUser(1) returned stream with UserID %d", st.UserID)
|
||||
}
|
||||
}
|
||||
|
||||
byUser := snap.ByUser()
|
||||
if len(byUser) != 3 {
|
||||
t.Fatalf("ByUser len = %d, want 3 (users 0,1,2)", len(byUser))
|
||||
}
|
||||
if len(byUser[1]) != 2 || len(byUser[2]) != 1 || len(byUser[0]) != 1 {
|
||||
t.Errorf("ByUser grouping wrong: %#v", map[int]int{0: len(byUser[0]), 1: len(byUser[1]), 2: len(byUser[2])})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTimestampParsing(t *testing.T) {
|
||||
infos := []nodesessions.SessionInfo{
|
||||
{SessionID: "s1", AuthUserID: 1, StartedAt: "2026-07-04T10:00:00Z", LastServedAt: "2026-07-04T10:05:00Z"},
|
||||
{SessionID: "s2", AuthUserID: 1, StartedAt: "not-a-time", LastServedAt: ""},
|
||||
}
|
||||
snap, err := NewFuncSource(fakeFn(infos)).Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Snapshot: %v", err)
|
||||
}
|
||||
bySession := map[string]LiveStream{}
|
||||
for _, st := range snap.Streams {
|
||||
bySession[st.SessionID] = st
|
||||
}
|
||||
|
||||
want := time.Date(2026, 7, 4, 10, 5, 0, 0, time.UTC)
|
||||
if !bySession["s1"].LastServedAt.Equal(want) {
|
||||
t.Errorf("s1 LastServedAt = %v, want %v", bySession["s1"].LastServedAt, want)
|
||||
}
|
||||
if !bySession["s2"].StartedAt.IsZero() {
|
||||
t.Errorf("s2 StartedAt = %v, want zero (unparseable)", bySession["s2"].StartedAt)
|
||||
}
|
||||
if !bySession["s2"].LastServedAt.IsZero() {
|
||||
t.Errorf("s2 LastServedAt = %v, want zero (empty)", bySession["s2"].LastServedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeKeepsNewest(t *testing.T) {
|
||||
// Same SessionID observed on two nodes (proxy vs transcode node). Keep the
|
||||
// record with the most recent LastServedAt.
|
||||
infos := []nodesessions.SessionInfo{
|
||||
{SessionID: "dup", AuthUserID: 5, NodeName: "proxy", LastServedAt: "2026-07-04T10:00:00Z", BytesServed: 100},
|
||||
{SessionID: "dup", AuthUserID: 5, NodeName: "transcode", LastServedAt: "2026-07-04T10:09:00Z", BytesServed: 900},
|
||||
{SessionID: "other", AuthUserID: 5, NodeName: "proxy", LastServedAt: "2026-07-04T10:03:00Z"},
|
||||
}
|
||||
snap, err := NewFuncSource(fakeFn(infos)).Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Snapshot: %v", err)
|
||||
}
|
||||
|
||||
if got := len(snap.Streams); got != 2 {
|
||||
t.Fatalf("Streams len = %d, want 2 (deduped)", got)
|
||||
}
|
||||
if got := snap.CountByUser(5); got != 2 {
|
||||
t.Errorf("CountByUser(5) = %d, want 2", got)
|
||||
}
|
||||
|
||||
var dup LiveStream
|
||||
found := false
|
||||
for _, st := range snap.Streams {
|
||||
if st.SessionID == "dup" {
|
||||
dup = st
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatal("deduped session 'dup' not present")
|
||||
}
|
||||
if dup.NodeName != "transcode" {
|
||||
t.Errorf("dedupe kept NodeName %q, want transcode (newest LastServedAt)", dup.NodeName)
|
||||
}
|
||||
if dup.BytesServed != 900 {
|
||||
t.Errorf("dedupe kept BytesServed %d, want 900", dup.BytesServed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeStreamsDirect(t *testing.T) {
|
||||
// Direct unit test of the merge helper, mirroring RedisSource dedupe.
|
||||
in := []LiveStream{
|
||||
{SessionID: "a", NodeName: "n1", LastServedAt: time.Unix(100, 0)},
|
||||
{SessionID: "a", NodeName: "n2", LastServedAt: time.Unix(200, 0)},
|
||||
{SessionID: "b", NodeName: "n1", LastServedAt: time.Unix(150, 0)},
|
||||
}
|
||||
out := mergeStreams(in)
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("mergeStreams len = %d, want 2", len(out))
|
||||
}
|
||||
for _, st := range out {
|
||||
if st.SessionID == "a" && st.NodeName != "n2" {
|
||||
t.Errorf("merge kept %q for session a, want n2 (newest)", st.NodeName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeStreamsCarriesOwnershipForward(t *testing.T) {
|
||||
// The transcode node's own start record has no resolved owner (UserID 0) and
|
||||
// can be the freshest copy of a session. Taking it wholesale would bucket the
|
||||
// session under user 0, which the enforcer skips — exempting it from the cap.
|
||||
// The merge must recover the owner from the proxy's (staler) owned record.
|
||||
in := []LiveStream{
|
||||
{SessionID: "s", NodeName: "proxy", UserID: 42, ProfileID: "p1", MediaFileID: 7, LastServedAt: time.Unix(100, 0)},
|
||||
{SessionID: "s", NodeName: "transcode", UserID: 0, LastServedAt: time.Unix(200, 0)},
|
||||
}
|
||||
out := mergeStreams(in)
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("mergeStreams len = %d, want 1", len(out))
|
||||
}
|
||||
got := out[0]
|
||||
if got.NodeName != "transcode" {
|
||||
t.Errorf("kept NodeName %q, want transcode (freshest)", got.NodeName)
|
||||
}
|
||||
if got.UserID != 42 {
|
||||
t.Errorf("UserID = %d, want 42 (recovered from the proxy record)", got.UserID)
|
||||
}
|
||||
if got.ProfileID != "p1" || got.MediaFileID != 7 {
|
||||
t.Errorf("ownership not fully recovered: ProfileID=%q MediaFileID=%d", got.ProfileID, got.MediaFileID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestToLiveStreamCarriesRouteAndClient(t *testing.T) {
|
||||
// The normalized view must surface route (native vs jellycompat) and client
|
||||
// identity so the monitor is first-class, not just session id + method.
|
||||
info := nodesessions.SessionInfo{
|
||||
SessionID: "s1",
|
||||
AuthUserID: 9,
|
||||
Type: "transcode",
|
||||
Route: "jellycompat",
|
||||
ClientIP: "203.0.113.7",
|
||||
ClientName: "Infuse",
|
||||
Position: 61.5,
|
||||
HWAccel: "vaapi",
|
||||
}
|
||||
got := toLiveStream(info)
|
||||
if got.Route != "jellycompat" || got.ClientIP != "203.0.113.7" || got.ClientName != "Infuse" {
|
||||
t.Fatalf("route/client not carried: %+v", got)
|
||||
}
|
||||
if got.Position != 61.5 || got.HWAccel != "vaapi" {
|
||||
t.Fatalf("position/hwaccel not carried: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeStreamsBackfillsAttribution(t *testing.T) {
|
||||
// A fresher-but-thinner record (e.g. an ownerless transcode-node start record)
|
||||
// must not drop route/client that a staler record for the same session carries.
|
||||
in := []LiveStream{
|
||||
{SessionID: "s", NodeName: "proxy", UserID: 5, Route: "native", ClientName: "SiloTV", ClientIP: "10.0.0.9", LastServedAt: time.Unix(100, 0)},
|
||||
{SessionID: "s", NodeName: "transcode", UserID: 0, LastServedAt: time.Unix(200, 0)},
|
||||
}
|
||||
out := mergeStreams(in)
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("merge len = %d, want 1", len(out))
|
||||
}
|
||||
got := out[0]
|
||||
if got.NodeName != "transcode" {
|
||||
t.Errorf("kept NodeName %q, want transcode (freshest)", got.NodeName)
|
||||
}
|
||||
if got.UserID != 5 || got.Route != "native" || got.ClientName != "SiloTV" || got.ClientIP != "10.0.0.9" {
|
||||
t.Errorf("attribution not backfilled from staler record: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeStreamsKeepsLargestObservedByteTotal(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
older, newer int64
|
||||
want int64
|
||||
}{
|
||||
{name: "freshest has zero", older: 8192, newer: 0, want: 8192},
|
||||
{name: "stale has zero", older: 0, newer: 8192, want: 8192},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
out := mergeStreams([]LiveStream{
|
||||
{SessionID: "s", BytesServed: tc.older, LastServedAt: time.Unix(100, 0)},
|
||||
{SessionID: "s", BytesServed: tc.newer, LastServedAt: time.Unix(200, 0)},
|
||||
})
|
||||
if len(out) != 1 || out[0].BytesServed != tc.want {
|
||||
t.Fatalf("merge = %+v, want bytes %d", out, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFuncSourceNilFn(t *testing.T) {
|
||||
snap, err := NewFuncSource(nil).Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Snapshot: %v", err)
|
||||
}
|
||||
if len(snap.Streams) != 0 {
|
||||
t.Errorf("nil fn Streams len = %d, want 0", len(snap.Streams))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisSourceNilClient(t *testing.T) {
|
||||
snap, err := NewRedisSource(nil).Snapshot(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("Snapshot: %v", err)
|
||||
}
|
||||
if len(snap.Streams) != 0 {
|
||||
t.Errorf("nil client Streams len = %d, want 0", len(snap.Streams))
|
||||
}
|
||||
}
|
||||
|
||||
// TestDedupeSessionInfos mirrors the mergeStreams rules on the raw SessionInfo
|
||||
// shape used by the admin session list: one row per session id, freshest copy
|
||||
// wins, a resolved owner and missing attribution are carried across the merge
|
||||
// so the operator view matches the enforcer's picture (no double-counted
|
||||
// streams, no user-0 rows when any copy knows the owner).
|
||||
func TestDedupeSessionInfos(t *testing.T) {
|
||||
newer := time.Now().UTC().Format(time.RFC3339)
|
||||
older := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)
|
||||
infos := []nodesessions.SessionInfo{
|
||||
{SessionID: "s1", NodeName: "central", AuthUserID: 7, ProfileID: "p1", MediaFileID: 42,
|
||||
Route: "native", ClientName: "SiloTV", Position: 130, LastServedAt: older},
|
||||
// The same stream, seen from the edge serving it: freshest but ownerless.
|
||||
{SessionID: "s1", NodeName: "edge-1", LastServedAt: newer, BytesServed: 9000},
|
||||
{SessionID: "s2", NodeName: "edge-1", AuthUserID: 8, LastServedAt: newer},
|
||||
}
|
||||
|
||||
out := DedupeSessionInfos(infos)
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("dedupe len = %d, want 2", len(out))
|
||||
}
|
||||
byID := make(map[string]nodesessions.SessionInfo, len(out))
|
||||
for _, info := range out {
|
||||
byID[info.SessionID] = info
|
||||
}
|
||||
s1, ok := byID["s1"]
|
||||
if !ok {
|
||||
t.Fatalf("s1 missing from dedupe output")
|
||||
}
|
||||
if s1.NodeName != "edge-1" || s1.BytesServed != 9000 {
|
||||
t.Errorf("s1 freshest copy not kept: %+v", s1)
|
||||
}
|
||||
if s1.AuthUserID != 7 || s1.ProfileID != "p1" || s1.MediaFileID != 42 {
|
||||
t.Errorf("s1 owner not carried across merge: %+v", s1)
|
||||
}
|
||||
if s1.Route != "native" || s1.ClientName != "SiloTV" || s1.Position != 130 {
|
||||
t.Errorf("s1 attribution not backfilled: %+v", s1)
|
||||
}
|
||||
if s2 := byID["s2"]; s2.AuthUserID != 8 {
|
||||
t.Errorf("s2 mangled by dedupe: %+v", s2)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDedupeSessionInfosKeepsLargestObservedByteTotal(t *testing.T) {
|
||||
older := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)
|
||||
newer := time.Now().UTC().Format(time.RFC3339Nano)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
older, newer int64
|
||||
want int64
|
||||
}{
|
||||
{name: "freshest has zero", older: 4096, newer: 0, want: 4096},
|
||||
{name: "stale has zero", older: 0, newer: 4096, want: 4096},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
out := DedupeSessionInfos([]nodesessions.SessionInfo{
|
||||
{SessionID: "s", BytesServed: tc.older, LastServedAt: older},
|
||||
{SessionID: "s", BytesServed: tc.newer, LastServedAt: newer},
|
||||
})
|
||||
if len(out) != 1 || out[0].BytesServed != tc.want {
|
||||
t.Fatalf("dedupe = %+v, want bytes %d", out, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogicalAndTransportRecordsMergeWithOwnerResolved(t *testing.T) {
|
||||
out := mergeStreams([]LiveStream{
|
||||
{SessionID: "logical", UserID: 17, ProfileID: "p", LastServedAt: time.Unix(100, 0)},
|
||||
{SessionID: "transport-a", LogicalSessionID: "logical", LastServedAt: time.Unix(200, 0)},
|
||||
})
|
||||
if len(out) != 1 {
|
||||
t.Fatalf("merge len = %d, want 1", len(out))
|
||||
}
|
||||
if out[0].SessionID != "logical" || out[0].UserID != 17 || out[0].ProfileID != "p" {
|
||||
t.Fatalf("canonical merged stream = %+v", out[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestDistinctLogicalStreamsStayDistinct(t *testing.T) {
|
||||
out := mergeStreams([]LiveStream{
|
||||
{SessionID: "transport-a", LogicalSessionID: "logical-a"},
|
||||
{SessionID: "transport-b", LogicalSessionID: "logical-b"},
|
||||
})
|
||||
if len(out) != 2 {
|
||||
t.Fatalf("distinct logical streams merged: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTransportGenerationsOfOneLogicalSessionMerge(t *testing.T) {
|
||||
out := DedupeSessionInfos([]nodesessions.SessionInfo{
|
||||
{SessionID: "transport-a", LogicalSessionID: "logical", NodeName: "old", LastServedAt: "2026-07-30T01:00:00Z"},
|
||||
{SessionID: "transport-b", LogicalSessionID: "logical", NodeName: "new", LastServedAt: "2026-07-30T02:00:00Z"},
|
||||
})
|
||||
if len(out) != 1 || out[0].SessionID != "transport-b" || out[0].LogicalSessionID != "logical" {
|
||||
t.Fatalf("transport generation dedupe = %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveLocalSessionsMapping(t *testing.T) {
|
||||
sm := playback.NewSessionManager(0, 0)
|
||||
ctx := playback.WithClientInfo(context.Background(), playback.ClientInfo{
|
||||
Name: "Silo TV",
|
||||
})
|
||||
session, err := sm.StartSessionWithContext(ctx, 42, "profile-1", 9, playback.PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSessionWithContext: %v", err)
|
||||
}
|
||||
session.ClientIP = "192.0.2.10"
|
||||
if err := sm.BeginTransport(session.ID); err != nil {
|
||||
t.Fatalf("BeginTransport: %v", err)
|
||||
}
|
||||
if err := sm.EndTransport(session.ID); err != nil {
|
||||
t.Fatalf("EndTransport: %v", err)
|
||||
}
|
||||
session, err = sm.GetSession(session.ID)
|
||||
if err != nil {
|
||||
t.Fatalf("GetSession: %v", err)
|
||||
}
|
||||
got := LiveLocalSessions(sm, "local")
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("sessions = %+v", got)
|
||||
}
|
||||
info := got[0]
|
||||
if info.SessionID != session.ID || info.NodeName != "local" ||
|
||||
info.AuthUserID != 42 || info.ProfileID != "profile-1" ||
|
||||
info.MediaFileID != 9 || info.Type != string(playback.PlayDirect) ||
|
||||
info.Route != session.Origin() || info.ClientName != "Silo TV" ||
|
||||
info.ClientIP != "192.0.2.10" {
|
||||
t.Fatalf("mapped session = %+v", info)
|
||||
}
|
||||
if info.LastServedAt != session.LastServedAt.UTC().Format(time.RFC3339) {
|
||||
t.Fatalf("LastServedAt = %q, want server-observed timestamp", info.LastServedAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLiveLocalSessionsDoesNotProjectClientActivityAsLastServed(t *testing.T) {
|
||||
sm := playback.NewSessionManager(0, 0)
|
||||
session, err := sm.StartSession(42, "profile-1", 9, playback.PlayDirect, false)
|
||||
if err != nil {
|
||||
t.Fatalf("StartSession: %v", err)
|
||||
}
|
||||
if err := sm.UpdateProgress(session.ID, 12, false); err != nil {
|
||||
t.Fatalf("UpdateProgress: %v", err)
|
||||
}
|
||||
|
||||
got := LiveLocalSessions(sm, "local")
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("sessions = %+v", got)
|
||||
}
|
||||
if got[0].LastServedAt != "" {
|
||||
t.Fatalf("LastServedAt = %q, want empty for a never-served session", got[0].LastServedAt)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
package streamrevoke
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// permanentExpiry is the far-future sentinel written when a Revocation has a
|
||||
// zero-value ExpiresAt. The hot path treats a zero ExpiresAt as "never expires"
|
||||
// (a permanent kill), but the DB column is NOT NULL and Prune/List compare
|
||||
// expires_at <= now(): a literal zero time (0001-01-01) would be excluded by
|
||||
// List and deleted by the very next Prune, silently evaporating a
|
||||
// permanent kill. Writing a year-2999 sentinel preserves the intent durably.
|
||||
var permanentExpiry = time.Date(2999, 1, 1, 0, 0, 0, 0, time.UTC)
|
||||
|
||||
// PostgresDurableStore is the Postgres-backed DurableStore: a durable mirror of
|
||||
// the kill list so revocations survive a Redis flush or a server restart. It is
|
||||
// never on the hot path — Store consults it only on write (Upsert), on
|
||||
// warm/reconcile (List), and on trim (Prune).
|
||||
//
|
||||
// Rows are keyed by (kind, id) so re-revoking the same session/user (the async
|
||||
// over-cap enforcer does this every pass) UPSERTs the same row rather than
|
||||
// accumulating duplicates; physical growth is reclaimed by Prune.
|
||||
type PostgresDurableStore struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
const durableUpsertSQL = `
|
||||
INSERT INTO stream_revocations (kind, id, reason, revoked_at, expires_at)
|
||||
VALUES ($1, $2, $3, $4, $5)
|
||||
ON CONFLICT (kind, id) DO UPDATE SET
|
||||
reason = CASE WHEN EXCLUDED.revoked_at > stream_revocations.revoked_at
|
||||
THEN EXCLUDED.reason ELSE stream_revocations.reason END,
|
||||
revoked_at = GREATEST(stream_revocations.revoked_at, EXCLUDED.revoked_at),
|
||||
expires_at = CASE WHEN stream_revocations.unrevoked_at IS NOT NULL
|
||||
THEN EXCLUDED.expires_at
|
||||
ELSE GREATEST(stream_revocations.expires_at, EXCLUDED.expires_at) END,
|
||||
unrevoked_at = NULL,
|
||||
tombstone_expires_at = NULL
|
||||
WHERE stream_revocations.unrevoked_at IS NULL
|
||||
OR stream_revocations.tombstone_expires_at <= now()
|
||||
OR EXCLUDED.revoked_at > stream_revocations.unrevoked_at`
|
||||
|
||||
const durableTombstoneUpsertSQL = `
|
||||
INSERT INTO stream_revocations (
|
||||
kind, id, reason, revoked_at, expires_at, unrevoked_at, tombstone_expires_at
|
||||
)
|
||||
VALUES ($1, $2, '', $3, $4, $3, $4)
|
||||
ON CONFLICT (kind, id) DO UPDATE SET
|
||||
unrevoked_at = EXCLUDED.unrevoked_at,
|
||||
tombstone_expires_at = EXCLUDED.tombstone_expires_at
|
||||
WHERE EXCLUDED.unrevoked_at >= stream_revocations.revoked_at
|
||||
AND (
|
||||
stream_revocations.unrevoked_at IS NULL
|
||||
OR EXCLUDED.unrevoked_at >= stream_revocations.unrevoked_at
|
||||
)`
|
||||
|
||||
// NewPostgresDurableStore builds a DurableStore from a pgx pool. It returns a
|
||||
// nil DurableStore interface when pool is nil so callers can pass the result
|
||||
// straight into Options.Durable and a Redis-less/DB-less mode degrades to a
|
||||
// true nil interface (avoiding the "non-nil interface wrapping a nil pointer"
|
||||
// trap that would make Store.durable != nil erroneously true).
|
||||
func NewPostgresDurableStore(pool *pgxpool.Pool) DurableStore {
|
||||
if pool == nil {
|
||||
return nil
|
||||
}
|
||||
return &PostgresDurableStore{pool: pool}
|
||||
}
|
||||
|
||||
// Upsert writes or refreshes a revocation, keyed by (kind, id).
|
||||
func (s *PostgresDurableStore) Upsert(ctx context.Context, r Revocation) error {
|
||||
// A zero ExpiresAt means "permanent" on the hot path; persist it as a
|
||||
// far-future sentinel so Prune/List don't immediately reap the row.
|
||||
expiresAt := r.ExpiresAt
|
||||
if expiresAt.IsZero() {
|
||||
expiresAt = permanentExpiry
|
||||
}
|
||||
// Merge cutoff and expiry independently, matching applyLocal: the later
|
||||
// RevokedAt (and its reason) wins while expiry remains monotonic. A live
|
||||
// tombstone rejects stale revocations; a genuinely newer revocation replaces
|
||||
// the tombstone and starts a fresh expiry horizon.
|
||||
_, err := s.pool.Exec(ctx, durableUpsertSQL,
|
||||
string(r.Kind), r.ID, r.Reason, r.RevokedAt, expiresAt)
|
||||
if err != nil {
|
||||
return fmt.Errorf("streamrevoke upsert: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UpsertTombstone durably records an explicit unrevoke. A stale tombstone
|
||||
// cannot erase a newer revocation, and a stale replica cannot overwrite a live
|
||||
// tombstone through Upsert.
|
||||
func (s *PostgresDurableStore) UpsertTombstone(ctx context.Context, t Tombstone) error {
|
||||
if _, err := s.pool.Exec(ctx, durableTombstoneUpsertSQL,
|
||||
string(t.Kind), t.ID, t.UnrevokedAt, t.ExpiresAt); err != nil {
|
||||
return fmt.Errorf("streamrevoke tombstone upsert: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// List returns one snapshot of every active revocation and live tombstone.
|
||||
// Tombstoned rows are never surfaced as revocations.
|
||||
func (s *PostgresDurableStore) List(ctx context.Context) (DurableState, error) {
|
||||
rows, err := s.pool.Query(ctx, `
|
||||
SELECT kind, id, reason, revoked_at, expires_at,
|
||||
unrevoked_at, tombstone_expires_at
|
||||
FROM stream_revocations
|
||||
WHERE (unrevoked_at IS NULL AND expires_at > now())
|
||||
OR (unrevoked_at IS NOT NULL AND tombstone_expires_at > now())`)
|
||||
if err != nil {
|
||||
return DurableState{}, fmt.Errorf("streamrevoke list: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var state DurableState
|
||||
for rows.Next() {
|
||||
var r Revocation
|
||||
var kind string
|
||||
var unrevokedAt, tombstoneExpiresAt *time.Time
|
||||
if err := rows.Scan(
|
||||
&kind, &r.ID, &r.Reason, &r.RevokedAt, &r.ExpiresAt,
|
||||
&unrevokedAt, &tombstoneExpiresAt,
|
||||
); err != nil {
|
||||
return DurableState{}, fmt.Errorf("streamrevoke scan: %w", err)
|
||||
}
|
||||
r.Kind = Kind(kind)
|
||||
if unrevokedAt != nil {
|
||||
if tombstoneExpiresAt != nil {
|
||||
state.Tombstones = append(state.Tombstones, Tombstone{
|
||||
Kind: r.Kind, ID: r.ID, UnrevokedAt: *unrevokedAt, ExpiresAt: *tombstoneExpiresAt,
|
||||
})
|
||||
}
|
||||
continue
|
||||
}
|
||||
state.Revocations = append(state.Revocations, r)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return DurableState{}, fmt.Errorf("streamrevoke list rows: %w", err)
|
||||
}
|
||||
return state, nil
|
||||
}
|
||||
|
||||
// Prune physically deletes expired revocations and expired tombstones so
|
||||
// neither state grows without bound.
|
||||
func (s *PostgresDurableStore) Prune(ctx context.Context) error {
|
||||
if _, err := s.pool.Exec(ctx, `
|
||||
DELETE FROM stream_revocations
|
||||
WHERE (unrevoked_at IS NULL AND expires_at <= now())
|
||||
OR (unrevoked_at IS NOT NULL AND tombstone_expires_at <= now())`); err != nil {
|
||||
return fmt.Errorf("streamrevoke prune: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package streamrevoke
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPostgresUpsertMergesCutoffAndExpiryIndependently(t *testing.T) {
|
||||
required := []string{
|
||||
"reason = CASE WHEN EXCLUDED.revoked_at > stream_revocations.revoked_at",
|
||||
"revoked_at = GREATEST(stream_revocations.revoked_at, EXCLUDED.revoked_at)",
|
||||
"GREATEST(stream_revocations.expires_at, EXCLUDED.expires_at)",
|
||||
"stream_revocations.tombstone_expires_at <= now()",
|
||||
"EXCLUDED.revoked_at > stream_revocations.unrevoked_at",
|
||||
}
|
||||
for _, clause := range required {
|
||||
if !strings.Contains(durableUpsertSQL, clause) {
|
||||
t.Fatalf("durable upsert is missing independent merge clause %q", clause)
|
||||
}
|
||||
}
|
||||
if strings.Contains(durableUpsertSQL, "EXCLUDED.expires_at >= stream_revocations.expires_at") {
|
||||
t.Fatal("durable upsert still selects cutoff/reason according to expiry")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPostgresTombstoneUpsertRejectsStaleState(t *testing.T) {
|
||||
required := []string{
|
||||
"EXCLUDED.unrevoked_at >= stream_revocations.revoked_at",
|
||||
"EXCLUDED.unrevoked_at >= stream_revocations.unrevoked_at",
|
||||
}
|
||||
for _, clause := range required {
|
||||
if !strings.Contains(durableTombstoneUpsertSQL, clause) {
|
||||
t.Fatalf("durable tombstone upsert is missing ordering clause %q", clause)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,992 @@
|
||||
// Package streamrevoke implements a stream-revocation "kill switch": a small
|
||||
// set of revoked stream sessions/users that edge nodes consult on the hot path
|
||||
// via an in-memory cache, backed by Redis for multi-node propagation with an
|
||||
// optional durable Postgres mirror.
|
||||
//
|
||||
// The hot path (IsRevoked) is a pure in-memory read with no I/O. Revocations
|
||||
// are propagated to other nodes over Redis pub/sub for immediate application
|
||||
// and mirrored to Redis keys (with a TTL) so late-joining or restarting nodes
|
||||
// can reconcile via a periodic SCAN. A durable mirror, when configured, lets
|
||||
// kills survive a Redis flush.
|
||||
package streamrevoke
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/cache"
|
||||
"github.com/Silo-Server/silo-server/internal/httpstream"
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
const (
|
||||
// keyPrefix is the Redis key namespace for revocation mirror keys, e.g.
|
||||
// silo:revoked:sess:{id} and silo:revoked:user:{id}.
|
||||
keyPrefix = "silo:revoked:"
|
||||
|
||||
// scanPattern matches every revocation mirror key for cache warming and
|
||||
// periodic reconciliation.
|
||||
scanPattern = keyPrefix + "*"
|
||||
|
||||
// EventStreamRevoked is the cache.Event.Type published on cache.ChannelAdmin
|
||||
// when a revocation is created so other nodes apply it immediately.
|
||||
EventStreamRevoked = "stream_revoked"
|
||||
// EventStreamUnrevoked is published when an operator removes a revocation.
|
||||
EventStreamUnrevoked = "stream_unrevoked"
|
||||
|
||||
defaultPollInterval = 60 * time.Second
|
||||
defaultPropagationTimeout = 5 * time.Second
|
||||
// defaultTTL is intentionally >= the playback recipe-card lifetime
|
||||
// (playback.MaxTokenTTL, 24h): a kill must never expire before the session it
|
||||
// kills can be reconstructed, or PR #174's restart-resilient playback could
|
||||
// rebuild and re-serve a stream whose revocation had already lapsed. This is
|
||||
// an invariant, stated in words to avoid importing the playback package here
|
||||
// (which would create an import cycle); keep the two values coupled.
|
||||
defaultTTL = 24 * time.Hour
|
||||
)
|
||||
|
||||
var mergeMirrorScript = redis.NewScript(`
|
||||
local incoming = cjson.decode(ARGV[1])
|
||||
local existing_raw = redis.call('GET', KEYS[1])
|
||||
local merged = incoming
|
||||
|
||||
if existing_raw ~= false then
|
||||
local ok, existing = pcall(cjson.decode, existing_raw)
|
||||
if ok
|
||||
and type(existing.revoked_at_sec) == 'number'
|
||||
and type(existing.revoked_at_nsec) == 'number'
|
||||
and type(existing.expires_at_sec) == 'number'
|
||||
and type(existing.expires_at_nsec) == 'number' then
|
||||
merged = existing
|
||||
|
||||
local incoming_permanent = incoming.expires_at_sec == 0
|
||||
local existing_permanent = existing.expires_at_sec == 0
|
||||
local incoming_expiry_later = incoming.expires_at_sec > existing.expires_at_sec
|
||||
or (incoming.expires_at_sec == existing.expires_at_sec
|
||||
and incoming.expires_at_nsec > existing.expires_at_nsec)
|
||||
if incoming_permanent or (not existing_permanent and incoming_expiry_later) then
|
||||
merged.expires_at = incoming.expires_at
|
||||
merged.expires_at_sec = incoming.expires_at_sec
|
||||
merged.expires_at_nsec = incoming.expires_at_nsec
|
||||
end
|
||||
|
||||
local incoming_cutoff_later = incoming.revoked_at_sec > existing.revoked_at_sec
|
||||
or (incoming.revoked_at_sec == existing.revoked_at_sec
|
||||
and incoming.revoked_at_nsec > existing.revoked_at_nsec)
|
||||
if incoming_cutoff_later then
|
||||
merged.revoked_at = incoming.revoked_at
|
||||
merged.revoked_at_sec = incoming.revoked_at_sec
|
||||
merged.revoked_at_nsec = incoming.revoked_at_nsec
|
||||
merged.reason = incoming.reason
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
local permanent = merged.expires_at_sec == 0
|
||||
local expired = not permanent and (
|
||||
merged.expires_at_sec < tonumber(ARGV[2])
|
||||
or (merged.expires_at_sec == tonumber(ARGV[2])
|
||||
and merged.expires_at_nsec <= tonumber(ARGV[3]))
|
||||
)
|
||||
if expired then
|
||||
redis.call('DEL', KEYS[1])
|
||||
return 0
|
||||
end
|
||||
|
||||
local encoded = cjson.encode(merged)
|
||||
if permanent then
|
||||
redis.call('SET', KEYS[1], encoded)
|
||||
else
|
||||
local ttl_ms = math.ceil(
|
||||
(merged.expires_at_sec - tonumber(ARGV[2])) * 1000
|
||||
+ (merged.expires_at_nsec - tonumber(ARGV[3])) / 1000000
|
||||
)
|
||||
redis.call('SET', KEYS[1], encoded, 'PX', math.max(1, ttl_ms))
|
||||
end
|
||||
return 1
|
||||
`)
|
||||
|
||||
// Kind identifies what a revocation targets.
|
||||
type Kind string
|
||||
|
||||
const (
|
||||
// KindSession revokes a single stream session id.
|
||||
KindSession Kind = "sess"
|
||||
// KindUser revokes every stream belonging to a user id.
|
||||
KindUser Kind = "user"
|
||||
)
|
||||
|
||||
// Key uniquely identifies a revocation in the in-memory cache and Redis.
|
||||
type Key struct {
|
||||
Kind Kind
|
||||
ID string
|
||||
}
|
||||
|
||||
// Revocation is a single kill-switch entry.
|
||||
type Revocation struct {
|
||||
Kind Kind `json:"kind"`
|
||||
ID string `json:"id"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
RevokedAt time.Time `json:"revoked_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
// Tombstone is a bounded durable unrevocation marker. It prevents a stale
|
||||
// replica or restart warm from resurrecting a revocation that an operator
|
||||
// explicitly removed.
|
||||
type Tombstone struct {
|
||||
Kind Kind
|
||||
ID string
|
||||
UnrevokedAt time.Time
|
||||
ExpiresAt time.Time
|
||||
}
|
||||
|
||||
func (t Tombstone) key() Key {
|
||||
return Key{Kind: t.Kind, ID: t.ID}
|
||||
}
|
||||
|
||||
// DurableState is one consistent snapshot of the durable state for every key.
|
||||
// A key appears as either a live revocation or a live tombstone, never both.
|
||||
type DurableState struct {
|
||||
Revocations []Revocation
|
||||
Tombstones []Tombstone
|
||||
}
|
||||
|
||||
// key returns the in-memory cache key for this revocation.
|
||||
func (r Revocation) key() Key {
|
||||
return Key{Kind: r.Kind, ID: r.ID}
|
||||
}
|
||||
|
||||
// expired reports whether the revocation is no longer in force at now.
|
||||
func (r Revocation) expired(now time.Time) bool {
|
||||
return !r.ExpiresAt.IsZero() && !now.Before(r.ExpiresAt)
|
||||
}
|
||||
|
||||
// DurableStore is an optional Postgres-backed mirror so kills survive a Redis
|
||||
// flush. A concrete implementation lives elsewhere; this package only consumes
|
||||
// the interface when one is provided.
|
||||
type DurableStore interface {
|
||||
Upsert(ctx context.Context, r Revocation) error
|
||||
UpsertTombstone(ctx context.Context, t Tombstone) error
|
||||
List(ctx context.Context) (DurableState, error)
|
||||
Prune(ctx context.Context) error
|
||||
}
|
||||
|
||||
// Options configures a Store.
|
||||
type Options struct {
|
||||
Redis *redis.Client // nil => memory-only (integrated single-node)
|
||||
Bus cache.EventBus // nil => no push propagation
|
||||
Durable DurableStore // nil => no durable mirror
|
||||
PollInterval time.Duration // default 60s
|
||||
WatchInterval time.Duration // default 5s
|
||||
DefaultTTL time.Duration // default 24h
|
||||
PropagationTimeout time.Duration // default 5s
|
||||
}
|
||||
|
||||
// Store holds the in-memory revocation cache and its propagation plumbing.
|
||||
type Store struct {
|
||||
rdb *redis.Client
|
||||
bus cache.EventBus
|
||||
durable DurableStore
|
||||
pollInterval time.Duration
|
||||
defaultTTL time.Duration
|
||||
watchInterval time.Duration
|
||||
propagationTimeout time.Duration
|
||||
|
||||
opMu sync.Mutex
|
||||
mu sync.RWMutex
|
||||
items map[Key]Revocation
|
||||
tombstones map[Key]time.Time
|
||||
tombstoneExpires map[Key]time.Time
|
||||
mirrorWarnOnce sync.Once
|
||||
}
|
||||
|
||||
// New builds a Store, applying defaults for any unset Options.
|
||||
func New(opts Options) *Store {
|
||||
if opts.PollInterval <= 0 {
|
||||
opts.PollInterval = defaultPollInterval
|
||||
}
|
||||
if opts.DefaultTTL <= 0 {
|
||||
opts.DefaultTTL = defaultTTL
|
||||
}
|
||||
if opts.WatchInterval <= 0 {
|
||||
opts.WatchInterval = 5 * time.Second
|
||||
}
|
||||
if opts.PropagationTimeout <= 0 {
|
||||
opts.PropagationTimeout = defaultPropagationTimeout
|
||||
}
|
||||
return &Store{
|
||||
rdb: opts.Redis,
|
||||
bus: opts.Bus,
|
||||
durable: opts.Durable,
|
||||
pollInterval: opts.PollInterval,
|
||||
defaultTTL: opts.DefaultTTL,
|
||||
watchInterval: opts.WatchInterval,
|
||||
propagationTimeout: opts.PropagationTimeout,
|
||||
items: make(map[Key]Revocation),
|
||||
tombstones: make(map[Key]time.Time),
|
||||
tombstoneExpires: make(map[Key]time.Time),
|
||||
}
|
||||
}
|
||||
|
||||
// redisKey returns the Redis mirror key for a revocation key.
|
||||
func redisKey(k Key) string {
|
||||
return keyPrefix + string(k.Kind) + ":" + k.ID
|
||||
}
|
||||
|
||||
// userKey returns the cache key for a numeric user id.
|
||||
func userKey(userID int) Key {
|
||||
return Key{Kind: KindUser, ID: strconv.Itoa(userID)}
|
||||
}
|
||||
|
||||
// Refuse is the single shared enforcement point for every serve surface (edge
|
||||
// proxy, transcode node, native api, jellycompat): if the session or user is
|
||||
// revoked it writes a 403 and returns true, and the caller must stop serving.
|
||||
// Centralizing the check + response here keeps one section per concern instead
|
||||
// of duplicating "IsRevoked → 403" at each surface. It does NOT hang up an
|
||||
// in-flight connection — long-pour paths pair this with a connection-cut helper.
|
||||
// startedAt is when the request's stream credential was issued (see IsRevoked).
|
||||
func (s *Store) Refuse(w http.ResponseWriter, sessionID string, userID int, startedAt time.Time) bool {
|
||||
if s == nil || !s.IsRevoked(sessionID, userID, startedAt) {
|
||||
return false
|
||||
}
|
||||
http.Error(w, "stream revoked", http.StatusForbidden)
|
||||
return true
|
||||
}
|
||||
|
||||
// WatchAndCut watches a long-lived pour (a single long-GET direct-play/remux) and
|
||||
// once the session/user is revoked, forces the in-flight write to fail via
|
||||
// SetWriteDeadline — hanging up the socket even though the 24h token is still
|
||||
// valid (cutting an open connection is a socket action, not a token revocation).
|
||||
// It checks immediately on entry (so a pour that began the instant before the
|
||||
// kill is cut without waiting a tick) and then every 5s. Returns a stop func the
|
||||
// caller defers when the request finishes normally. This is the shared in-flight
|
||||
// cut used by every long-pour serve surface (edge proxy, native api,
|
||||
// jellycompat), so the cut logic lives in one place. HLS/transcode paths don't
|
||||
// need it — per-segment Refuse stops them within one segment.
|
||||
//
|
||||
// If the ResponseWriter chain doesn't support write deadlines, the failure is
|
||||
// logged and the stream still stops on its next request via Refuse. A context
|
||||
// cut latch also prevents rolling deadline writers from re-arming the socket.
|
||||
// This helper never wraps the writer, so it does not disable sendfile.
|
||||
// startedAt follows IsRevoked's contract; a pour in flight when a user kill
|
||||
// lands always predates that kill, so passing the request's credential/entry
|
||||
// time makes mid-pour user kills cut correctly on every surface.
|
||||
func (s *Store) WatchAndCut(w http.ResponseWriter, sessionID string, userID int, startedAt time.Time) func() {
|
||||
return s.WatchAndCutContext(context.Background(), w, sessionID, userID, startedAt)
|
||||
}
|
||||
|
||||
// WatchAndCutContext is WatchAndCut with the request context used for logging
|
||||
// and for resolving the rolling-deadline cut latch.
|
||||
func (s *Store) WatchAndCutContext(ctx context.Context, w http.ResponseWriter, sessionID string, userID int, startedAt time.Time) func() {
|
||||
if s == nil {
|
||||
return func() {}
|
||||
}
|
||||
latch := httpstream.CutLatchFrom(ctx)
|
||||
var warnOnce sync.Once
|
||||
cut := func() {
|
||||
latch.Cut()
|
||||
if err := http.NewResponseController(w).SetWriteDeadline(time.Now()); err != nil {
|
||||
warnOnce.Do(func() {
|
||||
slog.WarnContext(ctx, "stream cut could not set write deadline; in-flight pour continues until its next request",
|
||||
"component", "streamrevoke",
|
||||
"session", sessionID,
|
||||
"user", userID,
|
||||
"error", err,
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
if s.IsRevoked(sessionID, userID, startedAt) {
|
||||
cut()
|
||||
}
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
ticker := time.NewTicker(s.watchInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-done:
|
||||
return
|
||||
case <-ticker.C:
|
||||
if s.IsRevoked(sessionID, userID, startedAt) {
|
||||
cut()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
var once sync.Once
|
||||
return func() { once.Do(func() { close(done) }) }
|
||||
}
|
||||
|
||||
// IsRevoked is the HOT PATH: a pure in-memory cache read with no I/O. It
|
||||
// returns true if the session id is currently revoked, or if the user id is
|
||||
// revoked AND the stream's credential predates that user revocation.
|
||||
//
|
||||
// startedAt is when the request's stream credential was issued: the stream
|
||||
// token's iat on token-bearing surfaces (edge proxy, transcode node, native
|
||||
// ?st=, ABS bearer), the persisted playback-session start for public ABS
|
||||
// tracks, or the feed creation time for public RSS capabilities. A user
|
||||
// revocation is a CUTOFF, not a ban:
|
||||
// it kills streams whose credential predates it, while a stream authorized
|
||||
// after it (which required passing auth that the revocation just reset) plays
|
||||
// normally. Without the cutoff, the OnUserSessionsRevoked hook — fired by any
|
||||
// admin edit of password/role/enabled/permissions/quality — would 403 the
|
||||
// user's playback for the full 24h TTL even after they re-authenticate unless
|
||||
// an operator explicitly unrevoked it. A zero startedAt never
|
||||
// matches a user revocation (fail open, matching the enforcer's "never kill on
|
||||
// uncertainty"); session revocations are exact-id kills and ignore startedAt.
|
||||
func (s *Store) IsRevoked(sessionID string, userID int, startedAt time.Time) bool {
|
||||
now := time.Now()
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
if sessionID != "" {
|
||||
if r, ok := s.items[Key{Kind: KindSession, ID: sessionID}]; ok && !r.expired(now) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
// userID <= 0 is the "no resolved owner" sentinel (e.g. the transcode node's
|
||||
// session-only check). Never match it against a KindUser entry: a stray
|
||||
// user:"0" revocation must not read as "every ownerless request is revoked".
|
||||
if userID > 0 {
|
||||
if r, ok := s.items[userKey(userID)]; ok && !r.expired(now) &&
|
||||
!startedAt.IsZero() && startedAt.Before(r.RevokedAt) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// effectiveExpiry returns the instant a revocation lapses, treating a zero
|
||||
// ExpiresAt as "never" (far future) so monotonic comparisons order a permanent
|
||||
// kill above any bounded one.
|
||||
func (r Revocation) effectiveExpiry() time.Time {
|
||||
if r.ExpiresAt.IsZero() {
|
||||
return permanentExpiry
|
||||
}
|
||||
return r.ExpiresAt
|
||||
}
|
||||
|
||||
// mergeRevocations is the Go definition of the two independent monotonic
|
||||
// dimensions also applied atomically by mergeMirrorScript.
|
||||
func mergeRevocations(existing, incoming Revocation) Revocation {
|
||||
if incoming.effectiveExpiry().After(existing.effectiveExpiry()) {
|
||||
existing.ExpiresAt = incoming.ExpiresAt
|
||||
}
|
||||
if incoming.RevokedAt.After(existing.RevokedAt) {
|
||||
existing.RevokedAt = incoming.RevokedAt
|
||||
existing.Reason = incoming.Reason
|
||||
}
|
||||
return existing
|
||||
}
|
||||
|
||||
// applyLocal independently merges a revocation's two monotonic dimensions:
|
||||
// expiry never moves earlier, while the user-cutoff RevokedAt never moves
|
||||
// backward. Reason follows the record that supplied the newer cutoff. Keeping
|
||||
// the fields independent matters when a newer admin cutoff has a shorter
|
||||
// horizon than an existing kill. The caller must not hold s.mu.
|
||||
func (s *Store) applyLocal(r Revocation) {
|
||||
s.mu.Lock()
|
||||
if issuedAt, ok := s.tombstones[r.key()]; ok {
|
||||
if expiresAt := s.tombstoneExpires[r.key()]; !time.Now().Before(expiresAt) {
|
||||
delete(s.tombstones, r.key())
|
||||
delete(s.tombstoneExpires, r.key())
|
||||
} else if !r.RevokedAt.After(issuedAt) {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
} else {
|
||||
delete(s.tombstones, r.key())
|
||||
delete(s.tombstoneExpires, r.key())
|
||||
}
|
||||
}
|
||||
if existing, ok := s.items[r.key()]; ok {
|
||||
s.items[r.key()] = mergeRevocations(existing, r)
|
||||
} else {
|
||||
s.items[r.key()] = r
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// effective returns the currently-stored revocation for a key — the
|
||||
// independently merged copy applyLocal kept. Zero value if absent.
|
||||
func (s *Store) effective(k Key) Revocation {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.items[k]
|
||||
}
|
||||
|
||||
func (s *Store) isTombstoned(k Key) bool {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
_, ok := s.tombstones[k]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Revoke adds a revocation to the local cache, writes it to Redis (with a TTL
|
||||
// of until-now) when configured, mirrors it to the durable store when
|
||||
// configured, and publishes a pub/sub event so other nodes apply it
|
||||
// immediately.
|
||||
func (s *Store) Revoke(ctx context.Context, key Key, reason string, until time.Time) error {
|
||||
_, err := s.RevokeWithWarnings(ctx, key, reason, until)
|
||||
return err
|
||||
}
|
||||
|
||||
// RevokeWithWarnings applies a revocation locally and reports propagation
|
||||
// failures without weakening the local kill. Revoke is the compatibility
|
||||
// wrapper used by existing callers that only need local-success semantics.
|
||||
func (s *Store) RevokeWithWarnings(ctx context.Context, key Key, reason string, until time.Time) ([]string, error) {
|
||||
s.opMu.Lock()
|
||||
defer s.opMu.Unlock()
|
||||
return s.revokeWithWarningsLocked(ctx, key, reason, until)
|
||||
}
|
||||
|
||||
// revokeWithWarningsLocked performs RevokeWithWarnings while the caller holds
|
||||
// opMu, allowing revoke-if-absent to make its presence check and write atomic.
|
||||
func (s *Store) revokeWithWarningsLocked(ctx context.Context, key Key, reason string, until time.Time) ([]string, error) {
|
||||
now := time.Now()
|
||||
r := Revocation{
|
||||
Kind: key.Kind,
|
||||
ID: key.ID,
|
||||
Reason: reason,
|
||||
RevokedAt: now,
|
||||
ExpiresAt: until,
|
||||
}
|
||||
|
||||
// Local cache first: the hot path must reflect the kill immediately even if
|
||||
// downstream propagation fails.
|
||||
s.applyLocal(r)
|
||||
var warnings []string
|
||||
|
||||
// Propagation must not die with the caller: an admin terminate rides its
|
||||
// HTTP request's context, and an abort after applyLocal would strand the
|
||||
// kill in this process's memory — never reaching Redis (edges), pub/sub, or
|
||||
// the durable mirror (lost on restart). Bound the detached work so one slow
|
||||
// dependency cannot hold opMu and block every later operation indefinitely.
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.propagationTimeout)
|
||||
defer cancel()
|
||||
merged := s.effective(r.key())
|
||||
|
||||
// Redis mirror for multi-node propagation to edges. Mirror the merged copy
|
||||
// applyLocal kept; Redis atomically merges it with writes from other central
|
||||
// replicas.
|
||||
if err := s.mirrorToRedis(ctx, merged); err != nil {
|
||||
warnings = append(warnings, "Redis mirror was not updated")
|
||||
}
|
||||
|
||||
// Pub/sub push so already-connected nodes apply the kill immediately (the
|
||||
// warm loops deliberately skip this — SCAN reconcile handles late-joiners).
|
||||
// Publish the merged record too, so pub/sub-only delivery preserves both the
|
||||
// newest cutoff and the longest expiry when Redis is unavailable.
|
||||
if s.bus != nil {
|
||||
data, err := json.Marshal(merged)
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke marshal failed", "error", err, "kind", r.Kind, "id", r.ID)
|
||||
return warnings, err
|
||||
}
|
||||
evt := cache.Event{Type: EventStreamRevoked, Payload: string(data)}
|
||||
if err := s.bus.Publish(ctx, cache.ChannelAdmin, evt); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke publish failed", "error", err, "kind", r.Kind, "id", r.ID)
|
||||
warnings = append(warnings, "revocation event was not published")
|
||||
}
|
||||
}
|
||||
|
||||
// Durable mirror last: an unhealthy Postgres must not delay the urgent edge
|
||||
// Redis kill or its pub/sub push.
|
||||
if s.durable != nil {
|
||||
if err := s.durable.Upsert(ctx, merged); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke durable upsert failed", "error", err, "kind", r.Kind, "id", r.ID)
|
||||
warnings = append(warnings, "durable mirror was not updated")
|
||||
}
|
||||
}
|
||||
|
||||
return warnings, nil
|
||||
}
|
||||
|
||||
type mirrorPayload struct {
|
||||
Revocation
|
||||
RevokedAtSec int64 `json:"revoked_at_sec"`
|
||||
RevokedAtNSec int64 `json:"revoked_at_nsec"`
|
||||
ExpiresAtSec int64 `json:"expires_at_sec"`
|
||||
ExpiresAtNSec int64 `json:"expires_at_nsec"`
|
||||
}
|
||||
|
||||
func newMirrorPayload(r Revocation) mirrorPayload {
|
||||
payload := mirrorPayload{
|
||||
Revocation: r,
|
||||
RevokedAtSec: r.RevokedAt.Unix(),
|
||||
RevokedAtNSec: int64(r.RevokedAt.Nanosecond()),
|
||||
}
|
||||
if !r.ExpiresAt.IsZero() {
|
||||
payload.ExpiresAtSec = r.ExpiresAt.Unix()
|
||||
payload.ExpiresAtNSec = int64(r.ExpiresAt.Nanosecond())
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
// mirrorToRedis atomically merges the Redis mirror with a revocation and sets
|
||||
// its TTL to the time remaining until the merged expiry. It is the single shared
|
||||
// Redis-arming path, used both by Revoke and by the durable warm loops: edge
|
||||
// nodes (proxy/transcode) have no durable store and learn kills ONLY from
|
||||
// Redis, so re-arming Redis from the durable source of truth after a Redis flush
|
||||
// + central restart is what lets edges reconverge via their SCAN reconcile.
|
||||
// Best-effort: failures are logged and returned as propagation warnings. No-op
|
||||
// when Redis is absent.
|
||||
func (s *Store) mirrorToRedis(ctx context.Context, r Revocation) error {
|
||||
if s.rdb == nil {
|
||||
return nil
|
||||
}
|
||||
data, err := json.Marshal(newMirrorPayload(r))
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke marshal failed", "error", err, "kind", r.Kind, "id", r.ID)
|
||||
return err
|
||||
}
|
||||
now := time.Now()
|
||||
if err := mergeMirrorScript.Run(
|
||||
ctx,
|
||||
s.rdb,
|
||||
[]string{redisKey(r.key())},
|
||||
data,
|
||||
now.Unix(),
|
||||
now.Nanosecond(),
|
||||
).Err(); err == nil {
|
||||
return nil
|
||||
} else {
|
||||
s.mirrorWarnOnce.Do(func() {
|
||||
slog.WarnContext(ctx, "streamrevoke atomic Redis merge unavailable; falling back to plain mirror writes",
|
||||
"error", err)
|
||||
})
|
||||
}
|
||||
return s.mirrorToRedisFallback(ctx, r, data)
|
||||
}
|
||||
|
||||
func (s *Store) mirrorToRedisFallback(ctx context.Context, r Revocation, data []byte) error {
|
||||
if r.ExpiresAt.IsZero() {
|
||||
if err := s.rdb.Set(ctx, redisKey(r.key()), data, 0).Err(); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke redis set failed", "error", err, "kind", r.Kind, "id", r.ID)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
ttl := time.Until(r.ExpiresAt)
|
||||
if ttl <= 0 {
|
||||
if err := s.rdb.Del(ctx, redisKey(r.key())).Err(); err != nil {
|
||||
slog.DebugContext(ctx, "streamrevoke redis del failed", "error", err, "kind", r.Kind, "id", r.ID)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := s.rdb.Set(ctx, redisKey(r.key()), data, ttl).Err(); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke redis set failed", "error", err, "kind", r.Kind, "id", r.ID)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type unrevocation struct {
|
||||
Kind Kind `json:"kind"`
|
||||
ID string `json:"id"`
|
||||
IssuedAt time.Time `json:"issued_at"`
|
||||
ExpiresAt time.Time `json:"expires_at"`
|
||||
}
|
||||
|
||||
func (u unrevocation) key() Key {
|
||||
return Key{Kind: u.Kind, ID: u.ID}
|
||||
}
|
||||
|
||||
func (s *Store) normalizeUnrevocation(u unrevocation) unrevocation {
|
||||
maxExpiry := u.IssuedAt.Add(s.defaultTTL)
|
||||
if u.ExpiresAt.IsZero() || u.ExpiresAt.After(maxExpiry) {
|
||||
u.ExpiresAt = maxExpiry
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// applyUnrevocation installs a bounded tombstone before deleting the local
|
||||
// kill. A delayed unrevoke event never removes a newer revocation.
|
||||
func (s *Store) applyUnrevocation(u unrevocation) {
|
||||
u = s.normalizeUnrevocation(u)
|
||||
k := u.key()
|
||||
s.mu.Lock()
|
||||
if existing, ok := s.items[k]; ok && existing.RevokedAt.After(u.IssuedAt) {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if previous, ok := s.tombstones[k]; ok && previous.After(u.IssuedAt) {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
s.tombstones[k] = u.IssuedAt
|
||||
s.tombstoneExpires[k] = u.ExpiresAt
|
||||
delete(s.items, k)
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// Unrevoke removes a kill locally and from every configured mirror. A failed
|
||||
// publish fails safe: other processes may retain the kill until expiry.
|
||||
func (s *Store) Unrevoke(ctx context.Context, key Key) error {
|
||||
_, err := s.UnrevokeWithWarnings(ctx, key)
|
||||
return err
|
||||
}
|
||||
|
||||
// UnrevokeWithWarnings is the admin-facing form of Unrevoke.
|
||||
func (s *Store) UnrevokeWithWarnings(ctx context.Context, key Key) ([]string, error) {
|
||||
s.opMu.Lock()
|
||||
defer s.opMu.Unlock()
|
||||
|
||||
issuedAt := time.Now()
|
||||
s.mu.RLock()
|
||||
existing := s.items[key]
|
||||
s.mu.RUnlock()
|
||||
u := s.normalizeUnrevocation(unrevocation{Kind: key.Kind, ID: key.ID, IssuedAt: issuedAt, ExpiresAt: existing.ExpiresAt})
|
||||
s.applyUnrevocation(u)
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.propagationTimeout)
|
||||
defer cancel()
|
||||
var warnings []string
|
||||
if s.durable != nil {
|
||||
t := Tombstone{Kind: key.Kind, ID: key.ID, UnrevokedAt: u.IssuedAt, ExpiresAt: u.ExpiresAt}
|
||||
if err := s.durable.UpsertTombstone(ctx, t); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke durable tombstone upsert failed", "error", err, "kind", key.Kind, "id", key.ID)
|
||||
warnings = append(warnings, "durable unrevocation tombstone was not updated")
|
||||
}
|
||||
}
|
||||
if s.rdb != nil {
|
||||
if err := s.rdb.Del(ctx, redisKey(key)).Err(); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke redis delete failed", "error", err, "kind", key.Kind, "id", key.ID)
|
||||
warnings = append(warnings, "Redis mirror was not deleted")
|
||||
}
|
||||
}
|
||||
if s.bus != nil {
|
||||
data, err := json.Marshal(u)
|
||||
if err != nil {
|
||||
return warnings, err
|
||||
}
|
||||
if err := s.bus.Publish(ctx, cache.ChannelAdmin, cache.Event{Type: EventStreamUnrevoked, Payload: string(data)}); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke unrevoke publish failed", "error", err, "kind", key.Kind, "id", key.ID)
|
||||
warnings = append(warnings, "unrevocation event was not published; other processes may retain the kill until it expires")
|
||||
}
|
||||
}
|
||||
return warnings, nil
|
||||
}
|
||||
|
||||
// RevokeSession revokes a single session id for DefaultTTL.
|
||||
func (s *Store) RevokeSession(ctx context.Context, sessionID, reason string) error {
|
||||
return s.Revoke(ctx, Key{Kind: KindSession, ID: sessionID}, reason, time.Now().Add(s.defaultTTL))
|
||||
}
|
||||
|
||||
// RevokeSessionWithWarnings is the admin-facing form of RevokeSession.
|
||||
func (s *Store) RevokeSessionWithWarnings(ctx context.Context, sessionID, reason string) ([]string, error) {
|
||||
return s.RevokeWithWarnings(ctx, Key{Kind: KindSession, ID: sessionID}, reason, time.Now().Add(s.defaultTTL))
|
||||
}
|
||||
|
||||
// RevokeSessionFor revokes a single session id for a caller-chosen TTL. The
|
||||
// async enforcer uses a short TTL so a transient over-count (e.g. a ghost
|
||||
// session lingering next to a fresh reconnect) self-heals; a persistent abuser
|
||||
// is simply re-revoked on the next evaluation pass. ttl <= 0 falls back to
|
||||
// DefaultTTL.
|
||||
func (s *Store) RevokeSessionFor(ctx context.Context, sessionID, reason string, ttl time.Duration) error {
|
||||
if ttl <= 0 {
|
||||
ttl = s.defaultTTL
|
||||
}
|
||||
return s.Revoke(ctx, Key{Kind: KindSession, ID: sessionID}, reason, time.Now().Add(ttl))
|
||||
}
|
||||
|
||||
// RevokeSessionForIfAbsent adds a bounded session revocation only when that key
|
||||
// has no live revocation. It is for recurring automated enforcement: repeated
|
||||
// observations of the same over-cap session must not slide a false positive's
|
||||
// expiry forward forever. Admin Revoke methods intentionally retain their
|
||||
// monotonic merge behavior. The bool reports whether a new kill was written.
|
||||
func (s *Store) RevokeSessionForIfAbsent(
|
||||
ctx context.Context,
|
||||
sessionID, reason string,
|
||||
ttl time.Duration,
|
||||
) (bool, error) {
|
||||
if ttl <= 0 {
|
||||
ttl = s.defaultTTL
|
||||
}
|
||||
key := Key{Kind: KindSession, ID: sessionID}
|
||||
s.opMu.Lock()
|
||||
defer s.opMu.Unlock()
|
||||
|
||||
now := time.Now()
|
||||
s.mu.RLock()
|
||||
existing, ok := s.items[key]
|
||||
s.mu.RUnlock()
|
||||
if ok && !existing.expired(now) {
|
||||
return false, nil
|
||||
}
|
||||
_, err := s.revokeWithWarningsLocked(ctx, key, reason, now.Add(ttl))
|
||||
return err == nil, err
|
||||
}
|
||||
|
||||
// RevokeUser revokes the user's streams for DefaultTTL. It is a CUTOFF, not a
|
||||
// ban: enforcement (IsRevoked) only matches streams whose credential was issued
|
||||
// before this revocation, so playback the user starts after re-authenticating
|
||||
// is unaffected. This is what makes it safe for OnUserSessionsRevoked to call
|
||||
// on every auth-session revocation (admin edits included), while still cutting
|
||||
// every stream that rode a pre-revocation credential.
|
||||
func (s *Store) RevokeUser(ctx context.Context, userID int, reason string) error {
|
||||
// IsRevoked never matches userID <= 0 against a KindUser entry, so a kill for
|
||||
// such an id would be silently ineffective. Refuse it rather than report a
|
||||
// success that has no teeth.
|
||||
if userID <= 0 {
|
||||
return fmt.Errorf("streamrevoke: invalid userID %d", userID)
|
||||
}
|
||||
return s.Revoke(ctx, userKey(userID), reason, time.Now().Add(s.defaultTTL))
|
||||
}
|
||||
|
||||
// RevokeUserWithWarnings applies the default TTL while preserving propagation
|
||||
// warnings for the admin API.
|
||||
func (s *Store) RevokeUserWithWarnings(ctx context.Context, userID int, reason string) ([]string, error) {
|
||||
if userID <= 0 {
|
||||
return nil, fmt.Errorf("streamrevoke: invalid userID %d", userID)
|
||||
}
|
||||
return s.RevokeWithWarnings(ctx, userKey(userID), reason, time.Now().Add(s.defaultTTL))
|
||||
}
|
||||
|
||||
// List returns the currently-active revocations, pruning expired entries on
|
||||
// read.
|
||||
func (s *Store) List() []Revocation {
|
||||
now := time.Now()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := make([]Revocation, 0, len(s.items))
|
||||
for k, r := range s.items {
|
||||
if r.expired(now) {
|
||||
delete(s.items, k)
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// pruneExpired removes expired entries from the in-memory cache.
|
||||
func (s *Store) pruneExpired() {
|
||||
now := time.Now()
|
||||
s.mu.Lock()
|
||||
for k, r := range s.items {
|
||||
if r.expired(now) {
|
||||
delete(s.items, k)
|
||||
}
|
||||
}
|
||||
for k, expiresAt := range s.tombstoneExpires {
|
||||
if !now.Before(expiresAt) {
|
||||
delete(s.tombstones, k)
|
||||
delete(s.tombstoneExpires, k)
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// StartSync warms the cache (durable store then a Redis SCAN of silo:revoked:*)
|
||||
// and subscribes to cache.ChannelAdmin to apply EventStreamRevoked events. The
|
||||
// initial warm and subscribe run SYNCHRONOUSLY (inline) before StartSync
|
||||
// returns, so kills recorded durably or in Redis are already enforced before
|
||||
// the first stream is served. Only the PollInterval reconcile/prune loop runs
|
||||
// in a spawned goroutine. With a nil Redis client the Store is memory-only and
|
||||
// only the local prune (plus durable maintenance, if configured) runs on the
|
||||
// tick.
|
||||
func (s *Store) StartSync(ctx context.Context) {
|
||||
// Warm from the durable mirror first so kills survive a Redis flush. Each
|
||||
// non-expired entry is also re-armed into Redis (mirrorToRedis no-ops when
|
||||
// Redis is absent) so edge nodes — which learn kills only from Redis —
|
||||
// reconverge via their SCAN reconcile after a Redis flush + central restart.
|
||||
if s.durable != nil {
|
||||
warmCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.propagationTimeout)
|
||||
state, err := s.warmFromDurable(warmCtx)
|
||||
cancel()
|
||||
if err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke durable warm failed", "error", err)
|
||||
} else {
|
||||
for _, t := range state.Tombstones {
|
||||
s.applyUnrevocation(unrevocation{
|
||||
Kind: t.Kind, ID: t.ID, IssuedAt: t.UnrevokedAt, ExpiresAt: t.ExpiresAt,
|
||||
})
|
||||
}
|
||||
now := time.Now()
|
||||
for _, r := range state.Revocations {
|
||||
if !r.expired(now) {
|
||||
s.applyLocal(r)
|
||||
if !s.isTombstoned(r.key()) {
|
||||
_ = s.mirrorToRedis(ctx, s.effective(r.key()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Warm from Redis and subscribe for push updates.
|
||||
if s.rdb != nil {
|
||||
s.reconcileFromRedis(ctx)
|
||||
}
|
||||
if s.bus != nil {
|
||||
if err := s.bus.Subscribe(ctx, cache.ChannelAdmin, s.handleEvent); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke subscribe failed", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
go func() {
|
||||
ticker := time.NewTicker(s.pollInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.maintain(ctx)
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// warmFromDurable loads the active key state for the boot-time cache warm,
|
||||
// retrying a bounded number of times with short backoff. A single transient DB
|
||||
// error at startup (with Redis also flushed) would otherwise leave the kill
|
||||
// list empty until the first poll tick 60s later; the retry closes that window.
|
||||
// It is strictly bounded and honors ctx cancellation, so startup never blocks
|
||||
// indefinitely. Only the boot warm needs this — the recurring maintain tick is
|
||||
// its own backstop.
|
||||
func (s *Store) warmFromDurable(ctx context.Context) (DurableState, error) {
|
||||
backoffs := []time.Duration{200 * time.Millisecond, 400 * time.Millisecond}
|
||||
var state DurableState
|
||||
var err error
|
||||
for attempt := 0; ; attempt++ {
|
||||
if state, err = s.durable.List(ctx); err == nil {
|
||||
return state, nil
|
||||
}
|
||||
if attempt >= len(backoffs) {
|
||||
return DurableState{}, err
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return DurableState{}, err
|
||||
case <-time.After(backoffs[attempt]):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// maintain runs one reconcile/prune pass: it is the body of the poll tick,
|
||||
// factored out so tests can exercise it directly without a live ticker. All
|
||||
// steps are best-effort — a failure in one is logged and does not block the
|
||||
// others or the next tick.
|
||||
func (s *Store) maintain(ctx context.Context) {
|
||||
if s.rdb != nil {
|
||||
s.reconcileFromRedis(ctx)
|
||||
}
|
||||
// Durable maintenance: re-warm from the durable mirror so a mid-life Redis
|
||||
// flush self-heals, re-arming Redis for edge nodes too, then physically
|
||||
// reclaim expired rows.
|
||||
if s.durable != nil {
|
||||
s.opMu.Lock()
|
||||
maintainCtx, cancel := context.WithTimeout(ctx, s.propagationTimeout)
|
||||
|
||||
if state, err := s.durable.List(maintainCtx); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke durable reconcile failed", "error", err)
|
||||
} else {
|
||||
for _, t := range state.Tombstones {
|
||||
s.applyUnrevocation(unrevocation{
|
||||
Kind: t.Kind, ID: t.ID, IssuedAt: t.UnrevokedAt, ExpiresAt: t.ExpiresAt,
|
||||
})
|
||||
}
|
||||
now := time.Now()
|
||||
durable := make(map[Key]Revocation, len(state.Revocations))
|
||||
for _, r := range state.Revocations {
|
||||
if !r.expired(now) {
|
||||
durable[r.key()] = r
|
||||
s.applyLocal(r)
|
||||
if !s.isTombstoned(r.key()) {
|
||||
_ = s.mirrorToRedis(maintainCtx, s.effective(r.key()))
|
||||
}
|
||||
}
|
||||
}
|
||||
s.mu.RLock()
|
||||
local := make([]Revocation, 0, len(s.items))
|
||||
for k, r := range s.items {
|
||||
if _, tombstoned := s.tombstones[k]; !tombstoned && !r.expired(now) {
|
||||
local = append(local, r)
|
||||
}
|
||||
}
|
||||
s.mu.RUnlock()
|
||||
for _, r := range local {
|
||||
durableCopy, ok := durable[r.key()]
|
||||
if !ok ||
|
||||
durableCopy.effectiveExpiry().Before(r.effectiveExpiry()) ||
|
||||
durableCopy.RevokedAt.Before(r.RevokedAt) {
|
||||
if err := s.durable.Upsert(maintainCtx, r); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke durable self-heal failed", "error", err, "kind", r.Kind, "id", r.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := s.durable.Prune(maintainCtx); err != nil {
|
||||
slog.WarnContext(ctx, "streamrevoke durable prune failed", "error", err)
|
||||
}
|
||||
cancel()
|
||||
s.opMu.Unlock()
|
||||
}
|
||||
s.pruneExpired()
|
||||
}
|
||||
|
||||
// handleEvent applies an EventStreamRevoked event to the local cache. Other
|
||||
// event types on the channel are ignored.
|
||||
func (s *Store) handleEvent(evt cache.Event) {
|
||||
switch evt.Type {
|
||||
case EventStreamRevoked:
|
||||
var r Revocation
|
||||
if err := json.Unmarshal([]byte(evt.Payload), &r); err != nil {
|
||||
slog.Debug("streamrevoke event unmarshal failed", "error", err)
|
||||
return
|
||||
}
|
||||
if !r.expired(time.Now()) {
|
||||
s.applyLocal(r)
|
||||
}
|
||||
case EventStreamUnrevoked:
|
||||
var u unrevocation
|
||||
if err := json.Unmarshal([]byte(evt.Payload), &u); err != nil {
|
||||
slog.Debug("streamrevoke unrevoke event unmarshal failed", "error", err)
|
||||
return
|
||||
}
|
||||
s.applyUnrevocation(u)
|
||||
}
|
||||
}
|
||||
|
||||
// reconcileFromRedis SCANs the revocation namespace and applies every live
|
||||
// entry to the local cache. Redis is authoritative for entries it holds; TTL
|
||||
// expiry there is mirrored by prune-on-read locally.
|
||||
func (s *Store) reconcileFromRedis(ctx context.Context) {
|
||||
var cursor uint64
|
||||
now := time.Now()
|
||||
for {
|
||||
keys, next, err := s.rdb.Scan(ctx, cursor, scanPattern, 256).Result()
|
||||
if err != nil {
|
||||
slog.Warn("streamrevoke redis scan failed", "error", err)
|
||||
return
|
||||
}
|
||||
for _, k := range keys {
|
||||
val, err := s.rdb.Get(ctx, k).Result()
|
||||
if err != nil {
|
||||
// Key may have expired between SCAN and GET; skip it.
|
||||
continue
|
||||
}
|
||||
var r Revocation
|
||||
if err := json.Unmarshal([]byte(val), &r); err != nil {
|
||||
slog.DebugContext(ctx, "streamrevoke redis unmarshal failed", "error", err, "key", k)
|
||||
continue
|
||||
}
|
||||
if !r.expired(now) {
|
||||
s.applyLocal(r)
|
||||
}
|
||||
}
|
||||
cursor = next
|
||||
if cursor == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
package streamrevoke
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
)
|
||||
|
||||
func realRedis(t *testing.T) *redis.Client {
|
||||
t.Helper()
|
||||
addr := os.Getenv("SILO_TEST_REDIS_ADDR")
|
||||
if addr == "" {
|
||||
t.Skip("SILO_TEST_REDIS_ADDR is not set")
|
||||
}
|
||||
rdb := redis.NewClient(&redis.Options{Addr: addr})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
if err := rdb.Ping(context.Background()).Err(); err != nil {
|
||||
t.Fatalf("ping test Redis: %v", err)
|
||||
}
|
||||
return rdb
|
||||
}
|
||||
|
||||
func uniqueRedisRevocation(t *testing.T) (Key, string) {
|
||||
t.Helper()
|
||||
id := t.Name() + ":" + time.Now().UTC().Format("20060102150405.000000000")
|
||||
key := Key{Kind: KindSession, ID: id}
|
||||
return key, redisKey(key)
|
||||
}
|
||||
|
||||
func readMirror(t *testing.T, rdb *redis.Client, key string) mirrorPayload {
|
||||
t.Helper()
|
||||
data, err := rdb.Get(context.Background(), key).Bytes()
|
||||
if err != nil {
|
||||
t.Fatalf("get mirror: %v", err)
|
||||
}
|
||||
var payload mirrorPayload
|
||||
if err := json.Unmarshal(data, &payload); err != nil {
|
||||
t.Fatalf("decode mirror %q: %v", data, err)
|
||||
}
|
||||
return payload
|
||||
}
|
||||
|
||||
func TestRedisMirrorConvergesConcurrentIndependentDimensions(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
rdb := realRedis(t)
|
||||
key, redisMirrorKey := uniqueRedisRevocation(t)
|
||||
t.Cleanup(func() { _ = rdb.Del(ctx, redisMirrorKey).Err() })
|
||||
base := time.Unix(time.Now().Add(time.Hour).Unix(), 123456789).UTC()
|
||||
longer := Revocation{
|
||||
Kind: key.Kind, ID: key.ID, Reason: "older-cutoff",
|
||||
RevokedAt: base, ExpiresAt: base.Add(2 * time.Hour),
|
||||
}
|
||||
later := Revocation{
|
||||
Kind: key.Kind, ID: key.ID, Reason: "later-cutoff",
|
||||
RevokedAt: base.Add(time.Nanosecond), ExpiresAt: base.Add(time.Hour),
|
||||
}
|
||||
|
||||
stores := []*Store{New(Options{Redis: rdb}), New(Options{Redis: rdb})}
|
||||
revocations := []Revocation{longer, later}
|
||||
errs := make(chan error, len(stores))
|
||||
var wg sync.WaitGroup
|
||||
for i := range stores {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
errs <- stores[i].mirrorToRedis(ctx, revocations[i])
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
got := readMirror(t, rdb, redisMirrorKey)
|
||||
if !got.RevokedAt.Equal(later.RevokedAt) || got.Reason != later.Reason {
|
||||
t.Fatalf("cutoff=%v reason=%q, want %v %q", got.RevokedAt, got.Reason, later.RevokedAt, later.Reason)
|
||||
}
|
||||
if !got.ExpiresAt.Equal(longer.ExpiresAt) {
|
||||
t.Fatalf("expiry=%v, want %v", got.ExpiresAt, longer.ExpiresAt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisMirrorPermanentKillIsNeverDowngraded(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
rdb := realRedis(t)
|
||||
key, redisMirrorKey := uniqueRedisRevocation(t)
|
||||
t.Cleanup(func() { _ = rdb.Del(ctx, redisMirrorKey).Err() })
|
||||
base := time.Now().UTC()
|
||||
store := New(Options{Redis: rdb})
|
||||
if err := store.mirrorToRedis(ctx, Revocation{
|
||||
Kind: key.Kind, ID: key.ID, Reason: "permanent",
|
||||
RevokedAt: base,
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bounded := Revocation{
|
||||
Kind: key.Kind, ID: key.ID, Reason: "new-cutoff",
|
||||
RevokedAt: base.Add(time.Nanosecond), ExpiresAt: base.Add(time.Hour),
|
||||
}
|
||||
if err := store.mirrorToRedis(ctx, bounded); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := readMirror(t, rdb, redisMirrorKey)
|
||||
if !got.ExpiresAt.IsZero() {
|
||||
t.Fatalf("expiry=%v, want permanent", got.ExpiresAt)
|
||||
}
|
||||
if !got.RevokedAt.Equal(bounded.RevokedAt) || got.Reason != bounded.Reason {
|
||||
t.Fatalf("cutoff=%v reason=%q, want %v %q", got.RevokedAt, got.Reason, bounded.RevokedAt, bounded.Reason)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisMirrorLapsedIncomingDoesNotDeleteStrongerKill(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
rdb := realRedis(t)
|
||||
key, redisMirrorKey := uniqueRedisRevocation(t)
|
||||
t.Cleanup(func() { _ = rdb.Del(ctx, redisMirrorKey).Err() })
|
||||
now := time.Now().UTC()
|
||||
store := New(Options{Redis: rdb})
|
||||
live := Revocation{
|
||||
Kind: key.Kind, ID: key.ID, Reason: "live",
|
||||
RevokedAt: now.Add(-time.Minute), ExpiresAt: now.Add(time.Hour),
|
||||
}
|
||||
if err := store.mirrorToRedis(ctx, live); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.mirrorToRedis(ctx, Revocation{
|
||||
Kind: key.Kind, ID: key.ID, Reason: "lapsed-new-cutoff",
|
||||
RevokedAt: now, ExpiresAt: now.Add(-time.Second),
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := readMirror(t, rdb, redisMirrorKey)
|
||||
if !got.ExpiresAt.Equal(live.ExpiresAt) || got.Reason != "lapsed-new-cutoff" {
|
||||
t.Fatalf("merged mirror=%+v, want live expiry and newer cutoff reason", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisMirrorOverwritesLegacyPayload(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
rdb := realRedis(t)
|
||||
key, redisMirrorKey := uniqueRedisRevocation(t)
|
||||
t.Cleanup(func() { _ = rdb.Del(ctx, redisMirrorKey).Err() })
|
||||
legacy := Revocation{
|
||||
Kind: key.Kind, ID: key.ID, Reason: "legacy",
|
||||
RevokedAt: time.Now().Add(-time.Hour), ExpiresAt: time.Now().Add(time.Hour),
|
||||
}
|
||||
legacyData, err := json.Marshal(legacy)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := rdb.Set(ctx, redisMirrorKey, legacyData, time.Hour).Err(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
incoming := Revocation{
|
||||
Kind: key.Kind, ID: key.ID, Reason: "replacement",
|
||||
RevokedAt: time.Now().UTC(), ExpiresAt: time.Now().Add(2 * time.Hour).UTC(),
|
||||
}
|
||||
if err := New(Options{Redis: rdb}).mirrorToRedis(ctx, incoming); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := readMirror(t, rdb, redisMirrorKey)
|
||||
if !got.RevokedAt.Equal(incoming.RevokedAt) || !got.ExpiresAt.Equal(incoming.ExpiresAt) ||
|
||||
got.RevokedAtSec == 0 || got.ExpiresAtSec == 0 {
|
||||
t.Fatalf("mirror=%+v, want numeric replacement payload for %+v", got, incoming)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedisMirrorEqualNanosecondCutoffKeepsExistingReason(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
rdb := realRedis(t)
|
||||
key, redisMirrorKey := uniqueRedisRevocation(t)
|
||||
t.Cleanup(func() { _ = rdb.Del(ctx, redisMirrorKey).Err() })
|
||||
cutoff := time.Unix(time.Now().Unix(), 123456789).UTC()
|
||||
first := Revocation{
|
||||
Kind: key.Kind, ID: key.ID, Reason: "original-reason",
|
||||
RevokedAt: cutoff, ExpiresAt: cutoff.Add(time.Hour),
|
||||
}
|
||||
second := Revocation{
|
||||
Kind: key.Kind, ID: key.ID, Reason: "second",
|
||||
RevokedAt: cutoff, ExpiresAt: cutoff.Add(2 * time.Hour),
|
||||
}
|
||||
store := New(Options{Redis: rdb})
|
||||
if err := store.mirrorToRedis(ctx, first); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := store.mirrorToRedis(ctx, second); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
got := readMirror(t, rdb, redisMirrorKey)
|
||||
if !got.RevokedAt.Equal(cutoff) || got.Reason != first.Reason {
|
||||
t.Fatalf("cutoff=%v reason=%q, want stable %v %q", got.RevokedAt, got.Reason, cutoff, first.Reason)
|
||||
}
|
||||
if !got.ExpiresAt.Equal(second.ExpiresAt) {
|
||||
t.Fatalf("expiry=%v, want independently merged %v", got.ExpiresAt, second.ExpiresAt)
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -7,6 +7,10 @@ import (
|
||||
"github.com/golang-jwt/jwt/v5"
|
||||
)
|
||||
|
||||
// MaxTTL is the absolute lifetime of a stream token and therefore the longest
|
||||
// a session can remain reconstructable.
|
||||
const MaxTTL = 24 * time.Hour
|
||||
|
||||
// Claims holds everything a stateless proxy or transcode node needs
|
||||
// to serve a streaming session without database access.
|
||||
//
|
||||
@@ -43,6 +47,13 @@ type Claims struct {
|
||||
ProfileID string `json:"pid,omitempty"`
|
||||
MediaFileID int `json:"mfid,omitempty"`
|
||||
|
||||
// Monitoring attribution (not byte-affecting, not a trust assertion): the
|
||||
// origin protocol ("native" | "jellycompat") and the reported client/app
|
||||
// name, carried so an edge node — which never sees the originating API path —
|
||||
// can stamp them onto its live-session record for the first-class monitor view.
|
||||
Origin string `json:"org,omitempty"`
|
||||
ClientName string `json:"cn,omitempty"`
|
||||
|
||||
// Reconstruction recipe — the byte-affecting encode parameters, mirroring the
|
||||
// former playback.RecipeCard. Zero for direct/remux tokens, which reconstruct
|
||||
// from identity alone plus the client-supplied position.
|
||||
@@ -69,6 +80,17 @@ type Claims struct {
|
||||
jwt.RegisteredClaims
|
||||
}
|
||||
|
||||
// IssuedTime returns the token's iat as a time.Time, zero when absent. It is
|
||||
// the credential-issue instant the revocation store's user-kill cutoff compares
|
||||
// against (streamrevoke.Store.IsRevoked): a user revocation kills streams whose
|
||||
// token predates it and spares ones minted after re-authentication.
|
||||
func (c *Claims) IssuedTime() time.Time {
|
||||
if c == nil || c.IssuedAt == nil {
|
||||
return time.Time{}
|
||||
}
|
||||
return c.IssuedAt.Time
|
||||
}
|
||||
|
||||
// Sign creates a signed JWT string from the given claims.
|
||||
func Sign(c Claims, secret string, ttl time.Duration) (string, error) {
|
||||
now := time.Now()
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
// TranscodeStartRequest is the JSON body for POST /transcode/start.
|
||||
type TranscodeStartRequest struct {
|
||||
SessionID string `json:"session_id"`
|
||||
LogicalSessionID string `json:"logical_session_id,omitempty"`
|
||||
InputPath string `json:"input_path"`
|
||||
SourceVideoCodec string `json:"source_video_codec"`
|
||||
VideoBitstreamFilter string `json:"video_bitstream_filter,omitempty"`
|
||||
@@ -48,6 +49,15 @@ type TranscodeStartRequest struct {
|
||||
SubtitleCodec string `json:"subtitle_codec,omitempty"`
|
||||
TotalDuration float64 `json:"total_duration"`
|
||||
RequireReady bool `json:"require_ready,omitempty"`
|
||||
|
||||
// Monitoring attribution copied from the originating session/token so the
|
||||
// node's live-session record is owner-attributed and route-tagged even when
|
||||
// no proxy record fronts it (closes the ownerless-record window).
|
||||
AuthUserID int `json:"auth_user_id,omitempty"`
|
||||
ProfileID string `json:"profile_id,omitempty"`
|
||||
MediaFileID int `json:"media_file_id,omitempty"`
|
||||
Route string `json:"route,omitempty"`
|
||||
ClientName string `json:"client_name,omitempty"`
|
||||
}
|
||||
|
||||
// TranscodeStartResponse is the JSON response for POST /transcode/start.
|
||||
@@ -89,6 +99,7 @@ type Server struct {
|
||||
reaperOnce sync.Once
|
||||
mu sync.RWMutex
|
||||
activeJobs atomic.Int32
|
||||
revocation revocationStore
|
||||
|
||||
// reconstructGroup single-flights node-side session reconstruction per session
|
||||
// id so a post-restart wave of concurrent manifest/segment requests for the same
|
||||
@@ -109,6 +120,11 @@ type Server struct {
|
||||
lifecycleMu sync.Mutex
|
||||
lifecycleLocks map[string]*sessionLifecycleLock
|
||||
|
||||
// runTracker is a deterministic scheduling seam for lifecycle tests.
|
||||
// Production leaves it nil and launches each write directly in a goroutine;
|
||||
// this is not a projection queue.
|
||||
runTracker func(func())
|
||||
|
||||
// recipeStore is the control-plane recipe store consulted when a forwarded
|
||||
// token carries no recipe (the jellycompat node hop). Nil disables that path.
|
||||
recipeStore recipeStore
|
||||
@@ -150,6 +166,29 @@ func (s *Server) lockSessionLifecycle(sessionID string) func() {
|
||||
}
|
||||
}
|
||||
|
||||
// trackIfCurrent serializes the delayed monitoring write with stop/reap/start
|
||||
// lifecycle transitions. Once it owns the lifecycle lock, it verifies that the
|
||||
// exact session generation is still registered before writing; a stopped or
|
||||
// replaced generation can therefore never recreate a ghost record.
|
||||
func (s *Server) trackIfCurrent(ctx context.Context, sessionID string, expected *playback.TranscodeSession, info nodesessions.SessionInfo) {
|
||||
run := func() {
|
||||
unlock := s.lockSessionLifecycle(sessionID)
|
||||
defer unlock()
|
||||
s.mu.RLock()
|
||||
current, ok := s.sessions[sessionID]
|
||||
s.mu.RUnlock()
|
||||
if !ok || current != expected {
|
||||
return
|
||||
}
|
||||
s.tracker.Track(ctx, info)
|
||||
}
|
||||
if s.runTracker != nil {
|
||||
s.runTracker(run)
|
||||
return
|
||||
}
|
||||
go run()
|
||||
}
|
||||
|
||||
// restartSessionLocked re-spawns session under the per-session lifecycle lock so
|
||||
// a segment-recovery restart can never race a fresh start, reconstruct, or
|
||||
// another restart into the same output directory. It holds the lock only across
|
||||
@@ -369,6 +408,31 @@ func (s *Server) SetRecipeStore(store recipeStore) {
|
||||
s.recipeStore = store
|
||||
}
|
||||
|
||||
// revocationStore is the subset of *streamrevoke.Store the node consults to
|
||||
// refuse serving (and rebuilding) killed sessions. IsRevoked is a pure in-memory
|
||||
// lookup. Nil disables enforcement. Defense-in-depth: the proxy in front already
|
||||
// enforces, but this guards the reconstruct path so a killed session is never
|
||||
// re-spawned after a node restart.
|
||||
type revocationStore interface {
|
||||
IsRevoked(sessionID string, userID int, startedAt time.Time) bool
|
||||
Refuse(w http.ResponseWriter, sessionID string, userID int, startedAt time.Time) bool
|
||||
}
|
||||
|
||||
// SetRevocationStore wires the kill switch this node consults per request.
|
||||
func (s *Server) SetRevocationStore(store revocationStore) {
|
||||
s.revocation = store
|
||||
}
|
||||
|
||||
// refuseIfRevoked refuses (403) a serve request for a revoked session. It checks
|
||||
// the session key only (no per-segment token parse): the node is always fronted
|
||||
// by the proxy/central, which enforce per-user revocations, and the enforcer +
|
||||
// admin terminate revoke by session id. The reconstruct guard uses the verified
|
||||
// token's user id directly, so per-user kills still block a rebuild after a
|
||||
// node restart.
|
||||
func (s *Server) refuseIfRevoked(w http.ResponseWriter, sessionID string) bool {
|
||||
return s.revocation != nil && s.revocation.Refuse(w, sessionID, 0, time.Time{})
|
||||
}
|
||||
|
||||
// Handler returns the chi.Router with all transcode routes.
|
||||
func (s *Server) Handler() http.Handler {
|
||||
s.startIdleReaper()
|
||||
@@ -566,16 +630,22 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) {
|
||||
// tracking write is monitoring-only.
|
||||
effectiveHWAccel := session.Opts().HWAccel
|
||||
trackCtx := context.WithoutCancel(r.Context())
|
||||
go s.tracker.Track(trackCtx, nodesessions.SessionInfo{
|
||||
SessionID: req.SessionID,
|
||||
NodeURL: s.tracker.NodeURL(),
|
||||
NodeName: s.tracker.NodeName(),
|
||||
Type: "transcode",
|
||||
CodecVideo: req.TargetCodecVideo,
|
||||
CodecAudio: req.TargetCodecAudio,
|
||||
Resolution: req.TargetResolution,
|
||||
HWAccel: effectiveHWAccel,
|
||||
StartedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
s.trackIfCurrent(trackCtx, req.SessionID, session, nodesessions.SessionInfo{
|
||||
SessionID: req.SessionID,
|
||||
LogicalSessionID: req.LogicalSessionID,
|
||||
NodeURL: s.tracker.NodeURL(),
|
||||
NodeName: s.tracker.NodeName(),
|
||||
Type: "transcode",
|
||||
CodecVideo: req.TargetCodecVideo,
|
||||
CodecAudio: req.TargetCodecAudio,
|
||||
Resolution: req.TargetResolution,
|
||||
HWAccel: effectiveHWAccel,
|
||||
StartedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
AuthUserID: req.AuthUserID,
|
||||
ProfileID: req.ProfileID,
|
||||
MediaFileID: req.MediaFileID,
|
||||
Route: req.Route,
|
||||
ClientName: req.ClientName,
|
||||
})
|
||||
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
@@ -609,6 +679,11 @@ func (s *Server) reconstructFromToken(r *http.Request, sessionID string, request
|
||||
"session", sessionID, "playback_session_id", sessionID)
|
||||
return nil
|
||||
}
|
||||
// Never rebuild a killed session: without this guard a node restart would let
|
||||
// reconstruction re-spawn ffmpeg for a session the kill switch already revoked.
|
||||
if s.revocation != nil && s.revocation.IsRevoked(claims.SessionID, claims.UserID, claims.IssuedTime()) {
|
||||
return nil
|
||||
}
|
||||
card := playback.RecipeCardFromClaims(claims)
|
||||
// The token's recipe must be a transcode card for the session id in the URL: a
|
||||
// mismatch is a forged or stale request, and direct/remux cards carry no encode
|
||||
@@ -651,7 +726,7 @@ func (s *Server) reconstructFromToken(r *http.Request, sessionID string, request
|
||||
}
|
||||
resolved = *fetched
|
||||
}
|
||||
return s.spawnReconstruct(r, sessionID, requestedSegment, resolved), nil
|
||||
return s.spawnReconstruct(r, sessionID, requestedSegment, resolved, claims.Origin, claims.ClientName), nil
|
||||
})
|
||||
if session, _ := v.(*playback.TranscodeSession); session != nil {
|
||||
return session
|
||||
@@ -663,7 +738,7 @@ func (s *Server) reconstructFromToken(r *http.Request, sessionID string, request
|
||||
// registers it in the live map. It is only ever called inside the per-session
|
||||
// single-flight in reconstructFromToken, so it is the sole writer racing to
|
||||
// register sessionID. Returns nil if the spawn fails or the slot wait is canceled.
|
||||
func (s *Server) spawnReconstruct(r *http.Request, sessionID string, requestedSegment int, card playback.RecipeCard) *playback.TranscodeSession {
|
||||
func (s *Server) spawnReconstruct(r *http.Request, sessionID string, requestedSegment int, card playback.RecipeCard, route, clientName string) *playback.TranscodeSession {
|
||||
// Pace the cold-start burst so a node restart that loses many sessions does not
|
||||
// launch every ffmpeg at once. A client that disconnects while waiting releases
|
||||
// its slot rather than queueing dead work.
|
||||
@@ -736,19 +811,22 @@ func (s *Server) spawnReconstruct(r *http.Request, sessionID string, requestedSe
|
||||
s.activeJobs.Add(1)
|
||||
|
||||
trackCtx := context.WithoutCancel(r.Context())
|
||||
go s.tracker.Track(trackCtx, nodesessions.SessionInfo{
|
||||
SessionID: sessionID,
|
||||
NodeURL: s.tracker.NodeURL(),
|
||||
NodeName: s.tracker.NodeName(),
|
||||
Type: "transcode",
|
||||
CodecVideo: card.TargetCodecVideo,
|
||||
CodecAudio: card.TargetCodecAudio,
|
||||
Resolution: card.TargetResolution,
|
||||
HWAccel: session.Opts().HWAccel,
|
||||
StartedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
AuthUserID: card.UserID,
|
||||
ProfileID: card.ProfileID,
|
||||
MediaFileID: card.MediaFileID,
|
||||
s.trackIfCurrent(trackCtx, sessionID, session, nodesessions.SessionInfo{
|
||||
SessionID: sessionID,
|
||||
LogicalSessionID: card.SessionID,
|
||||
NodeURL: s.tracker.NodeURL(),
|
||||
NodeName: s.tracker.NodeName(),
|
||||
Type: "transcode",
|
||||
CodecVideo: card.TargetCodecVideo,
|
||||
CodecAudio: card.TargetCodecAudio,
|
||||
Resolution: card.TargetResolution,
|
||||
HWAccel: session.Opts().HWAccel,
|
||||
StartedAt: time.Now().UTC().Format(time.RFC3339),
|
||||
AuthUserID: card.UserID,
|
||||
ProfileID: card.ProfileID,
|
||||
MediaFileID: card.MediaFileID,
|
||||
Route: route,
|
||||
ClientName: clientName,
|
||||
})
|
||||
|
||||
slog.InfoContext(r.Context(), "transcode node session reconstructed from token", "component", "transcodenode",
|
||||
@@ -825,6 +903,10 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) {
|
||||
func (s *Server) handleManifest(w http.ResponseWriter, r *http.Request) {
|
||||
sessionID := chi.URLParam(r, "session_id")
|
||||
|
||||
if s.refuseIfRevoked(w, sessionID) {
|
||||
return
|
||||
}
|
||||
|
||||
// Lookup and liveness refresh happen atomically so the idle reaper can
|
||||
// never unregister the job between them and tear down a session this
|
||||
// request is about to serve from.
|
||||
@@ -854,12 +936,20 @@ func (s *Server) handleManifest(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Cache-Control", "no-store, max-age=0")
|
||||
w.Header().Set("Pragma", "no-cache")
|
||||
w.Write(manifest)
|
||||
// Serve-activity mark so the node's own record reflects real serving instead
|
||||
// of a LastServedAt frozen at start time (bytes are measured by the fronting
|
||||
// proxy; wrapping ServeFile here would cost sendfile for a duplicate count).
|
||||
s.tracker.MarkServed(sessionID)
|
||||
}
|
||||
|
||||
func (s *Server) handleSegment(w http.ResponseWriter, r *http.Request) {
|
||||
sessionID := chi.URLParam(r, "session_id")
|
||||
name := chi.URLParam(r, "name")
|
||||
|
||||
if s.refuseIfRevoked(w, sessionID) {
|
||||
return
|
||||
}
|
||||
|
||||
// Lookup and liveness refresh happen atomically so the idle reaper can
|
||||
// never unregister the job between them and tear down a session this
|
||||
// request is about to serve from.
|
||||
@@ -983,6 +1073,8 @@ func (s *Server) handleSegment(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Cache-Control", "no-store, max-age=0")
|
||||
w.Header().Set("Pragma", "no-cache")
|
||||
http.ServeFile(w, r, segPath)
|
||||
// Serve-activity mark: see handleManifest.
|
||||
s.tracker.MarkServed(sessionID)
|
||||
}
|
||||
|
||||
func (s *Server) handleForceReload(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
package transcodenode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/nodesessions"
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
)
|
||||
|
||||
type transcodeRedisHook struct {
|
||||
mu sync.Mutex
|
||||
keys map[string]bool
|
||||
}
|
||||
|
||||
func (h *transcodeRedisHook) DialHook(next redis.DialHook) redis.DialHook { return next }
|
||||
func (h *transcodeRedisHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
|
||||
return func(ctx context.Context, cmd redis.Cmder) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
switch cmd.Name() {
|
||||
case "set":
|
||||
h.keys[cmd.Args()[1].(string)] = true
|
||||
if status, ok := cmd.(*redis.StatusCmd); ok {
|
||||
status.SetVal("OK")
|
||||
}
|
||||
return nil
|
||||
case "del":
|
||||
for _, arg := range cmd.Args()[1:] {
|
||||
delete(h.keys, arg.(string))
|
||||
}
|
||||
if count, ok := cmd.(*redis.IntCmd); ok {
|
||||
count.SetVal(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return next(ctx, cmd)
|
||||
}
|
||||
}
|
||||
func (h *transcodeRedisHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
|
||||
return next
|
||||
}
|
||||
|
||||
func newTranscodeLifecycleTracker(t *testing.T) (*nodesessions.Tracker, *transcodeRedisHook) {
|
||||
t.Helper()
|
||||
rdb := redis.NewClient(&redis.Options{Dialer: func(context.Context, string, string) (net.Conn, error) {
|
||||
t.Fatal("unexpected Redis dial")
|
||||
return nil, nil
|
||||
}})
|
||||
t.Cleanup(func() { _ = rdb.Close() })
|
||||
hook := &transcodeRedisHook{keys: make(map[string]bool)}
|
||||
rdb.AddHook(hook)
|
||||
return nodesessions.NewTracker(rdb, "http://node", "node", "transcode"), hook
|
||||
}
|
||||
|
||||
func (h *transcodeRedisHook) has(key string) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.keys[key]
|
||||
}
|
||||
|
||||
func TestDelayedTrackSkipsStoppedSession(t *testing.T) {
|
||||
tracker, redisState := newTranscodeLifecycleTracker(t)
|
||||
session := &playback.TranscodeSession{}
|
||||
var queued []func()
|
||||
s := &Server{
|
||||
tracker: tracker,
|
||||
sessions: map[string]*playback.TranscodeSession{"transport": session},
|
||||
runTracker: func(fn func()) {
|
||||
queued = append(queued, fn)
|
||||
},
|
||||
}
|
||||
s.trackIfCurrent(context.Background(), "transport", session, nodesessions.SessionInfo{
|
||||
SessionID: "transport", LogicalSessionID: "logical",
|
||||
})
|
||||
|
||||
unlock := s.lockSessionLifecycle("transport")
|
||||
s.mu.Lock()
|
||||
delete(s.sessions, "transport")
|
||||
s.mu.Unlock()
|
||||
tracker.Remove(context.Background(), "transport")
|
||||
unlock()
|
||||
queued[0]()
|
||||
|
||||
if got := tracker.Snapshot(); len(got) != 0 {
|
||||
t.Fatalf("delayed track recreated stopped record: %+v", got)
|
||||
}
|
||||
key := nodesessions.KeyPrefix + tracker.NodeHash() + ":transport"
|
||||
if redisState.has(key) {
|
||||
t.Fatal("delayed track recreated stopped Redis key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDelayedTrackSkipsReplacedSession(t *testing.T) {
|
||||
tracker, _ := newTranscodeLifecycleTracker(t)
|
||||
oldSession := &playback.TranscodeSession{}
|
||||
newSession := &playback.TranscodeSession{}
|
||||
var queued []func()
|
||||
s := &Server{
|
||||
tracker: tracker,
|
||||
sessions: map[string]*playback.TranscodeSession{"transport": oldSession},
|
||||
runTracker: func(fn func()) {
|
||||
queued = append(queued, fn)
|
||||
},
|
||||
}
|
||||
s.trackIfCurrent(context.Background(), "transport", oldSession, nodesessions.SessionInfo{
|
||||
SessionID: "transport", LogicalSessionID: "old",
|
||||
})
|
||||
|
||||
unlock := s.lockSessionLifecycle("transport")
|
||||
s.mu.Lock()
|
||||
s.sessions["transport"] = newSession
|
||||
s.mu.Unlock()
|
||||
tracker.Remove(context.Background(), "transport")
|
||||
tracker.Track(context.Background(), nodesessions.SessionInfo{
|
||||
SessionID: "transport", LogicalSessionID: "new",
|
||||
})
|
||||
unlock()
|
||||
queued[0]()
|
||||
|
||||
got := tracker.Snapshot()
|
||||
if len(got) != 1 || got[0].LogicalSessionID != "new" {
|
||||
t.Fatalf("old delayed track replaced new record: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,279 @@
|
||||
// Package transfers tracks active download-class file pours in memory.
|
||||
package transfers
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultMaxEntries = 10_000
|
||||
defaultMaxPerUser = 24
|
||||
fullWarningInterval = time.Minute
|
||||
maxDownloadIDLength = 256
|
||||
maxProfileIDLength = 256
|
||||
maxRouteLength = 64
|
||||
maxClientIPLength = 128
|
||||
maxClientNameLength = 256
|
||||
)
|
||||
|
||||
var (
|
||||
ErrRegistryFull = errors.New("transfer registry is full")
|
||||
ErrUserTransferLimit = errors.New("user transfer limit reached")
|
||||
ErrInvalidID = errors.New("transfer id is required")
|
||||
ErrDuplicateID = errors.New("transfer id is already active")
|
||||
)
|
||||
|
||||
const MaxPerUserSetting = "playback.max_user_concurrent_transfers"
|
||||
|
||||
// Transfer describes one active HTTP file pour. DownloadID is correlation
|
||||
// metadata only: ID uniquely identifies this request, including concurrent
|
||||
// Range requests for the same download row.
|
||||
type Transfer struct {
|
||||
ID string `json:"id"`
|
||||
DownloadID string `json:"download_id"`
|
||||
UserID int `json:"user_id"`
|
||||
ProfileID string `json:"profile_id"`
|
||||
MediaFileID int `json:"media_file_id"`
|
||||
Route string `json:"route"`
|
||||
ClientIP string `json:"client_ip"`
|
||||
ClientName string `json:"client_name"`
|
||||
StartedAt time.Time `json:"started_at"`
|
||||
LastServedAt time.Time `json:"last_served_at"`
|
||||
BytesServed int64 `json:"bytes_served"`
|
||||
}
|
||||
|
||||
// Options customizes a Registry. Zero values use production defaults.
|
||||
type Options struct {
|
||||
MaxEntries int
|
||||
MaxPerUser *int
|
||||
Now func() time.Time
|
||||
Logger *slog.Logger
|
||||
}
|
||||
|
||||
type userTransferState struct {
|
||||
count int
|
||||
lastWarning time.Time
|
||||
}
|
||||
|
||||
// Registry is a bounded, process-local collection of active transfers.
|
||||
type Registry struct {
|
||||
mu sync.RWMutex
|
||||
items map[string]Transfer
|
||||
perUser map[int]userTransferState
|
||||
maxEntries int
|
||||
maxPerUser int
|
||||
now func() time.Time
|
||||
logger *slog.Logger
|
||||
lastFullWarning time.Time
|
||||
}
|
||||
|
||||
// New creates a Registry with production limits.
|
||||
func New() *Registry {
|
||||
return NewWithOptions(Options{})
|
||||
}
|
||||
|
||||
// NewWithOptions creates a Registry with explicit limits, primarily for tests.
|
||||
func NewWithOptions(opts Options) *Registry {
|
||||
if opts.MaxEntries <= 0 {
|
||||
opts.MaxEntries = defaultMaxEntries
|
||||
}
|
||||
if opts.Now == nil {
|
||||
opts.Now = time.Now
|
||||
}
|
||||
maxPerUser := defaultMaxPerUser
|
||||
if opts.MaxPerUser != nil {
|
||||
maxPerUser = max(0, *opts.MaxPerUser)
|
||||
}
|
||||
if opts.Logger == nil {
|
||||
opts.Logger = slog.Default()
|
||||
}
|
||||
return &Registry{
|
||||
items: make(map[string]Transfer),
|
||||
perUser: make(map[int]userTransferState),
|
||||
maxEntries: opts.MaxEntries,
|
||||
maxPerUser: maxPerUser,
|
||||
now: opts.Now,
|
||||
logger: opts.Logger,
|
||||
}
|
||||
}
|
||||
|
||||
// ParseMaxPerUser parses the process-start setting. Empty input selects the
|
||||
// production default; zero explicitly disables the per-user cap.
|
||||
func ParseMaxPerUser(raw string) (int, error) {
|
||||
raw = strings.TrimSpace(raw)
|
||||
if raw == "" {
|
||||
return defaultMaxPerUser, nil
|
||||
}
|
||||
value, err := strconv.Atoi(raw)
|
||||
if err != nil || value < 0 || value > defaultMaxEntries {
|
||||
return 0, fmt.Errorf("%s must be an integer between 0 and %d", MaxPerUserSetting, defaultMaxEntries)
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
// MaxPerUser returns the configured per-user concurrent-transfer cap. Zero
|
||||
// means unlimited.
|
||||
func (r *Registry) MaxPerUser() int {
|
||||
if r == nil {
|
||||
return 0
|
||||
}
|
||||
return r.maxPerUser
|
||||
}
|
||||
|
||||
// Begin records an active transfer. Strings derived from request metadata are
|
||||
// normalized and clamped so the entry limit also provides a useful memory bound.
|
||||
func (r *Registry) Begin(t Transfer) error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
t.ID = strings.TrimSpace(t.ID)
|
||||
if t.ID == "" {
|
||||
return ErrInvalidID
|
||||
}
|
||||
t.DownloadID = clamp(t.DownloadID, maxDownloadIDLength)
|
||||
t.ProfileID = clamp(t.ProfileID, maxProfileIDLength)
|
||||
t.Route = clamp(t.Route, maxRouteLength)
|
||||
t.ClientIP = clamp(t.ClientIP, maxClientIPLength)
|
||||
t.ClientName = clamp(t.ClientName, maxClientNameLength)
|
||||
if t.StartedAt.IsZero() {
|
||||
t.StartedAt = r.now()
|
||||
}
|
||||
t.StartedAt = t.StartedAt.UTC()
|
||||
t.LastServedAt = t.StartedAt
|
||||
t.BytesServed = 0
|
||||
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
if _, exists := r.items[t.ID]; exists {
|
||||
return ErrDuplicateID
|
||||
}
|
||||
userState := r.perUser[t.UserID]
|
||||
if r.maxPerUser > 0 && userState.count >= r.maxPerUser {
|
||||
now := r.now()
|
||||
if userState.lastWarning.IsZero() || now.Sub(userState.lastWarning) >= fullWarningInterval {
|
||||
userState.lastWarning = now
|
||||
r.perUser[t.UserID] = userState
|
||||
r.logger.Warn("user transfer limit reached",
|
||||
"component", "transfers",
|
||||
"route", t.Route,
|
||||
"user_id", t.UserID,
|
||||
"max_per_user", r.maxPerUser,
|
||||
)
|
||||
}
|
||||
return ErrUserTransferLimit
|
||||
}
|
||||
if len(r.items) >= r.maxEntries {
|
||||
now := r.now()
|
||||
if r.lastFullWarning.IsZero() || now.Sub(r.lastFullWarning) >= fullWarningInterval {
|
||||
r.lastFullWarning = now
|
||||
r.logger.Warn("transfer registry full",
|
||||
"component", "transfers",
|
||||
"route", t.Route,
|
||||
"user_id", t.UserID,
|
||||
"max_entries", r.maxEntries,
|
||||
)
|
||||
}
|
||||
return ErrRegistryFull
|
||||
}
|
||||
r.items[t.ID] = t
|
||||
userState.count++
|
||||
r.perUser[t.UserID] = userState
|
||||
return nil
|
||||
}
|
||||
|
||||
// Annotate adds correlation metadata after a handler-owned Begin. Unknown IDs
|
||||
// are ignored so late resolution cannot create an unbounded registry entry.
|
||||
func (r *Registry) Annotate(id, downloadID string, mediaFileID int) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
t, ok := r.items[id]
|
||||
if ok {
|
||||
t.DownloadID = clamp(downloadID, maxDownloadIDLength)
|
||||
t.MediaFileID = mediaFileID
|
||||
r.items[id] = t
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// AddServedBytes implements playback.ServedBytesRecorder. Updates for unknown
|
||||
// transfers are deliberately discarded, and non-positive values are ignored.
|
||||
// The metered writer reports in coarse chunks (currently 1 MiB and on Close),
|
||||
// so LastServedAt is intentionally a coarse monitoring timestamp.
|
||||
func (r *Registry) AddServedBytes(id string, n int64) error {
|
||||
if r == nil || n <= 0 {
|
||||
return nil
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
t, ok := r.items[id]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if n > math.MaxInt64-t.BytesServed {
|
||||
t.BytesServed = math.MaxInt64
|
||||
} else {
|
||||
t.BytesServed += n
|
||||
}
|
||||
t.LastServedAt = r.now().UTC()
|
||||
r.items[id] = t
|
||||
return nil
|
||||
}
|
||||
|
||||
// End removes an active transfer. It is idempotent.
|
||||
func (r *Registry) End(id string) {
|
||||
if r == nil {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
if t, ok := r.items[id]; ok {
|
||||
delete(r.items, id)
|
||||
state := r.perUser[t.UserID]
|
||||
state.count--
|
||||
if state.count <= 0 {
|
||||
delete(r.perUser, t.UserID)
|
||||
} else {
|
||||
r.perUser[t.UserID] = state
|
||||
}
|
||||
}
|
||||
r.mu.Unlock()
|
||||
}
|
||||
|
||||
// Snapshot returns a stable copy of the active transfers.
|
||||
func (r *Registry) Snapshot() []Transfer {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
r.mu.RLock()
|
||||
out := make([]Transfer, 0, len(r.items))
|
||||
for _, t := range r.items {
|
||||
out = append(out, t)
|
||||
}
|
||||
r.mu.RUnlock()
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].StartedAt.Equal(out[j].StartedAt) {
|
||||
return out[i].ID < out[j].ID
|
||||
}
|
||||
return out[i].StartedAt.Before(out[j].StartedAt)
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func clamp(value string, maxRunes int) string {
|
||||
value = strings.TrimSpace(value)
|
||||
if utf8.RuneCountInString(value) <= maxRunes {
|
||||
return value
|
||||
}
|
||||
runes := []rune(value)
|
||||
return string(runes[:maxRunes])
|
||||
}
|
||||
@@ -0,0 +1,395 @@
|
||||
package transfers
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/Silo-Server/silo-server/internal/playback"
|
||||
)
|
||||
|
||||
const testPourID = "pour"
|
||||
|
||||
func TestRegistryLifecycleAndAccounting(t *testing.T) {
|
||||
now := time.Date(2026, 7, 29, 12, 0, 0, 0, time.UTC)
|
||||
r := NewWithOptions(Options{Now: func() time.Time { return now }})
|
||||
started := now.Add(-time.Minute)
|
||||
if err := r.Begin(Transfer{ID: "pour-1", UserID: 7, DownloadID: "download-1", StartedAt: started}); err != nil {
|
||||
t.Fatalf("Begin: %v", err)
|
||||
}
|
||||
|
||||
got := r.Snapshot()
|
||||
if len(got) != 1 || got[0].LastServedAt != started || got[0].BytesServed != 0 {
|
||||
t.Fatalf("initial snapshot = %+v", got)
|
||||
}
|
||||
if err := r.AddServedBytes("pour-1", 25); err != nil {
|
||||
t.Fatalf("AddServedBytes: %v", err)
|
||||
}
|
||||
got = r.Snapshot()
|
||||
if got[0].BytesServed != 25 || got[0].LastServedAt != now {
|
||||
t.Fatalf("accounted snapshot = %+v", got)
|
||||
}
|
||||
|
||||
r.End("pour-1")
|
||||
r.End("pour-1")
|
||||
if got := r.Snapshot(); len(got) != 0 {
|
||||
t.Fatalf("snapshot after End = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryUnknownAndNonPositiveUpdatesAreNoOps(t *testing.T) {
|
||||
r := New()
|
||||
if err := r.AddServedBytes("missing", 20); err != nil {
|
||||
t.Fatalf("unknown AddServedBytes: %v", err)
|
||||
}
|
||||
if len(r.Snapshot()) != 0 {
|
||||
t.Fatal("unknown update created an entry")
|
||||
}
|
||||
if err := r.Begin(Transfer{ID: testPourID}); err != nil {
|
||||
t.Fatalf("Begin: %v", err)
|
||||
}
|
||||
for _, n := range []int64{0, -1, -100} {
|
||||
if err := r.AddServedBytes(testPourID, n); err != nil {
|
||||
t.Fatalf("AddServedBytes(%d): %v", n, err)
|
||||
}
|
||||
}
|
||||
if got := r.Snapshot()[0].BytesServed; got != 0 {
|
||||
t.Fatalf("BytesServed = %d, want 0", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryConcurrentPoursForSameDownloadDoNotCollide(t *testing.T) {
|
||||
r := New()
|
||||
for _, id := range []string{"range-a", "range-b"} {
|
||||
if err := r.Begin(Transfer{ID: id, DownloadID: "same-download"}); err != nil {
|
||||
t.Fatalf("Begin(%s): %v", id, err)
|
||||
}
|
||||
}
|
||||
_ = r.AddServedBytes("range-a", 10)
|
||||
_ = r.AddServedBytes("range-b", 20)
|
||||
got := r.Snapshot()
|
||||
if len(got) != 2 || got[0].ID == got[1].ID {
|
||||
t.Fatalf("snapshot = %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryConcurrentLifecycle(t *testing.T) {
|
||||
unlimited := 0
|
||||
r := NewWithOptions(Options{MaxEntries: 256, MaxPerUser: &unlimited})
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < 200; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
id := time.Unix(0, int64(i)+1).String()
|
||||
if err := r.Begin(Transfer{ID: id}); err != nil {
|
||||
t.Errorf("Begin(%d): %v", i, err)
|
||||
return
|
||||
}
|
||||
for j := 0; j < 10; j++ {
|
||||
_ = r.AddServedBytes(id, 1)
|
||||
_ = r.Snapshot()
|
||||
}
|
||||
r.End(id)
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
if got := r.Snapshot(); len(got) != 0 {
|
||||
t.Fatalf("leaked transfers = %d", len(got))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryCapHoldsUnderRacingBegin(t *testing.T) {
|
||||
const cap = 7
|
||||
unlimited := 0
|
||||
r := NewWithOptions(Options{MaxEntries: cap, MaxPerUser: &unlimited})
|
||||
var wg sync.WaitGroup
|
||||
var accepted atomic.Int64
|
||||
for i := 0; i < 100; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) {
|
||||
defer wg.Done()
|
||||
err := r.Begin(Transfer{ID: time.Unix(0, int64(i)+1).String()})
|
||||
if err == nil {
|
||||
accepted.Add(1)
|
||||
} else if !errors.Is(err, ErrRegistryFull) {
|
||||
t.Errorf("Begin(%d): %v", i, err)
|
||||
}
|
||||
}(i)
|
||||
}
|
||||
wg.Wait()
|
||||
if got := accepted.Load(); got != cap {
|
||||
t.Fatalf("accepted = %d, want %d", got, cap)
|
||||
}
|
||||
if got := len(r.Snapshot()); got != cap {
|
||||
t.Fatalf("snapshot size = %d, want %d", got, cap)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryPerUserLimitAndAnnotation(t *testing.T) {
|
||||
now := time.Date(2026, 7, 31, 0, 0, 0, 0, time.UTC)
|
||||
maxPerUser := 2
|
||||
logs := &countingLogHandler{}
|
||||
r := NewWithOptions(Options{
|
||||
MaxPerUser: &maxPerUser,
|
||||
Now: func() time.Time { return now },
|
||||
Logger: slog.New(logs),
|
||||
})
|
||||
for _, id := range []string{"one", "two"} {
|
||||
if err := r.Begin(Transfer{ID: id, UserID: 7, Route: "native_download"}); err != nil {
|
||||
t.Fatalf("Begin(%q): %v", id, err)
|
||||
}
|
||||
}
|
||||
if err := r.Begin(Transfer{ID: "over", UserID: 7, Route: "native_download"}); !errors.Is(err, ErrUserTransferLimit) {
|
||||
t.Fatalf("Begin at per-user cap = %v, want ErrUserTransferLimit", err)
|
||||
}
|
||||
if err := r.Begin(Transfer{ID: "other", UserID: 8}); err != nil {
|
||||
t.Fatalf("other user Begin: %v", err)
|
||||
}
|
||||
|
||||
r.End("never-began")
|
||||
if err := r.Begin(Transfer{ID: "still-over", UserID: 7}); !errors.Is(err, ErrUserTransferLimit) {
|
||||
t.Fatalf("Begin after unknown End = %v, want ErrUserTransferLimit", err)
|
||||
}
|
||||
if got := logs.count.Load(); got != 1 {
|
||||
t.Fatalf("warnings before interval = %d, want 1", got)
|
||||
}
|
||||
now = now.Add(fullWarningInterval)
|
||||
if err := r.Begin(Transfer{ID: "warn-again", UserID: 7}); !errors.Is(err, ErrUserTransferLimit) {
|
||||
t.Fatalf("Begin after warning interval = %v, want ErrUserTransferLimit", err)
|
||||
}
|
||||
if got := logs.count.Load(); got != 2 {
|
||||
t.Fatalf("warnings after interval = %d, want 2", got)
|
||||
}
|
||||
|
||||
r.End("one")
|
||||
if err := r.Begin(Transfer{ID: "replacement", UserID: 7}); err != nil {
|
||||
t.Fatalf("Begin after End: %v", err)
|
||||
}
|
||||
r.Annotate("replacement", " download ", 42)
|
||||
r.Annotate("unknown", "ignored", 99)
|
||||
var replacement Transfer
|
||||
for _, transfer := range r.Snapshot() {
|
||||
if transfer.ID == "replacement" {
|
||||
replacement = transfer
|
||||
}
|
||||
}
|
||||
if replacement.DownloadID != "download" || replacement.MediaFileID != 42 {
|
||||
t.Fatalf("annotated transfer = %+v", replacement)
|
||||
}
|
||||
if len(r.Snapshot()) != 3 {
|
||||
t.Fatalf("Annotate unknown changed registry: %+v", r.Snapshot())
|
||||
}
|
||||
|
||||
r.End("two")
|
||||
r.End("replacement")
|
||||
r.End("other")
|
||||
if len(r.perUser) != 0 {
|
||||
t.Fatalf("perUser retained zero-count state: %+v", r.perUser)
|
||||
}
|
||||
|
||||
unlimited := 0
|
||||
unlimitedRegistry := NewWithOptions(Options{MaxEntries: 30, MaxPerUser: &unlimited})
|
||||
for i := 0; i < 30; i++ {
|
||||
if err := unlimitedRegistry.Begin(Transfer{ID: strconv.Itoa(i), UserID: 1}); err != nil {
|
||||
t.Fatalf("explicit unlimited Begin(%d): %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
defaultRegistry := New()
|
||||
if defaultRegistry.MaxPerUser() != defaultMaxPerUser {
|
||||
t.Fatalf("unset MaxPerUser = %d, want %d", defaultRegistry.MaxPerUser(), defaultMaxPerUser)
|
||||
}
|
||||
for i := 0; i < defaultMaxPerUser; i++ {
|
||||
if err := defaultRegistry.Begin(Transfer{ID: strconv.Itoa(i), UserID: 1}); err != nil {
|
||||
t.Fatalf("default Begin(%d): %v", i, err)
|
||||
}
|
||||
}
|
||||
if err := defaultRegistry.Begin(Transfer{ID: "default-over", UserID: 1}); !errors.Is(err, ErrUserTransferLimit) {
|
||||
t.Fatalf("Begin at default cap = %v, want ErrUserTransferLimit", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseMaxPerUser(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
raw string
|
||||
want int
|
||||
wantErr bool
|
||||
}{
|
||||
{raw: "", want: defaultMaxPerUser},
|
||||
{raw: "0", want: 0},
|
||||
{raw: "48", want: 48},
|
||||
{raw: "-1", wantErr: true},
|
||||
{raw: "10001", wantErr: true},
|
||||
{raw: "many", wantErr: true},
|
||||
} {
|
||||
got, err := ParseMaxPerUser(tc.raw)
|
||||
if (err != nil) != tc.wantErr || got != tc.want {
|
||||
t.Errorf("ParseMaxPerUser(%q) = %d, %v; want %d, error=%v", tc.raw, got, err, tc.want, tc.wantErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type countingLogHandler struct {
|
||||
count atomic.Int64
|
||||
}
|
||||
|
||||
func (h *countingLogHandler) Enabled(context.Context, slog.Level) bool { return true }
|
||||
func (h *countingLogHandler) Handle(context.Context, slog.Record) error {
|
||||
h.count.Add(1)
|
||||
return nil
|
||||
}
|
||||
func (h *countingLogHandler) WithAttrs([]slog.Attr) slog.Handler { return h }
|
||||
func (h *countingLogHandler) WithGroup(string) slog.Handler { return h }
|
||||
|
||||
func TestRegistryClampsAndNormalizesRequestStrings(t *testing.T) {
|
||||
r := New()
|
||||
long := strings.Repeat("界", 400)
|
||||
if err := r.Begin(Transfer{
|
||||
ID: " pour ",
|
||||
DownloadID: " " + long + " ",
|
||||
ProfileID: long,
|
||||
Route: long,
|
||||
ClientIP: long,
|
||||
ClientName: long,
|
||||
}); err != nil {
|
||||
t.Fatalf("Begin: %v", err)
|
||||
}
|
||||
got := r.Snapshot()[0]
|
||||
if got.ID != testPourID {
|
||||
t.Fatalf("ID = %q", got.ID)
|
||||
}
|
||||
for name, tc := range map[string]struct {
|
||||
value string
|
||||
max int
|
||||
}{
|
||||
"download": {got.DownloadID, maxDownloadIDLength},
|
||||
"profile": {got.ProfileID, maxProfileIDLength},
|
||||
"route": {got.Route, maxRouteLength},
|
||||
"ip": {got.ClientIP, maxClientIPLength},
|
||||
"client": {got.ClientName, maxClientNameLength},
|
||||
} {
|
||||
if n := len([]rune(tc.value)); n != tc.max {
|
||||
t.Errorf("%s length = %d, want %d", name, n, tc.max)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMeterCloseFlushesSubMiBTailBeforeRegistryEnd(t *testing.T) {
|
||||
var accountingCalls atomic.Int64
|
||||
r := NewWithOptions(Options{Now: func() time.Time {
|
||||
accountingCalls.Add(1)
|
||||
return time.Now()
|
||||
}})
|
||||
if err := r.Begin(Transfer{ID: testPourID, StartedAt: time.Now()}); err != nil {
|
||||
t.Fatalf("Begin: %v", err)
|
||||
}
|
||||
|
||||
func() {
|
||||
defer r.End(testPourID) // registered first, runs last
|
||||
metered := playback.NewSessionMeteredWriter(httptest.NewRecorder(), r, testPourID)
|
||||
defer func() { _ = metered.Close() }() // registered second, runs first
|
||||
if _, err := metered.Write([]byte("smaller than one MiB")); err != nil {
|
||||
t.Fatalf("Write: %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
if got := accountingCalls.Load(); got != 1 {
|
||||
t.Fatalf("accounting calls = %d, want 1; tail was not flushed before End", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDirectPlayByteAccounting(t *testing.T) {
|
||||
payload := []byte("0123456789abcdef")
|
||||
path := filepath.Join(t.TempDir(), "sample.mp4")
|
||||
if err := os.WriteFile(path, payload, 0o600); err != nil {
|
||||
t.Fatalf("WriteFile: %v", err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
method string
|
||||
headers map[string]string
|
||||
writer func() http.ResponseWriter
|
||||
wantBytes int64
|
||||
wantStatus int
|
||||
}{
|
||||
{name: "full body", method: http.MethodGet, writer: func() http.ResponseWriter { return httptest.NewRecorder() }, wantBytes: int64(len(payload)), wantStatus: http.StatusOK},
|
||||
{name: "range", method: http.MethodGet, headers: map[string]string{"Range": "bytes=2-5"}, writer: func() http.ResponseWriter { return httptest.NewRecorder() }, wantBytes: 4, wantStatus: http.StatusPartialContent},
|
||||
{name: "head", method: http.MethodHead, writer: func() http.ResponseWriter { return httptest.NewRecorder() }, wantStatus: http.StatusOK},
|
||||
{name: "conditional", method: http.MethodGet, headers: map[string]string{"If-Modified-Since": time.Now().Add(time.Hour).UTC().Format(http.TimeFormat)}, writer: func() http.ResponseWriter { return httptest.NewRecorder() }, wantStatus: http.StatusNotModified},
|
||||
{name: "client disconnect", method: http.MethodGet, writer: func() http.ResponseWriter { return &disconnectWriter{limit: 5, header: make(http.Header)} }, wantBytes: 5, wantStatus: http.StatusOK},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
r := New()
|
||||
if err := r.Begin(Transfer{ID: testPourID}); err != nil {
|
||||
t.Fatalf("Begin: %v", err)
|
||||
}
|
||||
base := tt.writer()
|
||||
metered := playback.NewSessionMeteredWriter(base, r, testPourID)
|
||||
req := httptest.NewRequest(tt.method, "/file", nil)
|
||||
for key, value := range tt.headers {
|
||||
req.Header.Set(key, value)
|
||||
}
|
||||
if err := playback.ServeDirectPlay(metered, req, path); err != nil {
|
||||
t.Fatalf("ServeDirectPlay: %v", err)
|
||||
}
|
||||
if err := metered.Close(); err != nil {
|
||||
t.Fatalf("Close: %v", err)
|
||||
}
|
||||
got := r.Snapshot()[0]
|
||||
if got.BytesServed != tt.wantBytes {
|
||||
t.Fatalf("BytesServed = %d, want %d", got.BytesServed, tt.wantBytes)
|
||||
}
|
||||
switch w := base.(type) {
|
||||
case *httptest.ResponseRecorder:
|
||||
if w.Code != tt.wantStatus {
|
||||
t.Fatalf("status = %d, want %d", w.Code, tt.wantStatus)
|
||||
}
|
||||
case *disconnectWriter:
|
||||
if w.status != tt.wantStatus {
|
||||
t.Fatalf("status = %d, want %d", w.status, tt.wantStatus)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
type disconnectWriter struct {
|
||||
header http.Header
|
||||
status int
|
||||
limit int
|
||||
}
|
||||
|
||||
func (w *disconnectWriter) Header() http.Header { return w.header }
|
||||
|
||||
func (w *disconnectWriter) WriteHeader(status int) {
|
||||
if w.status == 0 {
|
||||
w.status = status
|
||||
}
|
||||
}
|
||||
|
||||
func (w *disconnectWriter) Write(p []byte) (int, error) {
|
||||
if w.status == 0 {
|
||||
w.status = http.StatusOK
|
||||
}
|
||||
n := min(len(p), w.limit)
|
||||
w.limit -= n
|
||||
if n < len(p) || w.limit == 0 {
|
||||
return n, io.ErrClosedPipe
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
-- +goose Up
|
||||
-- +goose StatementBegin
|
||||
CREATE TABLE public.stream_revocations (
|
||||
kind text NOT NULL,
|
||||
id text NOT NULL,
|
||||
reason text NOT NULL DEFAULT '',
|
||||
revoked_at timestamptz NOT NULL DEFAULT now(),
|
||||
expires_at timestamptz NOT NULL,
|
||||
CONSTRAINT stream_revocations_pkey PRIMARY KEY (kind, id),
|
||||
CONSTRAINT stream_revocations_kind_check CHECK (kind IN ('sess', 'user'))
|
||||
);
|
||||
|
||||
CREATE INDEX stream_revocations_expires_at_idx
|
||||
ON public.stream_revocations (expires_at);
|
||||
-- +goose StatementEnd
|
||||
|
||||
-- +goose Down
|
||||
-- +goose StatementBegin
|
||||
DROP TABLE IF EXISTS public.stream_revocations;
|
||||
-- +goose StatementEnd
|
||||
@@ -0,0 +1,15 @@
|
||||
-- +goose Up
|
||||
ALTER TABLE public.stream_revocations
|
||||
ADD COLUMN unrevoked_at timestamptz NULL,
|
||||
ADD COLUMN tombstone_expires_at timestamptz NULL;
|
||||
|
||||
CREATE INDEX stream_revocations_tombstone_expires_at_idx
|
||||
ON public.stream_revocations (tombstone_expires_at)
|
||||
WHERE unrevoked_at IS NOT NULL;
|
||||
|
||||
-- +goose Down
|
||||
DROP INDEX IF EXISTS public.stream_revocations_tombstone_expires_at_idx;
|
||||
|
||||
ALTER TABLE public.stream_revocations
|
||||
DROP COLUMN IF EXISTS tombstone_expires_at,
|
||||
DROP COLUMN IF EXISTS unrevoked_at;
|
||||
Reference in New Issue
Block a user