Files
silo-server/internal/streamenforcer/enforcer_test.go
T
CoffeeKnyte fc65dcaf16 fix(playback): make session liveness server-observed
Client progress reports could keep a zero-byte phantom session alive and,
worse, make it outrank a genuinely-serving stream when the enforcer picked
over-cap victims.

`streammonitor.LiveLocalSessions` substituted `Session.LastActivityAt` for a
zero `LastServedAt`, and `LastActivityAt` is advanced by `UpdateProgress` and
by the realtime WebSocket hello/ack/result handlers. Since
`streamenforcer.selectVictims` keeps the `limit` most-recently-served streams,
a progress-only phantom sorted ahead of a real stream and the real one was
trimmed instead. Reaping had the same root cause: `sessionIsInactiveLocked`
keyed idleness on `LastActivityAt`, so a client that kept pinging held a
session open forever.

Per decision A5 (Option C), client progress is now UI metadata only and never
feeds enforcement or reaping:

- `LiveLocalSessions` projects `LastServedAt` verbatim, emitting an empty
  timestamp when the session has never served, so it sorts as the stalest
  over-cap victim.
- `sessionIsInactiveLocked` measures idleness from `LastServedAt`, falling back
  only to `StartedAt`.
- A configurable never-served window (`DefaultUnservedSessionGrace`, 2m, via
  `SetUnservedSessionGrace`) keeps a legitimately slow start from being reaped
  before its first byte, without granting a phantom unbounded life. It is a
  separate knob rather than a hardcoded floor so it cannot silently override
  `SetLivenessGracePeriods`.

In-flight transports remain exempt, so direct-play and remux long pours and
per-segment HLS serves are unaffected.

Paused sessions with an open realtime/WebSocket connection are exempt from
reaping. That preserves the issue #243 fix (reaping a paused transcode froze
clients) while staying within Option C: an open, ping-checked connection is
server-observed, unlike a client's reported progress, and the session still
consumes one of the user's cap slots.

Part 1 of 3 for the Batch 4 liveness/replica work.

Part of #305
2026-07-31 00:03:44 +00:00

249 lines
8.3 KiB
Go

package streamenforcer
import (
"context"
"errors"
"sort"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/nodesessions"
"github.com/Silo-Server/silo-server/internal/playback"
"github.com/Silo-Server/silo-server/internal/streammonitor"
)
type fakeSource struct {
snap streammonitor.Snapshot
err error
}
func (f fakeSource) Snapshot(context.Context) (streammonitor.Snapshot, error) {
return f.snap, f.err
}
type fakeRevoker struct {
revoked []string
err error
seen map[string]struct{}
ttls []time.Duration
}
func (f *fakeRevoker) RevokeSessionForIfAbsent(
_ context.Context,
sessionID, _ string,
ttl time.Duration,
) (bool, error) {
if f.err != nil {
return false, f.err
}
if f.seen == nil {
f.seen = make(map[string]struct{})
}
if _, ok := f.seen[sessionID]; ok {
return false, nil
}
f.seen[sessionID] = struct{}{}
f.revoked = append(f.revoked, sessionID)
f.ttls = append(f.ttls, ttl)
return true, nil
}
// stream builds a LiveStream whose StartedAt and LastServedAt are both ts.
func stream(id string, user int, ts time.Time) streammonitor.LiveStream {
return streammonitor.LiveStream{SessionID: id, UserID: user, StartedAt: ts, LastServedAt: ts}
}
// served builds a LiveStream started at `started` but last served at `lastServed`
// (to model a ghost: old activity) — StartedAt independent from real liveness.
func served(id string, user int, started, lastServed time.Time) streammonitor.LiveStream {
return streammonitor.LiveStream{SessionID: id, UserID: user, StartedAt: started, LastServedAt: lastServed}
}
func TestEvaluateOnce(t *testing.T) {
base := time.Date(2026, 7, 4, 12, 0, 0, 0, time.UTC)
limit3 := func(context.Context, int) (int, error) { return 3, nil }
tests := []struct {
name string
streams []streammonitor.LiveStream
limits LimitFunc
wantRevoke []string
}{
{
name: "under cap: no revokes",
streams: []streammonitor.LiveStream{stream("a", 1, base), stream("b", 1, base.Add(time.Minute))},
limits: limit3,
wantRevoke: nil,
},
{
name: "over cap: least-recently-served trimmed, freshest kept",
streams: []streammonitor.LiveStream{
stream("stale", 1, base),
stream("old", 1, base.Add(1*time.Minute)),
stream("keep1", 1, base.Add(2*time.Minute)),
stream("keep2", 1, base.Add(3*time.Minute)),
stream("keep3", 1, base.Add(4*time.Minute)),
},
limits: limit3,
wantRevoke: []string{"stale", "old"},
},
{
name: "ghost trimmed, fresh reconnect survives",
// Two fresh (served just now) and two ghosts (served long ago), limit 2.
streams: []streammonitor.LiveStream{
served("ghost1", 1, base, base),
served("ghost2", 1, base.Add(time.Second), base.Add(time.Second)),
served("fresh1", 1, base.Add(10*time.Minute), base.Add(10*time.Minute)),
served("fresh2", 1, base.Add(10*time.Minute), base.Add(11*time.Minute)),
},
limits: func(context.Context, int) (int, error) { return 2, nil },
wantRevoke: []string{"ghost1", "ghost2"},
},
{
name: "limit lookup error: fail open",
streams: []streammonitor.LiveStream{stream("a", 1, base), stream("b", 1, base), stream("c", 1, base), stream("d", 1, base)},
limits: func(context.Context, int) (int, error) { return 0, errors.New("db down") },
wantRevoke: nil,
},
{
name: "unlimited (limit<=0): no revokes",
streams: []streammonitor.LiveStream{stream("a", 1, base), stream("b", 1, base), stream("c", 1, base)},
limits: func(context.Context, int) (int, error) { return 0, nil },
wantRevoke: nil,
},
{
name: "user 0 never enforced",
streams: []streammonitor.LiveStream{stream("a", 0, base), stream("b", 0, base), stream("c", 0, base), stream("d", 0, base)},
limits: limit3,
wantRevoke: nil,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
rev := &fakeRevoker{}
e := New(fakeSource{snap: streammonitor.Snapshot{Streams: tc.streams}}, tc.limits, rev, 0)
if err := e.EvaluateOnce(context.Background()); err != nil {
t.Fatalf("EvaluateOnce: %v", err)
}
got := append([]string{}, rev.revoked...)
sort.Strings(got)
want := append([]string{}, tc.wantRevoke...)
sort.Strings(want)
if len(got) != len(want) {
t.Fatalf("revoked = %v, want %v", got, want)
}
for i := range got {
if got[i] != want[i] {
t.Fatalf("revoked = %v, want %v", got, want)
}
}
})
}
}
func TestEvaluateOnceProgressOnlySessionIsVictimBeforeServingStreams(t *testing.T) {
sm := playback.NewSessionManager(0, 0)
phantom, err := sm.StartSession(7, "profile", 1, playback.PlayDirect, false)
if err != nil {
t.Fatalf("StartSession(phantom): %v", err)
}
for i := 0; i < 2; i++ {
session, startErr := sm.StartSession(7, "profile", 10+i, playback.PlayDirect, false)
if startErr != nil {
t.Fatalf("StartSession(serving %d): %v", i, startErr)
}
if beginErr := sm.BeginTransport(session.ID); beginErr != nil {
t.Fatalf("BeginTransport(serving %d): %v", i, beginErr)
}
if bytesErr := sm.AddServedBytes(session.ID, 1024); bytesErr != nil {
t.Fatalf("AddServedBytes(serving %d): %v", i, bytesErr)
}
if endErr := sm.EndTransport(session.ID); endErr != nil {
t.Fatalf("EndTransport(serving %d): %v", i, endErr)
}
}
time.Sleep(time.Until(time.Now().Truncate(time.Second).Add(time.Second)) + 10*time.Millisecond)
if err := sm.UpdateProgress(phantom.ID, 30, false); err != nil {
t.Fatalf("UpdateProgress(phantom): %v", err)
}
source := streammonitor.NewFuncSource(func(context.Context) ([]nodesessions.SessionInfo, error) {
return streammonitor.LiveLocalSessions(sm, "local"), nil
})
rev := &fakeRevoker{}
e := New(source, func(context.Context, int) (int, error) { return 2, nil }, rev, 0)
if err := e.EvaluateOnce(context.Background()); err != nil {
t.Fatalf("EvaluateOnce: %v", err)
}
if len(rev.revoked) != 1 || rev.revoked[0] != phantom.ID {
t.Fatalf("revoked = %v, want progress-only phantom %q", rev.revoked, phantom.ID)
}
}
func TestEvaluateOnceSourceError(t *testing.T) {
rev := &fakeRevoker{}
e := New(fakeSource{err: errors.New("scan failed")}, func(context.Context, int) (int, error) { return 1, nil }, rev, 0)
if err := e.EvaluateOnce(context.Background()); err == nil {
t.Fatal("expected error from source")
}
if len(rev.revoked) != 0 {
t.Fatalf("expected no revokes on source error, got %v", rev.revoked)
}
}
func TestEvaluateOnceRevokesCanonicalLogicalSessionID(t *testing.T) {
source := streammonitor.NewFuncSource(func(context.Context) ([]nodesessions.SessionInfo, error) {
return []nodesessions.SessionInfo{
{SessionID: "transport-a", LogicalSessionID: "logical-a", AuthUserID: 7, LastServedAt: "2026-07-30T01:00:00Z"},
{SessionID: "transport-b", LogicalSessionID: "logical-b", AuthUserID: 7, LastServedAt: "2026-07-30T02:00:00Z"},
}, nil
})
rev := &fakeRevoker{}
e := New(source, func(context.Context, int) (int, error) { return 1, nil }, rev, 0)
if err := e.EvaluateOnce(context.Background()); err != nil {
t.Fatalf("EvaluateOnce: %v", err)
}
if len(rev.revoked) != 1 || rev.revoked[0] != "logical-a" {
t.Fatalf("revoked = %v, want logical-a", rev.revoked)
}
}
func TestEvaluateOnceDoesNotExtendExistingAutomatedRevocation(t *testing.T) {
base := time.Now()
source := fakeSource{snap: streammonitor.Snapshot{Streams: []streammonitor.LiveStream{
stream("keep", 7, base.Add(time.Minute)),
stream("victim", 7, base),
}}}
rev := &fakeRevoker{}
e := New(source, func(context.Context, int) (int, error) { return 1, nil }, rev, 0)
if err := e.EvaluateOnce(context.Background()); err != nil {
t.Fatal(err)
}
if err := e.EvaluateOnce(context.Background()); err != nil {
t.Fatal(err)
}
if len(rev.revoked) != 1 {
t.Fatalf("revocation writes = %d, want 1", len(rev.revoked))
}
if len(rev.ttls) != 1 || rev.ttls[0] != playback.MaxTokenTTL {
t.Fatalf("revocation TTLs = %v, want [%v]", rev.ttls, playback.MaxTokenTTL)
}
}
func TestParseRevocationTTL(t *testing.T) {
if got, err := ParseRevocationTTL(""); err != nil || got != playback.MaxTokenTTL {
t.Fatalf("default = %v, %v; want %v", got, err, playback.MaxTokenTTL)
}
if got, err := ParseRevocationTTL("6h"); err != nil || got != 6*time.Hour {
t.Fatalf("valid = %v, %v; want 6h", got, err)
}
for _, raw := range []string{"bad", "4m59s", "24h1s"} {
if _, err := ParseRevocationTTL(raw); err == nil {
t.Fatalf("ParseRevocationTTL(%q) accepted invalid value", raw)
}
}
}