Files
silo-server/internal/streammonitor/monitor_test.go
T
CoffeeKnyte fc65dcaf16 fix(playback): make session liveness server-observed
Client progress reports could keep a zero-byte phantom session alive and,
worse, make it outrank a genuinely-serving stream when the enforcer picked
over-cap victims.

`streammonitor.LiveLocalSessions` substituted `Session.LastActivityAt` for a
zero `LastServedAt`, and `LastActivityAt` is advanced by `UpdateProgress` and
by the realtime WebSocket hello/ack/result handlers. Since
`streamenforcer.selectVictims` keeps the `limit` most-recently-served streams,
a progress-only phantom sorted ahead of a real stream and the real one was
trimmed instead. Reaping had the same root cause: `sessionIsInactiveLocked`
keyed idleness on `LastActivityAt`, so a client that kept pinging held a
session open forever.

Per decision A5 (Option C), client progress is now UI metadata only and never
feeds enforcement or reaping:

- `LiveLocalSessions` projects `LastServedAt` verbatim, emitting an empty
  timestamp when the session has never served, so it sorts as the stalest
  over-cap victim.
- `sessionIsInactiveLocked` measures idleness from `LastServedAt`, falling back
  only to `StartedAt`.
- A configurable never-served window (`DefaultUnservedSessionGrace`, 2m, via
  `SetUnservedSessionGrace`) keeps a legitimately slow start from being reaped
  before its first byte, without granting a phantom unbounded life. It is a
  separate knob rather than a hardcoded floor so it cannot silently override
  `SetLivenessGracePeriods`.

In-flight transports remain exempt, so direct-play and remux long pours and
per-segment HLS serves are unaffected.

Paused sessions with an open realtime/WebSocket connection are exempt from
reaping. That preserves the issue #243 fix (reaping a paused transcode froze
clients) while staying within Option C: an open, ping-checked connection is
server-observed, unlike a client's reported progress, and the session still
consumes one of the user's cap slots.

Part 1 of 3 for the Batch 4 liveness/replica work.

Part of #305
2026-07-31 00:03:44 +00:00

411 lines
14 KiB
Go

package streammonitor
import (
"context"
"testing"
"time"
"github.com/Silo-Server/silo-server/internal/nodesessions"
"github.com/Silo-Server/silo-server/internal/playback"
)
func fakeFn(infos []nodesessions.SessionInfo) func(ctx context.Context) ([]nodesessions.SessionInfo, error) {
return func(ctx context.Context) ([]nodesessions.SessionInfo, error) {
return infos, nil
}
}
func TestFuncSourceGrouping(t *testing.T) {
infos := []nodesessions.SessionInfo{
{SessionID: "s1", AuthUserID: 1, Type: "direct_play", StartedAt: "2026-07-04T10:00:00Z", LastServedAt: "2026-07-04T10:05:00Z"},
{SessionID: "s2", AuthUserID: 1, Type: "transcode", StartedAt: "2026-07-04T10:01:00Z", LastServedAt: "2026-07-04T10:06:00Z"},
{SessionID: "s3", AuthUserID: 2, Type: "remux", StartedAt: "2026-07-04T10:02:00Z", LastServedAt: "2026-07-04T10:07:00Z"},
{SessionID: "s4", AuthUserID: 0, Type: "direct_play"}, // no user, no timestamps
}
snap, err := NewFuncSource(fakeFn(infos)).Snapshot(context.Background())
if err != nil {
t.Fatalf("Snapshot: %v", err)
}
if got := len(snap.Streams); got != 4 {
t.Fatalf("Streams len = %d, want 4", got)
}
if got := snap.CountByUser(1); got != 2 {
t.Errorf("CountByUser(1) = %d, want 2", got)
}
if got := snap.CountByUser(2); got != 1 {
t.Errorf("CountByUser(2) = %d, want 1", got)
}
if got := snap.CountByUser(0); got != 1 {
t.Errorf("CountByUser(0) = %d, want 1", got)
}
if got := snap.CountByUser(99); got != 0 {
t.Errorf("CountByUser(99) = %d, want 0", got)
}
u1 := snap.StreamsForUser(1)
if len(u1) != 2 {
t.Fatalf("StreamsForUser(1) len = %d, want 2", len(u1))
}
for _, st := range u1 {
if st.UserID != 1 {
t.Errorf("StreamsForUser(1) returned stream with UserID %d", st.UserID)
}
}
byUser := snap.ByUser()
if len(byUser) != 3 {
t.Fatalf("ByUser len = %d, want 3 (users 0,1,2)", len(byUser))
}
if len(byUser[1]) != 2 || len(byUser[2]) != 1 || len(byUser[0]) != 1 {
t.Errorf("ByUser grouping wrong: %#v", map[int]int{0: len(byUser[0]), 1: len(byUser[1]), 2: len(byUser[2])})
}
}
func TestTimestampParsing(t *testing.T) {
infos := []nodesessions.SessionInfo{
{SessionID: "s1", AuthUserID: 1, StartedAt: "2026-07-04T10:00:00Z", LastServedAt: "2026-07-04T10:05:00Z"},
{SessionID: "s2", AuthUserID: 1, StartedAt: "not-a-time", LastServedAt: ""},
}
snap, err := NewFuncSource(fakeFn(infos)).Snapshot(context.Background())
if err != nil {
t.Fatalf("Snapshot: %v", err)
}
bySession := map[string]LiveStream{}
for _, st := range snap.Streams {
bySession[st.SessionID] = st
}
want := time.Date(2026, 7, 4, 10, 5, 0, 0, time.UTC)
if !bySession["s1"].LastServedAt.Equal(want) {
t.Errorf("s1 LastServedAt = %v, want %v", bySession["s1"].LastServedAt, want)
}
if !bySession["s2"].StartedAt.IsZero() {
t.Errorf("s2 StartedAt = %v, want zero (unparseable)", bySession["s2"].StartedAt)
}
if !bySession["s2"].LastServedAt.IsZero() {
t.Errorf("s2 LastServedAt = %v, want zero (empty)", bySession["s2"].LastServedAt)
}
}
func TestDedupeKeepsNewest(t *testing.T) {
// Same SessionID observed on two nodes (proxy vs transcode node). Keep the
// record with the most recent LastServedAt.
infos := []nodesessions.SessionInfo{
{SessionID: "dup", AuthUserID: 5, NodeName: "proxy", LastServedAt: "2026-07-04T10:00:00Z", BytesServed: 100},
{SessionID: "dup", AuthUserID: 5, NodeName: "transcode", LastServedAt: "2026-07-04T10:09:00Z", BytesServed: 900},
{SessionID: "other", AuthUserID: 5, NodeName: "proxy", LastServedAt: "2026-07-04T10:03:00Z"},
}
snap, err := NewFuncSource(fakeFn(infos)).Snapshot(context.Background())
if err != nil {
t.Fatalf("Snapshot: %v", err)
}
if got := len(snap.Streams); got != 2 {
t.Fatalf("Streams len = %d, want 2 (deduped)", got)
}
if got := snap.CountByUser(5); got != 2 {
t.Errorf("CountByUser(5) = %d, want 2", got)
}
var dup LiveStream
found := false
for _, st := range snap.Streams {
if st.SessionID == "dup" {
dup = st
found = true
}
}
if !found {
t.Fatal("deduped session 'dup' not present")
}
if dup.NodeName != "transcode" {
t.Errorf("dedupe kept NodeName %q, want transcode (newest LastServedAt)", dup.NodeName)
}
if dup.BytesServed != 900 {
t.Errorf("dedupe kept BytesServed %d, want 900", dup.BytesServed)
}
}
func TestMergeStreamsDirect(t *testing.T) {
// Direct unit test of the merge helper, mirroring RedisSource dedupe.
in := []LiveStream{
{SessionID: "a", NodeName: "n1", LastServedAt: time.Unix(100, 0)},
{SessionID: "a", NodeName: "n2", LastServedAt: time.Unix(200, 0)},
{SessionID: "b", NodeName: "n1", LastServedAt: time.Unix(150, 0)},
}
out := mergeStreams(in)
if len(out) != 2 {
t.Fatalf("mergeStreams len = %d, want 2", len(out))
}
for _, st := range out {
if st.SessionID == "a" && st.NodeName != "n2" {
t.Errorf("merge kept %q for session a, want n2 (newest)", st.NodeName)
}
}
}
func TestMergeStreamsCarriesOwnershipForward(t *testing.T) {
// The transcode node's own start record has no resolved owner (UserID 0) and
// can be the freshest copy of a session. Taking it wholesale would bucket the
// session under user 0, which the enforcer skips — exempting it from the cap.
// The merge must recover the owner from the proxy's (staler) owned record.
in := []LiveStream{
{SessionID: "s", NodeName: "proxy", UserID: 42, ProfileID: "p1", MediaFileID: 7, LastServedAt: time.Unix(100, 0)},
{SessionID: "s", NodeName: "transcode", UserID: 0, LastServedAt: time.Unix(200, 0)},
}
out := mergeStreams(in)
if len(out) != 1 {
t.Fatalf("mergeStreams len = %d, want 1", len(out))
}
got := out[0]
if got.NodeName != "transcode" {
t.Errorf("kept NodeName %q, want transcode (freshest)", got.NodeName)
}
if got.UserID != 42 {
t.Errorf("UserID = %d, want 42 (recovered from the proxy record)", got.UserID)
}
if got.ProfileID != "p1" || got.MediaFileID != 7 {
t.Errorf("ownership not fully recovered: ProfileID=%q MediaFileID=%d", got.ProfileID, got.MediaFileID)
}
}
func TestToLiveStreamCarriesRouteAndClient(t *testing.T) {
// The normalized view must surface route (native vs jellycompat) and client
// identity so the monitor is first-class, not just session id + method.
info := nodesessions.SessionInfo{
SessionID: "s1",
AuthUserID: 9,
Type: "transcode",
Route: "jellycompat",
ClientIP: "203.0.113.7",
ClientName: "Infuse",
Position: 61.5,
HWAccel: "vaapi",
}
got := toLiveStream(info)
if got.Route != "jellycompat" || got.ClientIP != "203.0.113.7" || got.ClientName != "Infuse" {
t.Fatalf("route/client not carried: %+v", got)
}
if got.Position != 61.5 || got.HWAccel != "vaapi" {
t.Fatalf("position/hwaccel not carried: %+v", got)
}
}
func TestMergeStreamsBackfillsAttribution(t *testing.T) {
// A fresher-but-thinner record (e.g. an ownerless transcode-node start record)
// must not drop route/client that a staler record for the same session carries.
in := []LiveStream{
{SessionID: "s", NodeName: "proxy", UserID: 5, Route: "native", ClientName: "SiloTV", ClientIP: "10.0.0.9", LastServedAt: time.Unix(100, 0)},
{SessionID: "s", NodeName: "transcode", UserID: 0, LastServedAt: time.Unix(200, 0)},
}
out := mergeStreams(in)
if len(out) != 1 {
t.Fatalf("merge len = %d, want 1", len(out))
}
got := out[0]
if got.NodeName != "transcode" {
t.Errorf("kept NodeName %q, want transcode (freshest)", got.NodeName)
}
if got.UserID != 5 || got.Route != "native" || got.ClientName != "SiloTV" || got.ClientIP != "10.0.0.9" {
t.Errorf("attribution not backfilled from staler record: %+v", got)
}
}
func TestMergeStreamsKeepsLargestObservedByteTotal(t *testing.T) {
for _, tc := range []struct {
name string
older, newer int64
want int64
}{
{name: "freshest has zero", older: 8192, newer: 0, want: 8192},
{name: "stale has zero", older: 0, newer: 8192, want: 8192},
} {
t.Run(tc.name, func(t *testing.T) {
out := mergeStreams([]LiveStream{
{SessionID: "s", BytesServed: tc.older, LastServedAt: time.Unix(100, 0)},
{SessionID: "s", BytesServed: tc.newer, LastServedAt: time.Unix(200, 0)},
})
if len(out) != 1 || out[0].BytesServed != tc.want {
t.Fatalf("merge = %+v, want bytes %d", out, tc.want)
}
})
}
}
func TestFuncSourceNilFn(t *testing.T) {
snap, err := NewFuncSource(nil).Snapshot(context.Background())
if err != nil {
t.Fatalf("Snapshot: %v", err)
}
if len(snap.Streams) != 0 {
t.Errorf("nil fn Streams len = %d, want 0", len(snap.Streams))
}
}
func TestRedisSourceNilClient(t *testing.T) {
snap, err := NewRedisSource(nil).Snapshot(context.Background())
if err != nil {
t.Fatalf("Snapshot: %v", err)
}
if len(snap.Streams) != 0 {
t.Errorf("nil client Streams len = %d, want 0", len(snap.Streams))
}
}
// TestDedupeSessionInfos mirrors the mergeStreams rules on the raw SessionInfo
// shape used by the admin session list: one row per session id, freshest copy
// wins, a resolved owner and missing attribution are carried across the merge
// so the operator view matches the enforcer's picture (no double-counted
// streams, no user-0 rows when any copy knows the owner).
func TestDedupeSessionInfos(t *testing.T) {
newer := time.Now().UTC().Format(time.RFC3339)
older := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339)
infos := []nodesessions.SessionInfo{
{SessionID: "s1", NodeName: "central", AuthUserID: 7, ProfileID: "p1", MediaFileID: 42,
Route: "native", ClientName: "SiloTV", Position: 130, LastServedAt: older},
// The same stream, seen from the edge serving it: freshest but ownerless.
{SessionID: "s1", NodeName: "edge-1", LastServedAt: newer, BytesServed: 9000},
{SessionID: "s2", NodeName: "edge-1", AuthUserID: 8, LastServedAt: newer},
}
out := DedupeSessionInfos(infos)
if len(out) != 2 {
t.Fatalf("dedupe len = %d, want 2", len(out))
}
byID := make(map[string]nodesessions.SessionInfo, len(out))
for _, info := range out {
byID[info.SessionID] = info
}
s1, ok := byID["s1"]
if !ok {
t.Fatalf("s1 missing from dedupe output")
}
if s1.NodeName != "edge-1" || s1.BytesServed != 9000 {
t.Errorf("s1 freshest copy not kept: %+v", s1)
}
if s1.AuthUserID != 7 || s1.ProfileID != "p1" || s1.MediaFileID != 42 {
t.Errorf("s1 owner not carried across merge: %+v", s1)
}
if s1.Route != "native" || s1.ClientName != "SiloTV" || s1.Position != 130 {
t.Errorf("s1 attribution not backfilled: %+v", s1)
}
if s2 := byID["s2"]; s2.AuthUserID != 8 {
t.Errorf("s2 mangled by dedupe: %+v", s2)
}
}
func TestDedupeSessionInfosKeepsLargestObservedByteTotal(t *testing.T) {
older := time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)
newer := time.Now().UTC().Format(time.RFC3339Nano)
for _, tc := range []struct {
name string
older, newer int64
want int64
}{
{name: "freshest has zero", older: 4096, newer: 0, want: 4096},
{name: "stale has zero", older: 0, newer: 4096, want: 4096},
} {
t.Run(tc.name, func(t *testing.T) {
out := DedupeSessionInfos([]nodesessions.SessionInfo{
{SessionID: "s", BytesServed: tc.older, LastServedAt: older},
{SessionID: "s", BytesServed: tc.newer, LastServedAt: newer},
})
if len(out) != 1 || out[0].BytesServed != tc.want {
t.Fatalf("dedupe = %+v, want bytes %d", out, tc.want)
}
})
}
}
func TestLogicalAndTransportRecordsMergeWithOwnerResolved(t *testing.T) {
out := mergeStreams([]LiveStream{
{SessionID: "logical", UserID: 17, ProfileID: "p", LastServedAt: time.Unix(100, 0)},
{SessionID: "transport-a", LogicalSessionID: "logical", LastServedAt: time.Unix(200, 0)},
})
if len(out) != 1 {
t.Fatalf("merge len = %d, want 1", len(out))
}
if out[0].SessionID != "logical" || out[0].UserID != 17 || out[0].ProfileID != "p" {
t.Fatalf("canonical merged stream = %+v", out[0])
}
}
func TestDistinctLogicalStreamsStayDistinct(t *testing.T) {
out := mergeStreams([]LiveStream{
{SessionID: "transport-a", LogicalSessionID: "logical-a"},
{SessionID: "transport-b", LogicalSessionID: "logical-b"},
})
if len(out) != 2 {
t.Fatalf("distinct logical streams merged: %+v", out)
}
}
func TestTransportGenerationsOfOneLogicalSessionMerge(t *testing.T) {
out := DedupeSessionInfos([]nodesessions.SessionInfo{
{SessionID: "transport-a", LogicalSessionID: "logical", NodeName: "old", LastServedAt: "2026-07-30T01:00:00Z"},
{SessionID: "transport-b", LogicalSessionID: "logical", NodeName: "new", LastServedAt: "2026-07-30T02:00:00Z"},
})
if len(out) != 1 || out[0].SessionID != "transport-b" || out[0].LogicalSessionID != "logical" {
t.Fatalf("transport generation dedupe = %+v", out)
}
}
func TestLiveLocalSessionsMapping(t *testing.T) {
sm := playback.NewSessionManager(0, 0)
ctx := playback.WithClientInfo(context.Background(), playback.ClientInfo{
Name: "Silo TV",
})
session, err := sm.StartSessionWithContext(ctx, 42, "profile-1", 9, playback.PlayDirect, false)
if err != nil {
t.Fatalf("StartSessionWithContext: %v", err)
}
session.ClientIP = "192.0.2.10"
if err := sm.BeginTransport(session.ID); err != nil {
t.Fatalf("BeginTransport: %v", err)
}
if err := sm.EndTransport(session.ID); err != nil {
t.Fatalf("EndTransport: %v", err)
}
session, err = sm.GetSession(session.ID)
if err != nil {
t.Fatalf("GetSession: %v", err)
}
got := LiveLocalSessions(sm, "local")
if len(got) != 1 {
t.Fatalf("sessions = %+v", got)
}
info := got[0]
if info.SessionID != session.ID || info.NodeName != "local" ||
info.AuthUserID != 42 || info.ProfileID != "profile-1" ||
info.MediaFileID != 9 || info.Type != string(playback.PlayDirect) ||
info.Route != session.Origin() || info.ClientName != "Silo TV" ||
info.ClientIP != "192.0.2.10" {
t.Fatalf("mapped session = %+v", info)
}
if info.LastServedAt != session.LastServedAt.UTC().Format(time.RFC3339) {
t.Fatalf("LastServedAt = %q, want server-observed timestamp", info.LastServedAt)
}
}
func TestLiveLocalSessionsDoesNotProjectClientActivityAsLastServed(t *testing.T) {
sm := playback.NewSessionManager(0, 0)
session, err := sm.StartSession(42, "profile-1", 9, playback.PlayDirect, false)
if err != nil {
t.Fatalf("StartSession: %v", err)
}
if err := sm.UpdateProgress(session.ID, 12, false); err != nil {
t.Fatalf("UpdateProgress: %v", err)
}
got := LiveLocalSessions(sm, "local")
if len(got) != 1 {
t.Fatalf("sessions = %+v", got)
}
if got[0].LastServedAt != "" {
t.Fatalf("LastServedAt = %q, want empty for a never-served session", got[0].LastServedAt)
}
}