fix(playback): make edge tracking lifecycle-safe and session identity canonical

Five defects that all produced a WRONG over-cap count, which is why they land before the revocation batch: decision A1 raises the over-cap revocation TTL from 5m to ~24h, removing the self-healing that currently limits the damage of a miscount. A false positive after A1 blocks a legitimate stream for a day, so the count has to be trustworthy first. #1 -- overlapping edge requests deleted a live stream. Tracker.sessions was a set and Remove tore down all state plus the Redis key, while both proxy pour handlers deferred removal unconditionally. Two overlapping Range GETs on one session id -- ordinary seek behaviour -- meant the first to finish deleted the record while the second was still pouring, and later AddBytes calls were then dropped because AddBytes ignores bytes for a session with no live record. The stream went invisible to authoritative monitoring while still serving. Track now returns a Lease that the request-scoped caller releases exactly once; teardown happens when the last live lease is released. A plain refcount would have been wrong: Track(A) -> Remove -> Track(B) -> Release(A) decrements B, and clamping at zero does not help because the count legitimately belongs to B. That is not hypothetical -- the transcode node deliberately replaces sessions under the same id so a quality switch does not orphan ffmpeg, and it calls unconditional Remove from its reaper and stop paths. So each generation carries an epoch, Remove and Cleanup bump it, and a release from a superseded generation is a logged no-op. Lease identity is a set rather than a counter, which makes a duplicate release detectable instead of silently destructive. The transcode node keeps using Remove: its Track calls are not request-scoped and are correctly owned by session lifecycle. "Every Track needs a paired Release" is true only of the request-scoped callers. #8 -- async transcode tracking could leave a permanent ghost. The tracking write ran as a bare goroutine with a WithoutCancel context, so if stop won the race the delayed Track recreated the record after cleanup -- and because it landed in sessions, Snapshot treated it as live until Remove and it NEVER idle-expired. A permanent phantom inflating its owner's count, able to trigger false over-cap kills of that user's real streams. The write now takes the per-session lifecycle lock that stop and reap already hold, and re-checks session pointer identity before writing, so a stopped or replaced generation cannot resurrect a record. Pointer identity rather than id equality is what makes same-id replacement safe. The write stays off the request path -- the API server and the playback client are blocked on the 202. #9 + M3 -- protocol-v3 counted one stream twice. The stream token carries a transport id distinct from the logical session id, and the node tracked under the transport id while the API/proxy record used the logical one, so mergeStreams saw two streams. M3 was the reason this had not yet bitten: the v3 fresh-start caller sent no owner attribution at all, so the transport record landed under user 0, which the enforcer skips -- silently exempting the stream from the cap entirely. Fresh v3 starts now carry the logical session id and full owner attribution (both were already in scope at the call site), and merging is keyed on logical identity where present via one shared helper used by both merge functions, which had already drifted apart once. The enforcer view resolves SessionID to the logical id so a kill targets the real session rather than a replaceable transport generation. The raw admin view keeps the transport id and exposes logical_session_id as an additive omitempty field, advertised on the node-sessions capability endpoint, so the v1 response shape is unchanged. GAP-15 -- edge transcode liveness was request-observed. touchTranscodeSession fired before proxying, so hammering dead segment URLs advanced LastServedAt with zero bytes served. Visibility and liveness are now separate operations: EnsureEphemeral makes a session visible without claiming bytes were served, and served-byte liveness advances only from a 2xx/206 upstream response. Previously the proxy metered every upstream body regardless of status, so a node 404's error body counted as served bytes -- moving the touch later would not have fixed it. S4 -- LiveLocalSessions moved from the HTTP handlers package to streammonitor, which owns monitoring. A background enforcer importing api/handlers was backwards. Pure move; its existing mapping assertions moved with it. The LastActivityAt fallback inside it is left as-is -- decision A5 removes it in the liveness batch. Verified with go test -race across nodesessions, proxy and transcodenode; the overlap regression test was confirmed to fail under the old unconditional teardown. Part of #305.
2026-07-30 09:09:35 +00:00
parent 8a9cf04a41
commit ecb4555eec
18 changed files with 1033 additions and 206 deletions
@@ -28,6 +28,7 @@ import (
 // TranscodeStartRequest is the JSON body for POST /transcode/start.
 type TranscodeStartRequest struct {
 	SessionID              string  `json:"session_id"`
+	LogicalSessionID       string  `json:"logical_session_id,omitempty"`
 	InputPath              string  `json:"input_path"`
 	SourceVideoCodec       string  `json:"source_video_codec"`
 	VideoBitstreamFilter   string  `json:"video_bitstream_filter,omitempty"`
@@ -119,6 +120,11 @@ type Server struct {
 	lifecycleMu    sync.Mutex
 	lifecycleLocks map[string]*sessionLifecycleLock

+	// runTracker is a deterministic scheduling seam for lifecycle tests.
+	// Production leaves it nil and launches each write directly in a goroutine;
+	// this is not a projection queue.
+	runTracker func(func())
+
 	// recipeStore is the control-plane recipe store consulted when a forwarded
 	// token carries no recipe (the jellycompat node hop). Nil disables that path.
 	recipeStore recipeStore
@@ -160,6 +166,29 @@ func (s *Server) lockSessionLifecycle(sessionID string) func() {
 	}
 }

+// trackIfCurrent serializes the delayed monitoring write with stop/reap/start
+// lifecycle transitions. Once it owns the lifecycle lock, it verifies that the
+// exact session generation is still registered before writing; a stopped or
+// replaced generation can therefore never recreate a ghost record.
+func (s *Server) trackIfCurrent(ctx context.Context, sessionID string, expected *playback.TranscodeSession, info nodesessions.SessionInfo) {
+	run := func() {
+		unlock := s.lockSessionLifecycle(sessionID)
+		defer unlock()
+		s.mu.RLock()
+		current, ok := s.sessions[sessionID]
+		s.mu.RUnlock()
+		if !ok || current != expected {
+			return
+		}
+		s.tracker.Track(ctx, info)
+	}
+	if s.runTracker != nil {
+		s.runTracker(run)
+		return
+	}
+	go run()
+}
+
 // restartSessionLocked re-spawns session under the per-session lifecycle lock so
 // a segment-recovery restart can never race a fresh start, reconstruct, or
 // another restart into the same output directory. It holds the lock only across
@@ -601,21 +630,22 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) {
 	// tracking write is monitoring-only.
 	effectiveHWAccel := session.Opts().HWAccel
 	trackCtx := context.WithoutCancel(r.Context())
-	go s.tracker.Track(trackCtx, nodesessions.SessionInfo{
-		SessionID:   req.SessionID,
-		NodeURL:     s.tracker.NodeURL(),
-		NodeName:    s.tracker.NodeName(),
-		Type:        "transcode",
-		CodecVideo:  req.TargetCodecVideo,
-		CodecAudio:  req.TargetCodecAudio,
-		Resolution:  req.TargetResolution,
-		HWAccel:     effectiveHWAccel,
-		StartedAt:   time.Now().UTC().Format(time.RFC3339),
-		AuthUserID:  req.AuthUserID,
-		ProfileID:   req.ProfileID,
-		MediaFileID: req.MediaFileID,
-		Route:       req.Route,
-		ClientName:  req.ClientName,
+	s.trackIfCurrent(trackCtx, req.SessionID, session, nodesessions.SessionInfo{
+		SessionID:        req.SessionID,
+		LogicalSessionID: req.LogicalSessionID,
+		NodeURL:          s.tracker.NodeURL(),
+		NodeName:         s.tracker.NodeName(),
+		Type:             "transcode",
+		CodecVideo:       req.TargetCodecVideo,
+		CodecAudio:       req.TargetCodecAudio,
+		Resolution:       req.TargetResolution,
+		HWAccel:          effectiveHWAccel,
+		StartedAt:        time.Now().UTC().Format(time.RFC3339),
+		AuthUserID:       req.AuthUserID,
+		ProfileID:        req.ProfileID,
+		MediaFileID:      req.MediaFileID,
+		Route:            req.Route,
+		ClientName:       req.ClientName,
 	})

 	w.WriteHeader(http.StatusAccepted)
@@ -781,21 +811,22 @@ func (s *Server) spawnReconstruct(r *http.Request, sessionID string, requestedSe
 	s.activeJobs.Add(1)

 	trackCtx := context.WithoutCancel(r.Context())
-	go s.tracker.Track(trackCtx, nodesessions.SessionInfo{
-		SessionID:   sessionID,
-		NodeURL:     s.tracker.NodeURL(),
-		NodeName:    s.tracker.NodeName(),
-		Type:        "transcode",
-		CodecVideo:  card.TargetCodecVideo,
-		CodecAudio:  card.TargetCodecAudio,
-		Resolution:  card.TargetResolution,
-		HWAccel:     session.Opts().HWAccel,
-		StartedAt:   time.Now().UTC().Format(time.RFC3339),
-		AuthUserID:  card.UserID,
-		ProfileID:   card.ProfileID,
-		MediaFileID: card.MediaFileID,
-		Route:       route,
-		ClientName:  clientName,
+	s.trackIfCurrent(trackCtx, sessionID, session, nodesessions.SessionInfo{
+		SessionID:        sessionID,
+		LogicalSessionID: card.SessionID,
+		NodeURL:          s.tracker.NodeURL(),
+		NodeName:         s.tracker.NodeName(),
+		Type:             "transcode",
+		CodecVideo:       card.TargetCodecVideo,
+		CodecAudio:       card.TargetCodecAudio,
+		Resolution:       card.TargetResolution,
+		HWAccel:          session.Opts().HWAccel,
+		StartedAt:        time.Now().UTC().Format(time.RFC3339),
+		AuthUserID:       card.UserID,
+		ProfileID:        card.ProfileID,
+		MediaFileID:      card.MediaFileID,
+		Route:            route,
+		ClientName:       clientName,
 	})

 	slog.InfoContext(r.Context(), "transcode node session reconstructed from token", "component", "transcodenode",
@@ -0,0 +1,129 @@
+package transcodenode
+
+import (
+	"context"
+	"net"
+	"sync"
+	"testing"
+
+	"github.com/redis/go-redis/v9"
+
+	"github.com/Silo-Server/silo-server/internal/nodesessions"
+	"github.com/Silo-Server/silo-server/internal/playback"
+)
+
+type transcodeRedisHook struct {
+	mu   sync.Mutex
+	keys map[string]bool
+}
+
+func (h *transcodeRedisHook) DialHook(next redis.DialHook) redis.DialHook { return next }
+func (h *transcodeRedisHook) ProcessHook(next redis.ProcessHook) redis.ProcessHook {
+	return func(ctx context.Context, cmd redis.Cmder) error {
+		h.mu.Lock()
+		defer h.mu.Unlock()
+		switch cmd.Name() {
+		case "set":
+			h.keys[cmd.Args()[1].(string)] = true
+			if status, ok := cmd.(*redis.StatusCmd); ok {
+				status.SetVal("OK")
+			}
+			return nil
+		case "del":
+			for _, arg := range cmd.Args()[1:] {
+				delete(h.keys, arg.(string))
+			}
+			if count, ok := cmd.(*redis.IntCmd); ok {
+				count.SetVal(1)
+			}
+			return nil
+		}
+		return next(ctx, cmd)
+	}
+}
+func (h *transcodeRedisHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook {
+	return next
+}
+
+func newTranscodeLifecycleTracker(t *testing.T) (*nodesessions.Tracker, *transcodeRedisHook) {
+	t.Helper()
+	rdb := redis.NewClient(&redis.Options{Dialer: func(context.Context, string, string) (net.Conn, error) {
+		t.Fatal("unexpected Redis dial")
+		return nil, nil
+	}})
+	t.Cleanup(func() { _ = rdb.Close() })
+	hook := &transcodeRedisHook{keys: make(map[string]bool)}
+	rdb.AddHook(hook)
+	return nodesessions.NewTracker(rdb, "http://node", "node", "transcode"), hook
+}
+
+func (h *transcodeRedisHook) has(key string) bool {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	return h.keys[key]
+}
+
+func TestDelayedTrackSkipsStoppedSession(t *testing.T) {
+	tracker, redisState := newTranscodeLifecycleTracker(t)
+	session := &playback.TranscodeSession{}
+	var queued []func()
+	s := &Server{
+		tracker:  tracker,
+		sessions: map[string]*playback.TranscodeSession{"transport": session},
+		runTracker: func(fn func()) {
+			queued = append(queued, fn)
+		},
+	}
+	s.trackIfCurrent(context.Background(), "transport", session, nodesessions.SessionInfo{
+		SessionID: "transport", LogicalSessionID: "logical",
+	})
+
+	unlock := s.lockSessionLifecycle("transport")
+	s.mu.Lock()
+	delete(s.sessions, "transport")
+	s.mu.Unlock()
+	tracker.Remove(context.Background(), "transport")
+	unlock()
+	queued[0]()
+
+	if got := tracker.Snapshot(); len(got) != 0 {
+		t.Fatalf("delayed track recreated stopped record: %+v", got)
+	}
+	key := nodesessions.KeyPrefix + tracker.NodeHash() + ":transport"
+	if redisState.has(key) {
+		t.Fatal("delayed track recreated stopped Redis key")
+	}
+}
+
+func TestDelayedTrackSkipsReplacedSession(t *testing.T) {
+	tracker, _ := newTranscodeLifecycleTracker(t)
+	oldSession := &playback.TranscodeSession{}
+	newSession := &playback.TranscodeSession{}
+	var queued []func()
+	s := &Server{
+		tracker:  tracker,
+		sessions: map[string]*playback.TranscodeSession{"transport": oldSession},
+		runTracker: func(fn func()) {
+			queued = append(queued, fn)
+		},
+	}
+	s.trackIfCurrent(context.Background(), "transport", oldSession, nodesessions.SessionInfo{
+		SessionID: "transport", LogicalSessionID: "old",
+	})
+
+	unlock := s.lockSessionLifecycle("transport")
+	s.mu.Lock()
+	s.sessions["transport"] = newSession
+	s.mu.Unlock()
+	tracker.Remove(context.Background(), "transport")
+	tracker.Track(context.Background(), nodesessions.SessionInfo{
+		SessionID: "transport", LogicalSessionID: "new",
+	})
+	unlock()
+	queued[0]()
+
+	got := tracker.Snapshot()
+	if len(got) != 1 || got[0].LogicalSessionID != "new" {
+		t.Fatalf("old delayed track replaced new record: %+v", got)
+	}
+}