Files
silo-server/internal/streamrevoke/store_test.go
T
CoffeeKnyte 0217cce2df feat(playback): stream kill switch + async over-cap enforcer
Add the enforcement layer on top of server-observed monitoring: a revocation
kill switch that stops any stream within ~120s and keeps it dead, plus an async
over-cap enforcer that drives kills off the live monitoring picture — entirely
off the per-segment hot path and with no client-protocol change.

- internal/streamrevoke: the central kill list. IsRevoked is a pure in-memory
  lookup safe on the request hot path; a Redis pub/sub + poll mirror keeps edge
  caches current, and a Postgres durable mirror lets kills survive a server
  restart AND a Redis flush so a restart-resilient stream cannot be reconstructed
  and re-served after being killed. A user revocation is a cutoff (kills tokens
  minted before it, spares post-reauth tokens), not a 24h ban.
- internal/streamenforcer: async over-cap brain — reads the monitoring snapshot
  and per-user limits, selects victims, and collapses every reason (exceeded
  limit, admin terminate, abuse) to the same action: write a revocation.
- Edge + native + jellycompat enforcement: proxy refuses revoked sessions on
  every request and cuts long direct-play/remux pours mid-stream; the transcode
  node guards both serve and the reconstruct path so a killed session is never
  re-spawned after a node restart; jellycompat serve surfaces close their
  kill-switch coverage holes.
- streamtoken.IssuedTime exposes the token iat the user-kill cutoff compares
  against; token IssuedTime + revocation guards wire through router, downloads,
  and admin terminate-by-id (with admin-list dedupe).
- Restore sendfile zero-copy on direct-play/remux byte counting so the monitor's
  served-byte accounting does not cost the sendfile fast path.
- migrations/sql: stream_revocations durable table.

Part of the stream monitoring & kill-switch epic.
2026-07-29 12:26:03 +00:00

472 lines
14 KiB
Go

package streamrevoke
import (
"context"
"errors"
"sync"
"testing"
"time"
)
// errFakeDurable is the error fakeDurable.ListActive returns while failNext > 0.
var errFakeDurable = errors.New("fake durable failure")
// newMemStore returns a memory-only Store (no Redis, no bus, no durable mirror).
func newMemStore() *Store {
return New(Options{})
}
// fakeDurable is an in-memory DurableStore double for exercising the durable
// wiring (Upsert on revoke, warm on start, Prune on the poll tick) without a
// live Postgres. StartSync's warm is synchronous, but the poll goroutine calls
// ListActive/Prune concurrently with the test's own reads, so every field is
// mutex-guarded. failNext, when > 0, makes ListActive return an error that many
// times before succeeding, to exercise the bounded boot-warm retry.
type fakeDurable struct {
mu sync.Mutex
rows map[Key]Revocation
upserts int
prunes int
failNext int
lists int
}
func newFakeDurable() *fakeDurable {
return &fakeDurable{rows: make(map[Key]Revocation)}
}
func (f *fakeDurable) Upsert(_ context.Context, r Revocation) error {
f.mu.Lock()
defer f.mu.Unlock()
f.rows[r.key()] = r
f.upserts++
return nil
}
func (f *fakeDurable) ListActive(_ context.Context) ([]Revocation, error) {
f.mu.Lock()
defer f.mu.Unlock()
f.lists++
if f.failNext > 0 {
f.failNext--
return nil, errFakeDurable
}
now := time.Now()
out := make([]Revocation, 0, len(f.rows))
for _, r := range f.rows {
if !r.expired(now) {
out = append(out, r)
}
}
return out, nil
}
func (f *fakeDurable) Prune(_ context.Context) error {
f.mu.Lock()
defer f.mu.Unlock()
now := time.Now()
for k, r := range f.rows {
if r.expired(now) {
delete(f.rows, k)
}
}
f.prunes++
return nil
}
func (f *fakeDurable) upsertCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return f.upserts
}
func (f *fakeDurable) pruneCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return f.prunes
}
func (f *fakeDurable) listCount() int {
f.mu.Lock()
defer f.mu.Unlock()
return f.lists
}
// TestRevokeMirrorsToDurable asserts Revoke writes through to the durable store.
func TestRevokeMirrorsToDurable(t *testing.T) {
ctx := context.Background()
fake := newFakeDurable()
s := New(Options{Durable: fake})
if err := s.RevokeSession(ctx, "sess-1", "abuse"); err != nil {
t.Fatalf("RevokeSession: %v", err)
}
if got := fake.upsertCount(); got != 1 {
t.Fatalf("durable upserts = %d, want 1", got)
}
// Re-revoking the same session upserts the same row (bounded growth).
if err := s.RevokeSession(ctx, "sess-1", "abuse again"); err != nil {
t.Fatalf("RevokeSession: %v", err)
}
fake.mu.Lock()
rows := len(fake.rows)
fake.mu.Unlock()
if rows != 1 {
t.Fatalf("durable rows after re-revoke = %d, want 1", rows)
}
}
// TestStartSyncWarmsFromDurable simulates a restart: a fresh Store must
// repopulate its hot-path map from the durable mirror, and skip expired rows.
func TestStartSyncWarmsFromDurable(t *testing.T) {
// Cancel the poll goroutine StartSync spawns so it does not leak past the test.
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
fake := newFakeDurable()
// Seed the durable store as if a previous process had written these.
active := Revocation{Kind: KindSession, ID: "sess-live", Reason: "x", RevokedAt: time.Now(), ExpiresAt: time.Now().Add(time.Hour)}
expired := Revocation{Kind: KindUser, ID: "9", Reason: "x", RevokedAt: time.Now().Add(-2 * time.Hour), ExpiresAt: time.Now().Add(-time.Hour)}
_ = fake.Upsert(ctx, active)
_ = fake.Upsert(ctx, expired)
s := New(Options{Durable: fake})
s.StartSync(ctx)
if !s.IsRevoked("sess-live", 0, time.Time{}) {
t.Fatalf("expected sess-live to be revoked after warm from durable")
}
// startedAt predates the (expired) user revocation, so only expiry decides.
if s.IsRevoked("whatever", 9, time.Now().Add(-3*time.Hour)) {
t.Fatalf("expected expired user 9 not to be warmed into the cache")
}
}
// TestWarmFromDurableRetries proves the bounded boot-warm retry (Fix 2): a
// transient ListActive failure at startup is retried rather than leaving the
// kill list empty until the first poll tick.
func TestWarmFromDurableRetries(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
t.Cleanup(cancel)
fake := newFakeDurable()
live := Revocation{Kind: KindSession, ID: "sess-retry", Reason: "x", RevokedAt: time.Now(), ExpiresAt: time.Now().Add(time.Hour)}
_ = fake.Upsert(ctx, live)
// Fail the first two ListActive calls; the third (within the 3-attempt
// bound) succeeds.
fake.mu.Lock()
fake.failNext = 2
fake.lists = 0
fake.mu.Unlock()
s := New(Options{Durable: fake})
s.StartSync(ctx)
if !s.IsRevoked("sess-retry", 0, time.Time{}) {
t.Fatalf("expected sess-retry warmed into cache after retried ListActive")
}
if got := fake.listCount(); got != 3 {
t.Fatalf("ListActive calls = %d, want 3 (two failures + one success)", got)
}
}
// TestWarmFromDurableGivesUpBounded proves the retry is bounded: when every
// attempt fails, warmFromDurable stops after the fixed attempt budget instead
// of blocking startup.
func TestWarmFromDurableGivesUpBounded(t *testing.T) {
fake := newFakeDurable()
fake.mu.Lock()
fake.failNext = 100 // always fail
fake.mu.Unlock()
s := New(Options{Durable: fake})
if _, err := s.warmFromDurable(context.Background()); err == nil {
t.Fatalf("expected warmFromDurable to return an error when all attempts fail")
}
if got := fake.listCount(); got != 3 {
t.Fatalf("ListActive attempts = %d, want 3 (bounded)", got)
}
}
// TestMaintainPrunesAndReWarmsFromDurable asserts the poll-tick body calls the
// durable Prune and re-warms live rows into the hot-path map (self-healing a
// Redis flush that cleared the local cache).
func TestMaintainPrunesAndReWarmsFromDurable(t *testing.T) {
ctx := context.Background()
fake := newFakeDurable()
live := Revocation{Kind: KindSession, ID: "sess-heal", Reason: "x", RevokedAt: time.Now(), ExpiresAt: time.Now().Add(time.Hour)}
dead := Revocation{Kind: KindSession, ID: "sess-dead", Reason: "x", RevokedAt: time.Now().Add(-2 * time.Hour), ExpiresAt: time.Now().Add(-time.Hour)}
_ = fake.Upsert(ctx, live)
_ = fake.Upsert(ctx, dead)
s := New(Options{Durable: fake})
// Local cache starts empty (as after a Redis flush + local reconcile miss).
if s.IsRevoked("sess-heal", 0, time.Time{}) {
t.Fatalf("did not expect sess-heal in a fresh empty cache")
}
s.maintain(ctx)
if got := fake.pruneCount(); got != 1 {
t.Fatalf("durable prune calls = %d, want 1", got)
}
if !s.IsRevoked("sess-heal", 0, time.Time{}) {
t.Fatalf("expected sess-heal re-warmed into cache by maintain")
}
fake.mu.Lock()
_, deadStillThere := fake.rows[dead.key()]
fake.mu.Unlock()
if deadStillThere {
t.Fatalf("expected expired durable row to be pruned")
}
}
func TestIsRevoked(t *testing.T) {
ctx := context.Background()
// preCutoff stands in for a stream credential issued before any revocation
// a test writes; user-kind matching requires the credential to predate the
// revocation (the cutoff), session-kind matching ignores it.
preCutoff := time.Now().Add(-time.Minute)
tests := []struct {
name string
setup func(t *testing.T, s *Store)
sessionID string
userID int
startedAt time.Time
wantRevoke bool
}{
{
name: "revoked session is revoked",
setup: func(t *testing.T, s *Store) {
if err := s.RevokeSession(ctx, "sess-1", "abuse"); err != nil {
t.Fatalf("RevokeSession: %v", err)
}
},
sessionID: "sess-1",
userID: 42,
startedAt: preCutoff,
wantRevoke: true,
},
{
name: "revoked user revokes that user's pre-revocation streams",
setup: func(t *testing.T, s *Store) {
if err := s.RevokeUser(ctx, 7, "banned"); err != nil {
t.Fatalf("RevokeUser: %v", err)
}
},
sessionID: "any-session-for-user-7",
userID: 7,
startedAt: preCutoff,
wantRevoke: true,
},
{
name: "user revocation is a cutoff: a post-revocation credential plays",
setup: func(t *testing.T, s *Store) {
if err := s.RevokeUser(ctx, 7, "sessions_revoked"); err != nil {
t.Fatalf("RevokeUser: %v", err)
}
},
sessionID: "fresh-session-for-user-7",
userID: 7,
startedAt: time.Now().Add(time.Second),
wantRevoke: false,
},
{
name: "user revocation never matches an unknown credential time (fail open)",
setup: func(t *testing.T, s *Store) {
if err := s.RevokeUser(ctx, 7, "banned"); err != nil {
t.Fatalf("RevokeUser: %v", err)
}
},
sessionID: "sess-without-start-time",
userID: 7,
startedAt: time.Time{},
wantRevoke: false,
},
{
name: "session revocation ignores the credential time",
setup: func(t *testing.T, s *Store) {
if err := s.RevokeSession(ctx, "sess-exact", "admin_terminate"); err != nil {
t.Fatalf("RevokeSession: %v", err)
}
},
sessionID: "sess-exact",
userID: 42,
startedAt: time.Now().Add(time.Hour),
wantRevoke: true,
},
{
name: "expired revocation is not revoked",
setup: func(t *testing.T, s *Store) {
past := time.Now().Add(-time.Minute)
if err := s.Revoke(ctx, Key{Kind: KindSession, ID: "sess-old"}, "stale", past); err != nil {
t.Fatalf("Revoke: %v", err)
}
},
sessionID: "sess-old",
userID: 1,
startedAt: preCutoff,
wantRevoke: false,
},
{
name: "unrelated session and user are not revoked",
setup: func(t *testing.T, s *Store) {},
sessionID: "unknown",
userID: 999,
startedAt: preCutoff,
wantRevoke: false,
},
{
name: "unrelated session with a different revoked user is not revoked",
setup: func(t *testing.T, s *Store) {
if err := s.RevokeUser(ctx, 7, "banned"); err != nil {
t.Fatalf("RevokeUser: %v", err)
}
},
sessionID: "sess-other",
userID: 8,
startedAt: preCutoff,
wantRevoke: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := newMemStore()
tt.setup(t, s)
if got := s.IsRevoked(tt.sessionID, tt.userID, tt.startedAt); got != tt.wantRevoke {
t.Fatalf("IsRevoked(%q, %d, %v) = %v, want %v", tt.sessionID, tt.userID, tt.startedAt, got, tt.wantRevoke)
}
})
}
}
func TestList(t *testing.T) {
ctx := context.Background()
s := newMemStore()
if got := s.List(); len(got) != 0 {
t.Fatalf("List on empty store = %d entries, want 0", len(got))
}
if err := s.RevokeSession(ctx, "sess-a", "r1"); err != nil {
t.Fatalf("RevokeSession: %v", err)
}
if err := s.RevokeUser(ctx, 5, "r2"); err != nil {
t.Fatalf("RevokeUser: %v", err)
}
// An expired entry must not appear in List.
if err := s.Revoke(ctx, Key{Kind: KindSession, ID: "sess-exp"}, "r3", time.Now().Add(-time.Second)); err != nil {
t.Fatalf("Revoke: %v", err)
}
got := s.List()
if len(got) != 2 {
t.Fatalf("List = %d active entries, want 2: %+v", len(got), got)
}
seen := make(map[Key]Revocation, len(got))
for _, r := range got {
seen[Key{Kind: r.Kind, ID: r.ID}] = r
}
if _, ok := seen[Key{Kind: KindSession, ID: "sess-a"}]; !ok {
t.Errorf("List missing revoked session sess-a")
}
if _, ok := seen[Key{Kind: KindUser, ID: "5"}]; !ok {
t.Errorf("List missing revoked user 5")
}
if _, ok := seen[Key{Kind: KindSession, ID: "sess-exp"}]; ok {
t.Errorf("List returned expired entry sess-exp")
}
}
func TestExpiryPrunesFromCache(t *testing.T) {
ctx := context.Background()
s := newMemStore()
// Revoke with a short future TTL, then confirm it lapses.
until := time.Now().Add(30 * time.Millisecond)
if err := s.Revoke(ctx, Key{Kind: KindUser, ID: "3"}, "temp", until); err != nil {
t.Fatalf("Revoke: %v", err)
}
preCutoff := time.Now().Add(-time.Minute)
if !s.IsRevoked("whatever", 3, preCutoff) {
t.Fatalf("expected user 3 to be revoked before expiry")
}
time.Sleep(50 * time.Millisecond)
if s.IsRevoked("whatever", 3, preCutoff) {
t.Fatalf("expected user 3 to no longer be revoked after expiry")
}
if got := s.List(); len(got) != 0 {
t.Fatalf("List after expiry = %d entries, want 0", len(got))
}
}
// TestMonotonicExpiry guards the invariant that a re-revoke with a SHORTER TTL
// can never shorten a longer-lived kill. The async over-cap enforcer re-revokes
// with a short self-healing TTL; without monotonic expiry it would shrink an
// admin's 24h RevokeSession on the same session key and reopen the
// restart-resurrection window.
func TestMonotonicExpiry(t *testing.T) {
ctx := context.Background()
s := newMemStore()
long := time.Now().Add(24 * time.Hour)
if err := s.Revoke(ctx, Key{Kind: KindSession, ID: "sess-1"}, "admin_terminate", long); err != nil {
t.Fatalf("Revoke long: %v", err)
}
// A shorter re-revoke (the enforcer's 5m self-heal TTL) must not win.
short := time.Now().Add(5 * time.Minute)
if err := s.Revoke(ctx, Key{Kind: KindSession, ID: "sess-1"}, "over_cap", short); err != nil {
t.Fatalf("Revoke short: %v", err)
}
got := s.List()
if len(got) != 1 {
t.Fatalf("List = %d entries, want 1", len(got))
}
if !got[0].ExpiresAt.Equal(long) {
t.Fatalf("ExpiresAt = %v, want the longer %v (shorter re-revoke must not shorten the kill)", got[0].ExpiresAt, long)
}
// A LONGER re-revoke does extend the kill.
longer := time.Now().Add(48 * time.Hour)
if err := s.Revoke(ctx, Key{Kind: KindSession, ID: "sess-1"}, "extended", longer); err != nil {
t.Fatalf("Revoke longer: %v", err)
}
got = s.List()
if len(got) != 1 || !got[0].ExpiresAt.Equal(longer) {
t.Fatalf("ExpiresAt after longer re-revoke = %v, want %v", got[0].ExpiresAt, longer)
}
}
// TestUserZeroNeverMatches guards the "no resolved owner" sentinel: a session-
// only check (userID 0, e.g. from the transcode node) must never be caught by a
// stray user:"0" revocation, which would read as "every ownerless request is
// revoked".
func TestUserZeroNeverMatches(t *testing.T) {
ctx := context.Background()
s := newMemStore()
// RevokeUser(0) is rejected outright: IsRevoked never matches a user:0 entry,
// so creating one would be a silently-ineffective kill. Guarding here means no
// stray user:0 revocation can exist to be misread as "every ownerless request
// is revoked".
if err := s.RevokeUser(ctx, 0, "should-not-nuke-everything"); err == nil {
t.Fatalf("RevokeUser(0) must be rejected, got nil error")
}
if s.IsRevoked("some-unrelated-session", 0, time.Now().Add(-time.Minute)) {
t.Fatalf("userID 0 sentinel must not match a user:0 revocation")
}
// A real session revocation still works with a 0 owner id.
if err := s.RevokeSession(ctx, "sess-x", "abuse"); err != nil {
t.Fatalf("RevokeSession: %v", err)
}
if !s.IsRevoked("sess-x", 0, time.Time{}) {
t.Fatalf("expected sess-x to be revoked even with owner id 0")
}
}