haeaeae/plezy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
0fbf2e205c8e8d797ed6f3da45db62e28f1d283f
plezy/server/main_test.go
T
edde746 6ab46ced54 fix(relay): claim OAuth callbacks before the code exchange and bound poster and poll traffic per IP 
Two callbacks with the same state could both exchange the authorization code upstream (concurrently or by replay); the session is now atomically claimed under the proxy lock before the exchange.

GET /posters/ and the /auth/result long-poll had no per-IP limiting or concurrency bound, and every poster lookup serialized through an exclusive store lock; both endpoints now use the established limiter and non-expired hits take a read lock. The limiter also tracks active transfers per IP with caps below the global limits (4 for fetches, 2 for uploads), because one unauthenticated client could otherwise take all 16 global poster-fetch slots and hold them through slow ServeContent reads, starving everyone else with 429s; concurrency checks precede bucket charges so a capped denial consumes no admission tokens.

/auth/result gets its own per-IP budget instead of sharing /auth/start's burst-3 bucket, where two concurrent NAT'd sign-ins 429'd on the fourth request. It is charged only after the poll secret validates (bogus requests keep the generic 410 and cost nothing), denials carry an honest Retry-After, and the Dart poller - which treated 429 as terminal and abandoned a valid session - retries them until the session's 10-minute lifetime expires.
2026-08-21 19:23:43 +02:00

7973 lines
240 KiB
Go
Raw Blame History

package main
import (
"bufio"
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/gorilla/websocket"
)
func TestGeneratedRelayProtocolVersionsMatchSpec(t *testing.T) {
data, err := os.ReadFile(filepath.Join("..", "relay_protocol.json"))
if err != nil {
t.Fatalf("read relay protocol spec: %v", err)
}
var spec struct {
ProtocolVersion int `json:"protocolVersion"`
LegacyProtocolVersion int `json:"legacyProtocolVersion"`
}
if err := json.Unmarshal(data, &spec); err != nil {
t.Fatalf("decode relay protocol spec: %v", err)
}
if relayProtocolVersion != spec.ProtocolVersion ||
legacyRelayProtocolVersion != spec.LegacyProtocolVersion {
t.Fatalf(
"generated versions=(%d,%d), spec=(%d,%d)",
relayProtocolVersion,
legacyRelayProtocolVersion,
spec.ProtocolVersion,
spec.LegacyProtocolVersion,
)
}
}
// newTestServer builds a goroutine-free, network-free test server.
func newTestServer(t *testing.T, stateFile string) *Server {
t.Helper()
s := &Server{
rooms: make(map[string]*Room),
logs: newLogStore(t.TempDir()),
posters: newPosterStore(t.TempDir(), maxPosterStoreSize, posterMaxAge),
posterUploads: newPosterUploadLimiter(posterPerIPRateBurst, posterPerIPRateSustained, posterGlobalRateBurst, posterGlobalRateSustained, maxConcurrentPosterUploads, maxConcurrentPosterUploadsPerIP, time.Now()),
posterFetches: newPosterUploadLimiter(posterFetchPerIPRateBurst, posterFetchPerIPRateSustained, posterFetchGlobalRateBurst, posterFetchGlobalRateSustained, maxConcurrentPosterFetches, maxConcurrentPosterFetchesPerIP, time.Now()),
conns: newConnTracker(),
clientIPs: newClientIPResolver(nil),
}
s.snap = newSnapshotter(stateFile, s.buildSnapshot)
return s
}
func mustReconnectToken(t *testing.T) (string, reconnectVerifier) {
t.Helper()
token, verifier, err := mintReconnectToken()
if err != nil {
t.Fatalf("mint reconnect token: %v", err)
}
return token, verifier
}
func makeRoomSnapshots(count int, maximumLengthIDs bool, now time.Time) []roomSnapshot {
rooms := make([]roomSnapshot, 0, count)
for i := range count {
suffix := fmt.Sprintf("%04d", i)
sessionID := "S" + suffix
hostPeerID := "H"
if maximumLengthIDs {
sessionID = suffix + strings.Repeat("S", maxSessionIDLength-len(suffix))
hostPeerID = strings.Repeat("H", maxPeerIDLength)
}
rooms = append(rooms, roomSnapshot{
SessionID: sessionID,
HostPeerID: hostPeerID,
HostReconnectVerifier: encodeReconnectVerifier(reconnectVerifier{1}),
CreatedAt: now.Add(-time.Minute),
LastActivityAt: now,
})
}
return rooms
}
func TestSnapshotRoundTrip(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "rooms.json")
s := newTestServer(t, path)
now := time.Now().UTC().Truncate(time.Second)
_, verifier1 := mustReconnectToken(t)
_, verifier2 := mustReconnectToken(t)
s.rooms["ABC12"] = &Room{
SessionID: "ABC12",
HostPeerID: "host-1",
hostVerifier: verifier1,
peerReservations: make(map[string]peerReservation),
Peers: map[string]*Client{},
CreatedAt: now.Add(-time.Minute),
LastActivityAt: now,
quotaOwnerKey: "203.0.113.44",
}
s.rooms["XYZ99"] = &Room{
SessionID: "XYZ99",
HostPeerID: "host-2",
hostVerifier: verifier2,
peerReservations: make(map[string]peerReservation),
Peers: map[string]*Client{},
CreatedAt: now.Add(-time.Hour),
LastActivityAt: now.Add(-time.Second),
}
if err := s.snap.write(); err != nil {
t.Fatalf("write: %v", err)
}
data, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read snapshot: %v", err)
}
if bytes.Contains(data, []byte("203.0.113.44")) || bytes.Contains(data, []byte("quotaOwner")) {
t.Fatalf("snapshot persisted process-local client identity: %s", data)
}
// Reload from disk into a fresh server.
s2 := newTestServer(t, path)
if _, err := s2.loadSnapshot(path); err != nil {
t.Fatalf("loadSnapshot: %v", err)
}
if got := len(s2.rooms); got != 2 {
t.Fatalf("expected 2 rooms after reload, got %d", got)
}
for _, id := range []string{"ABC12", "XYZ99"} {
r, ok := s2.rooms[id]
if !ok {
t.Fatalf("room %s missing after reload", id)
}
orig := s.rooms[id]
if r.HostPeerID != orig.HostPeerID {
t.Errorf("%s: HostPeerID=%q want %q", id, r.HostPeerID, orig.HostPeerID)
}
if !reconnectVerifierMatches(r.hostVerifier, orig.hostVerifier) {
t.Errorf("%s: host reconnect verifier did not round-trip", id)
}
if !r.CreatedAt.Equal(orig.CreatedAt) {
t.Errorf("%s: CreatedAt=%v want %v", id, r.CreatedAt, orig.CreatedAt)
}
if !r.LastActivityAt.Equal(orig.LastActivityAt) {
t.Errorf("%s: LastActivityAt=%v want %v", id, r.LastActivityAt, orig.LastActivityAt)
}
if r.quotaOwnerKey != "" {
t.Errorf("%s: quotaOwnerKey=%q after reload, want empty", id, r.quotaOwnerKey)
}
if r.Peers == nil {
t.Errorf("%s: Peers map nil after reload", id)
}
if len(r.Peers) != 0 {
t.Errorf("%s: expected empty Peers, got %d", id, len(r.Peers))
}
}
s2.conns.mu.Lock()
defer s2.conns.mu.Unlock()
if len(s2.conns.roomsPerIP) != 0 {
t.Fatalf("reload restored process-local room quota: %v", s2.conns.roomsPerIP)
}
}
func TestLoadSkipsExpired(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "rooms.json")
now := time.Now()
_, verifier := mustReconnectToken(t)
encodedVerifier := encodeReconnectVerifier(verifier)
snap := stateSnapshot{
Version: snapshotFormatVersion,
SavedAt: now,
Rooms: []roomSnapshot{
{SessionID: "FRESH", HostPeerID: "h", HostReconnectVerifier: encodedVerifier, CreatedAt: now.Add(-time.Minute), LastActivityAt: now.Add(-30 * time.Second)},
{SessionID: "OLD24", HostPeerID: "h", HostReconnectVerifier: encodedVerifier, CreatedAt: now.Add(-25 * time.Hour), LastActivityAt: now.Add(-time.Second)},
{SessionID: "IDLE6", HostPeerID: "h", HostReconnectVerifier: encodedVerifier, CreatedAt: now.Add(-2 * time.Hour), LastActivityAt: now.Add(-6 * time.Minute)},
{SessionID: "", HostPeerID: "h", HostReconnectVerifier: encodedVerifier, CreatedAt: now, LastActivityAt: now},
{SessionID: "NOHOS", HostPeerID: "", HostReconnectVerifier: encodedVerifier, CreatedAt: now, LastActivityAt: now},
},
}
data, err := json.Marshal(snap)
if err != nil {
t.Fatalf("marshal: %v", err)
}
if err := os.WriteFile(path, data, 0644); err != nil {
t.Fatalf("write: %v", err)
}
s := newTestServer(t, path)
if _, err := s.loadSnapshot(path); err != nil {
t.Fatalf("loadSnapshot: %v", err)
}
if len(s.rooms) != 1 {
t.Fatalf("expected 1 room after load, got %d: %v", len(s.rooms), s.rooms)
}
if _, ok := s.rooms["FRESH"]; !ok {
t.Fatalf("FRESH room should have loaded")
}
}
func TestLoadHandlesCorrupt(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "rooms.json")
if err := os.WriteFile(path, []byte("not valid json {{{"), 0644); err != nil {
t.Fatalf("write: %v", err)
}
s := newTestServer(t, path)
if _, err := s.loadSnapshot(path); err != nil {
t.Fatalf("loadSnapshot returned error: %v", err)
}
if len(s.rooms) != 0 {
t.Fatalf("expected empty rooms after corrupt load, got %d", len(s.rooms))
}
// Corrupt snapshots remain available for diagnosis.
if _, err := os.Stat(path); err != nil {
t.Fatalf("corrupt file should NOT be deleted: %v", err)
}
}
func TestLoadHandlesMissing(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "does-not-exist.json")
s := newTestServer(t, path)
if _, err := s.loadSnapshot(path); err != nil {
t.Fatalf("loadSnapshot: %v", err)
}
if len(s.rooms) != 0 {
t.Fatalf("expected empty rooms, got %d", len(s.rooms))
}
}
func TestLoadRejectsSnapshotsWithoutHostAuthority(t *testing.T) {
for _, version := range []int{1, 99} {
t.Run(fmt.Sprintf("version_%d", version), func(t *testing.T) {
path := filepath.Join(t.TempDir(), "rooms.json")
data := []byte(fmt.Sprintf(`{"version":%d,"rooms":[{"sessionId":"X","hostPeerId":"H"}]}`, version))
if err := os.WriteFile(path, data, 0644); err != nil {
t.Fatalf("write: %v", err)
}
s := newTestServer(t, path)
if _, err := s.loadSnapshot(path); err != nil {
t.Fatalf("loadSnapshot: %v", err)
}
if len(s.rooms) != 0 {
t.Fatalf("expected empty rooms for version %d, got %d", version, len(s.rooms))
}
})
}
}
func TestCleanupUsesIdleNotAge(t *testing.T) {
s := newTestServer(t, filepath.Join(t.TempDir(), "rooms.json"))
now := time.Now()
// Recent activity keeps this old room.
s.rooms["KEEP"] = &Room{
SessionID: "KEEP",
HostPeerID: "h",
Peers: map[string]*Client{},
CreatedAt: now.Add(-2 * time.Hour),
LastActivityAt: now.Add(-1 * time.Minute),
}
// Idle rooms are removed.
s.rooms["GONE"] = &Room{
SessionID: "GONE",
HostPeerID: "h",
Peers: map[string]*Client{},
CreatedAt: now.Add(-2 * time.Hour),
LastActivityAt: now.Add(-10 * time.Minute),
}
// The absolute TTL removes this room despite recent activity.
s.rooms["OLD"] = &Room{
SessionID: "OLD",
HostPeerID: "h",
Peers: map[string]*Client{},
CreatedAt: now.Add(-25 * time.Hour),
LastActivityAt: now.Add(-10 * time.Second),
}
s.runCleanupStep(now)
if _, ok := s.rooms["KEEP"]; !ok {
t.Errorf("KEEP should still exist (recent activity)")
}
if _, ok := s.rooms["GONE"]; ok {
t.Errorf("GONE should have been cleaned (idle>5min)")
}
if _, ok := s.rooms["OLD"]; ok {
t.Errorf("OLD should have been cleaned (age>24h)")
}
}
func TestRemoveRoomLockedReleasesOwnedQuotaExactlyOnce(t *testing.T) {
s := newTestServer(t, filepath.Join(t.TempDir(), "rooms.json"))
ownerKey := "203.0.113.8"
room := &Room{
SessionID: "OWNED",
HostPeerID: "H",
Peers: map[string]*Client{},
quotaOwnerKey: ownerKey,
}
s.rooms[room.SessionID] = room
if !s.conns.tryCreateRoom(ownerKey) {
t.Fatal("reserve room quota")
}
s.mu.Lock()
firstRemoval := s.removeRoomLocked(room.SessionID, room)
secondRemoval := s.removeRoomLocked(room.SessionID, room)
s.mu.Unlock()
if !firstRemoval || secondRemoval {
t.Fatalf("first removal=%v second removal=%v, want true then false", firstRemoval, secondRemoval)
}
s.conns.mu.Lock()
remaining := s.conns.roomsPerIP[ownerKey]
s.conns.mu.Unlock()
if remaining != 0 {
t.Fatalf("quota after repeated removal=%d, want 0", remaining)
}
replacement := &Room{SessionID: room.SessionID, Peers: map[string]*Client{}}
s.mu.Lock()
s.rooms[room.SessionID] = replacement
staleRemoval := s.removeRoomLocked(room.SessionID, room)
authoritative := s.rooms[room.SessionID]
s.mu.Unlock()
if staleRemoval || authoritative != replacement {
t.Fatal("stale pointer removed the authoritative replacement")
}
}
func TestSnapshotAtomicWriteSurvivesRenameFailure(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "rooms.json")
s := newTestServer(t, path)
// Seed the on-disk snapshot.
s.rooms["ORIG"] = &Room{
SessionID: "ORIG",
HostPeerID: "h",
Peers: map[string]*Client{},
CreatedAt: time.Now(),
LastActivityAt: time.Now(),
}
if err := s.snap.write(); err != nil {
t.Fatalf("first write: %v", err)
}
origBytes, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read orig: %v", err)
}
// A directory at the temporary path makes the open fail before rename.
if err := os.Mkdir(path+".tmp", 0755); err != nil {
t.Fatalf("create blocking temporary directory: %v", err)
}
delete(s.rooms, "ORIG")
s.rooms["NEW"] = &Room{
SessionID: "NEW",
HostPeerID: "h",
Peers: map[string]*Client{},
CreatedAt: time.Now(),
LastActivityAt: time.Now(),
}
if err := s.snap.write(); err == nil {
t.Fatal("expected pre-rename snapshot write failure")
}
nowBytes, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read after failed write: %v", err)
}
if string(origBytes) != string(nowBytes) {
t.Fatalf("snapshot file was corrupted after failed write:\nbefore: %s\nafter: %s", origBytes, nowBytes)
}
}
func TestSnapshotRenameFailureIsPreCommit(t *testing.T) {
root := t.TempDir()
path := filepath.Join(root, "rooms.json")
if err := os.Mkdir(path, 0755); err != nil {
t.Fatalf("create conflicting snapshot directory: %v", err)
}
markerPath := filepath.Join(path, "marker")
if err := os.WriteFile(markerPath, []byte("preserved"), 0644); err != nil {
t.Fatalf("write destination marker: %v", err)
}
snapshot := stateSnapshot{
Version: snapshotFormatVersion,
SavedAt: time.Now(),
}
sn := newSnapshotter(path, func() stateSnapshot { return snapshot })
syncCalled := false
sn.syncDir = func(string) error {
syncCalled = true
return nil
}
if err := sn.write(); err == nil {
t.Fatal("snapshot rename over a directory succeeded")
}
if syncCalled {
t.Fatal("directory sync ran after failed rename")
}
marker, err := os.ReadFile(markerPath)
if err != nil {
t.Fatalf("read destination marker after failed rename: %v", err)
}
if string(marker) != "preserved" {
t.Fatalf("rename failure changed destination marker: %q", marker)
}
if _, err := os.Stat(path + ".tmp"); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("temporary snapshot remains after failed rename: %v", err)
}
}
func TestSnapshotWriteRejectsOversizeAndPreservesLastValidFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "rooms.json")
s := newTestServer(t, path)
now := time.Now()
s.rooms["ORIG"] = &Room{
SessionID: "ORIG",
HostPeerID: "H",
Peers: map[string]*Client{},
CreatedAt: now,
LastActivityAt: now,
}
if err := s.snap.write(); err != nil {
t.Fatalf("write valid snapshot: %v", err)
}
original, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read valid snapshot: %v", err)
}
s.snap.build = func() stateSnapshot {
return stateSnapshot{
Version: snapshotFormatVersion,
SavedAt: now,
Rooms: []roomSnapshot{{
SessionID: strings.Repeat("S", snapshotMaxFileSize),
HostPeerID: "H",
CreatedAt: now,
LastActivityAt: now,
}},
}
}
if err := s.snap.write(); err == nil {
t.Fatal("oversized snapshot write succeeded")
}
after, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read preserved snapshot: %v", err)
}
if !bytes.Equal(after, original) {
t.Fatal("oversized write replaced the last valid snapshot")
}
reloaded := newTestServer(t, path)
if _, err := reloaded.loadSnapshot(path); err != nil {
t.Fatalf("load preserved snapshot: %v", err)
}
if _, ok := reloaded.rooms["ORIG"]; !ok {
t.Fatal("last valid snapshot did not survive oversized write")
}
}
func TestSnapshotAtRetainedRoomCapFitsAndReloads(t *testing.T) {
path := filepath.Join(t.TempDir(), "rooms.json")
s := newTestServer(t, path)
now := time.Now().UTC()
for _, room := range makeRoomSnapshots(maxRetainedRooms, true, now) {
reservations := make(map[string]peerReservation, maxRoomSize-1)
for index := range maxRoomSize - 1 {
prefix := fmt.Sprintf("G%d", index)
peerID := prefix + strings.Repeat("G", maxPeerIDLength-len(prefix))
reservations[peerID] = peerReservation{
verifier: reconnectVerifier{byte(index + 1)},
absentSince: now.Add(-time.Minute),
}
}
s.rooms[room.SessionID] = &Room{
SessionID: room.SessionID,
HostPeerID: room.HostPeerID,
ProtocolVersion: relayProtocolVersion,
peerReservations: reservations,
Peers: map[string]*Client{},
CreatedAt: room.CreatedAt,
LastActivityAt: room.LastActivityAt,
}
}
snapshot := s.buildSnapshot()
data, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal maximum snapshot: %v", err)
}
if len(data) > snapshotMaxFileSize {
t.Fatalf("maximum admitted snapshot is %d bytes, exceeds %d", len(data), snapshotMaxFileSize)
}
if err := s.snap.write(); err != nil {
t.Fatalf("write maximum snapshot: %v", err)
}
reloaded := newTestServer(t, path)
if _, err := reloaded.loadSnapshot(path); err != nil {
t.Fatalf("load maximum snapshot: %v", err)
}
if got := len(reloaded.rooms); got != maxRetainedRooms {
t.Fatalf("reloaded rooms=%d, want %d", got, maxRetainedRooms)
}
for _, expected := range snapshot.Rooms {
room := reloaded.rooms[expected.SessionID]
if room == nil || room.Peers == nil {
t.Fatalf("room %q is not available for joins after reload", expected.SessionID)
}
if len(room.peerReservations) != maxRoomSize-1 {
t.Fatalf("room %q reloaded %d reservations, want %d", expected.SessionID, len(room.peerReservations), maxRoomSize-1)
}
}
}
func TestLoadRejectsSnapshotOverRetainedRoomCapWithoutPartialState(t *testing.T) {
path := filepath.Join(t.TempDir(), "rooms.json")
now := time.Now().UTC()
snapshot := stateSnapshot{
Version: snapshotFormatVersion,
SavedAt: now,
Rooms: makeRoomSnapshots(maxRetainedRooms+1, false, now),
}
data, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal over-count snapshot: %v", err)
}
if len(data) > snapshotMaxFileSize {
t.Fatalf("over-count fixture is %d bytes, must exercise count limit below %d", len(data), snapshotMaxFileSize)
}
if err := os.WriteFile(path, data, 0644); err != nil {
t.Fatalf("write over-count snapshot: %v", err)
}
s := newTestServer(t, path)
if _, err := s.loadSnapshot(path); err != nil {
t.Fatalf("loadSnapshot: %v", err)
}
if len(s.rooms) != 0 {
t.Fatalf("over-count snapshot partially loaded %d rooms", len(s.rooms))
}
s.conns.mu.Lock()
defer s.conns.mu.Unlock()
if len(s.conns.roomsPerIP) != 0 {
t.Fatalf("over-count snapshot changed process quota state: %v", s.conns.roomsPerIP)
}
if _, err := os.Stat(path); err != nil {
t.Fatalf("rejected snapshot was not preserved: %v", err)
}
}
func TestSnapshotDebounceCoalesces(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "rooms.json")
var (
buildCount int
countMu sync.Mutex
)
built := make(chan struct{}, 1)
sn := newSnapshotter(path, func() stateSnapshot {
countMu.Lock()
buildCount++
countMu.Unlock()
select {
case built <- struct{}{}:
default:
}
return stateSnapshot{Version: snapshotFormatVersion, SavedAt: time.Now(), Rooms: nil}
})
go sn.run()
t.Cleanup(func() { _ = sn.flushAndStop(time.Second) })
// A burst should collapse into one debounced write.
for i := 0; i < 20; i++ {
sn.recordMutation()
}
select {
case <-built:
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for debounced snapshot build")
}
quiet := time.NewTimer(5 * snapshotDebounce)
defer quiet.Stop()
select {
case <-built:
t.Fatal("debounced burst produced an unexpected trailing snapshot build")
case <-quiet.C:
}
countMu.Lock()
got := buildCount
countMu.Unlock()
if got != 1 {
t.Fatalf("expected 1 build from burst, got %d", got)
}
}
func newRunningSnapshotterForTest(
t *testing.T,
persist func([]byte) error,
) *snapshotter {
t.Helper()
sn := newSnapshotter(filepath.Join(t.TempDir(), "rooms.json"), func() stateSnapshot {
return stateSnapshot{Version: snapshotFormatVersion, SavedAt: time.Now()}
})
sn.persist = persist
go sn.run()
t.Cleanup(func() { _ = sn.flushAndStop(time.Second) })
return sn
}
func awaitTerminalOutcome(t *testing.T, sn *snapshotter, ticket *terminalMutationTicket) terminalMutationOutcome {
t.Helper()
outcome := make(chan terminalMutationOutcome, 1)
go func() {
outcome <- sn.waitForDurable(ticket)
}()
select {
case result := <-outcome:
return result
case <-time.After(2 * time.Second):
t.Fatal("timed out waiting for terminal snapshot outcome")
return terminalMutationOutcome{}
}
}
func TestSnapshotTerminalMutationBypassesDebounce(t *testing.T) {
var persistCalls atomic.Int64
sn := newSnapshotter(filepath.Join(t.TempDir(), "rooms.json"), func() stateSnapshot {
return stateSnapshot{Version: snapshotFormatVersion, SavedAt: time.Now()}
})
sn.debounce = time.Hour
sn.persist = func([]byte) error {
persistCalls.Add(1)
return nil
}
go sn.run()
t.Cleanup(func() { _ = sn.flushAndStop(time.Second) })
ticket := sn.recordTerminalMutation(nil)
if outcome := awaitTerminalOutcome(t, sn, ticket); outcome.err != nil || !outcome.deliver {
t.Fatalf("terminal outcome=%+v", outcome)
}
if got := persistCalls.Load(); got != 1 {
t.Fatalf("terminal persist calls=%d, want 1", got)
}
}
func TestSnapshotFlushInterruptsDebounce(t *testing.T) {
debounceStarted := make(chan struct{})
var debounceOnce sync.Once
var persistCalls atomic.Int64
sn := newSnapshotter(filepath.Join(t.TempDir(), "rooms.json"), func() stateSnapshot {
return stateSnapshot{Version: snapshotFormatVersion, SavedAt: time.Now()}
})
sn.debounce = time.Hour
sn.beforeDebounceWait = func() {
debounceOnce.Do(func() { close(debounceStarted) })
}
sn.persist = func([]byte) error {
persistCalls.Add(1)
return nil
}
go sn.run()
t.Cleanup(func() { _ = sn.flushAndStop(time.Second) })
sn.recordMutation()
select {
case <-debounceStarted:
case <-time.After(2 * time.Second):
t.Fatal("snapshot writer did not enter its debounce window")
}
if err := sn.flushAndStop(time.Second); err != nil {
t.Fatalf("flush during debounce: %v", err)
}
if got := persistCalls.Load(); got != 1 {
t.Fatalf("flush persist calls=%d, want 1", got)
}
}
func TestSnapshotDurableWaitersCoalesce(t *testing.T) {
started := make(chan struct{})
release := make(chan struct{})
var startOnce sync.Once
var persistCalls atomic.Int64
sn := newRunningSnapshotterForTest(t, func([]byte) error {
persistCalls.Add(1)
startOnce.Do(func() { close(started) })
<-release
return nil
})
captureReady := make(chan struct{})
releaseCapture := make(chan struct{})
t.Cleanup(func() {
select {
case <-releaseCapture:
default:
close(releaseCapture)
}
})
var captureOnce sync.Once
sn.stateMu.Lock()
sn.beforeCapture = func() {
captureOnce.Do(func() { close(captureReady) })
<-releaseCapture
}
sn.stateMu.Unlock()
tickets := make([]*terminalMutationTicket, 0, 4)
tickets = append(tickets, sn.recordTerminalMutation(nil))
select {
case <-captureReady:
case <-time.After(2 * time.Second):
t.Fatal("writer did not reach the coalescing capture barrier")
}
for range 3 {
tickets = append(tickets, sn.recordTerminalMutation(nil))
}
close(releaseCapture)
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("coalesced snapshot did not begin persistence")
}
for _, ticket := range tickets {
select {
case <-ticket.result:
t.Fatal("terminal waiter completed before persistence returned")
default:
}
}
close(release)
for _, ticket := range tickets {
outcome := awaitTerminalOutcome(t, sn, ticket)
if outcome.err != nil || !outcome.deliver {
t.Fatalf("coalesced terminal outcome=%+v", outcome)
}
}
if got := persistCalls.Load(); got != 1 {
t.Fatalf("coalesced persist calls=%d, want 1", got)
}
}
func TestSnapshotMutationAfterCaptureRequiresFollowUp(t *testing.T) {
firstStarted := make(chan struct{})
releaseFirst := make(chan struct{})
secondStarted := make(chan struct{})
releaseSecond := make(chan struct{})
var calls atomic.Int64
sn := newRunningSnapshotterForTest(t, func([]byte) error {
switch calls.Add(1) {
case 1:
close(firstStarted)
<-releaseFirst
case 2:
close(secondStarted)
<-releaseSecond
default:
return errors.New("unexpected extra snapshot persistence")
}
return nil
})
first := sn.recordTerminalMutation(nil)
select {
case <-firstStarted:
case <-time.After(2 * time.Second):
t.Fatal("first generation did not reach persistence")
}
second := sn.recordTerminalMutation(nil)
close(releaseFirst)
if outcome := awaitTerminalOutcome(t, sn, first); outcome.err != nil {
t.Fatalf("first generation outcome=%+v", outcome)
}
select {
case <-second.result:
t.Fatal("later generation was acknowledged by the earlier capture")
default:
}
select {
case <-secondStarted:
case <-time.After(2 * time.Second):
t.Fatal("later generation did not require a follow-up persistence")
}
select {
case <-second.result:
t.Fatal("later generation completed before its persistence returned")
default:
}
close(releaseSecond)
if outcome := awaitTerminalOutcome(t, sn, second); outcome.err != nil {
t.Fatalf("second generation outcome=%+v", outcome)
}
}
func TestSnapshotFailureCompletesOnlyCoveredWaiters(t *testing.T) {
injectedErr := errors.New("covered generation failed")
firstStarted := make(chan struct{})
failFirst := make(chan struct{})
secondStarted := make(chan struct{})
releaseSecond := make(chan struct{})
var calls atomic.Int64
sn := newRunningSnapshotterForTest(t, func([]byte) error {
switch calls.Add(1) {
case 1:
close(firstStarted)
<-failFirst
return injectedErr
case 2:
close(secondStarted)
<-releaseSecond
return nil
default:
return errors.New("unexpected extra snapshot persistence")
}
})
captureReady := make(chan struct{})
releaseCapture := make(chan struct{})
t.Cleanup(func() {
select {
case <-releaseCapture:
default:
close(releaseCapture)
}
})
var captureOnce sync.Once
sn.stateMu.Lock()
sn.beforeCapture = func() {
captureOnce.Do(func() { close(captureReady) })
<-releaseCapture
}
sn.stateMu.Unlock()
covered := sn.recordTerminalMutation(nil)
select {
case <-captureReady:
case <-time.After(2 * time.Second):
t.Fatal("writer did not reach the failure fan-out capture barrier")
}
coveredTwo := sn.recordTerminalMutation(nil)
close(releaseCapture)
select {
case <-firstStarted:
case <-time.After(2 * time.Second):
t.Fatal("covered generation did not reach persistence")
}
later := sn.recordTerminalMutation(nil)
close(failFirst)
if outcome := awaitTerminalOutcome(t, sn, covered); !errors.Is(outcome.err, injectedErr) {
t.Fatalf("covered waiter outcome=%+v, want injected error", outcome)
}
if outcome := awaitTerminalOutcome(t, sn, coveredTwo); !errors.Is(outcome.err, injectedErr) {
t.Fatalf("second covered waiter outcome=%+v, want injected error", outcome)
}
select {
case <-later.result:
t.Fatal("later waiter received the covered generation's failure")
default:
}
select {
case <-secondStarted:
case <-time.After(2 * time.Second):
t.Fatal("later generation did not remain retryable")
}
close(releaseSecond)
if outcome := awaitTerminalOutcome(t, sn, later); outcome.err != nil {
t.Fatalf("later waiter outcome=%+v", outcome)
}
}
func TestSnapshotFailureBeforeWaitRetainsOutcome(t *testing.T) {
injectedErr := errors.New("failed before handler waited")
completed := make(chan struct{})
sn := newRunningSnapshotterForTest(t, func([]byte) error {
return injectedErr
})
ticket := sn.recordTerminalMutation(func(err error) terminalMutationOutcome {
close(completed)
return terminalMutationOutcome{err: err, deliver: true}
})
select {
case <-completed:
case <-time.After(2 * time.Second):
t.Fatal("writer did not complete failure before waiter registration point")
}
if outcome := awaitTerminalOutcome(t, sn, ticket); !errors.Is(outcome.err, injectedErr) {
t.Fatalf("late waiter outcome=%+v, want retained failure", outcome)
}
}
func TestSnapshotCoveredTriggerDoesNotWriteAgain(t *testing.T) {
var persistCalls atomic.Int64
persisted := make(chan struct{}, 1)
sn := newRunningSnapshotterForTest(t, func([]byte) error {
persistCalls.Add(1)
persisted <- struct{}{}
return nil
})
sn.recordMutation()
ticket := sn.recordTerminalMutation(nil)
if outcome := awaitTerminalOutcome(t, sn, ticket); outcome.err != nil {
t.Fatalf("terminal outcome=%+v", outcome)
}
select {
case <-persisted:
default:
t.Fatal("covering persistence was not observed")
}
quiet := time.NewTimer(5 * snapshotDebounce)
defer quiet.Stop()
select {
case <-persisted:
t.Fatal("covered trigger caused a trailing persistence")
case <-quiet.C:
}
if got := persistCalls.Load(); got != 1 {
t.Fatalf("covered trigger persist calls=%d, want 1", got)
}
}
func TestSnapshotFlushAndStopIncludesMutationDuringWrite(t *testing.T) {
firstStarted := make(chan struct{})
releaseFirst := make(chan struct{})
secondPersisted := make(chan struct{})
var calls atomic.Int64
sn := newSnapshotter(filepath.Join(t.TempDir(), "rooms.json"), func() stateSnapshot {
return stateSnapshot{Version: snapshotFormatVersion, SavedAt: time.Now()}
})
sn.persist = func([]byte) error {
switch calls.Add(1) {
case 1:
close(firstStarted)
<-releaseFirst
case 2:
close(secondPersisted)
default:
return errors.New("shutdown performed an unexpected extra persistence")
}
return nil
}
go sn.run()
sn.recordMutation()
result := make(chan error, 1)
go func() {
result <- sn.flushAndStop(2 * time.Second)
}()
select {
case <-firstStarted:
case <-time.After(2 * time.Second):
t.Fatal("shutdown write did not begin")
}
sn.recordMutation()
close(releaseFirst)
select {
case <-secondPersisted:
case <-time.After(2 * time.Second):
t.Fatal("mutation accepted during shutdown write was not persisted")
}
select {
case err := <-result:
if err != nil {
t.Fatalf("flushAndStop: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("flushAndStop did not finish")
}
}
func TestSnapshotFlushAndStopFailureResolvesWaiter(t *testing.T) {
injectedErr := errors.New("shutdown persistence failed")
started := make(chan struct{})
release := make(chan struct{})
var startOnce sync.Once
sn := newSnapshotter(filepath.Join(t.TempDir(), "rooms.json"), func() stateSnapshot {
return stateSnapshot{Version: snapshotFormatVersion, SavedAt: time.Now()}
})
sn.persist = func([]byte) error {
startOnce.Do(func() { close(started) })
<-release
return injectedErr
}
go sn.run()
ticket := sn.recordTerminalMutation(nil)
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("terminal write did not begin")
}
flushResult := make(chan error, 1)
go func() {
flushResult <- sn.flushAndStop(2 * time.Second)
}()
close(release)
if outcome := awaitTerminalOutcome(t, sn, ticket); !errors.Is(outcome.err, injectedErr) {
t.Fatalf("shutdown waiter outcome=%+v", outcome)
}
select {
case err := <-flushResult:
if !errors.Is(err, injectedErr) {
t.Fatalf("flushAndStop error=%v, want %v", err, injectedErr)
}
case <-time.After(2 * time.Second):
t.Fatal("failed flushAndStop did not return")
}
}
func TestSnapshotDirectorySyncFailureIsBestEffort(t *testing.T) {
for _, failure := range []struct {
name string
err error
}{
{name: "open", err: &os.PathError{Op: "open", Path: "snapshot-dir", Err: errors.New("permission denied")}},
{name: "sync", err: &os.PathError{Op: "sync", Path: "snapshot-dir", Err: errors.New("unsupported")}},
{name: "close", err: &os.PathError{Op: "close", Path: "snapshot-dir", Err: errors.New("close failed")}},
} {
t.Run(failure.name, func(t *testing.T) {
path := filepath.Join(t.TempDir(), "rooms.json")
now := time.Now().UTC()
snapshot := stateSnapshot{
Version: snapshotFormatVersion,
SavedAt: now,
Rooms: []roomSnapshot{{
SessionID: "DIRSYNC",
HostPeerID: "H",
HostReconnectVerifier: encodeReconnectVerifier(reconnectVerifier{1}),
CreatedAt: now,
LastActivityAt: now,
}},
}
data, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal snapshot: %v", err)
}
sn := newSnapshotter(path, func() stateSnapshot { return snapshot })
sn.syncDir = func(string) error { return failure.err }
if err := sn.persistAtomic(data); err != nil {
t.Fatalf("post-rename directory sync was reported as uncommitted: %v", err)
}
replacement, err := os.ReadFile(path)
if err != nil {
t.Fatalf("read committed replacement: %v", err)
}
if !bytes.Equal(replacement, data) {
t.Fatal("committed replacement bytes changed")
}
if _, err := os.Stat(path + ".tmp"); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("temporary snapshot remains after commit: %v", err)
}
reloaded := newTestServer(t, path)
if _, err := reloaded.loadSnapshot(path); err != nil {
t.Fatalf("reload committed replacement: %v", err)
}
if reloaded.rooms["DIRSYNC"] == nil {
t.Fatal("committed replacement was not reloadable")
}
})
}
}
func TestSnapshotDirectorySyncWarningIsThrottled(t *testing.T) {
var output bytes.Buffer
previousOutput := log.Writer()
previousFlags := log.Flags()
previousPrefix := log.Prefix()
log.SetOutput(&output)
log.SetFlags(0)
log.SetPrefix("")
t.Cleanup(func() {
log.SetOutput(previousOutput)
log.SetFlags(previousFlags)
log.SetPrefix(previousPrefix)
})
path := filepath.Join(t.TempDir(), "rooms.json")
snapshot := stateSnapshot{Version: snapshotFormatVersion, SavedAt: time.Now()}
data, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal snapshot: %v", err)
}
sn := newSnapshotter(path, func() stateSnapshot { return snapshot })
dirErr := errors.New("directory sync unsupported")
sn.syncDir = func(string) error { return dirErr }
if err := sn.persistAtomic(data); err != nil {
t.Fatalf("first committed replacement: %v", err)
}
if err := sn.persistAtomic(data); err != nil {
t.Fatalf("second committed replacement: %v", err)
}
preCommitErr := errors.New("temporary file write failed")
sn.persist = func([]byte) error { return preCommitErr }
go sn.run()
ticket := sn.recordTerminalMutation(nil)
if outcome := awaitTerminalOutcome(t, sn, ticket); !errors.Is(outcome.err, preCommitErr) {
t.Fatalf("pre-commit outcome=%+v", outcome)
}
_ = sn.flushAndStop(time.Second)
logs := output.String()
if got := strings.Count(logs, "parent directory sync failed after rename commit"); got != 1 {
t.Fatalf("directory-sync warning count=%d, want 1; logs=%q", got, logs)
}
if !strings.Contains(logs, "write failed before rename commit") {
t.Fatalf("pre-commit write failure was suppressed: %q", logs)
}
}
type relayHarness struct {
srv *Server
httpSrv *httptest.Server
wsURL string
baseURL string
}
func mustClientIPResolver(t *testing.T, cidrs string) clientIPResolver {
t.Helper()
prefixes, err := parseTrustedProxyCIDRs(cidrs)
if err != nil {
t.Fatalf("parse trusted proxies: %v", err)
}
return newClientIPResolver(prefixes)
}
func newRelayHarness(t *testing.T) *relayHarness {
t.Helper()
tmpDir := t.TempDir()
return newRelayHarnessAt(t, tmpDir, filepath.Join(tmpDir, "rooms.json"))
}
func newRelayHarnessNoTrust(t *testing.T) *relayHarness {
t.Helper()
tmpDir := t.TempDir()
return newRelayHarnessAtWithResolver(
t,
tmpDir,
filepath.Join(tmpDir, "rooms.json"),
newClientIPResolver(nil),
)
}
// newRelayHarnessAt allows two harnesses to share a snapshot across restarts.
func newRelayHarnessAt(t *testing.T, logDir, stateFile string) *relayHarness {
t.Helper()
return newRelayHarnessAtWithResolver(t, logDir, stateFile, mustClientIPResolver(t, "127.0.0.0/8"))
}
func newRelayHarnessAtWithResolver(
t *testing.T,
logDir, stateFile string,
clientIPs clientIPResolver,
) *relayHarness {
t.Helper()
srv := newServer(logDir, stateFile, filepath.Join(t.TempDir(), "posters"), clientIPs)
return newRelayHarnessWithServer(t, srv, true)
}
func newStorageHarness(t *testing.T, logs *logStore, posters *posterStore) *relayHarness {
t.Helper()
srv := &Server{
rooms: make(map[string]*Room),
logs: logs,
posters: posters,
posterUploads: newPosterUploadLimiter(posterPerIPRateBurst, posterPerIPRateSustained, posterGlobalRateBurst, posterGlobalRateSustained, maxConcurrentPosterUploads, maxConcurrentPosterUploadsPerIP, time.Now()),
posterFetches: newPosterUploadLimiter(posterFetchPerIPRateBurst, posterFetchPerIPRateSustained, posterFetchGlobalRateBurst, posterFetchGlobalRateSustained, maxConcurrentPosterFetches, maxConcurrentPosterFetchesPerIP, time.Now()),
logLookups: make(chan struct{}, maxConcurrentLogLookups),
conns: newConnTracker(),
clientIPs: mustClientIPResolver(t, "127.0.0.0/8"),
}
return newRelayHarnessWithServer(t, srv, false)
}
func newRelayHarnessWithServer(t *testing.T, srv *Server, stopSnapshot bool) *relayHarness {
t.Helper()
mux := http.NewServeMux()
mux.HandleFunc("/relay", srv.handleWS)
mux.HandleFunc("/logs", srv.handlePostLogs)
mux.HandleFunc("/logs/", srv.handleGetLogs)
mux.HandleFunc("/posters", srv.handlePostPosters)
mux.HandleFunc("/posters/", srv.handleGetPosters)
httpSrv := httptest.NewServer(mux)
t.Cleanup(func() {
httpSrv.Close()
if stopSnapshot {
_ = srv.snap.flushAndStop(time.Second)
}
})
u, _ := url.Parse(httpSrv.URL)
wsURL := "ws://" + u.Host + "/relay"
return &relayHarness{srv: srv, httpSrv: httpSrv, wsURL: wsURL, baseURL: httpSrv.URL}
}
func createModernRoomWithGuest(
t *testing.T,
h *relayHarness,
sessionID, hostIP, guestIP string,
) (host, guest *testConn, hostToken, guestToken string) {
t.Helper()
hostToken, _ = mustReconnectToken(t)
host = h.dial(t, hostIP)
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: sessionID,
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
guestToken, _ = mustReconnectToken(t)
guest = h.dial(t, guestIP)
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: sessionID,
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
return host, guest, hostToken, guestToken
}
func injectSnapshotPersistenceFailure(t *testing.T, sn *snapshotter, injected error) {
t.Helper()
if err := sn.write(); err != nil {
t.Fatalf("persist pre-failure baseline: %v", err)
}
sn.writeMu.Lock()
original := sn.persist
sn.persist = func([]byte) error { return injected }
sn.writeMu.Unlock()
t.Cleanup(func() {
sn.writeMu.Lock()
sn.persist = original
sn.writeMu.Unlock()
})
}
func copySnapshotForRestart(t *testing.T, source string) string {
t.Helper()
data, err := os.ReadFile(source)
if err != nil {
t.Fatalf("read committed restart snapshot: %v", err)
}
path := filepath.Join(t.TempDir(), "rooms.json")
if err := os.WriteFile(path, data, 0644); err != nil {
t.Fatalf("copy committed restart snapshot: %v", err)
}
return path
}
type deterministicRemover struct {
mu sync.Mutex
failures map[string]error
calls map[string]int
}
func newDeterministicRemover() *deterministicRemover {
return &deterministicRemover{
failures: make(map[string]error),
calls: make(map[string]int),
}
}
func (r *deterministicRemover) remove(path string) error {
r.mu.Lock()
r.calls[path]++
err := r.failures[path]
r.mu.Unlock()
if err != nil {
return err
}
return os.Remove(path)
}
func (r *deterministicRemover) fail(path string, err error) {
r.mu.Lock()
defer r.mu.Unlock()
r.failures[path] = err
}
func (r *deterministicRemover) recover(path string) {
r.mu.Lock()
defer r.mu.Unlock()
delete(r.failures, path)
}
func (r *deterministicRemover) callCount(path string) int {
r.mu.Lock()
defer r.mu.Unlock()
return r.calls[path]
}
func (h *relayHarness) dial(t *testing.T, ip string) *testConn {
t.Helper()
headers := http.Header{}
if ip != "" {
headers.Set("X-Forwarded-For", ip)
}
conn, _, err := websocket.DefaultDialer.Dial(h.wsURL, headers)
if err != nil {
t.Fatalf("dial (ip=%s): %v", ip, err)
}
tc := &testConn{t: t, conn: conn}
t.Cleanup(func() { conn.Close() })
return tc
}
func (h *relayHarness) dialRaw(ip string) (*websocket.Conn, error) {
headers := http.Header{}
if ip != "" {
headers.Set("X-Forwarded-For", ip)
}
conn, _, err := websocket.DefaultDialer.Dial(h.wsURL, headers)
return conn, err
}
func newWebSocketPair(t *testing.T) (*websocket.Conn, *websocket.Conn) {
t.Helper()
serverConnCh := make(chan *websocket.Conn, 1)
upgradeErrCh := make(chan error, 1)
httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
upgradeErrCh <- err
return
}
serverConnCh <- conn
}))
u, err := url.Parse(httpServer.URL)
if err != nil {
httpServer.Close()
t.Fatalf("parse websocket pair URL: %v", err)
}
peerConn, _, err := websocket.DefaultDialer.Dial("ws://"+u.Host, nil)
if err != nil {
httpServer.Close()
t.Fatalf("dial websocket pair: %v", err)
}
var serverConn *websocket.Conn
select {
case serverConn = <-serverConnCh:
case err := <-upgradeErrCh:
peerConn.Close()
httpServer.Close()
t.Fatalf("upgrade websocket pair: %v", err)
case <-time.After(2 * time.Second):
peerConn.Close()
httpServer.Close()
t.Fatal("timed out waiting for websocket pair upgrade")
}
t.Cleanup(func() {
serverConn.Close()
peerConn.Close()
httpServer.Close()
})
return serverConn, peerConn
}
func (h *relayHarness) dialWithHeaders(headers http.Header) (*websocket.Conn, *http.Response, error) {
return websocket.DefaultDialer.Dial(h.wsURL, headers)
}
func (h *relayHarness) waitRoomPeers(t *testing.T, sessionID string, want int) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
h.srv.mu.RLock()
room := h.srv.rooms[sessionID]
h.srv.mu.RUnlock()
if room != nil {
room.mu.RLock()
got := len(room.Peers)
room.mu.RUnlock()
if got == want {
return
}
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("room %s never reached %d peers within 2s", sessionID, want)
}
func (h *relayHarness) waitIPConnections(t *testing.T, ip string, want int) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
h.srv.conns.mu.Lock()
got := h.srv.conns.perIP[ip]
h.srv.conns.mu.Unlock()
if got == want {
return
}
time.Sleep(20 * time.Millisecond)
}
t.Fatalf("IP %s never reached %d connections within 2s", ip, want)
}
type testConn struct {
t *testing.T
conn *websocket.Conn
}
func (c *testConn) send(msg clientMsg) {
c.t.Helper()
data, err := json.Marshal(msg)
if err != nil {
c.t.Fatalf("marshal: %v", err)
}
if err := c.conn.WriteMessage(websocket.TextMessage, data); err != nil {
c.t.Fatalf("write: %v", err)
}
}
func (c *testConn) sendRaw(data []byte) {
c.t.Helper()
if err := c.conn.WriteMessage(websocket.TextMessage, data); err != nil {
c.t.Fatalf("write raw: %v", err)
}
}
func (c *testConn) recv() serverMsg {
c.t.Helper()
c.conn.SetReadDeadline(time.Now().Add(2 * time.Second))
_, data, err := c.conn.ReadMessage()
if err != nil {
c.t.Fatalf("read: %v", err)
}
var m serverMsg
if err := json.Unmarshal(data, &m); err != nil {
c.t.Fatalf("unmarshal %q: %v", data, err)
}
return m
}
func (c *testConn) expect(typ string) serverMsg {
c.t.Helper()
m := c.recv()
if m.Type != typ {
c.t.Fatalf("expected type=%s, got type=%s code=%s message=%s", typ, m.Type, m.Code, m.Message)
}
return m
}
func (c *testConn) expectError(code string) serverMsg {
c.t.Helper()
m := c.expect("error")
if m.Code != code {
c.t.Fatalf("expected code=%s, got code=%s message=%s", code, m.Code, m.Message)
}
return m
}
func (c *testConn) expectAuthority(typ, hostPeerID string) serverMsg {
c.t.Helper()
message := c.expect(typ)
if message.HostPeerID != hostPeerID {
c.t.Fatalf("%s hostPeerId=%q, want %q", typ, message.HostPeerID, hostPeerID)
}
if _, ok := reconnectVerifierFromToken(message.ReconnectToken); !ok {
c.t.Fatalf("%s reconnectToken has invalid shape", typ)
}
return message
}
// recvNothing asserts that no frame arrives within the window.
func (c *testConn) recvNothing(within time.Duration) {
c.t.Helper()
c.conn.SetReadDeadline(time.Now().Add(within))
_, data, err := c.conn.ReadMessage()
if err == nil {
c.t.Fatalf("expected no message within %v, got %s", within, data)
}
if ne, ok := err.(net.Error); !ok || !ne.Timeout() {
c.t.Fatalf("expected read timeout, got %v", err)
}
}
// recvUntilClosed drains queued frames and waits for terminal closure.
func (c *testConn) recvUntilClosed(within time.Duration) ([]serverMsg, error) {
c.t.Helper()
if err := c.conn.SetReadDeadline(time.Now().Add(within)); err != nil {
return nil, fmt.Errorf("set close-read deadline: %w", err)
}
var messages []serverMsg
for {
messageType, data, err := c.conn.ReadMessage()
if err != nil {
if ne, ok := err.(net.Error); ok && ne.Timeout() {
return messages, fmt.Errorf("terminal closure not observed within %v: %w", within, err)
}
return messages, nil
}
if messageType != websocket.TextMessage {
return messages, fmt.Errorf("unexpected websocket message type %d before closure", messageType)
}
var message serverMsg
if err := json.Unmarshal(data, &message); err != nil {
return messages, fmt.Errorf("decode frame before closure %q: %w", data, err)
}
messages = append(messages, message)
}
}
func requireClientClosed(t *testing.T, client *Client) {
t.Helper()
select {
case <-client.done:
case <-time.After(2 * time.Second):
t.Fatal("client did not close")
}
}
func requirePeerClosed(t *testing.T, peer *websocket.Conn) {
t.Helper()
testPeer := &testConn{t: t, conn: peer}
if messages, err := testPeer.recvUntilClosed(2 * time.Second); err != nil {
t.Fatalf("peer remained open after client failure (messages=%v): %v", messages, err)
}
}
func TestClientQueueOverflowClosesConnection(t *testing.T) {
serverConn, peerConn := newWebSocketPair(t)
client := &Client{
conn: serverConn,
send: make(chan outboundFrame, 1),
done: make(chan struct{}),
}
if !client.enqueue([]byte(`{"sequence":1}`)) {
t.Fatal("first frame was not accepted")
}
if client.enqueue([]byte(`{"sequence":2}`)) {
t.Fatal("overflowing frame was accepted")
}
requireClientClosed(t, client)
requirePeerClosed(t, peerConn)
if client.enqueue([]byte(`{"sequence":3}`)) {
t.Fatal("frame was accepted after terminal close")
}
client.close()
}
func TestClientQueueOverflowBroadcastKeepsHealthyRecipient(t *testing.T) {
slowServerConn, slowPeerConn := newWebSocketPair(t)
slow := &Client{
conn: slowServerConn,
send: make(chan outboundFrame, 1),
done: make(chan struct{}),
}
if !slow.enqueue([]byte(`{"sequence":1}`)) {
t.Fatal("failed to prime slow client queue")
}
healthyServerConn, healthyPeerConn := newWebSocketPair(t)
healthy := newClient(healthyServerConn)
t.Cleanup(healthy.close)
room := &Room{
Peers: map[string]*Client{
"slow": slow,
"healthy": healthy,
},
}
payload := json.RawMessage(`{"sequence":2}`)
room.broadcastExcept("sender", serverMsg{
Type: relayTypeMessage,
From: "sender",
Payload: payload,
})
requireClientClosed(t, slow)
requirePeerClosed(t, slowPeerConn)
received := (&testConn{t: t, conn: healthyPeerConn}).expect(relayTypeMessage)
if received.From != "sender" {
t.Fatalf("healthy recipient sender=%q, want sender", received.From)
}
if string(received.Payload) != string(payload) {
t.Fatalf("healthy recipient payload=%s, want %s", received.Payload, payload)
}
}
func TestClientQueueOverflowDirectedTargetStillExists(t *testing.T) {
serverConn, peerConn := newWebSocketPair(t)
target := &Client{
conn: serverConn,
send: make(chan outboundFrame, 1),
done: make(chan struct{}),
}
if !target.enqueue([]byte(`{"sequence":1}`)) {
t.Fatal("failed to prime directed target queue")
}
sender := &Client{}
room := &Room{Peers: map[string]*Client{"sender": sender, "target": target}}
if result := room.sendFrom("sender", sender, "target", serverMsg{
Type: relayTypeMessage,
From: "sender",
Payload: json.RawMessage(`{"sequence":2}`),
}); result != directedTargetFound {
t.Fatalf("full existing target result=%v, want directedTargetFound", result)
}
requireClientClosed(t, target)
requirePeerClosed(t, peerConn)
if result := room.sendFrom("sender", sender, "missing", serverMsg{Type: relayTypeMessage}); result != directedTargetMissing {
t.Fatalf("missing target result=%v, want directedTargetMissing", result)
}
}
func TestClientWriteFailureClosesConnection(t *testing.T) {
serverConn, _ := newWebSocketPair(t)
client := &Client{
conn: serverConn,
send: make(chan outboundFrame, 1),
done: make(chan struct{}),
}
if err := serverConn.Close(); err != nil {
t.Fatalf("close writer connection: %v", err)
}
client.send <- outboundFrame{data: []byte(`{"type":"queued"}`)}
exited := make(chan struct{})
go func() {
client.writePump()
close(exited)
}()
requireClientClosed(t, client)
select {
case <-exited:
case <-time.After(2 * time.Second):
t.Fatal("write pump did not exit after write failure")
}
client.close()
}
func TestRateLimiterBurstExhausts(t *testing.T) {
rl := newRateLimiter(5, 10)
for i := 0; i < 5; i++ {
if !rl.allow() {
t.Fatalf("allow %d: expected true", i)
}
}
if rl.allow() {
t.Fatal("allow 6: expected false (burst exhausted)")
}
}
func TestRateLimiterRefillsOverTime(t *testing.T) {
rl := newRateLimiter(5, 10)
for i := 0; i < 5; i++ {
rl.allow()
}
if rl.allow() {
t.Fatal("burst should be exhausted before sleep")
}
time.Sleep(1200 * time.Millisecond)
count := 0
for rl.allow() {
count++
}
if count < 1 {
t.Fatalf("expected at least 1 token after 1.2s refill, got %d", count)
}
if count > 5 {
t.Fatalf("expected at most burst=5 after refill, got %d", count)
}
}
func TestRateLimiterAllowRace(t *testing.T) {
rl := newRateLimiter(100, 1000)
var wg sync.WaitGroup
var successes atomic.Int64
for i := 0; i < 10; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < 50; j++ {
if rl.allow() {
successes.Add(1)
}
}
}()
}
wg.Wait()
// Spot-check the result bounds; -race checks synchronization.
if got := successes.Load(); got <= 0 || got > 500 {
t.Fatalf("unexpected successes count %d (want 1..500)", got)
}
}
func TestRateLimiterReclaimableOnlyAfterFullRefill(t *testing.T) {
now := time.Now()
limiter := &rateLimiter{
tokens: 0,
maxTokens: 5,
refillRate: 1,
lastTime: now,
}
if limiter.reclaimable(now.Add(4 * time.Second)) {
t.Fatal("partially refilled limiter must retain its effective state")
}
if !limiter.reclaimable(now.Add(5 * time.Second)) {
t.Fatal("fully refilled limiter should be reclaimable")
}
}
func TestCleanupRateWindowsUsesWindowBoundary(t *testing.T) {
now := time.Now()
windows := map[string]time.Time{
"active": now.Add(-logRateInterval + time.Nanosecond),
"expired": now.Add(-logRateInterval),
}
cleanupRateWindows(windows, now, logRateInterval)
if _, ok := windows["active"]; !ok {
t.Fatal("active fixed-window limiter was removed early")
}
if _, ok := windows["expired"]; ok {
t.Fatal("expired fixed-window limiter was retained")
}
}
func TestConnTrackerPerIPLimit(t *testing.T) {
ct := newConnTracker()
for i := 0; i < maxConnsPerIP; i++ {
if !ct.tryConnect("10.0.0.1") {
t.Fatalf("tryConnect %d: expected true", i)
}
}
if ct.tryConnect("10.0.0.1") {
t.Fatalf("tryConnect %d from same IP: expected false", maxConnsPerIP+1)
}
}
func TestConnTrackerGlobalLimit(t *testing.T) {
ct := newConnTracker()
for i := 0; i < maxGlobalConns; i++ {
ip := fmt.Sprintf("10.0.%d.%d", i/256, i%256)
if !ct.tryConnect(ip) {
t.Fatalf("tryConnect %d (ip=%s): expected true", i, ip)
}
}
if ct.tryConnect("10.99.99.99") {
t.Fatal("tryConnect should fail once globalCount hits max")
}
}
func TestConnTrackerDisconnectFrees(t *testing.T) {
ct := newConnTracker()
ip := "10.0.0.2"
for i := 0; i < 5; i++ {
ct.tryConnect(ip)
}
for i := 0; i < 5; i++ {
ct.disconnect(ip)
}
ct.mu.Lock()
if _, ok := ct.perIP[ip]; ok {
t.Error("perIP entry should be deleted when count reaches 0")
}
if ct.globalCount != 0 {
t.Errorf("globalCount=%d, want 0", ct.globalCount)
}
ct.mu.Unlock()
// Extra disconnect is a no-op.
ct.disconnect(ip)
}
func TestConnTrackerRoomQuota(t *testing.T) {
ct := newConnTracker()
ip := "10.0.0.3"
for i := 0; i < maxRoomsPerIP; i++ {
if !ct.tryCreateRoom(ip) {
t.Fatalf("tryCreateRoom %d: expected true", i)
}
}
if ct.tryCreateRoom(ip) {
t.Fatalf("tryCreateRoom %d: expected false (quota)", maxRoomsPerIP+1)
}
ct.releaseRoom(ip)
if !ct.tryCreateRoom(ip) {
t.Fatal("tryCreateRoom after release: expected true")
}
}
func TestConnTrackerCleanupPreservesEffectiveRateLimits(t *testing.T) {
ct := newConnTracker()
ip := "10.0.1.1"
for i := range connRateBurst {
if !ct.tryConnect(ip) {
t.Fatalf("tryConnect %d: expected true", i)
}
}
for range connRateBurst {
ct.disconnect(ip)
}
ct.cleanup(time.Now())
if ct.tryConnect(ip) {
t.Fatal("cleanup reset a connection rate limit that was still effective")
}
ct.cleanup(time.Now().Add(10 * time.Second))
if !ct.tryConnect(ip) {
t.Fatal("fully refilled limiter should be reclaimable")
}
ct.cleanup(time.Now().Add(10 * time.Second))
ct.mu.Lock()
_, retainedWhileConnected := ct.ipRate[ip]
ct.mu.Unlock()
if !retainedWhileConnected {
t.Fatal("cleanup removed a limiter with an active connection")
}
}
func TestConnTrackerConnectRateLimit(t *testing.T) {
ct := newConnTracker()
ip := "10.0.0.4"
for i := range connRateBurst {
if !ct.tryConnect(ip) {
t.Fatalf("warmup tryConnect %d: expected true", i)
}
}
// Free a slot so the next denial comes from the rate limiter.
if ct.tryConnect(ip) {
t.Fatal("expected false from rate-limit bucket, not per-IP cap")
}
}
func TestPosterUploadLimiterAdmissionPolicy(t *testing.T) {
now := time.Unix(1_700_000_000, 0)
t.Run("per IP burst and independent clients", func(t *testing.T) {
limiter := newPosterUploadLimiter(2, 1, 10, 1, 10, 10, now)
for range 2 {
if !limiter.tryStart("203.0.113.1", now) {
t.Fatal("per-IP burst rejected early")
}
limiter.finish("203.0.113.1")
}
if limiter.tryStart("203.0.113.1", now) {
t.Fatal("request beyond per-IP burst succeeded")
}
if !limiter.tryStart("203.0.113.2", now) {
t.Fatal("independent IP was denied")
}
limiter.finish("203.0.113.2")
})
t.Run("global burst spans distinct clients", func(t *testing.T) {
limiter := newPosterUploadLimiter(10, 1, 2, 1, 10, 10, now)
for _, ip := range []string{"203.0.113.1", "203.0.113.2"} {
if !limiter.tryStart(ip, now) {
t.Fatalf("%s rejected before global burst exhausted", ip)
}
limiter.finish(ip)
}
if limiter.tryStart("203.0.113.3", now) {
t.Fatal("request beyond global burst succeeded")
}
if len(limiter.perIP) != 2 {
t.Fatalf("globally denied request allocated per-IP state: %d buckets", len(limiter.perIP))
}
})
t.Run("concurrency denial consumes no tokens", func(t *testing.T) {
limiter := newPosterUploadLimiter(1, 0, 2, 0, 1, 1, now)
if !limiter.tryStart("203.0.113.1", now) {
t.Fatal("first upload denied")
}
if limiter.tryStart("203.0.113.2", now) {
t.Fatal("upload above concurrency limit succeeded")
}
limiter.finish("203.0.113.1")
if !limiter.tryStart("203.0.113.2", now) {
t.Fatal("concurrency denial consumed admission tokens")
}
limiter.finish("203.0.113.2")
})
t.Run("per IP concurrency cap leaves slots for other clients", func(t *testing.T) {
limiter := newPosterUploadLimiter(3, 0, 10, 0, 8, 2, now)
for range 2 {
if !limiter.tryStart("203.0.113.1", now) {
t.Fatal("request within per-IP concurrency cap denied")
}
}
if limiter.tryStart("203.0.113.1", now) {
t.Fatal("request above per-IP concurrency cap succeeded")
}
if !limiter.tryStart("203.0.113.2", now) {
t.Fatal("saturated client starved an independent IP")
}
// The capped denial consumed no admission tokens: the IP's third and
// final burst token must still admit it once a slot frees up.
limiter.finish("203.0.113.1")
if !limiter.tryStart("203.0.113.1", now) {
t.Fatal("per-IP concurrency denial consumed admission tokens")
}
})
t.Run("finish releases the slot of the finishing IP only", func(t *testing.T) {
limiter := newPosterUploadLimiter(10, 0, 10, 0, 10, 1, now)
if !limiter.tryStart("203.0.113.1", now) {
t.Fatal("first client denied")
}
if !limiter.tryStart("203.0.113.2", now) {
t.Fatal("second client denied")
}
limiter.finish("203.0.113.1")
if !limiter.tryStart("203.0.113.1", now) {
t.Fatal("released client was still capped")
}
if limiter.tryStart("203.0.113.2", now) {
t.Fatal("finish released the wrong client's slot")
}
})
t.Run("per IP denial refunds global token", func(t *testing.T) {
limiter := newPosterUploadLimiter(1, 0, 2, 0, 2, 2, now)
if !limiter.tryStart("203.0.113.1", now) {
t.Fatal("first upload denied")
}
limiter.finish("203.0.113.1")
if limiter.tryStart("203.0.113.1", now) {
t.Fatal("exhausted IP unexpectedly admitted")
}
if !limiter.tryStart("203.0.113.2", now) {
t.Fatal("refunded global token was unavailable to another IP")
}
limiter.finish("203.0.113.2")
})
t.Run("finish restores only concurrency and time restores rate", func(t *testing.T) {
limiter := newPosterUploadLimiter(1, 1, 1, 1, 1, 1, now)
if !limiter.tryStart("203.0.113.1", now) {
t.Fatal("first upload denied")
}
limiter.finish("203.0.113.1")
if limiter.active != 0 {
t.Fatalf("active=%d, want 0", limiter.active)
}
if limiter.tryStart("203.0.113.1", now) {
t.Fatal("finish incorrectly refunded rate tokens")
}
if !limiter.tryStart("203.0.113.1", now.Add(time.Second)) {
t.Fatal("sustained refill did not restore capacity")
}
limiter.finish("203.0.113.1")
})
t.Run("cleanup retains effective buckets then reclaims full ones", func(t *testing.T) {
limiter := newPosterUploadLimiter(2, 1, 10, 1, 2, 2, now)
if !limiter.tryStart("203.0.113.1", now) {
t.Fatal("first upload denied")
}
limiter.finish("203.0.113.1")
limiter.cleanup(now)
if _, ok := limiter.perIP["203.0.113.1"]; !ok {
t.Fatal("cleanup removed effective per-IP limiter")
}
limiter.cleanup(now.Add(time.Second))
if _, ok := limiter.perIP["203.0.113.1"]; ok {
t.Fatal("cleanup retained fully refilled per-IP limiter")
}
})
}
func TestClientIPResolverTrustChains(t *testing.T) {
tests := []struct {
name string
trusted string
remote string
headers []string
want string
wantErr bool
}{
{name: "absent forwarding header", remote: "127.0.0.1:12345", want: "127.0.0.1"},
{name: "untrusted peer ignores spoof", remote: "198.51.100.10:12345", headers: []string{"203.0.113.5"}, want: "198.51.100.10"},
{name: "one trusted proxy", trusted: "10.0.0.0/8", remote: "10.0.0.2:8080", headers: []string{"203.0.113.5"}, want: "203.0.113.5"},
{name: "append chain ignores forged leftmost", trusted: "10.0.0.0/8", remote: "10.0.0.2:8080", headers: []string{"198.51.100.99, 203.0.113.5"}, want: "203.0.113.5"},
{name: "two trusted proxies", trusted: "10.0.0.0/8, 192.0.2.0/24", remote: "10.0.0.2:8080", headers: []string{"203.0.113.5, 192.0.2.10"}, want: "203.0.113.5"},
{name: "untrusted intermediate is client boundary", trusted: "10.0.0.0/8", remote: "10.0.0.2:8080", headers: []string{"203.0.113.5, 198.51.100.7"}, want: "198.51.100.7"},
{name: "repeated header lines preserve chain", trusted: "10.0.0.0/8, 192.0.2.0/24", remote: "10.0.0.2:8080", headers: []string{"203.0.113.5", "192.0.2.10"}, want: "203.0.113.5"},
{name: "trusted proxy without forwarding header", trusted: "10.0.0.0/8", remote: "10.0.0.2:8080", want: "10.0.0.2"},
{name: "IPv4 mapped peer is unmapped", remote: "[::ffff:192.0.2.4]:8080", want: "192.0.2.4"},
{name: "IPv4 mapped forwarded address is unmapped", trusted: "10.0.0.0/8", remote: "10.0.0.2:8080", headers: []string{"::ffff:203.0.113.5"}, want: "203.0.113.5"},
{name: "native IPv6 client is grouped to 64", trusted: "10.0.0.0/8", remote: "10.0.0.2:8080", headers: []string{"2001:db8:85a3:12::abcd"}, want: "2001:db8:85a3:12::"},
{name: "untrusted malformed header is ignored", remote: "198.51.100.10:12345", headers: []string{"bad,,host:123"}, want: "198.51.100.10"},
{name: "malformed immediate peer", remote: "not-an-address", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resolver := mustClientIPResolver(t, tt.trusted)
req := &http.Request{RemoteAddr: tt.remote, Header: make(http.Header)}
for _, value := range tt.headers {
req.Header.Add("X-Forwarded-For", value)
}
got, err := resolver.resolve(req)
if tt.wantErr {
if err == nil {
t.Fatalf("resolve()=%q, want error", got)
}
return
}
if err != nil {
t.Fatalf("resolve(): %v", err)
}
if got != tt.want {
t.Fatalf("resolve()=%q, want %q", got, tt.want)
}
})
}
}
func TestClientIPResolverRejectsMalformedTrustedChains(t *testing.T) {
resolver := mustClientIPResolver(t, "10.0.0.0/8")
for _, value := range []string{
"",
"203.0.113.5,",
"203.0.113.5,,192.0.2.1",
"not-an-ip",
"203.0.113.5:1234",
"fe80::1%eth0",
} {
t.Run(fmt.Sprintf("%q", value), func(t *testing.T) {
req := &http.Request{
RemoteAddr: "10.0.0.2:8080",
Header: http.Header{"X-Forwarded-For": []string{value}},
}
if got, err := resolver.resolve(req); err == nil {
t.Fatalf("resolve()=%q, want error", got)
}
})
}
}
func TestParseTrustedProxyCIDRs(t *testing.T) {
prefixes, err := parseTrustedProxyCIDRs(" ")
if err != nil || len(prefixes) != 0 {
t.Fatalf("empty config = %v, %v; want no prefixes", prefixes, err)
}
prefixes, err = parseTrustedProxyCIDRs(" 10.1.2.3/8, ::ffff:192.0.2.12/120, 2001:db8::1/32 ")
if err != nil {
t.Fatalf("valid config: %v", err)
}
got := make([]string, len(prefixes))
for i, prefix := range prefixes {
got[i] = prefix.String()
}
want := []string{"10.0.0.0/8", "192.0.2.0/24", "2001:db8::/32"}
if fmt.Sprint(got) != fmt.Sprint(want) {
t.Fatalf("prefixes=%v, want %v", got, want)
}
for _, value := range []string{"10.0.0.0/8,", "10.0.0.0/8,garbage", "::ffff:192.0.2.1/64"} {
if prefixes, err := parseTrustedProxyCIDRs(value); err == nil || prefixes != nil {
t.Fatalf("parseTrustedProxyCIDRs(%q)=(%v, %v), want nil error result", value, prefixes, err)
}
}
}
// Random IDs may repeat; this test checks shape. Collision retry is covered by
// TestLogStorePersistsAcrossRestartAndAvoidsIDCollisions.
func TestGenerateLogIDShape(t *testing.T) {
for range 200 {
id := generateLogID()
if len(id) != logIDLength {
t.Fatalf("len=%d want %d (id=%q)", len(id), logIDLength, id)
}
for _, c := range id {
if !strings.ContainsRune(idChars, c) {
t.Fatalf("id %q has unexpected char %q", id, c)
}
}
}
}
func TestCreateSucceeds(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "1.1.1.1")
c.send(clientMsg{Type: relayTypeCreate, SessionID: "ROOM1", PeerID: "host-a"})
message := c.expectAuthority(relayTypeCreated, "host-a")
if message.SessionID != "ROOM1" {
t.Errorf("SessionID=%q want ROOM1", message.SessionID)
}
h.waitRoomPeers(t, "ROOM1", 1)
}
func TestCreateMissingSessionIDRejected(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "1.1.1.2")
c.send(clientMsg{Type: "create", PeerID: "host-a"})
c.expectError("invalid_message")
}
func TestCreateMissingPeerIDRejected(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "1.1.1.3")
c.send(clientMsg{Type: "create", SessionID: "ROOM1"})
c.expectError("invalid_message")
}
func TestCreateDuplicateReturnsRoomExists(t *testing.T) {
h := newRelayHarness(t)
c1 := h.dial(t, "1.1.1.4")
c1.send(clientMsg{Type: "create", SessionID: "SAME", PeerID: "host-1"})
c1.expect("created")
// Use a different IP so the rooms quota does not interfere.
c2 := h.dial(t, "1.1.1.5")
c2.send(clientMsg{Type: "create", SessionID: "SAME", PeerID: "host-2"})
c2.expectError("room_exists")
}
func TestCreateNegotiatesModernProtocolWithClientKnownToken(t *testing.T) {
h := newRelayHarness(t)
hostToken, _ := mustReconnectToken(t)
host := h.dial(t, "1.1.1.40")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "MODERN_CREATE",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion + 1,
})
mismatch := host.expectError(relayErrorProtocolMismatch)
if mismatch.ProtocolVersion != relayProtocolVersion {
t.Fatalf("protocol mismatch advertised version=%d, want %d", mismatch.ProtocolVersion, relayProtocolVersion)
}
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "MODERN_CREATE",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: mismatch.ProtocolVersion,
})
created := host.expectAuthority(relayTypeCreated, "H")
if created.ReconnectToken != hostToken {
t.Fatal("modern create rotated the client-known reconnect token")
}
if created.ProtocolVersion != relayProtocolVersion {
t.Fatalf("created protocolVersion=%d, want %d", created.ProtocolVersion, relayProtocolVersion)
}
}
func TestModernCreateRetryAfterLostSetupResponseIsIdempotent(t *testing.T) {
h := newRelayHarness(t)
hostToken, _ := mustReconnectToken(t)
create := clientMsg{
Type: relayTypeCreate,
SessionID: "CREATE_RETRY",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
}
first := h.dial(t, "1.1.1.41")
first.send(create)
if err := first.conn.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
t.Fatalf("set discarded response deadline: %v", err)
}
messageType, _, err := first.conn.ReadMessage()
if err != nil {
t.Fatalf("read discarded setup response: %v", err)
}
if messageType != websocket.TextMessage {
t.Fatalf("discarded setup response type=%d, want text", messageType)
}
retry := h.dial(t, "1.1.1.42")
retry.send(create)
created := retry.expectAuthority(relayTypeCreated, "H")
if created.ReconnectToken != hostToken || created.ProtocolVersion != relayProtocolVersion {
t.Fatalf("retry authority changed: tokenMatch=%v protocol=%d", created.ReconnectToken == hostToken, created.ProtocolVersion)
}
if len(created.Peers) != 0 {
t.Fatalf("retry reported unexpected peers: %v", created.Peers)
}
if messages, err := first.recvUntilClosed(2 * time.Second); err != nil {
t.Fatalf("superseded create connection remained open: %v (frames=%v)", err, messages)
}
h.srv.mu.RLock()
roomCount := len(h.srv.rooms)
room := h.srv.rooms["CREATE_RETRY"]
h.srv.mu.RUnlock()
if roomCount != 1 || room == nil {
t.Fatalf("idempotent retry retained rooms=%d targetPresent=%v", roomCount, room != nil)
}
}
func TestIdempotentCreateReannouncesPreviouslyAbsentHost(t *testing.T) {
h := newRelayHarness(t)
hostToken, _ := mustReconnectToken(t)
create := clientMsg{
Type: relayTypeCreate,
SessionID: "CREATE_REANNOUNCE",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
}
host := h.dial(t, "1.1.1.43")
host.send(create)
host.expectAuthority(relayTypeCreated, "H")
guestToken, _ := mustReconnectToken(t)
guest := h.dial(t, "1.1.1.44")
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "CREATE_REANNOUNCE",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
if err := host.conn.Close(); err != nil {
t.Fatalf("close original host: %v", err)
}
left := guest.expect(relayTypePeerLeft)
if left.PeerID != "H" {
t.Fatalf("disconnected host event peerId=%q, want H", left.PeerID)
}
returning := h.dial(t, "1.1.1.45")
returning.send(create)
returning.expectAuthority(relayTypeCreated, "H")
reannounced := guest.expect(relayTypePeerJoined)
if reannounced.PeerID != "H" {
t.Fatalf("returning host event peerId=%q, want H", reannounced.PeerID)
}
}
func TestCreateReclaimsAbandonedEmptyRoom(t *testing.T) {
h := newRelayHarness(t)
hostToken, hostVerifier := mustReconnectToken(t)
original := &Room{
SessionID: "STALE",
HostPeerID: "old-host",
ProtocolVersion: relayProtocolVersion,
hostVerifier: hostVerifier,
peerReservations: make(map[string]peerReservation),
Peers: map[string]*Client{},
CreatedAt: time.Now().Add(-time.Minute),
LastActivityAt: time.Now(),
}
h.srv.mu.Lock()
h.srv.rooms["STALE"] = original
h.srv.mu.Unlock()
creatorToken, _ := mustReconnectToken(t)
creator := h.dial(t, "1.1.1.6")
creator.send(clientMsg{
Type: relayTypeCreate,
SessionID: "STALE",
PeerID: "new-host",
ReconnectToken: creatorToken,
ProtocolVersion: relayProtocolVersion,
})
creator.expectAuthority(relayTypeCreated, "new-host")
h.srv.mu.RLock()
current := h.srv.rooms["STALE"]
h.srv.mu.RUnlock()
if current == original {
t.Fatal("abandoned room identity survived the reclaim")
}
// The former capability cannot reclaim the live replacement.
former := h.dial(t, "1.1.1.60")
former.send(clientMsg{
Type: relayTypeCreate,
SessionID: "STALE",
PeerID: "old-host",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
former.expectError(relayErrorRoomExists)
}
// A restarted host presents a fresh capability for an abandoned room code.
func TestAbandonedCodeIsRecreatableByARestartedHost(t *testing.T) {
h := newRelayHarness(t)
firstToken, _ := mustReconnectToken(t)
host := h.dial(t, "6.4.0.1")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "REUSE",
PeerID: "H",
ReconnectToken: firstToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
host.conn.Close()
h.waitRoomPeers(t, "REUSE", 0)
// A fresh capability cannot prove previous ownership, even with the same peer ID.
restartToken, _ := mustReconnectToken(t)
restarted := h.dial(t, "6.4.0.2")
restarted.send(clientMsg{
Type: relayTypeCreate,
SessionID: "REUSE",
PeerID: "H",
ReconnectToken: restartToken,
ProtocolVersion: relayProtocolVersion,
})
recreated := restarted.expectAuthority(relayTypeCreated, "H")
if recreated.ReconnectToken != restartToken {
t.Fatalf("recreated room token=%q, want the presented capability", recreated.ReconnectToken)
}
guestToken, _ := mustReconnectToken(t)
guest := h.dial(t, "6.4.0.3")
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "REUSE",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
restarted.expect(relayTypePeerJoined)
}
func TestCreateReclaimsOwnedEmptyRoomWithoutDoubleCharging(t *testing.T) {
h := newRelayHarness(t)
ownerKey := "1.1.1.61"
now := time.Now()
replacementToken, replacementVerifier := mustReconnectToken(t)
h.srv.mu.Lock()
for i := range maxRoomsPerIP {
sessionID := fmt.Sprintf("OWNED%d", i)
h.srv.rooms[sessionID] = &Room{
SessionID: sessionID,
HostPeerID: "old-host",
hostVerifier: replacementVerifier,
peerReservations: make(map[string]peerReservation),
Peers: map[string]*Client{},
quotaOwnerKey: ownerKey,
CreatedAt: now.Add(-time.Hour),
LastActivityAt: now,
}
if !h.srv.conns.tryCreateRoom(ownerKey) {
h.srv.mu.Unlock()
t.Fatal("failed to seed retained quota")
}
}
h.srv.mu.Unlock()
c := h.dial(t, ownerKey)
c.send(clientMsg{
Type: relayTypeCreate,
SessionID: "OWNED0",
PeerID: "old-host",
ReconnectToken: replacementToken,
})
c.expect(relayTypeCreated)
h.srv.conns.mu.Lock()
quota := h.srv.conns.roomsPerIP[ownerKey]
h.srv.conns.mu.Unlock()
if quota != maxRoomsPerIP {
t.Fatalf("replacement quota=%d, want %d", quota, maxRoomsPerIP)
}
h.srv.mu.RLock()
roomCount := len(h.srv.rooms)
h.srv.mu.RUnlock()
if roomCount != maxRoomsPerIP {
t.Fatalf("replacement room count=%d, want %d", roomCount, maxRoomsPerIP)
}
}
func TestCreateHitsRoomsPerIPLimit(t *testing.T) {
h := newRelayHarness(t)
ip := "1.1.1.7"
for i := 0; i < maxRoomsPerIP; i++ {
c := h.dial(t, ip)
c.send(clientMsg{Type: "create", SessionID: fmt.Sprintf("R%d", i), PeerID: "host"})
c.expect("created")
}
// The fourth room from this IP exceeds the quota.
c := h.dial(t, ip)
c.send(clientMsg{Type: "create", SessionID: "ROVERFLOW", PeerID: "host"})
c.expectError("rate_limited")
}
func TestRetainedRoomQuotaSurvivesDisconnectAndReturnsOnRemoval(t *testing.T) {
h := newRelayHarness(t)
ownerKey := "1.1.1.70"
sessionIDs := make([]string, 0, maxRoomsPerIP)
for i := range maxRoomsPerIP {
sessionID := fmt.Sprintf("RETAIN%d", i)
host := h.dial(t, ownerKey)
host.send(clientMsg{Type: relayTypeCreate, SessionID: sessionID, PeerID: "H"})
host.expect(relayTypeCreated)
host.conn.Close()
h.waitRoomPeers(t, sessionID, 0)
sessionIDs = append(sessionIDs, sessionID)
}
for i, sessionID := range sessionIDs {
guest := h.dial(t, fmt.Sprintf("1.1.2.%d", i+1))
guest.send(clientMsg{Type: relayTypeJoin, SessionID: sessionID, PeerID: "G"})
guest.expect(relayTypeJoined)
guest.conn.Close()
h.waitRoomPeers(t, sessionID, 0)
}
blocked := h.dial(t, ownerKey)
blocked.send(clientMsg{Type: relayTypeCreate, SessionID: "RETAINX", PeerID: "H"})
blocked.expectError(relayErrorRateLimited)
h.srv.conns.mu.Lock()
retainedQuota := h.srv.conns.roomsPerIP[ownerKey]
h.srv.conns.mu.Unlock()
if retainedQuota != maxRoomsPerIP {
t.Fatalf("retained quota=%d, want %d after creator disconnects", retainedQuota, maxRoomsPerIP)
}
now := time.Now()
h.srv.mu.RLock()
idleRoom := h.srv.rooms[sessionIDs[0]]
h.srv.mu.RUnlock()
idleRoom.mu.Lock()
idleRoom.LastActivityAt = now.Add(-emptyRoomMaxAge - time.Second)
idleRoom.mu.Unlock()
h.srv.runCleanupStep(now)
h.srv.conns.mu.Lock()
afterIdleRemoval := h.srv.conns.roomsPerIP[ownerKey]
h.srv.conns.mu.Unlock()
if afterIdleRemoval != maxRoomsPerIP-1 {
t.Fatalf("quota after idle removal=%d, want %d", afterIdleRemoval, maxRoomsPerIP-1)
}
blocked.send(clientMsg{Type: relayTypeCreate, SessionID: "RETAIN3", PeerID: "H"})
blocked.expect(relayTypeCreated)
occupied := h.dial(t, "1.1.2.99")
occupied.send(clientMsg{Type: relayTypeJoin, SessionID: sessionIDs[1], PeerID: "G"})
occupied.expect(relayTypeJoined)
h.srv.mu.RLock()
expiringRoom := h.srv.rooms[sessionIDs[1]]
h.srv.mu.RUnlock()
expiringRoom.mu.Lock()
expiringRoom.CreatedAt = now.Add(-roomMaxAge - time.Second)
expiringRoom.mu.Unlock()
h.srv.runCleanupStep(now)
if _, err := occupied.recvUntilClosed(2 * time.Second); err != nil {
t.Fatalf("occupied expired room client remained connected: %v", err)
}
h.srv.conns.mu.Lock()
afterHardExpiry := h.srv.conns.roomsPerIP[ownerKey]
h.srv.conns.mu.Unlock()
if afterHardExpiry != maxRoomsPerIP-1 {
t.Fatalf("quota after hard expiry=%d, want %d", afterHardExpiry, maxRoomsPerIP-1)
}
recovered := h.dial(t, ownerKey)
recovered.send(clientMsg{Type: relayTypeCreate, SessionID: "RETAIN4", PeerID: "H"})
recovered.expect(relayTypeCreated)
h.srv.conns.mu.Lock()
finalQuota := h.srv.conns.roomsPerIP[ownerKey]
h.srv.conns.mu.Unlock()
if finalQuota != maxRoomsPerIP {
t.Fatalf("final retained quota=%d, want %d", finalQuota, maxRoomsPerIP)
}
}
func TestGlobalRetainedRoomCapBlocksCreateButPreservesJoin(t *testing.T) {
h := newRelayHarness(t)
now := time.Now()
h.srv.mu.Lock()
for _, persisted := range makeRoomSnapshots(maxRetainedRooms, false, now) {
h.srv.rooms[persisted.SessionID] = &Room{
SessionID: persisted.SessionID,
HostPeerID: persisted.HostPeerID,
Peers: map[string]*Client{},
CreatedAt: persisted.CreatedAt,
LastActivityAt: persisted.LastActivityAt,
}
}
h.srv.mu.Unlock()
ownerKey := "1.1.3.1"
client := h.dial(t, ownerKey)
client.send(clientMsg{Type: relayTypeCreate, SessionID: "OVERGLOBAL", PeerID: "H"})
client.expectError(relayErrorRateLimited)
h.srv.mu.RLock()
roomCount := len(h.srv.rooms)
h.srv.mu.RUnlock()
snapshotRoomCount := len(h.srv.buildSnapshot().Rooms)
if roomCount != maxRetainedRooms || snapshotRoomCount != maxRetainedRooms {
t.Fatalf("rejected create mutated retained state: rooms=%d snapshot=%d", roomCount, snapshotRoomCount)
}
h.srv.conns.mu.Lock()
reservation := h.srv.conns.roomsPerIP[ownerKey]
h.srv.conns.mu.Unlock()
if reservation != 0 {
t.Fatalf("global rejection reserved per-source quota: %d", reservation)
}
client.send(clientMsg{Type: relayTypeJoin, SessionID: "S0000", PeerID: "G"})
client.expect(relayTypeJoined)
client.conn.Close()
h.waitRoomPeers(t, "S0000", 0)
h.srv.mu.RLock()
expired := h.srv.rooms["S0001"]
h.srv.mu.RUnlock()
expired.mu.Lock()
expired.LastActivityAt = now.Add(-emptyRoomMaxAge - time.Second)
expired.mu.Unlock()
h.srv.runCleanupStep(now)
creator := h.dial(t, ownerKey)
creator.send(clientMsg{Type: relayTypeCreate, SessionID: "AFTERGLOBAL", PeerID: "H"})
creator.expect(relayTypeCreated)
h.srv.mu.RLock()
roomCount = len(h.srv.rooms)
h.srv.mu.RUnlock()
if roomCount != maxRetainedRooms {
t.Fatalf("room count after cleanup and create=%d, want %d", roomCount, maxRetainedRooms)
}
}
func TestConcurrentCreatesCannotExceedGlobalRetainedRoomCap(t *testing.T) {
for iteration := range 8 {
t.Run(fmt.Sprintf("iteration_%d", iteration), func(t *testing.T) {
h := newRelayHarness(t)
now := time.Now()
h.srv.mu.Lock()
for _, persisted := range makeRoomSnapshots(maxRetainedRooms-1, false, now) {
h.srv.rooms[persisted.SessionID] = &Room{
SessionID: persisted.SessionID,
HostPeerID: persisted.HostPeerID,
Peers: map[string]*Client{},
CreatedAt: persisted.CreatedAt,
LastActivityAt: persisted.LastActivityAt,
}
}
h.srv.mu.Unlock()
type createResult struct {
ownerKey string
message serverMsg
err error
}
connections := make([]*websocket.Conn, 2)
for i := range connections {
conn, err := h.dialRaw(fmt.Sprintf("1.1.4.%d", i+1))
if err != nil {
t.Fatalf("dial create contender %d: %v", i, err)
}
connections[i] = conn
t.Cleanup(func() { conn.Close() })
}
start := make(chan struct{})
results := make(chan createResult, len(connections))
for i, conn := range connections {
ownerKey := fmt.Sprintf("1.1.4.%d", i+1)
go func(conn *websocket.Conn, ownerKey string, index int) {
<-start
err := conn.WriteJSON(clientMsg{
Type: relayTypeCreate,
SessionID: fmt.Sprintf("RACE%d", index),
PeerID: "H",
})
var message serverMsg
if err == nil {
err = conn.ReadJSON(&message)
}
results <- createResult{ownerKey: ownerKey, message: message, err: err}
}(conn, ownerKey, i)
}
close(start)
created, rejected := 0, 0
acceptedOwner := ""
for range connections {
result := <-results
if result.err != nil {
t.Fatalf("concurrent create failed: %v", result.err)
}
switch {
case result.message.Type == relayTypeCreated:
created++
acceptedOwner = result.ownerKey
case result.message.Type == relayTypeError && result.message.Code == relayErrorRateLimited:
rejected++
default:
t.Fatalf("unexpected concurrent result: %+v", result.message)
}
}
if created != 1 || rejected != 1 {
t.Fatalf("created=%d rejected=%d, want one each", created, rejected)
}
h.srv.mu.RLock()
roomCount := len(h.srv.rooms)
h.srv.mu.RUnlock()
if roomCount != maxRetainedRooms {
t.Fatalf("room count=%d, want %d", roomCount, maxRetainedRooms)
}
h.srv.conns.mu.Lock()
acceptedQuota := h.srv.conns.roomsPerIP[acceptedOwner]
totalQuota := 0
for _, count := range h.srv.conns.roomsPerIP {
totalQuota += count
}
h.srv.conns.mu.Unlock()
if acceptedQuota != 1 || totalQuota != 1 {
t.Fatalf("accepted quota=%d total quota=%d, want 1 and 1", acceptedQuota, totalQuota)
}
})
}
}
func TestRelayUntrustedXFFCannotRotateConnectionOrRoomIdentity(t *testing.T) {
t.Run("connections", func(t *testing.T) {
h := newRelayHarnessNoTrust(t)
var conns []*websocket.Conn
for i := 0; i < maxConnsPerIP; i++ {
conn, err := h.dialRaw(fmt.Sprintf("203.0.113.%d", i+1))
if err != nil {
t.Fatalf("dial %d: %v", i, err)
}
conns = append(conns, conn)
}
t.Cleanup(func() {
for _, conn := range conns {
conn.Close()
}
})
headers := http.Header{"X-Forwarded-For": []string{"198.51.100.200"}}
conn, resp, err := h.dialWithHeaders(headers)
if conn != nil {
conn.Close()
t.Fatal("connection above direct peer limit unexpectedly succeeded")
}
if err == nil || resp == nil {
t.Fatalf("dial error=%v response=%v, want HTTP 429", err, resp)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusTooManyRequests {
t.Fatalf("status=%d, want 429", resp.StatusCode)
}
})
t.Run("rooms", func(t *testing.T) {
h := newRelayHarnessNoTrust(t)
for i := 0; i <= maxRoomsPerIP; i++ {
client := h.dial(t, fmt.Sprintf("203.0.113.%d", i+1))
client.send(clientMsg{Type: "create", SessionID: fmt.Sprintf("SPOOF%d", i), PeerID: "host"})
if i < maxRoomsPerIP {
client.expect("created")
} else {
client.expectError("rate_limited")
}
}
})
}
func TestRelayTrustedClientsHaveIndependentConnectionBuckets(t *testing.T) {
h := newRelayHarness(t)
var conns []*websocket.Conn
for i := 0; i < maxConnsPerIP; i++ {
conn, err := h.dialRaw("203.0.113.10")
if err != nil {
t.Fatalf("client A dial %d: %v", i, err)
}
conns = append(conns, conn)
}
conn, err := h.dialRaw("203.0.113.11")
if err != nil {
t.Fatalf("client B should have an independent bucket: %v", err)
}
conns = append(conns, conn)
t.Cleanup(func() {
for _, conn := range conns {
conn.Close()
}
})
}
func TestRelayMalformedTrustedChainDoesNotMutateAdmission(t *testing.T) {
h := newRelayHarness(t)
headers := http.Header{"X-Forwarded-For": []string{"203.0.113.5,"}}
conn, resp, err := h.dialWithHeaders(headers)
if conn != nil {
conn.Close()
t.Fatal("malformed trusted chain unexpectedly upgraded")
}
if err == nil || resp == nil {
t.Fatalf("dial error=%v response=%v, want HTTP 400", err, resp)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("status=%d, want 400", resp.StatusCode)
}
h.srv.conns.mu.Lock()
defer h.srv.conns.mu.Unlock()
if h.srv.conns.globalCount != 0 || len(h.srv.conns.perIP) != 0 || len(h.srv.conns.ipRate) != 0 {
t.Fatalf("malformed chain mutated connection admission: %+v", h.srv.conns)
}
}
func TestConnectionCannotRetainMultipleRoomMemberships(t *testing.T) {
h := newRelayHarness(t)
ip := "1.1.1.8"
client := h.dial(t, ip)
client.send(clientMsg{Type: "create", SessionID: "PRIMARY", PeerID: "host"})
client.expect("created")
otherHost := h.dial(t, "1.1.1.9")
otherHost.send(clientMsg{Type: "create", SessionID: "OTHER", PeerID: "other-host"})
otherHost.expect("created")
client.send(clientMsg{Type: "join", SessionID: "OTHER", PeerID: "ghost"})
client.expectError("already_in_room")
client.send(clientMsg{Type: "create", SessionID: "EXTRA", PeerID: "extra-host"})
client.expectError("already_in_room")
h.waitRoomPeers(t, "PRIMARY", 1)
h.waitRoomPeers(t, "OTHER", 1)
h.srv.mu.RLock()
_, extraExists := h.srv.rooms["EXTRA"]
h.srv.mu.RUnlock()
if extraExists {
t.Fatal("rejected create retained an extra room")
}
h.srv.conns.mu.Lock()
roomsForIP := h.srv.conns.roomsPerIP[ip]
h.srv.conns.mu.Unlock()
if roomsForIP != 1 {
t.Fatalf("roomsPerIP[%q]=%d, want 1", ip, roomsForIP)
}
}
func TestJoinSucceedsAndBroadcastsPeerJoined(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "2.0.0.1")
host.send(clientMsg{Type: relayTypeCreate, SessionID: "J1", PeerID: "H"})
host.expectAuthority(relayTypeCreated, "H")
guest := h.dial(t, "2.0.0.2")
guest.send(clientMsg{Type: relayTypeJoin, SessionID: "J1", PeerID: "G"})
joined := guest.expectAuthority(relayTypeJoined, "H")
if joined.SessionID != "J1" {
t.Errorf("SessionID=%q want J1", joined.SessionID)
}
if len(joined.Peers) != 1 || joined.Peers[0] != "H" {
t.Errorf("Peers=%v, want [H]", joined.Peers)
}
peerJoined := host.expect(relayTypePeerJoined)
if peerJoined.PeerID != "G" {
t.Errorf("peerJoined.PeerID=%q want G", peerJoined.PeerID)
}
}
func TestHostIdentityClaimsRequireReconnectCapability(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "2.0.1.1")
host.send(clientMsg{Type: relayTypeCreate, SessionID: "AUTH", PeerID: "HOST"})
created := host.expectAuthority(relayTypeCreated, "HOST")
guest := h.dial(t, "2.0.1.2")
guest.send(clientMsg{Type: relayTypeJoin, SessionID: "AUTH", PeerID: "GUEST"})
guest.expectAuthority(relayTypeJoined, "HOST")
host.expect(relayTypePeerJoined)
attacker := h.dial(t, "2.0.1.3")
attacker.send(clientMsg{Type: relayTypeJoin, SessionID: "AUTH", PeerID: "HOST"})
attacker.expectError(relayErrorPeerIdUnavailable)
wrongToken, _ := mustReconnectToken(t)
attacker.send(clientMsg{
Type: relayTypeJoin,
SessionID: "AUTH",
PeerID: "HOST",
ReconnectToken: wrongToken,
})
attacker.expectError(relayErrorPeerIdUnavailable)
attacker.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"forged":true}`)})
attacker.expectError(relayErrorNotInRoom)
host.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"real":true}`)})
message := guest.expect(relayTypeMessage)
if message.From != "HOST" {
t.Fatalf("message sender=%q, want HOST", message.From)
}
hostVerifier, ok := reconnectVerifierFromToken(created.ReconnectToken)
if !ok {
t.Fatal("created reconnect token became invalid")
}
h.srv.mu.RLock()
room := h.srv.rooms["AUTH"]
h.srv.mu.RUnlock()
room.mu.RLock()
currentVerifier := room.hostVerifier
currentHost := room.Peers["HOST"]
room.mu.RUnlock()
if !reconnectVerifierMatches(hostVerifier, currentVerifier) || currentHost == nil {
t.Fatal("failed claims mutated host authority")
}
}
func TestJoinMissingFieldsRejected(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "2.0.0.3")
c.send(clientMsg{Type: "join"})
c.expectError("invalid_message")
}
func TestRelayIdentifiersRejectUnsafeOrOversizedValues(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "2.0.0.30")
invalid := []string{"has space", "has/slash", strings.Repeat("x", maxSessionIDLength+1)}
for _, sessionID := range invalid {
c.send(clientMsg{Type: "create", SessionID: sessionID, PeerID: "H"})
c.expectError("invalid_message")
}
c.send(clientMsg{Type: "create", SessionID: "SAFE_ID-1", PeerID: "H"})
c.expect("created")
c.send(clientMsg{Type: "sendTo", To: "bad target", Payload: json.RawMessage(`{}`)})
c.expectError("invalid_message")
}
func TestJoinUnknownRoomFails(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "2.0.0.4")
c.send(clientMsg{Type: "join", SessionID: "NOPE", PeerID: "G"})
c.expectError("room_not_found")
}
func TestReleasedGuestProbesDoNotConsumeDurableCapacity(t *testing.T) {
h := newRelayHarness(t)
hostToken, _ := mustReconnectToken(t)
host := h.dial(t, "2.0.0.40")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "PROBE_CAPACITY",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
for i := range maxRoomSize * 2 {
peerID := fmt.Sprintf("P%d", i)
probeToken, _ := mustReconnectToken(t)
probe := h.dial(t, fmt.Sprintf("2.0.1.%d", i+1))
probe.send(clientMsg{
Type: relayTypeJoin,
SessionID: "PROBE_CAPACITY",
PeerID: peerID,
ReconnectToken: probeToken,
ProtocolVersion: relayProtocolVersion,
})
probe.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
probe.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: probeToken,
ProtocolVersion: relayProtocolVersion,
})
probe.expect(relayTypeLeft)
left := host.expect(relayTypePeerLeft)
if left.PeerID != peerID {
t.Fatalf("released probe event peerId=%q, want %q", left.PeerID, peerID)
}
}
h.srv.mu.RLock()
room := h.srv.rooms["PROBE_CAPACITY"]
h.srv.mu.RUnlock()
room.mu.RLock()
reservations := len(room.peerReservations)
room.mu.RUnlock()
if reservations != 0 {
t.Fatalf("released probes retained %d durable reservations", reservations)
}
for i := range maxRoomSize - 1 {
peerID := fmt.Sprintf("G%d", i)
token, _ := mustReconnectToken(t)
guest := h.dial(t, fmt.Sprintf("2.0.2.%d", i+1))
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "PROBE_CAPACITY",
PeerID: peerID,
ReconnectToken: token,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
}
}
func TestFullRoomAllowsOnlyAuthenticatedLiveReplacements(t *testing.T) {
h := newRelayHarness(t)
hostToken, _ := mustReconnectToken(t)
host := h.dial(t, "2.1.0.1")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "FULL",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
created := host.expectAuthority(relayTypeCreated, "H")
guests := make(map[string]*testConn)
guestTokens := make(map[string]string)
for i := 1; i < maxRoomSize; i++ {
peerID := fmt.Sprintf("G%d", i)
guestToken, _ := mustReconnectToken(t)
guest := h.dial(t, fmt.Sprintf("2.1.0.%d", 100+i))
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "FULL",
PeerID: peerID,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
joined := guest.expectAuthority(relayTypeJoined, "H")
if joined.ReconnectToken != guestToken {
t.Fatalf("%s join rotated its client-known token", peerID)
}
guests[peerID] = guest
guestTokens[peerID] = guestToken
}
for range maxRoomSize - 1 {
host.expect(relayTypePeerJoined)
}
for range maxRoomSize - 2 {
guests["G1"].expect(relayTypePeerJoined)
}
overflowToken, _ := mustReconnectToken(t)
overflow := h.dial(t, "2.1.0.250")
overflow.send(clientMsg{
Type: relayTypeJoin,
SessionID: "FULL",
PeerID: "LATE",
ReconnectToken: overflowToken,
ProtocolVersion: relayProtocolVersion,
})
overflow.expectError(relayErrorRoomFull)
overflow.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{}`)})
overflow.expectError(relayErrorNotInRoom)
unprovedHost := h.dial(t, "2.1.0.251")
unprovedHost.send(clientMsg{
Type: relayTypeJoin,
SessionID: "FULL",
PeerID: "H",
ProtocolVersion: relayProtocolVersion,
})
unprovedHost.expectError(relayErrorPeerIdUnavailable)
unprovedGuest := h.dial(t, "2.1.0.252")
unprovedGuest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "FULL",
PeerID: "G1",
ProtocolVersion: relayProtocolVersion,
})
unprovedGuest.expectError(relayErrorPeerIdUnavailable)
wrongGuestToken, _ := mustReconnectToken(t)
unprovedGuest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "FULL",
PeerID: "G1",
ReconnectToken: wrongGuestToken,
ProtocolVersion: relayProtocolVersion,
})
unprovedGuest.expectError(relayErrorPeerIdUnavailable)
newHost := h.dial(t, "2.1.0.253")
newHost.send(clientMsg{
Type: relayTypeJoin,
SessionID: "FULL",
PeerID: "H",
ReconnectToken: created.ReconnectToken,
ProtocolVersion: relayProtocolVersion,
})
hostJoined := newHost.expectAuthority(relayTypeJoined, "H")
if len(hostJoined.Peers) != maxRoomSize-1 {
t.Fatalf("replacement host peers=%v, want %d peers", hostJoined.Peers, maxRoomSize-1)
}
if messages, err := host.recvUntilClosed(2 * time.Second); err != nil {
t.Fatalf("displaced host did not close: %v (frames=%v)", err, messages)
}
hostReturn := guests["G1"].expect(relayTypePeerJoined)
if hostReturn.PeerID != "H" {
t.Fatalf("host replacement event peerId=%q", hostReturn.PeerID)
}
newGuest := h.dial(t, "2.1.0.254")
newGuest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "FULL",
PeerID: "G1",
ReconnectToken: guestTokens["G1"],
ProtocolVersion: relayProtocolVersion,
})
newGuest.expectAuthority(relayTypeJoined, "H")
if messages, err := guests["G1"].recvUntilClosed(2 * time.Second); err != nil {
t.Fatalf("displaced guest did not close: %v (frames=%v)", err, messages)
}
guestReturn := newHost.expect(relayTypePeerJoined)
if guestReturn.PeerID != "G1" {
t.Fatalf("guest replacement event peerId=%q", guestReturn.PeerID)
}
newHost.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"state":"current"}`)})
message := newGuest.expect(relayTypeMessage)
if message.From != "H" {
t.Fatalf("replacement sender=%q, want H", message.From)
}
}
func TestDisconnectedHostKeepsAReservedRoomSlot(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "2.1.1.1")
host.send(clientMsg{Type: relayTypeCreate, SessionID: "HOST_SLOT", PeerID: "H"})
created := host.expectAuthority(relayTypeCreated, "H")
for i := 1; i < maxRoomSize; i++ {
guest := h.dial(t, fmt.Sprintf("2.1.1.%d", 100+i))
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "HOST_SLOT",
PeerID: fmt.Sprintf("G%d", i),
})
guest.expectAuthority(relayTypeJoined, "H")
}
h.waitRoomPeers(t, "HOST_SLOT", maxRoomSize)
if err := host.conn.Close(); err != nil {
t.Fatalf("close host: %v", err)
}
h.waitRoomPeers(t, "HOST_SLOT", maxRoomSize-1)
lateGuest := h.dial(t, "2.1.1.250")
lateGuest.send(clientMsg{Type: relayTypeJoin, SessionID: "HOST_SLOT", PeerID: "LATE"})
lateGuest.expectError(relayErrorRoomFull)
returningHost := h.dial(t, "2.1.1.251")
returningHost.send(clientMsg{
Type: relayTypeJoin,
SessionID: "HOST_SLOT",
PeerID: "H",
ReconnectToken: created.ReconnectToken,
})
joined := returningHost.expectAuthority(relayTypeJoined, "H")
if len(joined.Peers) != maxRoomSize-1 {
t.Fatalf("returning host peers=%v, want %d peers", joined.Peers, maxRoomSize-1)
}
h.waitRoomPeers(t, "HOST_SLOT", maxRoomSize)
}
func TestLegacySameSourceHostReconnectAndModernTokenEnforcement(t *testing.T) {
t.Run("legacy same-source tokenless reconnect", func(t *testing.T) {
h := newRelayHarness(t)
source := "2.1.2.1"
host := h.dial(t, source)
host.send(clientMsg{Type: relayTypeCreate, SessionID: "LEGACY_RECONNECT", PeerID: "H"})
host.expectAuthority(relayTypeCreated, "H")
guest := h.dial(t, "2.1.2.2")
guest.send(clientMsg{Type: relayTypeJoin, SessionID: "LEGACY_RECONNECT", PeerID: "G"})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
if err := host.conn.Close(); err != nil {
t.Fatalf("close legacy host: %v", err)
}
left := guest.expect(relayTypePeerLeft)
if left.PeerID != "H" {
t.Fatalf("legacy disconnect peerId=%q, want H", left.PeerID)
}
returning := h.dial(t, source)
returning.send(clientMsg{Type: relayTypeJoin, SessionID: "LEGACY_RECONNECT", PeerID: "H"})
joined := returning.expect(relayTypeJoined)
if joined.HostPeerID != "H" || joined.ReconnectToken != "" || joined.ProtocolVersion != legacyRelayProtocolVersion {
t.Fatalf("legacy reconnect authority=%+v", joined)
}
rejoined := guest.expect(relayTypePeerJoined)
if rejoined.PeerID != "H" {
t.Fatalf("legacy reconnect event peerId=%q, want H", rejoined.PeerID)
}
})
t.Run("modern same-source reconnect requires token", func(t *testing.T) {
h := newRelayHarness(t)
source := "2.1.3.1"
hostToken, _ := mustReconnectToken(t)
host := h.dial(t, source)
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "MODERN_RECONNECT",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
guestToken, _ := mustReconnectToken(t)
guest := h.dial(t, "2.1.3.2")
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "MODERN_RECONNECT",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
if err := host.conn.Close(); err != nil {
t.Fatalf("close modern host: %v", err)
}
left := guest.expect(relayTypePeerLeft)
if left.PeerID != "H" {
t.Fatalf("modern disconnect peerId=%q, want H", left.PeerID)
}
unproved := h.dial(t, source)
unproved.send(clientMsg{
Type: relayTypeJoin,
SessionID: "MODERN_RECONNECT",
PeerID: "H",
ProtocolVersion: relayProtocolVersion,
})
unproved.expectError(relayErrorPeerIdUnavailable)
returning := h.dial(t, source)
returning.send(clientMsg{
Type: relayTypeJoin,
SessionID: "MODERN_RECONNECT",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
joined := returning.expectAuthority(relayTypeJoined, "H")
if joined.ReconnectToken != hostToken || joined.ProtocolVersion != relayProtocolVersion {
t.Fatalf("modern reconnect authority changed: %+v", joined)
}
})
}
func TestJoinAdmissionIsAtomicWithEmptyRoomCleanup(t *testing.T) {
h := newRelayHarness(t)
_, hostVerifier := mustReconnectToken(t)
now := time.Now()
room := &Room{
SessionID: "ATOMIC_CLEANUP",
HostPeerID: "H",
hostVerifier: hostVerifier,
peerReservations: make(map[string]peerReservation),
Peers: make(map[string]*Client),
CreatedAt: now.Add(-time.Hour),
LastActivityAt: now.Add(-emptyRoomMaxAge - time.Second),
}
h.srv.mu.Lock()
h.srv.rooms[room.SessionID] = room
h.srv.mu.Unlock()
reached := make(chan struct{})
release := make(chan struct{})
var once sync.Once
h.srv.beforeJoinRoomLock = func() {
once.Do(func() {
close(reached)
<-release
})
}
joiner := h.dial(t, "2.2.0.1")
joiner.send(clientMsg{Type: relayTypeJoin, SessionID: room.SessionID, PeerID: "G1"})
<-reached
cleanupDone := make(chan struct{})
go func() {
h.srv.runCleanupStep(now)
close(cleanupDone)
}()
select {
case <-cleanupDone:
t.Fatal("cleanup passed a join that still owns the server read lock")
case <-time.After(100 * time.Millisecond):
}
close(release)
joiner.expectAuthority(relayTypeJoined, "H")
<-cleanupDone
h.srv.mu.RLock()
current := h.srv.rooms[room.SessionID]
h.srv.mu.RUnlock()
if current != room {
t.Fatal("successful join committed to a detached room")
}
room.mu.RLock()
_, present := room.Peers["G1"]
room.mu.RUnlock()
if !present {
t.Fatal("joining peer missing from authoritative room")
}
second := h.dial(t, "2.2.0.2")
second.send(clientMsg{Type: relayTypeJoin, SessionID: room.SessionID, PeerID: "G2"})
second.expectAuthority(relayTypeJoined, "H")
joiner.expect(relayTypePeerJoined)
second.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"atomic":true}`)})
if message := joiner.expect(relayTypeMessage); message.From != "G2" {
t.Fatalf("message sender=%q, want G2", message.From)
}
}
func TestJoinAdmissionIsAtomicWithReservedRoomCreate(t *testing.T) {
h := newRelayHarness(t)
_, hostVerifier := mustReconnectToken(t)
now := time.Now()
room := &Room{
SessionID: "ATOMIC_CREATE",
HostPeerID: "H",
hostVerifier: hostVerifier,
peerReservations: make(map[string]peerReservation),
Peers: make(map[string]*Client),
CreatedAt: now,
LastActivityAt: now,
}
h.srv.mu.Lock()
h.srv.rooms[room.SessionID] = room
h.srv.mu.Unlock()
reached := make(chan struct{})
release := make(chan struct{})
var once sync.Once
h.srv.beforeJoinRoomLock = func() {
once.Do(func() {
close(reached)
<-release
})
}
joiner := h.dial(t, "2.3.0.1")
joiner.send(clientMsg{Type: relayTypeJoin, SessionID: room.SessionID, PeerID: "G"})
<-reached
creator := h.dial(t, "2.3.0.2")
creator.send(clientMsg{Type: relayTypeCreate, SessionID: room.SessionID, PeerID: "OTHER"})
type readResult struct {
message serverMsg
err error
}
createResult := make(chan readResult, 1)
go func() {
creator.conn.SetReadDeadline(time.Now().Add(2 * time.Second))
_, data, err := creator.conn.ReadMessage()
if err != nil {
createResult <- readResult{err: err}
return
}
var message serverMsg
err = json.Unmarshal(data, &message)
createResult <- readResult{message: message, err: err}
}()
select {
case result := <-createResult:
t.Fatalf("create completed before join admission committed: message=%+v err=%v", result.message, result.err)
case <-time.After(100 * time.Millisecond):
}
close(release)
joiner.expectAuthority(relayTypeJoined, "H")
result := <-createResult
if result.err != nil {
t.Fatalf("read create result: %v", result.err)
}
if result.message.Type != relayTypeError || result.message.Code != relayErrorRoomExists {
t.Fatalf("create result=%+v, want room_exists", result.message)
}
h.srv.mu.RLock()
current := h.srv.rooms[room.SessionID]
h.srv.mu.RUnlock()
if current != room {
t.Fatal("reserved room was replaced during admission")
}
}
func TestBroadcastDeliversToOthersNotSender(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "3.0.0.1")
host.send(clientMsg{Type: "create", SessionID: "B1", PeerID: "H"})
host.expect("created")
g1 := h.dial(t, "3.0.0.2")
g1.send(clientMsg{Type: "join", SessionID: "B1", PeerID: "G1"})
g1.expect("joined")
host.expect("peerJoined")
g2 := h.dial(t, "3.0.0.3")
g2.send(clientMsg{Type: "join", SessionID: "B1", PeerID: "G2"})
g2.expect("joined")
host.expect("peerJoined")
g1.expect("peerJoined")
payload := json.RawMessage(`{"hello":"world"}`)
g1.send(clientMsg{Type: "broadcast", Payload: payload})
hostMsg := host.expect("message")
if hostMsg.From != "G1" {
t.Errorf("host From=%q want G1", hostMsg.From)
}
if string(hostMsg.Payload) != string(payload) {
t.Errorf("host payload=%s want %s", hostMsg.Payload, payload)
}
g2Msg := g2.expect("message")
if g2Msg.From != "G1" {
t.Errorf("g2 From=%q want G1", g2Msg.From)
}
// Broadcasts exclude the sender.
g1.recvNothing(200 * time.Millisecond)
}
func TestBroadcastNotInRoomRejected(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "3.0.0.4")
c.send(clientMsg{Type: "broadcast", Payload: json.RawMessage(`{}`)})
c.expectError("not_in_room")
}
func TestSendToDeliversToTargetOnly(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "4.0.0.1")
host.send(clientMsg{Type: "create", SessionID: "S1", PeerID: "H"})
host.expect("created")
g1 := h.dial(t, "4.0.0.2")
g1.send(clientMsg{Type: "join", SessionID: "S1", PeerID: "G1"})
g1.expect("joined")
host.expect("peerJoined")
g2 := h.dial(t, "4.0.0.3")
g2.send(clientMsg{Type: "join", SessionID: "S1", PeerID: "G2"})
g2.expect("joined")
host.expect("peerJoined")
g1.expect("peerJoined")
payload := json.RawMessage(`{"direct":true}`)
host.send(clientMsg{Type: "sendTo", To: "G1", Payload: payload})
m := g1.expect("message")
if m.From != "H" {
t.Errorf("From=%q want H", m.From)
}
if string(m.Payload) != string(payload) {
t.Errorf("payload mismatch: %s", m.Payload)
}
g2.recvNothing(200 * time.Millisecond)
}
func TestRelayMessagesRefreshRoomActivity(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "4.0.0.7")
host.send(clientMsg{Type: "create", SessionID: "ACTIVE", PeerID: "H"})
host.expect("created")
guest := h.dial(t, "4.0.0.8")
guest.send(clientMsg{Type: "join", SessionID: "ACTIVE", PeerID: "G"})
guest.expect("joined")
host.expect("peerJoined")
h.srv.mu.RLock()
room := h.srv.rooms["ACTIVE"]
h.srv.mu.RUnlock()
old := time.Now().Add(-time.Hour)
room.mu.Lock()
room.LastActivityAt = old
room.mu.Unlock()
host.send(clientMsg{Type: "broadcast", Payload: json.RawMessage(`{"broadcast":true}`)})
guest.expect("message")
room.mu.RLock()
broadcastActivity := room.LastActivityAt
room.mu.RUnlock()
if !broadcastActivity.After(old) {
t.Fatalf("broadcast activity=%v, want after %v", broadcastActivity, old)
}
room.mu.Lock()
room.LastActivityAt = old
room.mu.Unlock()
host.send(clientMsg{Type: "sendTo", To: "G", Payload: json.RawMessage(`{"direct":true}`)})
guest.expect("message")
room.mu.RLock()
directActivity := room.LastActivityAt
room.mu.RUnlock()
if !directActivity.After(old) {
t.Fatalf("sendTo activity=%v, want after %v", directActivity, old)
}
}
func TestSendToUnknownTargetRejected(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "4.0.0.4")
host.send(clientMsg{Type: "create", SessionID: "S2", PeerID: "H"})
host.expect("created")
host.send(clientMsg{Type: "sendTo", To: "ghost", Payload: json.RawMessage(`{}`)})
host.expectError("not_in_room")
}
func TestSendToMissingToRejected(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "4.0.0.5")
host.send(clientMsg{Type: "create", SessionID: "S3", PeerID: "H"})
host.expect("created")
host.send(clientMsg{Type: "sendTo", Payload: json.RawMessage(`{}`)})
host.expectError("invalid_message")
}
func TestSendToNotInRoomRejected(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "4.0.0.6")
c.send(clientMsg{Type: "sendTo", To: "anyone", Payload: json.RawMessage(`{}`)})
c.expectError("not_in_room")
}
func TestPingReturnsPong(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "5.0.0.1")
c.send(clientMsg{Type: "ping"})
c.expect("pong")
}
func TestUnknownTypeRejected(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "5.0.0.2")
c.send(clientMsg{Type: "nope"})
c.expectError("invalid_message")
}
func TestInvalidJSONRejected(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "5.0.0.3")
c.sendRaw([]byte("not json {{{"))
c.expectError("invalid_message")
}
func TestPerConnectionMessageRateLimit(t *testing.T) {
h := newRelayHarness(t)
c := h.dial(t, "5.0.0.4")
c.send(clientMsg{Type: "create", SessionID: "RL", PeerID: "H"})
c.expect("created")
// Exceed the per-connection bucket and observe rate limiting.
sawRateLimit := false
for i := 0; i < rateBurst+10; i++ {
c.send(clientMsg{Type: "ping"})
}
for i := 0; i < rateBurst+10; i++ {
m := c.recv()
if m.Code == "rate_limited" {
sawRateLimit = true
break
}
}
if !sawRateLimit {
t.Fatal("expected to hit rate_limited within burst+10 messages")
}
}
func TestDisconnectBroadcastsPeerLeft(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "6.0.0.1")
host.send(clientMsg{Type: "create", SessionID: "D1", PeerID: "H"})
host.expect("created")
guest := h.dial(t, "6.0.0.2")
guest.send(clientMsg{Type: "join", SessionID: "D1", PeerID: "G"})
guest.expect("joined")
host.expect("peerJoined")
guest.conn.Close()
left := host.expect("peerLeft")
if left.PeerID != "G" {
t.Errorf("PeerID=%q, want G", left.PeerID)
}
}
func TestStalePeerSkipsCleanupBroadcast(t *testing.T) {
h := newRelayHarness(t)
hostToken, _ := mustReconnectToken(t)
host := h.dial(t, "6.1.0.1")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "D2",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
guestToken, _ := mustReconnectToken(t)
g1 := h.dial(t, "6.1.0.2")
g1.send(clientMsg{
Type: relayTypeJoin,
SessionID: "D2",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
g1.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
g2 := h.dial(t, "6.1.0.3")
g2.send(clientMsg{
Type: relayTypeJoin,
SessionID: "D2",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
g2.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
if messages, err := g1.recvUntilClosed(2 * time.Second); err != nil {
t.Fatalf("displaced guest did not close: %v (frames=%v)", err, messages)
}
h.srv.mu.RLock()
room := h.srv.rooms["D2"]
room.mu.RLock()
reservation := room.peerReservations["G"]
room.mu.RUnlock()
h.srv.mu.RUnlock()
if !reservation.absentSince.IsZero() {
t.Fatalf("stale displaced client stamped live replacement absent at %v", reservation.absentSince)
}
host.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"after":"replacement"}`)})
message := g2.expect(relayTypeMessage)
if message.From != "H" {
t.Fatalf("post-replacement sender=%q, want H", message.From)
}
}
func TestDisconnectedModernGuestIdentityRejectsTheftAndAcceptsRightfulReconnect(t *testing.T) {
h := newRelayHarness(t)
hostToken, _ := mustReconnectToken(t)
host := h.dial(t, "6.1.0.10")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "GUEST_RECONNECT",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
guestToken, guestVerifier := mustReconnectToken(t)
guest := h.dial(t, "6.1.0.11")
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "GUEST_RECONNECT",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
if err := guest.conn.Close(); err != nil {
t.Fatalf("close guest: %v", err)
}
left := host.expect(relayTypePeerLeft)
if left.PeerID != "G" {
t.Fatalf("disconnected peerId=%q, want G", left.PeerID)
}
h.srv.mu.RLock()
room := h.srv.rooms["GUEST_RECONNECT"]
room.mu.RLock()
disconnectedReservation, reserved := room.peerReservations["G"]
room.mu.RUnlock()
h.srv.mu.RUnlock()
if !reserved || disconnectedReservation.absentSince.IsZero() {
t.Fatalf("authoritative disconnect reservation=%+v present=%v, want stamped absence", disconnectedReservation, reserved)
}
if !reconnectVerifierMatches(disconnectedReservation.verifier, guestVerifier) {
t.Fatal("authoritative disconnect changed the retained guest verifier")
}
thiefToken, _ := mustReconnectToken(t)
thief := h.dial(t, "6.1.0.12")
thief.send(clientMsg{
Type: relayTypeJoin,
SessionID: "GUEST_RECONNECT",
PeerID: "G",
ReconnectToken: thiefToken,
ProtocolVersion: relayProtocolVersion,
})
thief.expectError(relayErrorPeerIdUnavailable)
thief.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"forged":true}`)})
thief.expectError(relayErrorNotInRoom)
rightful := h.dial(t, "6.1.0.13")
rightful.send(clientMsg{
Type: relayTypeJoin,
SessionID: "GUEST_RECONNECT",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
joined := rightful.expectAuthority(relayTypeJoined, "H")
if joined.ReconnectToken != guestToken {
t.Fatal("rightful reconnect rotated the retained guest token")
}
rejoined := host.expect(relayTypePeerJoined)
if rejoined.PeerID != "G" {
t.Fatalf("rightful reconnect event peerId=%q, want G", rejoined.PeerID)
}
h.srv.mu.RLock()
room = h.srv.rooms["GUEST_RECONNECT"]
room.mu.RLock()
reconnectedReservation := room.peerReservations["G"]
room.mu.RUnlock()
h.srv.mu.RUnlock()
if !reconnectedReservation.absentSince.IsZero() {
t.Fatalf("rightful reconnect retained absence timestamp %v", reconnectedReservation.absentSince)
}
if !reconnectVerifierMatches(reconnectedReservation.verifier, guestVerifier) {
t.Fatal("rightful reconnect changed the retained guest verifier")
}
}
func fillDisconnectedGuestReservations(
t *testing.T,
h *relayHarness,
sessionID string,
) (*testConn, *Room) {
t.Helper()
hostToken, _ := mustReconnectToken(t)
host := h.dial(t, "6.2.0.1")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: sessionID,
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
for index := range maxRoomSize - 1 {
guestToken, _ := mustReconnectToken(t)
guest := h.dial(t, fmt.Sprintf("6.2.1.%d", index+1))
peerID := fmt.Sprintf("G%d", index)
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: sessionID,
PeerID: peerID,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
if err := guest.conn.Close(); err != nil {
t.Fatalf("close guest %s: %v", peerID, err)
}
left := host.expect(relayTypePeerLeft)
if left.PeerID != peerID {
t.Fatalf("disconnect event peerId=%q, want %q", left.PeerID, peerID)
}
}
h.srv.mu.RLock()
room := h.srv.rooms[sessionID]
h.srv.mu.RUnlock()
return host, room
}
func expireDisconnectedReservations(room *Room, now time.Time) {
room.mu.Lock()
for peerID, reservation := range room.peerReservations {
reservation.absentSince = now.Add(-peerReservationGrace - time.Second)
room.peerReservations[peerID] = reservation
}
room.mu.Unlock()
}
func TestDisconnectedModernGuestReservationsExpireAndRestoreCapacity(t *testing.T) {
t.Run("admission prunes without cleanup tick", func(t *testing.T) {
h := newRelayHarness(t)
host, room := fillDisconnectedGuestReservations(t, h, "RESERVATION_ADMISSION")
freshToken, _ := mustReconnectToken(t)
fresh := h.dial(t, "6.2.2.1")
join := clientMsg{
Type: relayTypeJoin,
SessionID: "RESERVATION_ADMISSION",
PeerID: "FRESH",
ReconnectToken: freshToken,
ProtocolVersion: relayProtocolVersion,
}
fresh.send(join)
fresh.expectError(relayErrorRoomFull)
expireDisconnectedReservations(room, time.Now())
fresh.send(join)
joined := fresh.expectAuthority(relayTypeJoined, "H")
if joined.ReconnectToken != freshToken {
t.Fatal("fresh admission changed its reconnect capability")
}
host.expect(relayTypePeerJoined)
room.mu.RLock()
if len(room.peerReservations) != 1 {
t.Fatalf("admission-time prune retained %d reservations, want only fresh peer", len(room.peerReservations))
}
_, freshReserved := room.peerReservations["FRESH"]
room.mu.RUnlock()
if !freshReserved {
t.Fatal("fresh admission was not reserved")
}
})
t.Run("cleanup persists pruning and keeps active guest", func(t *testing.T) {
root := t.TempDir()
statePath := filepath.Join(root, "rooms.json")
h := newRelayHarnessAt(t, filepath.Join(root, "logs"), statePath)
host, room := fillDisconnectedGuestReservations(t, h, "RESERVATION_CLEANUP")
expireDisconnectedReservations(room, time.Now())
h.srv.runCleanupStep(time.Now())
room.mu.RLock()
if len(room.peerReservations) != 0 {
t.Fatalf("cleanup retained %d expired reservations", len(room.peerReservations))
}
room.mu.RUnlock()
freshToken, _ := mustReconnectToken(t)
fresh := h.dial(t, "6.2.2.2")
fresh.send(clientMsg{
Type: relayTypeJoin,
SessionID: "RESERVATION_CLEANUP",
PeerID: "ACTIVE",
ReconnectToken: freshToken,
ProtocolVersion: relayProtocolVersion,
})
fresh.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
room.mu.Lock()
active := room.peerReservations["ACTIVE"]
active.absentSince = time.Now().Add(-peerReservationGrace - time.Second)
room.peerReservations["ACTIVE"] = active
room.mu.Unlock()
h.srv.runCleanupStep(time.Now())
room.mu.RLock()
_, activeReserved := room.peerReservations["ACTIVE"]
room.mu.RUnlock()
if !activeReserved {
t.Fatal("cleanup pruned an authoritative connected guest")
}
room.mu.Lock()
active = room.peerReservations["ACTIVE"]
active.absentSince = time.Time{}
room.peerReservations["ACTIVE"] = active
h.srv.snap.recordMutation()
room.mu.Unlock()
if err := h.srv.snap.flushAndStop(2 * time.Second); err != nil {
t.Fatalf("flush cleanup mutation: %v", err)
}
data, err := os.ReadFile(statePath)
if err != nil {
t.Fatalf("read cleanup snapshot: %v", err)
}
var snapshot stateSnapshot
if err := json.Unmarshal(data, &snapshot); err != nil {
t.Fatalf("decode cleanup snapshot: %v", err)
}
for _, persistedRoom := range snapshot.Rooms {
if persistedRoom.SessionID != "RESERVATION_CLEANUP" {
continue
}
for index := range maxRoomSize - 1 {
if _, retained := persistedRoom.PeerReservations[fmt.Sprintf("G%d", index)]; retained {
t.Fatalf("cleanup snapshot retained expired reservation G%d", index)
}
}
}
})
}
func TestAdmissionPrunePersistsWhenJoinRejected(t *testing.T) {
root := t.TempDir()
statePath := filepath.Join(root, "rooms.json")
h := newRelayHarnessAt(t, filepath.Join(root, "logs"), statePath)
_, room := fillDisconnectedGuestReservations(t, h, "REJECTED_AFTER_PRUNE")
expireDisconnectedReservations(room, time.Now())
wrongHostToken, _ := mustReconnectToken(t)
rejected := h.dial(t, "6.2.2.3")
rejected.send(clientMsg{
Type: relayTypeJoin,
SessionID: "REJECTED_AFTER_PRUNE",
PeerID: "H",
ReconnectToken: wrongHostToken,
ProtocolVersion: relayProtocolVersion,
})
rejected.expectError(relayErrorPeerIdUnavailable)
room.mu.RLock()
if len(room.peerReservations) != 0 {
t.Fatalf("rejected admission retained %d expired reservations", len(room.peerReservations))
}
room.mu.RUnlock()
if err := h.srv.snap.flushAndStop(2 * time.Second); err != nil {
t.Fatalf("flush rejected-admission prune: %v", err)
}
data, err := os.ReadFile(statePath)
if err != nil {
t.Fatalf("read rejected-admission snapshot: %v", err)
}
var snapshot stateSnapshot
if err := json.Unmarshal(data, &snapshot); err != nil {
t.Fatalf("decode rejected-admission snapshot: %v", err)
}
if len(snapshot.Rooms) != 1 || len(snapshot.Rooms[0].PeerReservations) != 0 {
t.Fatalf("rejected-admission snapshot retained reservations: %+v", snapshot.Rooms)
}
}
func TestExpiredGuestReservationLosesExclusiveClaim(t *testing.T) {
h := newRelayHarness(t)
host, room := fillDisconnectedGuestReservations(t, h, "EXPIRED_CLAIM")
expireDisconnectedReservations(room, time.Now())
room.mu.Lock()
if !pruneExpiredPeerReservationsLocked(room, time.Now()) {
room.mu.Unlock()
t.Fatal("expired reservations were not physically pruned")
}
h.srv.snap.recordMutation()
if _, retained := room.peerReservations["G0"]; retained {
room.mu.Unlock()
t.Fatal("expired peer ID remained exclusively reserved")
}
room.mu.Unlock()
freshToken, _ := mustReconnectToken(t)
fresh := h.dial(t, "6.2.3.1")
fresh.send(clientMsg{
Type: relayTypeJoin,
SessionID: "EXPIRED_CLAIM",
PeerID: "G0",
ReconnectToken: freshToken,
ProtocolVersion: relayProtocolVersion,
})
joined := fresh.expectAuthority(relayTypeJoined, "H")
if joined.ReconnectToken != freshToken {
t.Fatal("expired peer ID was treated as a privileged reconnect")
}
host.expect(relayTypePeerJoined)
}
func TestGuestReservationSnapshotV4Migration(t *testing.T) {
statePath := filepath.Join(t.TempDir(), "rooms.json")
now := time.Now().UTC()
_, hostVerifier := mustReconnectToken(t)
_, guestVerifier := mustReconnectToken(t)
legacy := stateSnapshot{
Version: 3,
SavedAt: now,
Rooms: []roomSnapshot{{
SessionID: "V3_MIGRATION",
HostPeerID: "H",
ProtocolVersion: relayProtocolVersion,
HostReconnectVerifier: encodeReconnectVerifier(hostVerifier),
PeerReconnectVerifiers: map[string]string{"G": encodeReconnectVerifier(guestVerifier)},
CreatedAt: now.Add(-time.Minute),
LastActivityAt: now,
}},
}
data, err := json.Marshal(legacy)
if err != nil {
t.Fatalf("marshal v3 fixture: %v", err)
}
if err := os.WriteFile(statePath, data, 0644); err != nil {
t.Fatalf("write v3 fixture: %v", err)
}
h := newRelayHarnessAt(t, t.TempDir(), statePath)
rewritten, err := os.ReadFile(statePath)
if err != nil {
t.Fatalf("read synchronous v4 rewrite: %v", err)
}
var snapshot stateSnapshot
if err := json.Unmarshal(rewritten, &snapshot); err != nil {
t.Fatalf("decode v4 rewrite: %v", err)
}
if snapshot.Version != snapshotFormatVersion {
t.Fatalf("rewritten version=%d, want %d", snapshot.Version, snapshotFormatVersion)
}
reservation := snapshot.Rooms[0].PeerReservations["G"]
if reservation.AbsentSinceUnixNano == 0 ||
reservation.Verifier != encodeReconnectVerifier(guestVerifier) ||
len(snapshot.Rooms[0].PeerReconnectVerifiers) != 0 {
t.Fatalf("migrated reservation=%+v legacy=%v", reservation, snapshot.Rooms[0].PeerReconnectVerifiers)
}
h.srv.mu.RLock()
room := h.srv.rooms["V3_MIGRATION"]
if room == nil {
h.srv.mu.RUnlock()
t.Fatal("migrated room was not restored")
}
room.mu.RLock()
runtimeReservation := room.peerReservations["G"]
room.mu.RUnlock()
h.srv.mu.RUnlock()
if runtimeReservation.absentSince.IsZero() {
t.Fatal("legacy reservation was restored as connected")
}
}
func TestSnapshotV2LoadsAndRewritesV4(t *testing.T) {
statePath := filepath.Join(t.TempDir(), "rooms.json")
now := time.Now().UTC()
_, hostVerifier := mustReconnectToken(t)
legacy := stateSnapshot{
Version: 2,
SavedAt: now,
Rooms: []roomSnapshot{{
SessionID: "V2_MIGRATION",
HostPeerID: "H",
HostReconnectVerifier: encodeReconnectVerifier(hostVerifier),
CreatedAt: now.Add(-time.Minute),
LastActivityAt: now,
}},
}
data, err := json.Marshal(legacy)
if err != nil {
t.Fatalf("marshal v2 fixture: %v", err)
}
if err := os.WriteFile(statePath, data, 0644); err != nil {
t.Fatalf("write v2 fixture: %v", err)
}
h := newRelayHarnessAt(t, t.TempDir(), statePath)
h.srv.mu.RLock()
room := h.srv.rooms["V2_MIGRATION"]
h.srv.mu.RUnlock()
if room == nil || !reconnectVerifierMatches(room.hostVerifier, hostVerifier) {
t.Fatal("v2 host authority did not load")
}
rewritten, err := os.ReadFile(statePath)
if err != nil {
t.Fatalf("read v2 rewrite: %v", err)
}
var snapshot stateSnapshot
if err := json.Unmarshal(rewritten, &snapshot); err != nil {
t.Fatalf("decode v2 rewrite: %v", err)
}
if snapshot.Version != snapshotFormatVersion {
t.Fatalf("v2 rewrite version=%d, want %d", snapshot.Version, snapshotFormatVersion)
}
}
func TestSnapshotV4RetainsGuestAbsenceAcrossRestart(t *testing.T) {
statePath := filepath.Join(t.TempDir(), "rooms.json")
now := time.Now().UTC().Truncate(time.Millisecond)
absentSince := now.Add(-time.Minute)
_, hostVerifier := mustReconnectToken(t)
_, guestVerifier := mustReconnectToken(t)
snapshot := stateSnapshot{
Version: snapshotFormatVersion,
SavedAt: now,
Rooms: []roomSnapshot{{
SessionID: "V4_ABSENCE",
HostPeerID: "H",
ProtocolVersion: relayProtocolVersion,
HostReconnectVerifier: encodeReconnectVerifier(hostVerifier),
PeerReservations: map[string]peerReservationSnapshot{
"G": {
Verifier: encodeReconnectVerifier(guestVerifier),
AbsentSinceUnixNano: absentSince.UnixNano(),
},
},
CreatedAt: now.Add(-time.Minute),
LastActivityAt: now,
}},
}
data, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal v4 fixture: %v", err)
}
if err := os.WriteFile(statePath, data, 0644); err != nil {
t.Fatalf("write v4 fixture: %v", err)
}
first := newRelayHarnessAt(t, t.TempDir(), statePath)
first.srv.mu.RLock()
room := first.srv.rooms["V4_ABSENCE"]
if room == nil {
first.srv.mu.RUnlock()
t.Fatal("v4 room missing after first restart")
}
room.mu.RLock()
firstAbsence := room.peerReservations["G"].absentSince
room.mu.RUnlock()
first.srv.mu.RUnlock()
if !firstAbsence.Equal(absentSince) {
t.Fatalf("first restart absence=%v, want %v", firstAbsence, absentSince)
}
if err := first.srv.snap.flushAndStop(2 * time.Second); err != nil {
t.Fatalf("flush first restart: %v", err)
}
second := newRelayHarnessAt(t, t.TempDir(), statePath)
second.srv.mu.RLock()
room = second.srv.rooms["V4_ABSENCE"]
if room == nil {
second.srv.mu.RUnlock()
t.Fatal("v4 room missing after second restart")
}
room.mu.RLock()
secondAbsence := room.peerReservations["G"].absentSince
room.mu.RUnlock()
second.srv.mu.RUnlock()
if !secondAbsence.Equal(absentSince) {
t.Fatalf("second restart refreshed absence=%v, want %v", secondAbsence, absentSince)
}
}
func TestSnapshotV4InitializesAndPrunesReservationsBeforeServing(t *testing.T) {
now := time.Now().UTC()
_, hostVerifier := mustReconnectToken(t)
_, connectedVerifier := mustReconnectToken(t)
_, expiredVerifier := mustReconnectToken(t)
statePath := filepath.Join(t.TempDir(), "rooms.json")
snapshot := stateSnapshot{
Version: snapshotFormatVersion,
SavedAt: now,
Rooms: []roomSnapshot{{
SessionID: "V4_STARTUP",
HostPeerID: "H",
ProtocolVersion: relayProtocolVersion,
HostReconnectVerifier: encodeReconnectVerifier(hostVerifier),
PeerReservations: map[string]peerReservationSnapshot{
"CONNECTED": {Verifier: encodeReconnectVerifier(connectedVerifier)},
"EXPIRED": {
Verifier: encodeReconnectVerifier(expiredVerifier),
AbsentSinceUnixNano: now.Add(-peerReservationGrace - time.Second).UnixNano(),
},
},
CreatedAt: now.Add(-time.Minute),
LastActivityAt: now,
}},
}
data, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal startup fixture: %v", err)
}
if err := os.WriteFile(statePath, data, 0644); err != nil {
t.Fatalf("write startup fixture: %v", err)
}
h := newRelayHarnessAt(t, t.TempDir(), statePath)
h.srv.mu.RLock()
room := h.srv.rooms["V4_STARTUP"]
if room == nil {
h.srv.mu.RUnlock()
t.Fatal("v4 startup room was not restored")
}
room.mu.RLock()
connected := room.peerReservations["CONNECTED"]
_, expired := room.peerReservations["EXPIRED"]
room.mu.RUnlock()
h.srv.mu.RUnlock()
if connected.absentSince.IsZero() {
t.Fatal("connected-at-capture reservation was not marked absent at startup")
}
if expired {
t.Fatal("already-expired reservation survived startup pruning")
}
rewritten, err := os.ReadFile(statePath)
if err != nil {
t.Fatalf("read startup rewrite: %v", err)
}
var persisted stateSnapshot
if err := json.Unmarshal(rewritten, &persisted); err != nil {
t.Fatalf("decode startup rewrite: %v", err)
}
if persisted.Rooms[0].PeerReservations["CONNECTED"].AbsentSinceUnixNano == 0 {
t.Fatal("startup rewrite did not persist initialized absence")
}
if _, retained := persisted.Rooms[0].PeerReservations["EXPIRED"]; retained {
t.Fatal("startup rewrite retained expired reservation")
}
}
func TestTerminalSuccessFramesFollowCommittedSnapshot(t *testing.T) {
t.Run("guest leave releases persisted reservation", func(t *testing.T) {
root := t.TempDir()
statePath := filepath.Join(root, "rooms.json")
h := newRelayHarnessAt(t, filepath.Join(root, "logs"), statePath)
terminalReady := make(chan struct{})
releaseTerminal := make(chan struct{})
var releaseOnce sync.Once
var syncCalls atomic.Int64
var terminalSyncBaseline atomic.Int64
h.srv.snap.syncDir = func(string) error {
syncCalls.Add(1)
return nil
}
h.srv.beforeTerminalDelivery = func() {
if syncCalls.Load() <= terminalSyncBaseline.Load() {
t.Error("terminal delivery preceded the covering directory-sync attempt")
}
close(terminalReady)
<-releaseTerminal
}
t.Cleanup(func() {
releaseOnce.Do(func() { close(releaseTerminal) })
})
host, guest, _, guestToken := createModernRoomWithGuest(
t,
h,
"DURABLE_LEAVE",
"6.1.0.20",
"6.1.0.21",
)
terminalSyncBaseline.Store(syncCalls.Load())
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
select {
case <-terminalReady:
case <-time.After(2 * time.Second):
t.Fatal("leave did not reach the post-persistence delivery barrier")
}
restartPath := copySnapshotForRestart(t, statePath)
restarted := newRelayHarnessAt(t, t.TempDir(), restartPath)
replacementToken, _ := mustReconnectToken(t)
replacement := restarted.dial(t, "6.1.0.22")
replacement.send(clientMsg{
Type: relayTypeJoin,
SessionID: "DURABLE_LEAVE",
PeerID: "G",
ReconnectToken: replacementToken,
ProtocolVersion: relayProtocolVersion,
})
replacement.expectAuthority(relayTypeJoined, "H")
releaseOnce.Do(func() { close(releaseTerminal) })
guest.expect(relayTypeLeft)
left := host.expect(relayTypePeerLeft)
if left.PeerID != "G" {
t.Fatalf("released peer event peerId=%q, want G", left.PeerID)
}
})
t.Run("host end removes persisted room", func(t *testing.T) {
root := t.TempDir()
statePath := filepath.Join(root, "rooms.json")
h := newRelayHarnessAt(t, filepath.Join(root, "logs"), statePath)
terminalReady := make(chan struct{})
releaseTerminal := make(chan struct{})
var releaseOnce sync.Once
var syncCalls atomic.Int64
var terminalSyncBaseline atomic.Int64
h.srv.snap.syncDir = func(string) error {
syncCalls.Add(1)
return nil
}
h.srv.beforeTerminalDelivery = func() {
if syncCalls.Load() <= terminalSyncBaseline.Load() {
t.Error("terminal delivery preceded the covering directory-sync attempt")
}
close(terminalReady)
<-releaseTerminal
}
t.Cleanup(func() {
releaseOnce.Do(func() { close(releaseTerminal) })
})
host, guest, hostToken, _ := createModernRoomWithGuest(
t,
h,
"DURABLE_END",
"6.1.0.23",
"6.1.0.24",
)
terminalSyncBaseline.Store(syncCalls.Load())
host.send(clientMsg{
Type: relayTypeEndSession,
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
select {
case <-terminalReady:
case <-time.After(2 * time.Second):
t.Fatal("end did not reach the post-persistence delivery barrier")
}
restartPath := copySnapshotForRestart(t, statePath)
restarted := newRelayHarnessAt(t, t.TempDir(), restartPath)
probe := restarted.dial(t, "6.1.0.25")
probe.send(clientMsg{
Type: relayTypeJoin,
SessionID: "DURABLE_END",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
probe.expectError(relayErrorRoomNotFound)
releaseOnce.Do(func() { close(releaseTerminal) })
host.expect(relayTypeEnded)
messages, err := guest.recvUntilClosed(2 * time.Second)
if err != nil {
t.Fatalf("guest remained connected after durable end: %v (frames=%v)", err, messages)
}
if len(messages) != 1 || messages[0].Type != relayTypeEnded {
t.Fatalf("guest terminal frames=%+v, want one ended notification", messages)
}
})
}
func TestTerminalPersistenceFailureSuppressesSuccess(t *testing.T) {
injectedErr := errors.New("injected snapshot persistence failure")
t.Run("guest leave", func(t *testing.T) {
root := t.TempDir()
statePath := filepath.Join(root, "rooms.json")
h := newRelayHarnessAt(t, filepath.Join(root, "logs"), statePath)
host, guest, _, guestToken := createModernRoomWithGuest(
t,
h,
"FAILED_LEAVE",
"6.1.0.26",
"6.1.0.27",
)
injectSnapshotPersistenceFailure(t, h.srv.snap, injectedErr)
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
failure := guest.expectError(relayErrorInvalidMessage)
if !strings.Contains(failure.Message, "persist") {
t.Fatalf("leave persistence error message=%q", failure.Message)
}
h.srv.mu.RLock()
room := h.srv.rooms["FAILED_LEAVE"]
room.mu.RLock()
liveClient := room.Peers["G"]
reservation, reserved := room.peerReservations["G"]
room.mu.RUnlock()
h.srv.mu.RUnlock()
if liveClient == nil || !reserved || reservation.releasePending || !reservation.absentSince.IsZero() {
t.Fatalf("failed leave live state client=%p reservation=%+v present=%v", liveClient, reservation, reserved)
}
guest.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"after":"failed-leave"}`)})
message := host.expect(relayTypeMessage)
if message.From != "G" {
t.Fatalf("post-failure broadcast sender=%q, want G", message.From)
}
restartPath := copySnapshotForRestart(t, statePath)
restarted := newRelayHarnessAt(t, t.TempDir(), restartPath)
replacementToken, _ := mustReconnectToken(t)
replacement := restarted.dial(t, "6.1.0.28")
replacement.send(clientMsg{
Type: relayTypeJoin,
SessionID: "FAILED_LEAVE",
PeerID: "G",
ReconnectToken: replacementToken,
ProtocolVersion: relayProtocolVersion,
})
replacement.expectError(relayErrorPeerIdUnavailable)
})
t.Run("host end", func(t *testing.T) {
root := t.TempDir()
statePath := filepath.Join(root, "rooms.json")
h := newRelayHarnessAt(t, filepath.Join(root, "logs"), statePath)
host, guest, hostToken, _ := createModernRoomWithGuest(
t,
h,
"FAILED_END",
"6.1.0.29",
"6.1.0.30",
)
injectSnapshotPersistenceFailure(t, h.srv.snap, injectedErr)
host.send(clientMsg{
Type: relayTypeEndSession,
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
failure := host.expectError(relayErrorInvalidMessage)
if !strings.Contains(failure.Message, "persist") {
t.Fatalf("end persistence error message=%q", failure.Message)
}
messages, err := guest.recvUntilClosed(2 * time.Second)
if err != nil {
t.Fatalf("guest remained connected after failed end persistence: %v (frames=%v)", err, messages)
}
if len(messages) != 0 {
t.Fatalf("guest received success frames after persistence failure: %+v", messages)
}
restartPath := copySnapshotForRestart(t, statePath)
restarted := newRelayHarnessAt(t, t.TempDir(), restartPath)
reconnected := restarted.dial(t, "6.1.0.31")
reconnected.send(clientMsg{
Type: relayTypeJoin,
SessionID: "FAILED_END",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
reconnected.expectAuthority(relayTypeJoined, "H")
})
}
func waitForPendingReservation(t *testing.T, room *Room, peerID string) {
t.Helper()
deadline := time.Now().Add(2 * time.Second)
for {
room.mu.RLock()
pending := room.peerReservations[peerID].releasePending
room.mu.RUnlock()
if pending {
return
}
if time.Now().After(deadline) {
t.Fatalf("reservation %s did not enter pending release", peerID)
}
time.Sleep(time.Millisecond)
}
}
func makeCurrentSnapshotDurable(t *testing.T, sn *snapshotter) {
t.Helper()
ticket := sn.recordTerminalMutation(nil)
if outcome := awaitTerminalOutcome(t, sn, ticket); outcome.err != nil {
t.Fatalf("persist baseline generation: %v", outcome.err)
}
}
func TestLeavePersistenceFailureKeepsMembershipAuthoritative(t *testing.T) {
injectedErr := errors.New("leave commit failed")
h := newRelayHarness(t)
host, guest, _, guestToken := createModernRoomWithGuest(
t,
h,
"LEAVE_AUTHORITATIVE",
"6.3.0.1",
"6.3.0.2",
)
h.srv.mu.RLock()
room := h.srv.rooms["LEAVE_AUTHORITATIVE"]
room.mu.RLock()
expectedClient := room.Peers["G"]
expectedReservation := room.peerReservations["G"]
room.mu.RUnlock()
h.srv.mu.RUnlock()
makeCurrentSnapshotDurable(t, h.srv.snap)
h.srv.snap.writeMu.Lock()
originalPersist := h.srv.snap.persist
var calls atomic.Int64
h.srv.snap.persist = func(data []byte) error {
if calls.Add(1) == 1 {
return injectedErr
}
return originalPersist(data)
}
h.srv.snap.writeMu.Unlock()
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectError(relayErrorInvalidMessage)
h.srv.mu.RLock()
room = h.srv.rooms["LEAVE_AUTHORITATIVE"]
room.mu.RLock()
live := room.Peers["G"]
reservation := room.peerReservations["G"]
room.mu.RUnlock()
h.srv.mu.RUnlock()
if live != expectedClient ||
reservation.releasePending ||
!reservation.absentSince.Equal(expectedReservation.absentSince) ||
!reconnectVerifierMatches(reservation.verifier, expectedReservation.verifier) {
t.Fatalf("failed leave live client=%p want=%p reservation=%+v want=%+v", live, expectedClient, reservation, expectedReservation)
}
guest.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"after":"rollback"}`)})
message := host.expect(relayTypeMessage)
if message.From != "G" {
t.Fatalf("post-rollback sender=%q, want G", message.From)
}
}
func TestSnapshotCaptureBoundaryExcludesLaterLeave(t *testing.T) {
injectedErr := errors.New("later leave persistence failed")
root := t.TempDir()
statePath := filepath.Join(root, "rooms.json")
h := newRelayHarnessAt(t, filepath.Join(root, "logs"), statePath)
_, guest, _, guestToken := createModernRoomWithGuest(
t,
h,
"CAPTURE_BOUNDARY",
"6.3.0.3",
"6.3.0.4",
)
makeCurrentSnapshotDurable(t, h.srv.snap)
h.srv.mu.RLock()
room := h.srv.rooms["CAPTURE_BOUNDARY"]
room.mu.RLock()
expectedClient := room.Peers["G"]
room.mu.RUnlock()
h.srv.mu.RUnlock()
captureReady := make(chan struct{})
releaseCapture := make(chan struct{})
var captureOnce sync.Once
h.srv.snap.stateMu.Lock()
h.srv.snap.afterSequenceCapture = func() {
captureOnce.Do(func() {
close(captureReady)
<-releaseCapture
})
}
h.srv.snap.stateMu.Unlock()
t.Cleanup(func() {
select {
case <-releaseCapture:
default:
close(releaseCapture)
}
})
leaveReachedLock := make(chan struct{})
var leaveOnce sync.Once
h.srv.beforeLeaveRoomLock = func() {
leaveOnce.Do(func() { close(leaveReachedLock) })
}
firstPayload := make(chan []byte, 1)
var persistCalls atomic.Int64
h.srv.snap.writeMu.Lock()
originalPersist := h.srv.snap.persist
h.srv.snap.persist = func(data []byte) error {
switch persistCalls.Add(1) {
case 1:
firstPayload <- append([]byte(nil), data...)
return originalPersist(data)
case 2:
return injectedErr
default:
return originalPersist(data)
}
}
h.srv.snap.writeMu.Unlock()
coveringTicket := h.srv.snap.recordTerminalMutation(nil)
select {
case <-captureReady:
case <-time.After(2 * time.Second):
t.Fatal("covering generation did not reach sequence boundary")
}
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
select {
case <-leaveReachedLock:
case <-time.After(2 * time.Second):
t.Fatal("later leave did not reach the protected room mutation")
}
close(releaseCapture)
if outcome := awaitTerminalOutcome(t, h.srv.snap, coveringTicket); outcome.err != nil {
t.Fatalf("covering generation outcome=%+v", outcome)
}
guest.expectError(relayErrorInvalidMessage)
assertReservation := func(label string, data []byte) {
t.Helper()
var snapshot stateSnapshot
if err := json.Unmarshal(data, &snapshot); err != nil {
t.Fatalf("decode %s snapshot: %v", label, err)
}
for _, persistedRoom := range snapshot.Rooms {
if persistedRoom.SessionID == "CAPTURE_BOUNDARY" {
if _, ok := persistedRoom.PeerReservations["G"]; !ok {
t.Fatalf("%s snapshot omitted the later failed leave", label)
}
return
}
}
t.Fatalf("%s snapshot omitted the room", label)
}
select {
case data := <-firstPayload:
assertReservation("covering", data)
case <-time.After(2 * time.Second):
t.Fatal("covering persistence payload was not captured")
}
diskData, err := os.ReadFile(statePath)
if err != nil {
t.Fatalf("read snapshot after failed follow-up: %v", err)
}
assertReservation("disk", diskData)
h.srv.mu.RLock()
room = h.srv.rooms["CAPTURE_BOUNDARY"]
room.mu.RLock()
liveClient := room.Peers["G"]
reservation := room.peerReservations["G"]
room.mu.RUnlock()
h.srv.mu.RUnlock()
if liveClient != expectedClient || reservation.releasePending {
t.Fatalf("failed later leave live client=%p want=%p reservation=%+v", liveClient, expectedClient, reservation)
}
}
func TestLeavePendingReservationRejectsReplacement(t *testing.T) {
injectedErr := errors.New("first leave persistence failed")
root := t.TempDir()
statePath := filepath.Join(root, "rooms.json")
h := newRelayHarnessAt(t, filepath.Join(root, "logs"), statePath)
host, guest, _, guestToken := createModernRoomWithGuest(
t,
h,
"PENDING_RELEASE",
"6.3.1.1",
"6.3.1.2",
)
makeCurrentSnapshotDurable(t, h.srv.snap)
h.srv.mu.RLock()
room := h.srv.rooms["PENDING_RELEASE"]
h.srv.mu.RUnlock()
firstStarted := make(chan struct{})
failFirst := make(chan struct{})
secondPayload := make(chan []byte, 1)
releaseSecond := make(chan struct{})
secondCommitted := make(chan struct{})
var calls atomic.Int64
h.srv.snap.writeMu.Lock()
originalPersist := h.srv.snap.persist
h.srv.snap.persist = func(data []byte) error {
switch calls.Add(1) {
case 1:
close(firstStarted)
<-failFirst
return injectedErr
case 2:
secondPayload <- append([]byte(nil), data...)
<-releaseSecond
err := originalPersist(data)
close(secondCommitted)
return err
default:
return originalPersist(data)
}
}
h.srv.snap.writeMu.Unlock()
t.Cleanup(func() {
select {
case <-failFirst:
default:
close(failFirst)
}
select {
case <-releaseSecond:
default:
close(releaseSecond)
}
})
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
select {
case <-firstStarted:
case <-time.After(2 * time.Second):
t.Fatal("pending leave did not reach persistence")
}
waitForPendingReservation(t, room, "G")
matching := h.dial(t, "6.3.1.3")
matching.send(clientMsg{
Type: relayTypeJoin,
SessionID: "PENDING_RELEASE",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
matching.expectError(relayErrorPeerIdUnavailable)
wrongToken, _ := mustReconnectToken(t)
wrong := h.dial(t, "6.3.1.4")
wrong.send(clientMsg{
Type: relayTypeJoin,
SessionID: "PENDING_RELEASE",
PeerID: "G",
ReconnectToken: wrongToken,
ProtocolVersion: relayProtocolVersion,
})
wrong.expectError(relayErrorPeerIdUnavailable)
laterToken, _ := mustReconnectToken(t)
later := h.dial(t, "6.3.1.5")
later.send(clientMsg{
Type: relayTypeJoin,
SessionID: "PENDING_RELEASE",
PeerID: "LATER",
ReconnectToken: laterToken,
ProtocolVersion: relayProtocolVersion,
})
later.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
if joined := guest.expect(relayTypePeerJoined); joined.PeerID != "LATER" {
t.Fatalf("pending guest join notification peerId=%q, want LATER", joined.PeerID)
}
close(failFirst)
guest.expectError(relayErrorInvalidMessage)
var corrected []byte
select {
case corrected = <-secondPayload:
case <-time.After(2 * time.Second):
t.Fatal("queued generation did not capture after leave rollback")
}
var correctedSnapshot stateSnapshot
if err := json.Unmarshal(corrected, &correctedSnapshot); err != nil {
t.Fatalf("decode corrected snapshot: %v", err)
}
foundRestored := false
for _, persistedRoom := range correctedSnapshot.Rooms {
if persistedRoom.SessionID == "PENDING_RELEASE" {
_, foundRestored = persistedRoom.PeerReservations["G"]
}
}
if !foundRestored {
t.Fatal("queued generation captured pending omission before rollback")
}
close(releaseSecond)
select {
case <-secondCommitted:
case <-time.After(2 * time.Second):
t.Fatal("corrected follow-up snapshot did not commit")
}
h.srv.snap.writeMu.Lock()
h.srv.snap.persist = originalPersist
h.srv.snap.writeMu.Unlock()
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expect(relayTypeLeft)
left := host.expect(relayTypePeerLeft)
if left.PeerID != "G" {
t.Fatalf("retry peerLeft=%q, want G", left.PeerID)
}
if err := h.srv.snap.flushAndStop(2 * time.Second); err != nil {
t.Fatalf("flush retry result: %v", err)
}
restarted := newRelayHarnessAt(t, t.TempDir(), statePath)
freshToken, _ := mustReconnectToken(t)
fresh := restarted.dial(t, "6.3.1.6")
fresh.send(clientMsg{
Type: relayTypeJoin,
SessionID: "PENDING_RELEASE",
PeerID: "G",
ReconnectToken: freshToken,
ProtocolVersion: relayProtocolVersion,
})
joined := fresh.expectAuthority(relayTypeJoined, "H")
if joined.ReconnectToken != freshToken {
t.Fatal("restart restored the released reservation")
}
}
func TestLeaveRetryAfterPersistenceFailure(t *testing.T) {
injectedErr := errors.New("retryable leave failure")
h := newRelayHarness(t)
host, guest, _, guestToken := createModernRoomWithGuest(
t,
h,
"LEAVE_RETRY",
"6.3.2.1",
"6.3.2.2",
)
makeCurrentSnapshotDurable(t, h.srv.snap)
h.srv.snap.writeMu.Lock()
originalPersist := h.srv.snap.persist
var failed atomic.Bool
h.srv.snap.persist = func(data []byte) error {
if failed.CompareAndSwap(false, true) {
return injectedErr
}
return originalPersist(data)
}
h.srv.snap.writeMu.Unlock()
leave := clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
}
guest.send(leave)
guest.expectError(relayErrorInvalidMessage)
guest.send(leave)
guest.expect(relayTypeLeft)
left := host.expect(relayTypePeerLeft)
if left.PeerID != "G" {
t.Fatalf("retry peerLeft=%q, want G", left.PeerID)
}
}
func TestLeaveFailurePreservesNewerActivity(t *testing.T) {
injectedErr := errors.New("activity-preserving leave failure")
h := newRelayHarness(t)
host, guest, _, guestToken := createModernRoomWithGuest(
t,
h,
"LEAVE_ACTIVITY",
"6.3.3.1",
"6.3.3.2",
)
otherToken, _ := mustReconnectToken(t)
other := h.dial(t, "6.3.3.3")
other.send(clientMsg{
Type: relayTypeJoin,
SessionID: "LEAVE_ACTIVITY",
PeerID: "OTHER",
ReconnectToken: otherToken,
ProtocolVersion: relayProtocolVersion,
})
other.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
if joined := guest.expect(relayTypePeerJoined); joined.PeerID != "OTHER" {
t.Fatalf("leave guest join notification peerId=%q, want OTHER", joined.PeerID)
}
makeCurrentSnapshotDurable(t, h.srv.snap)
h.srv.mu.RLock()
room := h.srv.rooms["LEAVE_ACTIVITY"]
h.srv.mu.RUnlock()
started := make(chan struct{})
fail := make(chan struct{})
var first atomic.Bool
h.srv.snap.writeMu.Lock()
originalPersist := h.srv.snap.persist
h.srv.snap.persist = func(data []byte) error {
if first.CompareAndSwap(false, true) {
close(started)
<-fail
return injectedErr
}
return originalPersist(data)
}
h.srv.snap.writeMu.Unlock()
t.Cleanup(func() {
select {
case <-fail:
default:
close(fail)
}
})
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("leave did not reach blocked persistence")
}
time.Sleep(time.Millisecond)
host.send(clientMsg{
Type: relayTypeSendTo,
To: "OTHER",
Payload: json.RawMessage(`{"during":"leave"}`),
})
other.expect(relayTypeMessage)
room.mu.RLock()
newerActivity := room.LastActivityAt
room.mu.RUnlock()
close(fail)
guest.expectError(relayErrorInvalidMessage)
room.mu.RLock()
afterFailure := room.LastActivityAt
room.mu.RUnlock()
if !afterFailure.Equal(newerActivity) {
t.Fatalf("leave rollback activity=%v, want concurrent activity %v", afterFailure, newerActivity)
}
}
func TestLeaveEndSessionRaceDoesNotResurrectRoom(t *testing.T) {
injectedErr := errors.New("leave attempt failed before room end")
root := t.TempDir()
statePath := filepath.Join(root, "rooms.json")
h := newRelayHarnessAt(t, filepath.Join(root, "logs"), statePath)
host, guest, hostToken, guestToken := createModernRoomWithGuest(
t,
h,
"LEAVE_END_RACE",
"6.3.4.1",
"6.3.4.2",
)
makeCurrentSnapshotDurable(t, h.srv.snap)
started := make(chan struct{})
fail := make(chan struct{})
var first atomic.Bool
h.srv.snap.writeMu.Lock()
originalPersist := h.srv.snap.persist
h.srv.snap.persist = func(data []byte) error {
if first.CompareAndSwap(false, true) {
close(started)
<-fail
return injectedErr
}
return originalPersist(data)
}
h.srv.snap.writeMu.Unlock()
t.Cleanup(func() {
select {
case <-fail:
default:
close(fail)
}
})
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
select {
case <-started:
case <-time.After(2 * time.Second):
t.Fatal("leave did not reach blocked persistence")
}
host.send(clientMsg{
Type: relayTypeEndSession,
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
deadline := time.Now().Add(2 * time.Second)
for {
h.srv.mu.RLock()
_, discoverable := h.srv.rooms["LEAVE_END_RACE"]
h.srv.mu.RUnlock()
if !discoverable {
break
}
if time.Now().After(deadline) {
t.Fatal("endSession did not win room authority")
}
time.Sleep(time.Millisecond)
}
close(fail)
host.expect(relayTypeEnded)
messages, err := guest.recvUntilClosed(2 * time.Second)
if err != nil {
t.Fatalf("guest remained connected after endSession won: %v (frames=%v)", err, messages)
}
if len(messages) != 1 || messages[0].Type != relayTypeEnded {
t.Fatalf("leave/end race terminal frames=%+v, want only ended", messages)
}
h.srv.mu.RLock()
_, resurrected := h.srv.rooms["LEAVE_END_RACE"]
h.srv.mu.RUnlock()
if resurrected {
t.Fatal("leave rollback resurrected ended room")
}
restarted := newRelayHarnessAt(t, t.TempDir(), statePath)
probe := restarted.dial(t, "6.3.4.3")
probe.send(clientMsg{
Type: relayTypeJoin,
SessionID: "LEAVE_END_RACE",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
probe.expectError(relayErrorRoomNotFound)
}
func TestConcurrentTerminalMutationsShareCommittedSnapshot(t *testing.T) {
h := newRelayHarness(t)
hostOne, guestOne, _, guestOneToken := createModernRoomWithGuest(
t,
h,
"TERMINAL_ONE",
"6.3.5.1",
"6.3.5.2",
)
hostTwo, guestTwo, hostTwoToken, _ := createModernRoomWithGuest(
t,
h,
"TERMINAL_TWO",
"6.3.5.3",
"6.3.5.4",
)
makeCurrentSnapshotDurable(t, h.srv.snap)
captureReady := make(chan struct{})
releaseCapture := make(chan struct{})
var captureOnce sync.Once
h.srv.snap.stateMu.Lock()
h.srv.snap.beforeCapture = func() {
captureOnce.Do(func() { close(captureReady) })
<-releaseCapture
}
h.srv.snap.stateMu.Unlock()
persistedPayloads := make(chan []byte, 2)
var persistCalls atomic.Int64
h.srv.snap.writeMu.Lock()
originalPersist := h.srv.snap.persist
h.srv.snap.persist = func(data []byte) error {
persistCalls.Add(1)
persistedPayloads <- append([]byte(nil), data...)
return originalPersist(data)
}
h.srv.snap.writeMu.Unlock()
t.Cleanup(func() {
select {
case <-releaseCapture:
default:
close(releaseCapture)
}
})
guestOne.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestOneToken,
ProtocolVersion: relayProtocolVersion,
})
select {
case <-captureReady:
case <-time.After(2 * time.Second):
t.Fatal("first terminal mutation did not reach capture barrier")
}
hostTwo.send(clientMsg{
Type: relayTypeEndSession,
ReconnectToken: hostTwoToken,
ProtocolVersion: relayProtocolVersion,
})
deadline := time.Now().Add(2 * time.Second)
for {
h.srv.mu.RLock()
_, secondDiscoverable := h.srv.rooms["TERMINAL_TWO"]
h.srv.mu.RUnlock()
if !secondDiscoverable {
break
}
if time.Now().After(deadline) {
t.Fatal("second terminal mutation did not stage before capture")
}
time.Sleep(time.Millisecond)
}
close(releaseCapture)
guestOne.expect(relayTypeLeft)
if left := hostOne.expect(relayTypePeerLeft); left.PeerID != "G" {
t.Fatalf("coalesced leave peerId=%q, want G", left.PeerID)
}
hostTwo.expect(relayTypeEnded)
messages, err := guestTwo.recvUntilClosed(2 * time.Second)
if err != nil {
t.Fatalf("coalesced end did not close guest: %v (frames=%v)", err, messages)
}
if len(messages) != 1 || messages[0].Type != relayTypeEnded {
t.Fatalf("coalesced end frames=%+v", messages)
}
if got := persistCalls.Load(); got != 1 {
t.Fatalf("coalesced terminal persist calls=%d, want 1", got)
}
var persisted stateSnapshot
select {
case data := <-persistedPayloads:
if err := json.Unmarshal(data, &persisted); err != nil {
t.Fatalf("decode coalesced terminal snapshot: %v", err)
}
default:
t.Fatal("coalesced terminal persistence did not expose its payload")
}
foundFirst := false
for _, persistedRoom := range persisted.Rooms {
switch persistedRoom.SessionID {
case "TERMINAL_ONE":
foundFirst = true
if _, retained := persistedRoom.PeerReservations["G"]; retained {
t.Fatal("coalesced leave snapshot retained released reservation")
}
case "TERMINAL_TWO":
t.Fatal("coalesced end snapshot retained ended room")
}
}
if !foundFirst {
t.Fatal("coalesced snapshot omitted surviving first room")
}
}
func TestRelayTrafficDoesNotScheduleSnapshots(t *testing.T) {
h := newRelayHarness(t)
host, guest, _, _ := createModernRoomWithGuest(
t,
h,
"TRAFFIC_NO_CHURN",
"6.3.6.1",
"6.3.6.2",
)
makeCurrentSnapshotDurable(t, h.srv.snap)
var persistCalls atomic.Int64
h.srv.snap.writeMu.Lock()
originalPersist := h.srv.snap.persist
h.srv.snap.persist = func(data []byte) error {
persistCalls.Add(1)
return originalPersist(data)
}
h.srv.snap.writeMu.Unlock()
for index := range 20 {
host.send(clientMsg{
Type: relayTypeBroadcast,
Payload: json.RawMessage(fmt.Sprintf(`{"index":%d}`, index)),
})
message := guest.expect(relayTypeMessage)
if message.From != "H" {
t.Fatalf("traffic sender=%q, want H", message.From)
}
}
time.Sleep(5 * snapshotDebounce)
if got := persistCalls.Load(); got != 0 {
t.Fatalf("high-frequency relay traffic scheduled %d snapshot writes", got)
}
}
func TestLeaveDirectorySyncFailureCommitsRelease(t *testing.T) {
root := t.TempDir()
statePath := filepath.Join(root, "rooms.json")
h := newRelayHarnessAt(t, filepath.Join(root, "logs"), statePath)
host, guest, _, guestToken := createModernRoomWithGuest(
t,
h,
"DIRSYNC_LEAVE",
"6.3.7.1",
"6.3.7.2",
)
makeCurrentSnapshotDurable(t, h.srv.snap)
h.srv.snap.syncDir = func(string) error {
return errors.New("directory sync unsupported")
}
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expect(relayTypeLeft)
if left := host.expect(relayTypePeerLeft); left.PeerID != "G" {
t.Fatalf("directory-sync degraded leave peerId=%q, want G", left.PeerID)
}
if err := h.srv.snap.flushAndStop(2 * time.Second); err != nil {
t.Fatalf("post-rename directory-sync warning failed committed leave: %v", err)
}
restarted := newRelayHarnessAt(t, t.TempDir(), statePath)
freshToken, _ := mustReconnectToken(t)
fresh := restarted.dial(t, "6.3.7.3")
fresh.send(clientMsg{
Type: relayTypeJoin,
SessionID: "DIRSYNC_LEAVE",
PeerID: "G",
ReconnectToken: freshToken,
ProtocolVersion: relayProtocolVersion,
})
joined := fresh.expectAuthority(relayTypeJoined, "H")
if joined.ReconnectToken != freshToken {
t.Fatal("restart restored a release committed before directory-sync warning")
}
}
func TestAuthenticatedModernGuestLeaveReleasesIdentity(t *testing.T) {
h := newRelayHarness(t)
hostToken, _ := mustReconnectToken(t)
host := h.dial(t, "6.1.0.20")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "GUEST_LEAVE",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
guestToken, _ := mustReconnectToken(t)
guest := h.dial(t, "6.1.0.21")
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "GUEST_LEAVE",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
wrongToken, _ := mustReconnectToken(t)
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: wrongToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectError(relayErrorPeerIdUnavailable)
guest.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"still":"joined"}`)})
stillJoined := host.expect(relayTypeMessage)
if stillJoined.From != "G" {
t.Fatalf("message after rejected leave came from %q, want G", stillJoined.From)
}
guest.send(clientMsg{
Type: relayTypeLeave,
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
leftAck := guest.expect(relayTypeLeft)
if leftAck.SessionID != "GUEST_LEAVE" || leftAck.PeerID != "G" || leftAck.ProtocolVersion != relayProtocolVersion {
t.Fatalf("left acknowledgement=%+v", leftAck)
}
leftEvent := host.expect(relayTypePeerLeft)
if leftEvent.PeerID != "G" {
t.Fatalf("released peer event peerId=%q, want G", leftEvent.PeerID)
}
replacementToken, _ := mustReconnectToken(t)
replacement := h.dial(t, "6.1.0.22")
replacement.send(clientMsg{
Type: relayTypeJoin,
SessionID: "GUEST_LEAVE",
PeerID: "G",
ReconnectToken: replacementToken,
ProtocolVersion: relayProtocolVersion,
})
joined := replacement.expectAuthority(relayTypeJoined, "H")
if joined.ReconnectToken != replacementToken {
t.Fatal("released guest identity retained its old verifier")
}
rejoined := host.expect(relayTypePeerJoined)
if rejoined.PeerID != "G" {
t.Fatalf("replacement event peerId=%q, want G", rejoined.PeerID)
}
}
func TestAuthenticatedModernHostEndDeletesRoomAndIsRetrySafe(t *testing.T) {
h := newRelayHarness(t)
hostToken, _ := mustReconnectToken(t)
host := h.dial(t, "6.1.0.30")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "HOST_END",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
guestToken, _ := mustReconnectToken(t)
guest := h.dial(t, "6.1.0.31")
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "HOST_END",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
wrongToken, _ := mustReconnectToken(t)
host.send(clientMsg{
Type: relayTypeEndSession,
ReconnectToken: wrongToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectError(relayErrorPeerIdUnavailable)
host.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"room":"live"}`)})
stillLive := guest.expect(relayTypeMessage)
if stillLive.From != "H" {
t.Fatalf("message after rejected end came from %q, want H", stillLive.From)
}
host.send(clientMsg{
Type: relayTypeEndSession,
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
ended := host.expect(relayTypeEnded)
if ended.SessionID != "HOST_END" || ended.ProtocolVersion != relayProtocolVersion {
t.Fatalf("ended acknowledgement=%+v", ended)
}
messages, err := guest.recvUntilClosed(2 * time.Second)
if err != nil {
t.Fatalf("guest remained connected after room end: %v (frames=%v)", err, messages)
}
if len(messages) != 1 ||
messages[0].Type != relayTypeEnded ||
messages[0].SessionID != "HOST_END" ||
messages[0].ProtocolVersion != relayProtocolVersion {
t.Fatalf("guest terminal frames=%+v, want one ended notification", messages)
}
host.send(clientMsg{
Type: relayTypeEndSession,
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectError(relayErrorNotInRoom)
retry := h.dial(t, "6.1.0.32")
retry.send(clientMsg{
Type: relayTypeJoin,
SessionID: "HOST_END",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
retry.expectError(relayErrorRoomNotFound)
}
func TestHostEndDeliversEndedAfterConcurrentGuestTraffic(t *testing.T) {
h := newRelayHarness(t)
endDeliveryReady := make(chan struct{})
releaseEndDelivery := make(chan struct{})
var releaseOnce sync.Once
h.srv.beforeTerminalDelivery = func() {
close(endDeliveryReady)
<-releaseEndDelivery
}
t.Cleanup(func() {
releaseOnce.Do(func() { close(releaseEndDelivery) })
})
hostToken, _ := mustReconnectToken(t)
host := h.dial(t, "6.1.0.33")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "END_RACE",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
guestToken, _ := mustReconnectToken(t)
guest := h.dial(t, "6.1.0.34")
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "END_RACE",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
host.send(clientMsg{
Type: relayTypeEndSession,
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
select {
case <-endDeliveryReady:
case <-time.After(2 * time.Second):
t.Fatal("host end did not reach the terminal-delivery barrier")
}
h.srv.mu.RLock()
_, discoverable := h.srv.rooms["END_RACE"]
h.srv.mu.RUnlock()
if discoverable {
t.Fatal("ending room remained discoverable before terminal delivery")
}
// Ordered frames prove membership traffic completed while ended delivery waited.
guest.send(clientMsg{
Type: relayTypeBroadcast,
Payload: json.RawMessage(`{"during":"end"}`),
})
guest.send(clientMsg{
Type: relayTypeSendTo,
To: "H",
Payload: json.RawMessage(`{"also":"during-end"}`),
})
guest.send(clientMsg{Type: relayTypePing})
guest.expect(relayTypePong)
releaseOnce.Do(func() { close(releaseEndDelivery) })
endedAck := host.expect(relayTypeEnded)
if endedAck.SessionID != "END_RACE" || endedAck.ProtocolVersion != relayProtocolVersion {
t.Fatalf("host ended acknowledgement=%+v", endedAck)
}
messages, err := guest.recvUntilClosed(2 * time.Second)
if err != nil {
t.Fatalf("guest remained connected after terminal delivery: %v (frames=%v)", err, messages)
}
if len(messages) != 1 ||
messages[0].Type != relayTypeEnded ||
messages[0].SessionID != "END_RACE" ||
messages[0].ProtocolVersion != relayProtocolVersion {
t.Fatalf("guest terminal frames=%+v, want one ended notification", messages)
}
}
func TestHostReconnectReplacesStaleConnectionWithoutLeaving(t *testing.T) {
h := newRelayHarness(t)
oldHostIP := "6.1.1.1"
oldHost := h.dial(t, oldHostIP)
oldHost.send(clientMsg{Type: relayTypeCreate, SessionID: "REJOIN", PeerID: "H"})
created := oldHost.expectAuthority(relayTypeCreated, "H")
guest := h.dial(t, "6.1.1.2")
guest.send(clientMsg{Type: relayTypeJoin, SessionID: "REJOIN", PeerID: "G"})
guest.expectAuthority(relayTypeJoined, "H")
oldHost.expect(relayTypePeerJoined)
newHost := h.dial(t, "6.1.1.3")
newHost.send(clientMsg{
Type: relayTypeJoin,
SessionID: "REJOIN",
PeerID: "H",
ReconnectToken: created.ReconnectToken,
})
joined := newHost.expectAuthority(relayTypeJoined, "H")
if len(joined.Peers) != 1 || joined.Peers[0] != "G" {
t.Fatalf("reconnected host peers=%v, want [G]", joined.Peers)
}
guest.expect(relayTypePeerJoined)
if messages, err := oldHost.recvUntilClosed(2 * time.Second); err != nil {
t.Fatalf("displaced host did not close: %v (frames=%v)", err, messages)
}
h.waitIPConnections(t, oldHostIP, 0)
newHost.send(clientMsg{Type: relayTypeBroadcast, Payload: json.RawMessage(`{"state":"ready"}`)})
message := guest.expect(relayTypeMessage)
if message.From != "H" {
t.Fatalf("message sender=%q, want H", message.From)
}
}
func TestEmptyRoomRequiresHostProofUntilExpiryThenSupportsFallbackCreate(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "6.1.2.1")
host.send(clientMsg{Type: relayTypeCreate, SessionID: "EMPTY", PeerID: "H"})
created := host.expectAuthority(relayTypeCreated, "H")
host.conn.Close()
h.waitRoomPeers(t, "EMPTY", 0)
unproved := h.dial(t, "6.1.2.20")
unproved.send(clientMsg{Type: relayTypeJoin, SessionID: "EMPTY", PeerID: "H"})
unproved.expectError(relayErrorPeerIdUnavailable)
reconnected := h.dial(t, "6.1.2.2")
reconnected.send(clientMsg{
Type: relayTypeJoin,
SessionID: "EMPTY",
PeerID: "H",
ReconnectToken: created.ReconnectToken,
})
joined := reconnected.expectAuthority(relayTypeJoined, "H")
if joined.ReconnectToken != created.ReconnectToken {
t.Fatal("host reconnect rotated its capability")
}
if len(joined.Peers) != 0 {
t.Fatalf("empty-room reconnect peers=%v, want none", joined.Peers)
}
reconnected.conn.Close()
h.waitRoomPeers(t, "EMPTY", 0)
now := time.Now()
h.srv.mu.RLock()
room := h.srv.rooms["EMPTY"]
h.srv.mu.RUnlock()
room.mu.Lock()
room.LastActivityAt = now.Add(-emptyRoomMaxAge - time.Second)
room.mu.Unlock()
h.srv.runCleanupStep(now)
fallback := h.dial(t, "6.1.2.3")
fallback.send(clientMsg{
Type: relayTypeJoin,
SessionID: "EMPTY",
PeerID: "H",
ReconnectToken: created.ReconnectToken,
})
fallback.expectError(relayErrorRoomNotFound)
fallback.send(clientMsg{
Type: relayTypeCreate,
SessionID: "EMPTY",
PeerID: "H",
ReconnectToken: created.ReconnectToken,
})
recreated := fallback.expectAuthority(relayTypeCreated, "H")
if recreated.ReconnectToken != created.ReconnectToken {
t.Fatal("fallback create rotated its retained capability")
}
}
func TestCleanupDisconnectsPeersBeforeRemovingExpiredOccupiedRoom(t *testing.T) {
h := newRelayHarness(t)
host := h.dial(t, "6.2.0.1")
host.send(clientMsg{Type: "create", SessionID: "EXPIRED", PeerID: "H"})
host.expect("created")
guest := h.dial(t, "6.2.0.2")
guest.send(clientMsg{Type: "join", SessionID: "EXPIRED", PeerID: "G"})
guest.expect("joined")
host.expect("peerJoined")
now := time.Now()
h.srv.mu.RLock()
room := h.srv.rooms["EXPIRED"]
h.srv.mu.RUnlock()
room.mu.Lock()
room.CreatedAt = now.Add(-roomMaxAge - time.Second)
room.mu.Unlock()
h.srv.runCleanupStep(now)
h.srv.mu.RLock()
_, exists := h.srv.rooms["EXPIRED"]
h.srv.mu.RUnlock()
if exists {
t.Fatal("expired room still exists after cleanup")
}
room.mu.RLock()
closing := room.closing
remainingPeers := len(room.Peers)
room.mu.RUnlock()
if !closing {
t.Error("expired room was not marked closing")
}
if remainingPeers != 0 {
t.Errorf("expired room retained %d peers after cleanup", remainingPeers)
}
for name, connection := range map[string]*testConn{"host": host, "guest": guest} {
messages, err := connection.recvUntilClosed(2 * time.Second)
if err != nil {
t.Errorf("%s did not reach terminal closure: %v", name, err)
}
for _, message := range messages {
if message.Type == relayTypePeerLeft || message.Type == relayTypePeerJoined {
t.Errorf("%s received %s during expired-room teardown", name, message.Type)
}
}
}
}
func postLog(t *testing.T, baseURL, ip string, body []byte) *http.Response {
t.Helper()
req, err := http.NewRequest(http.MethodPost, baseURL+"/logs", bytes.NewReader(body))
if err != nil {
t.Fatalf("new request: %v", err)
}
if ip != "" {
req.Header.Set("X-Forwarded-For", ip)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post: %v", err)
}
return resp
}
func getLog(t *testing.T, baseURL, ip, id string) *http.Response {
t.Helper()
req, err := http.NewRequest(http.MethodGet, baseURL+"/logs/"+id, nil)
if err != nil {
t.Fatalf("new request: %v", err)
}
if ip != "" {
req.Header.Set("X-Forwarded-For", ip)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("get: %v", err)
}
return resp
}
func postLogAndGetID(t *testing.T, baseURL, ip string, body []byte) string {
t.Helper()
resp := postLog(t, baseURL, ip, body)
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("post status=%d", resp.StatusCode)
}
var out struct {
ID string `json:"id"`
}
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
t.Fatalf("decode: %v", err)
}
if len(out.ID) != logIDLength {
t.Fatalf("id=%q len=%d want %d", out.ID, len(out.ID), logIDLength)
}
return out.ID
}
type posterUploadResponse struct {
ID string `json:"id"`
URL string `json:"url"`
ExpiresIn int `json:"expiresIn"`
}
func postPoster(t *testing.T, baseURL, ip string, body []byte) *http.Response {
t.Helper()
req, err := http.NewRequest(http.MethodPost, baseURL+"/posters", bytes.NewReader(body))
if err != nil {
t.Fatalf("new request: %v", err)
}
if ip != "" {
req.Header.Set("X-Forwarded-For", ip)
}
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("post: %v", err)
}
return resp
}
func postPosterAndDecode(t *testing.T, baseURL, ip string, body []byte) posterUploadResponse {
t.Helper()
resp := postPoster(t, baseURL, ip, body)
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("post status=%d", resp.StatusCode)
}
var out posterUploadResponse
if err := json.NewDecoder(resp.Body).Decode(&out); err != nil {
t.Fatalf("decode: %v", err)
}
if len(out.ID) != posterIDLength {
t.Fatalf("id=%q len=%d want %d", out.ID, len(out.ID), posterIDLength)
}
if out.URL == "" {
t.Fatal("empty poster url")
}
return out
}
type blockingLogResponseWriter struct {
header http.Header
writeStarted chan struct{}
releaseWrite chan struct{}
writeErr error
startOnce sync.Once
mu sync.Mutex
deadlines []time.Time
}
func newBlockingLogResponseWriter(writeErr error) *blockingLogResponseWriter {
return &blockingLogResponseWriter{
header: make(http.Header),
writeStarted: make(chan struct{}),
releaseWrite: make(chan struct{}),
writeErr: writeErr,
}
}
func (w *blockingLogResponseWriter) Header() http.Header {
return w.header
}
func (w *blockingLogResponseWriter) WriteHeader(int) {}
func (w *blockingLogResponseWriter) Write(p []byte) (int, error) {
w.startOnce.Do(func() { close(w.writeStarted) })
<-w.releaseWrite
if w.writeErr != nil {
return 0, w.writeErr
}
return len(p), nil
}
func (w *blockingLogResponseWriter) SetWriteDeadline(deadline time.Time) error {
w.mu.Lock()
w.deadlines = append(w.deadlines, deadline)
w.mu.Unlock()
return nil
}
func TestLogResponseTransmissionReleasesLookupSlotAndUsesDeadline(t *testing.T) {
logs := newLogStore(t.TempDir())
id, _, err := logs.store([]byte("diagnostic"), time.Now())
if err != nil {
t.Fatalf("store log: %v", err)
}
srv := &Server{
logs: logs,
logLookups: make(chan struct{}, 1),
clientIPs: newClientIPResolver(nil),
}
writer := newBlockingLogResponseWriter(errors.New("synthetic write failure: " + id))
request := httptest.NewRequest(http.MethodGet, "/logs/"+id, nil)
var output bytes.Buffer
previousOutput := log.Writer()
previousFlags := log.Flags()
previousPrefix := log.Prefix()
log.SetOutput(&output)
log.SetFlags(0)
log.SetPrefix("")
t.Cleanup(func() {
log.SetOutput(previousOutput)
log.SetFlags(previousFlags)
log.SetPrefix(previousPrefix)
})
done := make(chan struct{})
go func() {
srv.handleGetLogs(writer, request)
close(done)
}()
select {
case <-writer.writeStarted:
case <-time.After(time.Second):
t.Fatal("log response did not reach blocked write")
}
if occupied := len(srv.logLookups); occupied != 0 {
t.Fatalf("blocked response retained %d lookup slots", occupied)
}
second := httptest.NewRecorder()
srv.handleGetLogs(second, httptest.NewRequest(http.MethodGet, "/logs/"+id, nil))
if second.Code != http.StatusOK {
t.Fatalf("lookup while first response blocked status=%d, want 200", second.Code)
}
writer.mu.Lock()
deadlines := append([]time.Time(nil), writer.deadlines...)
writer.mu.Unlock()
if len(deadlines) == 0 || deadlines[0].IsZero() {
t.Fatalf("response write deadline not applied: %v", deadlines)
}
remaining := time.Until(deadlines[0])
if remaining <= 0 || remaining > httpResponseWriteTimeout {
t.Fatalf("response write deadline remaining=%v, want within (0, %v]", remaining, httpResponseWriteTimeout)
}
close(writer.releaseWrite)
select {
case <-done:
case <-time.After(time.Second):
t.Fatal("log handler did not finish after writer release")
}
writer.mu.Lock()
deadlines = append(deadlines[:0], writer.deadlines...)
writer.mu.Unlock()
if len(deadlines) != 1 || deadlines[0].IsZero() {
t.Fatalf("handler changed response deadline after writing: %v", deadlines)
}
if !strings.Contains(output.String(), "logs: response write failed") {
t.Fatalf("write failure was not observed: %q", output.String())
}
if strings.Contains(output.String(), id) {
t.Fatalf("write failure leaked log capability %q", id)
}
}
type logDeadlineObservation struct {
sequence uint64
deadline time.Time
}
type logResponseWriteObservation struct {
handlerReturned bool
deadline time.Time
deadlineSequence uint64
}
type logDeadlineConn struct {
net.Conn
handlerReturned <-chan struct{}
writeStarted chan logResponseWriteObservation
releaseWrite <-chan struct{}
blockFirstWrite sync.Once
mu sync.Mutex
deadline time.Time
deadlineSequence uint64
deadlineChanged chan logDeadlineObservation
}
func (c *logDeadlineConn) SetWriteDeadline(deadline time.Time) error {
if err := c.Conn.SetWriteDeadline(deadline); err != nil {
return err
}
c.mu.Lock()
c.deadline = deadline
c.deadlineSequence++
observation := logDeadlineObservation{
sequence: c.deadlineSequence,
deadline: deadline,
}
c.mu.Unlock()
c.deadlineChanged <- observation
return nil
}
func (c *logDeadlineConn) Write(p []byte) (int, error) {
c.blockFirstWrite.Do(func() {
c.mu.Lock()
observation := logResponseWriteObservation{
deadline: c.deadline,
deadlineSequence: c.deadlineSequence,
}
c.mu.Unlock()
select {
case <-c.handlerReturned:
observation.handlerReturned = true
default:
}
c.writeStarted <- observation
<-c.releaseWrite
})
return c.Conn.Write(p)
}
type logDeadlineListener struct {
net.Listener
handlerReturned <-chan struct{}
accepted chan *logDeadlineConn
releaseWrite <-chan struct{}
}
func (l *logDeadlineListener) Accept() (net.Conn, error) {
conn, err := l.Listener.Accept()
if err != nil {
return nil, err
}
wrapped := &logDeadlineConn{
Conn: conn,
handlerReturned: l.handlerReturned,
writeStarted: make(chan logResponseWriteObservation, 1),
releaseWrite: l.releaseWrite,
deadlineChanged: make(chan logDeadlineObservation, 16),
}
l.accepted <- wrapped
return wrapped, nil
}
func assertLogResponseDeadlineLifecycle(
t *testing.T,
present bool,
wantStatus int,
wantBody string,
) {
t.Helper()
logs := newLogStore(t.TempDir())
id := strings.Repeat("a", logIDLength)
if present {
var err error
id, _, err = logs.store([]byte(wantBody), time.Now())
if err != nil {
t.Fatalf("store log: %v", err)
}
}
srv := &Server{
logs: logs,
logLookups: make(chan struct{}, 1),
clientIPs: newClientIPResolver(nil),
}
handlerReturned := make(chan struct{})
mux := http.NewServeMux()
mux.HandleFunc("/logs/", func(w http.ResponseWriter, r *http.Request) {
srv.handleGetLogs(w, r)
close(handlerReturned)
})
mux.HandleFunc("/health", func(w http.ResponseWriter, _ *http.Request) {
_, _ = io.WriteString(w, "ok")
})
baseListener, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
t.Fatalf("listen: %v", err)
}
releaseFirstWrite := make(chan struct{})
listener := &logDeadlineListener{
Listener: baseListener,
handlerReturned: handlerReturned,
accepted: make(chan *logDeadlineConn, 1),
releaseWrite: releaseFirstWrite,
}
httpServer := newHTTPServer(listener.Addr().String(), mux)
serveDone := make(chan error, 1)
go func() {
serveDone <- httpServer.Serve(listener)
}()
var releaseWrite sync.Once
release := func() {
releaseWrite.Do(func() { close(releaseFirstWrite) })
}
t.Cleanup(func() {
release()
_ = httpServer.Close()
_ = listener.Close()
<-serveDone
})
clientConn, err := net.Dial("tcp", listener.Addr().String())
if err != nil {
t.Fatalf("dial: %v", err)
}
defer clientConn.Close()
var trackedConn *logDeadlineConn
select {
case trackedConn = <-listener.accepted:
case <-time.After(time.Second):
t.Fatal("server did not accept HTTP connection")
}
request, err := http.NewRequest(
http.MethodGet,
"http://"+listener.Addr().String()+"/logs/"+id,
nil,
)
if err != nil {
t.Fatalf("new log request: %v", err)
}
if err := request.Write(clientConn); err != nil {
t.Fatalf("write log request: %v", err)
}
var writeObservation logResponseWriteObservation
select {
case writeObservation = <-trackedConn.writeStarted:
case <-time.After(time.Second):
t.Fatal("response did not reach blocked final flush")
}
if !writeObservation.handlerReturned {
t.Fatal("response reached network before log handler returned")
}
if writeObservation.deadline.IsZero() {
t.Fatal("final response flush had no write deadline")
}
remaining := time.Until(writeObservation.deadline)
if remaining <= 0 || remaining > httpResponseWriteTimeout {
t.Fatalf(
"final flush deadline remaining=%v, want within (0, %v]",
remaining,
httpResponseWriteTimeout,
)
}
release()
reader := bufio.NewReader(clientConn)
response, err := http.ReadResponse(reader, request)
if err != nil {
t.Fatalf("read log response: %v", err)
}
body, err := io.ReadAll(response.Body)
response.Body.Close()
if err != nil {
t.Fatalf("read log response body: %v", err)
}
if response.StatusCode != wantStatus || string(body) != wantBody {
t.Fatalf(
"log response=(status=%d, body=%q), want (%d, %q)",
response.StatusCode,
body,
wantStatus,
wantBody,
)
}
cleared := false
timer := time.NewTimer(time.Second)
defer timer.Stop()
for !cleared {
select {
case observation := <-trackedConn.deadlineChanged:
cleared = observation.sequence > writeObservation.deadlineSequence &&
observation.deadline.IsZero()
case <-timer.C:
t.Fatal("net/http did not clear the response write deadline after final flush")
}
}
healthRequest, err := http.NewRequest(
http.MethodGet,
"http://"+listener.Addr().String()+"/health",
nil,
)
if err != nil {
t.Fatalf("new keep-alive request: %v", err)
}
if err := healthRequest.Write(clientConn); err != nil {
t.Fatalf("write keep-alive request: %v", err)
}
healthResponse, err := http.ReadResponse(reader, healthRequest)
if err != nil {
t.Fatalf("read keep-alive response: %v", err)
}
healthBody, err := io.ReadAll(healthResponse.Body)
healthResponse.Body.Close()
if err != nil {
t.Fatalf("read keep-alive response body: %v", err)
}
if healthResponse.StatusCode != http.StatusOK || string(healthBody) != "ok" {
t.Fatalf(
"keep-alive response=(status=%d, body=%q), want (200, %q)",
healthResponse.StatusCode,
healthBody,
"ok",
)
}
}
func TestLogResponseDeadlineSurvivesFinalFlush(t *testing.T) {
assertLogResponseDeadlineLifecycle(t, true, http.StatusOK, "diagnostic")
}
func TestLogResponseDeadlineCoversBufferedErrorResponse(t *testing.T) {
assertLogResponseDeadlineLifecycle(t, false, http.StatusNotFound, "Not found\n")
}
func TestHTTPServerWriteTimeoutCoversOAuthResultLongPoll(t *testing.T) {
server := newHTTPServer("127.0.0.1:0", http.NewServeMux())
if server.WriteTimeout != httpResponseWriteTimeout || server.WriteTimeout <= 0 {
t.Fatalf("WriteTimeout=%v, want bounded timeout %v", server.WriteTimeout, httpResponseWriteTimeout)
}
if server.WriteTimeout <= oauthResultWait {
t.Fatalf("WriteTimeout=%v must exceed oauthResultWait=%v", server.WriteTimeout, oauthResultWait)
}
if margin := server.WriteTimeout - oauthResultWait; margin < httpResponseWriteMargin {
t.Fatalf("WriteTimeout margin=%v, want at least %v", margin, httpResponseWriteMargin)
}
}
func TestLogsRoundTrip(t *testing.T) {
h := newRelayHarness(t)
payload := []byte("hello log world")
id := postLogAndGetID(t, h.baseURL, "7.0.0.1", payload)
getResp, err := http.Get(h.baseURL + "/logs/" + id)
if err != nil {
t.Fatalf("get: %v", err)
}
defer getResp.Body.Close()
if getResp.StatusCode != http.StatusOK {
t.Fatalf("get status=%d", getResp.StatusCode)
}
got, _ := io.ReadAll(getResp.Body)
if !bytes.Equal(got, payload) {
t.Fatalf("round-tripped bytes mismatch: got %q want %q", got, payload)
}
if ct := getResp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
t.Errorf("Content-Type=%q", ct)
}
}
func TestLogsUploadDoesNotWriteCapabilityToOperationalLog(t *testing.T) {
h := newRelayHarness(t)
var output bytes.Buffer
previousOutput := log.Writer()
previousFlags := log.Flags()
previousPrefix := log.Prefix()
log.SetOutput(&output)
log.SetFlags(0)
log.SetPrefix("")
t.Cleanup(func() {
log.SetOutput(previousOutput)
log.SetFlags(previousFlags)
log.SetPrefix(previousPrefix)
})
id := postLogAndGetID(t, h.baseURL, "203.0.113.40", []byte("safe diagnostic"))
if strings.Contains(output.String(), id) {
t.Fatalf("operational log retained bearer capability %q", id)
}
if !strings.Contains(output.String(), "logs: stored 15 bytes from 203.0.113.40") {
t.Fatalf("successful upload was not observable: %q", output.String())
}
}
func TestLogStoreRetiresLegacyCapabilitiesOnStartup(t *testing.T) {
dir := t.TempDir()
now := time.Now().Add(-time.Minute)
legacyID := strings.Repeat("a", 25) // legacy capability length
currentID := strings.Repeat("a", logIDLength)
legacyPath := filepath.Join(dir, legacyID+".log")
currentPath := filepath.Join(dir, currentID+".log")
for path, body := range map[string]string{legacyPath: "legacy", currentPath: "current"} {
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatalf("seed %s: %v", path, err)
}
if err := os.Chtimes(path, now, now); err != nil {
t.Fatalf("chtimes %s: %v", path, err)
}
}
store := newLogStore(dir)
if _, err := os.Stat(legacyPath); !errors.Is(err, os.ErrNotExist) {
t.Fatalf("legacy capability file still exists: %v", err)
}
if _, ok, err := store.lookup(legacyID, time.Now()); err != nil || ok {
t.Fatalf("legacy capability lookup=(ok=%v, err=%v), want absent", ok, err)
}
if _, ok, err := store.lookup(currentID, time.Now()); err != nil || !ok {
t.Fatalf("current capability lookup=(ok=%v, err=%v), want indexed", ok, err)
}
restarted := newLogStore(dir)
if _, ok, err := restarted.lookup(currentID, time.Now()); err != nil || !ok {
t.Fatalf("restarted capability lookup=(ok=%v, err=%v), want indexed", ok, err)
}
}
func TestLogsFailedLookupsAreBoundedButValidCapabilitiesRemainAvailable(t *testing.T) {
h := newRelayHarness(t)
payload := []byte("retrievable")
validID := postLogAndGetID(t, h.baseURL, "203.0.113.1", payload)
source := "203.0.113.50"
for i := range logLookupRateBurst {
unknownID := strings.Repeat("z", logIDLength-2) + fmt.Sprintf("%02d", i)
resp := getLog(t, h.baseURL, source, unknownID)
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("failed lookup %d status=%d, want 404", i, resp.StatusCode)
}
if got := resp.Header.Get("Cache-Control"); got != "private, no-store" {
t.Fatalf("failed lookup Cache-Control=%q", got)
}
}
throttled := getLog(t, h.baseURL, source, strings.Repeat("y", logIDLength))
throttled.Body.Close()
if throttled.StatusCode != http.StatusTooManyRequests {
t.Fatalf("exhausted lookup status=%d, want 429", throttled.StatusCode)
}
if got := throttled.Header.Get("Cache-Control"); got != "private, no-store" {
t.Fatalf("throttled Cache-Control=%q", got)
}
success := getLog(t, h.baseURL, source, validID)
defer success.Body.Close()
if success.StatusCode != http.StatusOK {
t.Fatalf("valid capability after exhausted failures status=%d", success.StatusCode)
}
got, err := io.ReadAll(success.Body)
if err != nil || !bytes.Equal(got, payload) {
t.Fatalf("valid body=%q err=%v, want %q", got, err, payload)
}
if cache := success.Header.Get("Cache-Control"); cache != "private, no-store" {
t.Fatalf("success Cache-Control=%q", cache)
}
independent := getLog(t, h.baseURL, "203.0.113.51", strings.Repeat("x", logIDLength))
independent.Body.Close()
if independent.StatusCode != http.StatusNotFound {
t.Fatalf("independent source status=%d, want 404", independent.StatusCode)
}
}
func TestLogFailedLookupCleanupIsDeterministic(t *testing.T) {
store := newLogStore(t.TempDir())
now := time.Unix(1_700_000_000, 0)
id, _, err := store.store([]byte("keep"), now)
if err != nil {
t.Fatalf("store: %v", err)
}
for range logLookupRateBurst {
if !store.allowFailedLookup("203.0.113.1", now) {
t.Fatal("burst rejected early")
}
}
if store.allowFailedLookup("203.0.113.1", now) {
t.Fatal("lookup beyond burst unexpectedly allowed")
}
store.cleanup(now)
if _, ok := store.failedLookupRate["203.0.113.1"]; !ok {
t.Fatal("cleanup removed an effective limiter")
}
store.cleanup(now.Add(time.Duration(logLookupRateBurst) * time.Second))
if _, ok := store.failedLookupRate["203.0.113.1"]; ok {
t.Fatal("cleanup retained a fully refilled limiter")
}
if _, ok := store.entries[id]; !ok {
t.Fatal("limiter cleanup removed stored log")
}
}
func TestLogStorePersistsAcrossRestartAndAvoidsIDCollisions(t *testing.T) {
dir := t.TempDir()
now := time.Now().Add(-time.Second)
first := newLogStore(dir)
first.generateID = func() string { return strings.Repeat("a", logIDLength) }
firstID, _, err := first.store([]byte("original"), now)
if err != nil {
t.Fatalf("store original: %v", err)
}
restarted := newLogStore(dir)
if _, ok, err := restarted.lookup(firstID, time.Now()); err != nil || !ok {
t.Fatal("stored log was not restored after restart")
}
secondWant := strings.Repeat("b", logIDLength)
ids := []string{firstID, secondWant}
restarted.generateID = func() string {
id := ids[0]
ids = ids[1:]
return id
}
secondID, _, err := restarted.store([]byte("second"), time.Now())
if err != nil {
t.Fatalf("store after restart: %v", err)
}
if secondID != secondWant {
t.Fatalf("collision generated id %q, want %q", secondID, secondWant)
}
original, err := os.ReadFile(restarted.filePath(firstID))
if err != nil {
t.Fatalf("read original: %v", err)
}
if string(original) != "original" {
t.Fatalf("colliding store overwrote original: %q", original)
}
}
func TestLogsUploadRateLimitedPerIP(t *testing.T) {
h := newRelayHarness(t)
r1 := postLog(t, h.baseURL, "7.0.0.2", []byte("first"))
r1.Body.Close()
if r1.StatusCode != http.StatusOK {
t.Fatalf("first post status=%d", r1.StatusCode)
}
r2 := postLog(t, h.baseURL, "7.0.0.2", []byte("second"))
r2.Body.Close()
if r2.StatusCode != http.StatusTooManyRequests {
t.Fatalf("second post status=%d want 429", r2.StatusCode)
}
}
func TestLogsUseTrustedCanonicalClientIdentity(t *testing.T) {
t.Run("untrusted spoofing shares direct peer bucket", func(t *testing.T) {
h := newRelayHarnessNoTrust(t)
first := postLog(t, h.baseURL, "203.0.113.1", []byte("first"))
first.Body.Close()
if first.StatusCode != http.StatusOK {
t.Fatalf("first status=%d", first.StatusCode)
}
second := postLog(t, h.baseURL, "203.0.113.2", []byte("second"))
second.Body.Close()
if second.StatusCode != http.StatusTooManyRequests {
t.Fatalf("rotated spoof status=%d, want 429", second.StatusCode)
}
})
t.Run("trusted clients have independent buckets", func(t *testing.T) {
h := newRelayHarness(t)
for _, ip := range []string{"203.0.113.1", "203.0.113.2"} {
resp := postLog(t, h.baseURL, ip, []byte(ip))
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("client %s status=%d, want 200", ip, resp.StatusCode)
}
}
})
t.Run("malformed trusted chain mutates no log state", func(t *testing.T) {
h := newRelayHarness(t)
post := postLog(t, h.baseURL, "203.0.113.1,", []byte("body"))
post.Body.Close()
if post.StatusCode != http.StatusBadRequest {
t.Fatalf("post status=%d, want 400", post.StatusCode)
}
get := getLog(t, h.baseURL, "203.0.113.1,", strings.Repeat("a", logIDLength))
get.Body.Close()
if get.StatusCode != http.StatusBadRequest {
t.Fatalf("get status=%d, want 400", get.StatusCode)
}
h.srv.logs.mu.RLock()
defer h.srv.logs.mu.RUnlock()
if len(h.srv.logs.entries) != 0 || len(h.srv.logs.rateLimit) != 0 || len(h.srv.logs.failedLookupRate) != 0 {
t.Fatalf("malformed chain mutated log state: entries=%d uploads=%d failures=%d",
len(h.srv.logs.entries), len(h.srv.logs.rateLimit), len(h.srv.logs.failedLookupRate))
}
})
t.Run("untrusted malformed header is ignored", func(t *testing.T) {
h := newRelayHarnessNoTrust(t)
resp := postLog(t, h.baseURL, "bad,", []byte("body"))
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status=%d, want 200", resp.StatusCode)
}
})
}
func TestLogsUploadTooLargeRejected(t *testing.T) {
h := newRelayHarness(t)
body := make([]byte, maxLogSize+1)
resp := postLog(t, h.baseURL, "7.0.0.3", body)
resp.Body.Close()
if resp.StatusCode != http.StatusRequestEntityTooLarge {
t.Fatalf("status=%d want 413", resp.StatusCode)
}
}
func TestLogsUploadEmptyRejected(t *testing.T) {
h := newRelayHarness(t)
resp := postLog(t, h.baseURL, "7.0.0.4", nil)
resp.Body.Close()
if resp.StatusCode != http.StatusBadRequest {
t.Fatalf("status=%d want 400", resp.StatusCode)
}
}
func TestLogsUploadStoreFull(t *testing.T) {
h := newRelayHarness(t)
// Saturate the store with distinct IPs so per-IP rate limit doesn't bite.
for i := 0; i < maxLogEntries; i++ {
ip := fmt.Sprintf("7.1.%d.%d", i/256, i%256)
resp := postLog(t, h.baseURL, ip, []byte("x"))
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("warmup %d (ip=%s) status=%d", i, ip, resp.StatusCode)
}
}
resp := postLog(t, h.baseURL, "7.2.0.1", []byte("overflow"))
resp.Body.Close()
if resp.StatusCode != http.StatusServiceUnavailable {
t.Fatalf("overflow status=%d want 503", resp.StatusCode)
}
}
func TestLogsGetUnknownIDIs404(t *testing.T) {
h := newRelayHarness(t)
resp, err := http.Get(h.baseURL + "/logs/" + strings.Repeat("c", logIDLength))
if err != nil {
t.Fatalf("get: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("status=%d want 404", resp.StatusCode)
}
}
func TestLogsGetMalformedIDIs404(t *testing.T) {
h := newRelayHarness(t)
for _, id := range []string{"", "abc", strings.Repeat("a", logIDLength+1), strings.Repeat("a", 25), strings.Repeat("!", logIDLength)} {
resp, err := http.Get(h.baseURL + "/logs/" + id)
if err != nil {
t.Fatalf("get %q: %v", id, err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Errorf("id=%q status=%d want 404", id, resp.StatusCode)
}
if got := resp.Header.Get("Cache-Control"); got != "private, no-store" {
t.Errorf("id=%q Cache-Control=%q", id, got)
}
}
}
func TestLogsGetExpiredIs404(t *testing.T) {
h := newRelayHarness(t)
id := postLogAndGetID(t, h.baseURL, "7.3.0.1", []byte("temp"))
h.srv.logs.mu.Lock()
entry := h.srv.logs.entries[id]
entry.ExpiresAt = time.Now().Add(-time.Minute)
h.srv.logs.entries[id] = entry
h.srv.logs.mu.Unlock()
get, err := http.Get(h.baseURL + "/logs/" + id)
if err != nil {
t.Fatalf("get: %v", err)
}
get.Body.Close()
if get.StatusCode != http.StatusNotFound {
t.Fatalf("status=%d want 404", get.StatusCode)
}
}
func TestLogsMethodNotAllowed(t *testing.T) {
h := newRelayHarness(t)
resp, err := http.Get(h.baseURL + "/logs")
if err != nil {
t.Fatalf("get: %v", err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusMethodNotAllowed {
t.Errorf("GET /logs status=%d want 405", resp.StatusCode)
}
}
var minimalPNG = []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0x01, 0x02, 0x03}
type countingReadCloser struct {
reader *bytes.Reader
reads atomic.Int32
}
func newCountingReadCloser(data []byte) *countingReadCloser {
return &countingReadCloser{reader: bytes.NewReader(data)}
}
func (r *countingReadCloser) Read(p []byte) (int, error) {
r.reads.Add(1)
return r.reader.Read(p)
}
func (r *countingReadCloser) Close() error { return nil }
type blockingReadCloser struct {
data []byte
offset int
started chan struct{}
release <-chan struct{}
once sync.Once
}
func (r *blockingReadCloser) Read(p []byte) (int, error) {
r.once.Do(func() { close(r.started) })
<-r.release
if r.offset == len(r.data) {
return 0, io.EOF
}
n := copy(p, r.data[r.offset:])
r.offset += n
return n, nil
}
func (r *blockingReadCloser) Close() error { return nil }
type deadlineBlockingReadCloser struct {
started chan struct{}
closed chan struct{}
startOnce sync.Once
closeOnce sync.Once
}
func newDeadlineBlockingReadCloser() *deadlineBlockingReadCloser {
return &deadlineBlockingReadCloser{
started: make(chan struct{}),
closed: make(chan struct{}),
}
}
func (r *deadlineBlockingReadCloser) Read([]byte) (int, error) {
r.startOnce.Do(func() { close(r.started) })
<-r.closed
return 0, errors.New("body closed")
}
func (r *deadlineBlockingReadCloser) Close() error {
r.closeOnce.Do(func() { close(r.closed) })
return nil
}
type failingReadCloser struct{}
func (failingReadCloser) Read([]byte) (int, error) { return 0, errors.New("read failed") }
func (failingReadCloser) Close() error { return nil }
func servePosterUpload(s *Server, body io.ReadCloser, xff string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodPost, "/posters", body)
req.RemoteAddr = "198.51.100.10:1234"
if xff != "" {
req.Header.Set("X-Forwarded-For", xff)
}
recorder := httptest.NewRecorder()
s.handlePostPosters(recorder, req)
return recorder
}
func snapshotPosterStore(t *testing.T, store *posterStore) (int, int64, []string) {
t.Helper()
store.mu.RLock()
entryCount := len(store.entries)
totalBytes := store.used
store.mu.RUnlock()
files, err := os.ReadDir(store.dir)
if err != nil {
t.Fatalf("read poster dir: %v", err)
}
names := make([]string, len(files))
for i, file := range files {
names[i] = file.Name()
}
return entryCount, totalBytes, names
}
func TestPosterHandlerRejectsRateLimitedRequestBeforeReadingOrStoring(t *testing.T) {
s := newTestServer(t, filepath.Join(t.TempDir(), "rooms.json"))
now := time.Now()
s.posterUploads = newPosterUploadLimiter(1, 0, 10, 0, 2, 2, now)
first := servePosterUpload(s, io.NopCloser(bytes.NewReader(minimalPNG)), "203.0.113.1")
if first.Code != http.StatusOK {
t.Fatalf("first status=%d", first.Code)
}
beforeEntries, beforeBytes, beforeFiles := snapshotPosterStore(t, s.posters)
rejectedBody := newCountingReadCloser(minimalPNG)
rejected := servePosterUpload(s, rejectedBody, "203.0.113.2")
if rejected.Code != http.StatusTooManyRequests {
t.Fatalf("rejected status=%d, want 429", rejected.Code)
}
if rejectedBody.reads.Load() != 0 {
t.Fatalf("rate-limited body read %d times", rejectedBody.reads.Load())
}
afterEntries, afterBytes, afterFiles := snapshotPosterStore(t, s.posters)
if beforeEntries != afterEntries || beforeBytes != afterBytes || fmt.Sprint(beforeFiles) != fmt.Sprint(afterFiles) {
t.Fatalf("denial mutated poster store: before=(%d,%d,%v) after=(%d,%d,%v)",
beforeEntries, beforeBytes, beforeFiles, afterEntries, afterBytes, afterFiles)
}
}
func TestPosterHandlerConcurrencyRejectsBeforeReadAndRecovers(t *testing.T) {
s := newTestServer(t, filepath.Join(t.TempDir(), "rooms.json"))
s.posterUploads = newPosterUploadLimiter(20, 0, 20, 0, 2, 2, time.Now())
release := make(chan struct{})
recorders := make(chan *httptest.ResponseRecorder, 2)
for range 2 {
body := &blockingReadCloser{
data: minimalPNG,
started: make(chan struct{}),
release: release,
}
go func() {
recorders <- servePosterUpload(s, body, "")
}()
select {
case <-body.started:
case <-time.After(time.Second):
t.Fatal("admitted body was not read")
}
}
extraBody := newCountingReadCloser(minimalPNG)
extra := servePosterUpload(s, extraBody, "")
if extra.Code != http.StatusTooManyRequests {
t.Fatalf("extra status=%d, want 429", extra.Code)
}
if extraBody.reads.Load() != 0 {
t.Fatalf("concurrency-rejected body read %d times", extraBody.reads.Load())
}
close(release)
for range 2 {
select {
case recorder := <-recorders:
if recorder.Code != http.StatusOK {
t.Fatalf("admitted status=%d", recorder.Code)
}
case <-time.After(time.Second):
t.Fatal("admitted upload did not finish")
}
}
recovered := servePosterUpload(s, io.NopCloser(bytes.NewReader(minimalPNG)), "")
if recovered.Code != http.StatusOK {
t.Fatalf("post-completion status=%d, want 200", recovered.Code)
}
s.posterUploads.mu.Lock()
active := s.posterUploads.active
s.posterUploads.mu.Unlock()
if active != 0 {
t.Fatalf("active=%d after completion, want 0", active)
}
}
func TestPosterHandlerDeadlineReleasesStalledUploadSlot(t *testing.T) {
s := newTestServer(t, filepath.Join(t.TempDir(), "rooms.json"))
s.posterUploads = newPosterUploadLimiter(10, 0, 10, 0, 1, 1, time.Now())
s.posterBodyReadTimeout = 20 * time.Millisecond
stalled := newDeadlineBlockingReadCloser()
result := make(chan *httptest.ResponseRecorder, 1)
go func() {
result <- servePosterUpload(s, stalled, "")
}()
select {
case <-stalled.started:
case <-time.After(time.Second):
t.Fatal("stalled body was not read")
}
var timedOut *httptest.ResponseRecorder
select {
case timedOut = <-result:
case <-time.After(time.Second):
t.Fatal("stalled upload did not honor body deadline")
}
if timedOut.Code != http.StatusRequestTimeout {
t.Fatalf("stalled status=%d, want 408", timedOut.Code)
}
s.posterUploads.mu.Lock()
active := s.posterUploads.active
s.posterUploads.mu.Unlock()
if active != 0 {
t.Fatalf("active=%d after body timeout, want 0", active)
}
recovered := servePosterUpload(s, io.NopCloser(bytes.NewReader(minimalPNG)), "")
if recovered.Code != http.StatusOK {
t.Fatalf("post-timeout status=%d, want 200", recovered.Code)
}
}
func TestPosterHandlerSlowChunkedBodyDeadline(t *testing.T) {
s := newTestServer(t, filepath.Join(t.TempDir(), "rooms.json"))
s.posterUploads = newPosterUploadLimiter(10, 0, 10, 0, 1, 1, time.Now())
s.posterBodyReadTimeout = 30 * time.Millisecond
httpServer := httptest.NewServer(http.HandlerFunc(s.handlePostPosters))
t.Cleanup(httpServer.Close)
address := strings.TrimPrefix(httpServer.URL, "http://")
conn, err := net.DialTimeout("tcp", address, time.Second)
if err != nil {
t.Fatalf("dial: %v", err)
}
if _, err := fmt.Fprintf(
conn,
"POST /posters HTTP/1.1\r\nHost: %s\r\nTransfer-Encoding: chunked\r\n\r\n1\r\nx\r\n",
address,
); err != nil {
conn.Close()
t.Fatalf("write partial chunked request: %v", err)
}
if err := conn.SetReadDeadline(time.Now().Add(time.Second)); err != nil {
conn.Close()
t.Fatalf("set response deadline: %v", err)
}
response, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{Method: http.MethodPost})
if err != nil {
conn.Close()
t.Fatalf("read timeout response: %v", err)
}
response.Body.Close()
conn.Close()
if response.StatusCode != http.StatusRequestTimeout {
t.Fatalf("slow chunked status=%d, want 408", response.StatusCode)
}
recovered, err := http.Post(httpServer.URL, "image/png", bytes.NewReader(minimalPNG))
if err != nil {
t.Fatalf("post after timeout: %v", err)
}
recovered.Body.Close()
if recovered.StatusCode != http.StatusOK {
t.Fatalf("post-timeout status=%d, want 200", recovered.StatusCode)
}
}
func TestPosterHandlerReleasesConcurrencyOnEveryExit(t *testing.T) {
tests := []struct {
name string
body func() io.ReadCloser
wantStatus int
storeFailure bool
}{
{name: "read error", body: func() io.ReadCloser { return failingReadCloser{} }, wantStatus: http.StatusBadRequest},
{name: "oversized", body: func() io.ReadCloser {
return io.NopCloser(bytes.NewReader(make([]byte, maxPosterSize+1)))
}, wantStatus: http.StatusRequestEntityTooLarge},
{name: "empty", body: func() io.ReadCloser { return io.NopCloser(bytes.NewReader(nil)) }, wantStatus: http.StatusBadRequest},
{name: "unsupported", body: func() io.ReadCloser {
return io.NopCloser(strings.NewReader("not an image"))
}, wantStatus: http.StatusUnsupportedMediaType},
{name: "store failure", body: func() io.ReadCloser {
return io.NopCloser(bytes.NewReader(minimalPNG))
}, wantStatus: http.StatusInternalServerError, storeFailure: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := newTestServer(t, filepath.Join(t.TempDir(), "rooms.json"))
s.posterUploads = newPosterUploadLimiter(10, 0, 10, 0, 1, 1, time.Now())
originalDir := s.posters.dir
if tt.storeFailure {
s.posters.dir = filepath.Join(t.TempDir(), "missing", "posters")
}
failed := servePosterUpload(s, tt.body(), "")
if failed.Code != tt.wantStatus {
t.Fatalf("status=%d, want %d", failed.Code, tt.wantStatus)
}
s.posters.dir = originalDir
recovery := servePosterUpload(s, io.NopCloser(bytes.NewReader(minimalPNG)), "")
if recovery.Code != http.StatusOK {
t.Fatalf("recovery status=%d, want 200", recovery.Code)
}
s.posterUploads.mu.Lock()
active := s.posterUploads.active
s.posterUploads.mu.Unlock()
if active != 0 {
t.Fatalf("active=%d, want 0", active)
}
})
}
}
func TestPosterHandlerUsesTrustedCanonicalIdentityAndGlobalBudget(t *testing.T) {
t.Run("untrusted XFF rotation cannot bypass per-IP limit", func(t *testing.T) {
h := newRelayHarnessNoTrust(t)
h.srv.posterUploads = newPosterUploadLimiter(3, 0, 20, 0, 4, 4, time.Now())
for i := range 3 {
resp := postPoster(t, h.baseURL, fmt.Sprintf("203.0.113.%d", i+1), minimalPNG)
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("upload %d status=%d", i, resp.StatusCode)
}
}
beforeEntries, beforeBytes, beforeFiles := snapshotPosterStore(t, h.srv.posters)
denied := postPoster(t, h.baseURL, "203.0.113.99", minimalPNG)
denied.Body.Close()
if denied.StatusCode != http.StatusTooManyRequests {
t.Fatalf("rotated spoof status=%d, want 429", denied.StatusCode)
}
afterEntries, afterBytes, afterFiles := snapshotPosterStore(t, h.srv.posters)
if beforeEntries != afterEntries || beforeBytes != afterBytes || fmt.Sprint(beforeFiles) != fmt.Sprint(afterFiles) {
t.Fatal("per-IP denial mutated poster store")
}
})
t.Run("trusted clients are independent but share global budget", func(t *testing.T) {
h := newRelayHarness(t)
h.srv.posterUploads = newPosterUploadLimiter(3, 0, 8, 0, 4, 4, time.Now())
for range 3 {
resp := postPoster(t, h.baseURL, "203.0.113.1", minimalPNG)
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("client A status=%d", resp.StatusCode)
}
}
perIPDenied := postPoster(t, h.baseURL, "203.0.113.1", minimalPNG)
perIPDenied.Body.Close()
if perIPDenied.StatusCode != http.StatusTooManyRequests {
t.Fatalf("client A overflow status=%d, want 429", perIPDenied.StatusCode)
}
for _, ip := range []string{"203.0.113.2", "203.0.113.2", "203.0.113.2", "203.0.113.3", "203.0.113.3"} {
resp := postPoster(t, h.baseURL, ip, minimalPNG)
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("client %s status=%d before global exhaustion", ip, resp.StatusCode)
}
}
beforeEntries, beforeBytes, beforeFiles := snapshotPosterStore(t, h.srv.posters)
globalDenied := postPoster(t, h.baseURL, "203.0.113.4", minimalPNG)
globalDenied.Body.Close()
if globalDenied.StatusCode != http.StatusTooManyRequests {
t.Fatalf("global overflow status=%d, want 429", globalDenied.StatusCode)
}
afterEntries, afterBytes, afterFiles := snapshotPosterStore(t, h.srv.posters)
if beforeEntries != afterEntries || beforeBytes != afterBytes || fmt.Sprint(beforeFiles) != fmt.Sprint(afterFiles) {
t.Fatal("global denial mutated poster store")
}
})
}
func TestPosterHandlerMalformedTrustedChainMutatesNothing(t *testing.T) {
s := newTestServer(t, filepath.Join(t.TempDir(), "rooms.json"))
s.clientIPs = mustClientIPResolver(t, "10.0.0.0/8")
body := newCountingReadCloser(minimalPNG)
req := httptest.NewRequest(http.MethodPost, "/posters", body)
req.RemoteAddr = "10.0.0.2:1234"
req.Header.Set("X-Forwarded-For", "203.0.113.1,")
recorder := httptest.NewRecorder()
s.handlePostPosters(recorder, req)
if recorder.Code != http.StatusBadRequest {
t.Fatalf("status=%d, want 400", recorder.Code)
}
if body.reads.Load() != 0 {
t.Fatalf("malformed-chain body read %d times", body.reads.Load())
}
s.posterUploads.mu.Lock()
active := s.posterUploads.active
perIP := len(s.posterUploads.perIP)
s.posterUploads.global.mu.Lock()
globalTokens := s.posterUploads.global.tokens
s.posterUploads.global.mu.Unlock()
s.posterUploads.mu.Unlock()
if active != 0 || perIP != 0 || globalTokens != posterGlobalRateBurst {
t.Fatalf("malformed chain mutated admission: active=%d perIP=%d global=%v", active, perIP, globalTokens)
}
entries, total, files := snapshotPosterStore(t, s.posters)
if entries != 0 || total != 0 || len(files) != 0 {
t.Fatalf("malformed chain mutated store: entries=%d total=%d files=%v", entries, total, files)
}
}
func TestPosterHandlerIgnoresMalformedHeaderFromUntrustedPeer(t *testing.T) {
h := newRelayHarnessNoTrust(t)
resp := postPoster(t, h.baseURL, "bad,", minimalPNG)
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Fatalf("status=%d, want 200", resp.StatusCode)
}
}
func TestPostersRoundTrip(t *testing.T) {
h := newRelayHarness(t)
payload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 0x01, 0x02, 0x03}
out := postPosterAndDecode(t, h.baseURL, "9.0.0.1", payload)
if out.ExpiresIn != int(posterMaxAge.Seconds()) {
t.Fatalf("expiresIn=%d want %d", out.ExpiresIn, int(posterMaxAge.Seconds()))
}
if !strings.HasPrefix(out.URL, "/posters/") || !strings.HasSuffix(out.URL, ".png") {
t.Fatalf("url=%q should be a relative png poster path", out.URL)
}
if strings.Contains(out.URL, "://") {
t.Fatalf("url=%q should be relative", out.URL)
}
getResp, err := http.Get(h.baseURL + out.URL)
if err != nil {
t.Fatalf("get: %v", err)
}
defer getResp.Body.Close()
if getResp.StatusCode != http.StatusOK {
t.Fatalf("get status=%d", getResp.StatusCode)
}
got, _ := io.ReadAll(getResp.Body)
if !bytes.Equal(got, payload) {
t.Fatalf("round-tripped bytes mismatch: got %v want %v", got, payload)
}
if ct := getResp.Header.Get("Content-Type"); !strings.HasPrefix(ct, "image/png") {
t.Errorf("Content-Type=%q", ct)
}
}
func servePosterGet(s *Server, path, remoteAddr string) *httptest.ResponseRecorder {
req := httptest.NewRequest(http.MethodGet, path, nil)
req.RemoteAddr = remoteAddr
recorder := httptest.NewRecorder()
s.handleGetPosters(recorder, req)
return recorder
}
func TestPosterGetRateLimitedPerIPWithBoundedConcurrency(t *testing.T) {
s := newTestServer(t, filepath.Join(t.TempDir(), "rooms.json"))
now := time.Now()
_, entry, err := s.posters.store(minimalPNG, "image/png", now)
if err != nil {
t.Fatalf("store poster: %v", err)
}
path := "/posters/" + entry.Filename
t.Run("per-IP budget", func(t *testing.T) {
s.posterFetches = newPosterUploadLimiter(2, 0, 100, 0, 8, 4, now)
for i := range 2 {
if got := servePosterGet(s, path, "203.0.113.7:1234"); got.Code != http.StatusOK {
t.Fatalf("fetch %d status=%d want 200", i, got.Code)
}
}
limited := servePosterGet(s, path, "203.0.113.7:1234")
if limited.Code != http.StatusTooManyRequests {
t.Fatalf("status=%d want 429 once per-IP burst is exhausted", limited.Code)
}
other := servePosterGet(s, path, "203.0.113.8:1234")
if other.Code != http.StatusOK {
t.Fatalf("status=%d want 200 for an independent client", other.Code)
}
})
t.Run("concurrency guard shared with handler", func(t *testing.T) {
s.posterFetches = newPosterUploadLimiter(100, 0, 100, 0, 1, 1, now)
if !s.posterFetches.tryStart("in-flight", now) {
t.Fatal("could not occupy the single fetch slot")
}
blocked := servePosterGet(s, path, "203.0.113.9:1234")
if blocked.Code != http.StatusTooManyRequests {
t.Fatalf("status=%d want 429 while the slot is held", blocked.Code)
}
s.posterFetches.finish("in-flight")
if got := servePosterGet(s, path, "203.0.113.9:1234"); got.Code != http.StatusOK {
t.Fatalf("status=%d want 200 after the slot is released", got.Code)
}
})
t.Run("slot released on every handler exit", func(t *testing.T) {
s.posterFetches = newPosterUploadLimiter(100, 0, 100, 0, 1, 1, now)
missing := "/posters/" + strings.Repeat("z", posterIDLength) + ".png"
if got := servePosterGet(s, missing, "203.0.113.10:1234"); got.Code != http.StatusNotFound {
t.Fatalf("missing poster status=%d want 404", got.Code)
}
if got := servePosterGet(s, path, "203.0.113.10:1234"); got.Code != http.StatusOK {
t.Fatalf("status=%d want 200 after a 404 exit released the slot", got.Code)
}
if got := servePosterGet(s, path, "203.0.113.10:1234"); got.Code != http.StatusOK {
t.Fatalf("status=%d want 200 after a 200 exit released the slot", got.Code)
}
})
}
// Concurrent lookups must serve non-expired hits from the read lock and
// delete an expired entry exactly once under the write lock.
func TestPosterLookupConcurrentHitsAndExpiryAreRaceClean(t *testing.T) {
ps := newPosterStore(t.TempDir(), 1024, time.Hour)
now := time.Now()
liveID, liveEntry, err := ps.store([]byte{1, 2, 3}, "image/png", now)
if err != nil {
t.Fatalf("store live poster: %v", err)
}
expiredID, expiredEntry, err := ps.store([]byte{4, 5, 6, 7}, "image/png", now.Add(-2*time.Hour))
if err != nil {
t.Fatalf("store expired poster: %v", err)
}
var liveMisses, expiredHits, lookupErrs atomic.Int64
var wg sync.WaitGroup
for range 8 {
wg.Add(1)
go func() {
defer wg.Done()
for range 200 {
entry, ok, err := ps.lookupEntry(liveID, now, nil)
if err != nil {
lookupErrs.Add(1)
} else if !ok || entry.Filename != liveEntry.Filename {
liveMisses.Add(1)
}
if _, ok, err := ps.lookupEntry(expiredID, now, nil); err != nil {
lookupErrs.Add(1)
} else if ok {
expiredHits.Add(1)
}
}
}()
}
wg.Wait()
if n := lookupErrs.Load(); n != 0 {
t.Fatalf("%d lookups returned errors", n)
}
if n := liveMisses.Load(); n != 0 {
t.Fatalf("live entry missed %d times during concurrent lookups", n)
}
if n := expiredHits.Load(); n != 0 {
t.Fatalf("expired entry served %d times", n)
}
ps.mu.RLock()
used := ps.used
_, liveRetained := ps.entries[liveID]
_, expiredRetained := ps.entries[expiredID]
ps.mu.RUnlock()
if !liveRetained || expiredRetained {
t.Fatalf("entries after concurrent lookups: live=%v expired=%v", liveRetained, expiredRetained)
}
if used != int64(3) {
t.Fatalf("used=%d want 3: the expired entry must be deleted exactly once", used)
}
if _, err := os.Stat(ps.filePath(expiredEntry.Filename)); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("expired poster file not removed: %v", err)
}
}
func TestPostersRejectInvalidAndOversizedUploads(t *testing.T) {
h := newRelayHarness(t)
invalid := postPoster(t, h.baseURL, "9.0.0.2", []byte("not an image"))
invalid.Body.Close()
if invalid.StatusCode != http.StatusUnsupportedMediaType {
t.Fatalf("invalid status=%d want 415", invalid.StatusCode)
}
oversized := postPoster(t, h.baseURL, "9.0.0.3", make([]byte, maxPosterSize+1))
oversized.Body.Close()
if oversized.StatusCode != http.StatusRequestEntityTooLarge {
t.Fatalf("oversized status=%d want 413", oversized.StatusCode)
}
}
func TestPosterStoreEvictsOldestOverQuota(t *testing.T) {
ps := newPosterStore(t.TempDir(), 12, time.Hour)
now := time.Now()
payload := []byte{1, 2, 3, 4, 5, 6, 7}
id1, entry1, err := ps.store(payload, "image/png", now)
if err != nil {
t.Fatalf("store first: %v", err)
}
id2, entry2, err := ps.store(payload, "image/png", now.Add(time.Minute))
if err != nil {
t.Fatalf("store second: %v", err)
}
ps.mu.RLock()
_, hasFirst := ps.entries[id1]
_, hasSecond := ps.entries[id2]
total := ps.used
ps.mu.RUnlock()
if hasFirst {
t.Fatal("oldest poster should have been evicted")
}
if !hasSecond {
t.Fatal("newest poster should remain")
}
if total != int64(len(payload)) {
t.Fatalf("stored bytes=%d want %d", total, len(payload))
}
if _, err := os.Stat(ps.filePath(entry1.Filename)); !os.IsNotExist(err) {
t.Fatalf("oldest file still exists or stat failed unexpectedly: %v", err)
}
if _, err := os.Stat(ps.filePath(entry2.Filename)); err != nil {
t.Fatalf("newest file missing: %v", err)
}
}
func TestPosterStoreCleanupExpiresOldPosters(t *testing.T) {
ps := newPosterStore(t.TempDir(), 1024, time.Hour)
now := time.Now()
id, entry, err := ps.store([]byte{1, 2, 3}, "image/png", now.Add(-2*time.Hour))
if err != nil {
t.Fatalf("store: %v", err)
}
if err := ps.cleanup(now); err != nil {
t.Fatalf("cleanup: %v", err)
}
ps.mu.RLock()
_, exists := ps.entries[id]
total := ps.used
ps.mu.RUnlock()
if exists {
t.Fatal("expired poster should have been removed")
}
if total != 0 {
t.Fatalf("stored bytes=%d want 0", total)
}
if _, err := os.Stat(ps.filePath(entry.Filename)); !os.IsNotExist(err) {
t.Fatalf("expired file still exists or stat failed unexpectedly: %v", err)
}
}
func regularFileBytes(t *testing.T, dir string) int64 {
t.Helper()
files, err := os.ReadDir(dir)
if err != nil {
t.Fatalf("read directory: %v", err)
}
var total int64
for _, file := range files {
info, err := file.Info()
if err != nil {
t.Fatalf("stat %s: %v", file.Name(), err)
}
if info.Mode().IsRegular() {
total += info.Size()
}
}
return total
}
func TestLogStoreRemovalFailureRetainsEntryUntilRetry(t *testing.T) {
dir := t.TempDir()
remover := newDeterministicRemover()
ls := newLogStoreWithRemover(dir, remover.remove)
ls.generateID = func() string { return strings.Repeat("a", logIDLength) }
now := time.Now()
id, _, err := ls.store([]byte("retained"), now)
if err != nil {
t.Fatalf("store: %v", err)
}
path := ls.filePath(id)
ls.mu.Lock()
entry := ls.entries[id]
entry.ExpiresAt = now.Add(-time.Minute)
ls.entries[id] = entry
ls.mu.Unlock()
remover.fail(path, fs.ErrPermission)
if _, ok, err := ls.lookup(id, now); !errors.Is(err, fs.ErrPermission) || ok {
t.Fatalf("lookup=(ok=%v, err=%v), want unavailable permission error", ok, err)
}
ls.mu.RLock()
_, indexed := ls.entries[id]
artifacts := ls.accountedLocked()
ls.mu.RUnlock()
if !indexed || artifacts != 1 {
t.Fatalf("failed removal changed metadata: indexed=%v artifacts=%d", indexed, artifacts)
}
if _, err := os.Stat(path); err != nil {
t.Fatalf("failed removal lost file: %v", err)
}
remover.recover(path)
if err := ls.cleanup(now); err != nil {
t.Fatalf("retry cleanup: %v", err)
}
if remover.callCount(path) != 2 {
t.Fatalf("remove calls=%d want 2", remover.callCount(path))
}
if _, err := os.Stat(path); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("file remains after retry: %v", err)
}
if err := ls.cleanup(now); err != nil {
t.Fatalf("idempotent cleanup: %v", err)
}
if remover.callCount(path) != 2 {
t.Fatalf("already committed entry removed again: calls=%d", remover.callCount(path))
}
}
func TestRemovalFailureDoesNotBlockUploadsWhileCapacityRemains(t *testing.T) {
t.Run("logs", func(t *testing.T) {
dir := t.TempDir()
remover := newDeterministicRemover()
store := newLogStoreWithRemover(dir, remover.remove)
ids := []string{strings.Repeat("a", logIDLength), strings.Repeat("b", logIDLength)}
nextID := 0
store.generateID = func() string {
id := ids[nextID]
nextID++
return id
}
now := time.Now()
firstID, _, err := store.store([]byte("expired"), now)
if err != nil {
t.Fatalf("store expired log: %v", err)
}
store.mu.Lock()
entry := store.entries[firstID]
entry.ExpiresAt = now.Add(-time.Minute)
store.entries[firstID] = entry
store.mu.Unlock()
remover.fail(store.filePath(firstID), fs.ErrPermission)
secondID, _, err := store.store([]byte("new"), now)
if err != nil {
t.Fatalf("unrelated removal failure blocked log upload: %v", err)
}
store.mu.RLock()
_, firstRetained := store.entries[firstID]
_, secondStored := store.entries[secondID]
store.mu.RUnlock()
if !firstRetained || !secondStored {
t.Fatalf("log accounting lost entries: first=%v second=%v", firstRetained, secondStored)
}
})
t.Run("posters", func(t *testing.T) {
dir := t.TempDir()
remover := newDeterministicRemover()
store := newPosterStoreWithRemover(dir, 1024, time.Hour, remover.remove)
now := time.Now()
firstID, first, err := store.store([]byte{1, 2, 3}, "image/png", now.Add(-2*time.Hour))
if err != nil {
t.Fatalf("store expired poster: %v", err)
}
remover.fail(store.filePath(first.Filename), fs.ErrPermission)
secondID, _, err := store.store([]byte{4, 5, 6}, "image/png", now)
if err != nil {
t.Fatalf("unrelated removal failure blocked poster upload: %v", err)
}
store.mu.RLock()
_, firstRetained := store.entries[firstID]
_, secondStored := store.entries[secondID]
store.mu.RUnlock()
if !firstRetained || !secondStored {
t.Fatalf("poster accounting lost entries: first=%v second=%v", firstRetained, secondStored)
}
})
}
func TestLogStoreErrNotExistCommitsDeletionOnce(t *testing.T) {
remover := newDeterministicRemover()
ls := newLogStoreWithRemover(t.TempDir(), remover.remove)
ls.generateID = func() string { return strings.Repeat("a", logIDLength) }
now := time.Now()
id, _, err := ls.store([]byte("gone"), now)
if err != nil {
t.Fatalf("store: %v", err)
}
path := ls.filePath(id)
if err := os.Remove(path); err != nil {
t.Fatalf("external remove: %v", err)
}
ls.mu.Lock()
entry := ls.entries[id]
entry.ExpiresAt = now.Add(-time.Minute)
ls.entries[id] = entry
ls.mu.Unlock()
if _, ok, err := ls.lookup(id, now); err != nil || ok {
t.Fatalf("lookup=(ok=%v, err=%v), want clean miss", ok, err)
}
if err := ls.cleanup(now); err != nil {
t.Fatalf("repeat cleanup: %v", err)
}
if remover.callCount(path) != 1 {
t.Fatalf("remove calls=%d want 1", remover.callCount(path))
}
}
func TestLogStoreTracksFailedTempCleanup(t *testing.T) {
dir := t.TempDir()
remover := newDeterministicRemover()
ls := newLogStoreWithRemover(dir, remover.remove)
logID := strings.Repeat("a", logIDLength)
ls.generateID = func() string { return logID }
tmpPath := ls.filePath(logID) + ".tmp"
if err := os.Mkdir(tmpPath, 0755); err != nil {
t.Fatalf("seed temp directory: %v", err)
}
cleanupErr := errors.New("synthetic temp removal failure")
remover.fail(tmpPath, cleanupErr)
if _, _, err := ls.store([]byte("payload"), time.Now()); err == nil {
t.Fatal("store succeeded despite temp write failure")
}
ls.mu.RLock()
_, pending := ls.pendingRemovals[filepath.Base(tmpPath)]
artifacts := ls.accountedLocked()
ls.mu.RUnlock()
if !pending || artifacts != 1 {
t.Fatalf("temp cleanup not tracked: pending=%v artifacts=%d", pending, artifacts)
}
remover.recover(tmpPath)
if err := ls.cleanup(time.Now()); err != nil {
t.Fatalf("retry temp cleanup: %v", err)
}
if _, err := os.Stat(tmpPath); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("temp artifact remains: %v", err)
}
}
func TestLogStoreStartupReconcilesLiveAndPendingRemovals(t *testing.T) {
dir := t.TempDir()
now := time.Now()
expiredID := strings.Repeat("a", logIDLength)
expiredPath := filepath.Join(dir, expiredID+".log")
tempPath := filepath.Join(dir, "upload.log.tmp")
malformedPath := filepath.Join(dir, "malformed")
for path, data := range map[string][]byte{
expiredPath: []byte("expired"),
tempPath: []byte("partial"),
malformedPath: []byte("invalid"),
} {
if err := os.WriteFile(path, data, 0644); err != nil {
t.Fatalf("seed %s: %v", filepath.Base(path), err)
}
}
old := now.Add(-logMaxAge - time.Hour)
if err := os.Chtimes(expiredPath, old, old); err != nil {
t.Fatalf("age expired log: %v", err)
}
remover := newDeterministicRemover()
for _, path := range []string{expiredPath, tempPath, malformedPath} {
remover.fail(path, fs.ErrPermission)
}
ls := newLogStoreWithRemover(dir, remover.remove)
if ls.startupErr == nil {
t.Fatal("startup removal failures were not reported")
}
ls.mu.RLock()
_, live := ls.entries[expiredID]
pending := len(ls.pendingRemovals)
artifacts := ls.accountedLocked()
ls.mu.RUnlock()
if !live || pending != 2 || artifacts != 3 {
t.Fatalf("startup accounting: live=%v pending=%d artifacts=%d", live, pending, artifacts)
}
newID, _, err := ls.store([]byte("new"), now)
if err != nil {
t.Fatalf("startup cleanup failure blocked new log: %v", err)
}
for _, path := range []string{expiredPath, tempPath, malformedPath} {
remover.recover(path)
}
if err := ls.cleanup(now); err != nil {
t.Fatalf("startup retry cleanup: %v", err)
}
restarted := newLogStore(dir)
restarted.mu.RLock()
restartedArtifacts := restarted.accountedLocked()
_, newLogRestored := restarted.entries[newID]
restarted.mu.RUnlock()
if restartedArtifacts != 1 || !newLogRestored {
t.Fatalf("restart reconstructed %d artifacts, new log restored=%v", restartedArtifacts, newLogRestored)
}
}
func TestStoresReconcileConfinedNonEmptyStaleDirectories(t *testing.T) {
t.Run("logs", func(t *testing.T) {
dir := t.TempDir()
staleDir := filepath.Join(dir, "abandoned.log.tmp")
if err := os.MkdirAll(filepath.Join(staleDir, "nested"), 0755); err != nil {
t.Fatalf("seed stale log directory: %v", err)
}
if err := os.WriteFile(filepath.Join(staleDir, "nested", "partial"), []byte("stale"), 0644); err != nil {
t.Fatalf("seed stale log payload: %v", err)
}
store := newLogStore(dir)
if store.startupErr != nil {
t.Fatalf("startup reconciliation: %v", store.startupErr)
}
if _, err := os.Stat(staleDir); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("stale log directory remains: %v", err)
}
store.generateID = func() string { return strings.Repeat("a", logIDLength) }
if _, _, err := store.store([]byte("new log"), time.Now()); err != nil {
t.Fatalf("store after reconciliation: %v", err)
}
})
t.Run("posters", func(t *testing.T) {
dir := t.TempDir()
staleDir := filepath.Join(dir, "abandoned.tmp")
if err := os.MkdirAll(filepath.Join(staleDir, "nested"), 0755); err != nil {
t.Fatalf("seed stale poster directory: %v", err)
}
if err := os.WriteFile(filepath.Join(staleDir, "nested", "partial"), []byte("stale"), 0644); err != nil {
t.Fatalf("seed stale poster payload: %v", err)
}
store := newPosterStore(dir, 1024, time.Hour)
if store.startupErr != nil {
t.Fatalf("startup reconciliation: %v", store.startupErr)
}
if _, err := os.Stat(staleDir); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("stale poster directory remains: %v", err)
}
if _, _, err := store.store([]byte{1, 2, 3}, "image/png", time.Now()); err != nil {
t.Fatalf("store after reconciliation: %v", err)
}
})
}
func TestRecursiveArtifactRemovalRejectsOutsideStore(t *testing.T) {
root := t.TempDir()
outside := t.TempDir()
nested := filepath.Join(outside, "nested")
if err := os.Mkdir(nested, 0755); err != nil {
t.Fatalf("seed outside directory: %v", err)
}
if err := os.WriteFile(filepath.Join(nested, "keep"), []byte("keep"), 0644); err != nil {
t.Fatalf("seed outside payload: %v", err)
}
err := removeArtifact(os.Remove, root, nested)
if !errors.Is(err, errArtifactOutsideStore) {
t.Fatalf("outside removal error=%v, want confinement error", err)
}
if _, err := os.Stat(filepath.Join(nested, "keep")); err != nil {
t.Fatalf("outside artifact was removed: %v", err)
}
}
func TestPosterQuotaRemovalFailureDoesNotReclaimAccounting(t *testing.T) {
dir := t.TempDir()
remover := newDeterministicRemover()
ps := newPosterStoreWithRemover(dir, 12, time.Hour, remover.remove)
now := time.Now()
payload := []byte{1, 2, 3, 4, 5, 6, 7}
oldID, oldEntry, err := ps.store(payload, "image/png", now)
if err != nil {
t.Fatalf("store oldest: %v", err)
}
oldPath := ps.filePath(oldEntry.Filename)
remover.fail(oldPath, fs.ErrPermission)
newID, newEntry, err := ps.store(payload, "image/png", now.Add(time.Minute))
if !errors.Is(err, fs.ErrPermission) {
t.Fatalf("quota store error=%v want permission error", err)
}
if newID != "" || newEntry != (artifactEntry{}) {
t.Fatalf("failed store returned success values: id=%q entry=%+v", newID, newEntry)
}
ps.mu.RLock()
_, retained := ps.entries[oldID]
total := ps.used
pending := ps.pendingDebt
accounted := ps.accountedLocked()
ps.mu.RUnlock()
if !retained || total != int64(len(payload)) || pending != 0 {
t.Fatalf("failed eviction accounting: retained=%v total=%d pending=%d", retained, total, pending)
}
if physical := regularFileBytes(t, dir); physical != accounted {
t.Fatalf("accounted bytes=%d physical bytes=%d", accounted, physical)
}
remover.recover(oldPath)
retryID, retryEntry, err := ps.store(payload, "image/png", now.Add(time.Minute))
if err != nil {
t.Fatalf("retry store: %v", err)
}
if retryID == "" || retryEntry.Size != int64(len(payload)) {
t.Fatalf("retry result: id=%q entry=%+v", retryID, retryEntry)
}
if remover.callCount(oldPath) != 2 {
t.Fatalf("old poster remove calls=%d want 2", remover.callCount(oldPath))
}
ps.mu.RLock()
accounted = ps.accountedLocked()
total = ps.used
ps.mu.RUnlock()
if total != int64(len(payload)) || regularFileBytes(t, dir) != accounted {
t.Fatalf("retry accounting: total=%d accounted=%d physical=%d", total, accounted, regularFileBytes(t, dir))
}
}
func TestPosterExpiredRemovalFailureAndErrNotExistAreExactOnce(t *testing.T) {
t.Run("failure retains accounting for retry", func(t *testing.T) {
remover := newDeterministicRemover()
ps := newPosterStoreWithRemover(t.TempDir(), 1024, time.Hour, remover.remove)
now := time.Now()
id, entry, err := ps.store([]byte{1, 2, 3}, "image/png", now)
if err != nil {
t.Fatalf("store: %v", err)
}
path := ps.filePath(entry.Filename)
ps.mu.Lock()
expired := ps.entries[id]
expired.ExpiresAt = now.Add(-time.Minute)
ps.entries[id] = expired
ps.mu.Unlock()
remover.fail(path, fs.ErrPermission)
if _, ok, err := ps.lookup(entry.Filename, now); !errors.Is(err, fs.ErrPermission) || ok {
t.Fatalf("lookup=(ok=%v, err=%v), want unavailable permission error", ok, err)
}
ps.mu.RLock()
_, retained := ps.entries[id]
total := ps.used
ps.mu.RUnlock()
if !retained || total != entry.Size {
t.Fatalf("failed expiry accounting: retained=%v total=%d", retained, total)
}
remover.recover(path)
if err := ps.cleanup(now); err != nil {
t.Fatalf("retry cleanup: %v", err)
}
if err := ps.cleanup(now); err != nil {
t.Fatalf("repeat cleanup: %v", err)
}
if remover.callCount(path) != 2 {
t.Fatalf("remove calls=%d want 2", remover.callCount(path))
}
ps.mu.RLock()
total = ps.used
ps.mu.RUnlock()
if total != 0 {
t.Fatalf("stored bytes=%d want 0", total)
}
})
t.Run("not exist commits once", func(t *testing.T) {
remover := newDeterministicRemover()
ps := newPosterStoreWithRemover(t.TempDir(), 1024, time.Hour, remover.remove)
now := time.Now()
id, entry, err := ps.store([]byte{1, 2, 3}, "image/png", now)
if err != nil {
t.Fatalf("store: %v", err)
}
path := ps.filePath(entry.Filename)
if err := os.Remove(path); err != nil {
t.Fatalf("external remove: %v", err)
}
ps.mu.Lock()
expired := ps.entries[id]
expired.ExpiresAt = now.Add(-time.Minute)
ps.entries[id] = expired
ps.mu.Unlock()
if _, ok, err := ps.lookup(entry.Filename, now); err != nil || ok {
t.Fatalf("lookup=(ok=%v, err=%v), want clean miss", ok, err)
}
if err := ps.cleanup(now); err != nil {
t.Fatalf("repeat cleanup: %v", err)
}
if remover.callCount(path) != 1 {
t.Fatalf("remove calls=%d want 1", remover.callCount(path))
}
ps.mu.RLock()
total := ps.used
ps.mu.RUnlock()
if total != 0 {
t.Fatalf("stored bytes=%d want 0", total)
}
})
}
func TestPosterStoreKnownCleanupDebtConsumesCapacityAndRetries(t *testing.T) {
dir := t.TempDir()
stalePath := filepath.Join(dir, "poster.tmp")
if err := os.WriteFile(stalePath, []byte("1234"), 0644); err != nil {
t.Fatalf("seed stale poster: %v", err)
}
remover := newDeterministicRemover()
remover.fail(stalePath, fs.ErrPermission)
ps := newPosterStoreWithRemover(dir, 5, time.Hour, remover.remove)
if ps.startupErr == nil {
t.Fatal("startup removal failure was not reported")
}
if _, _, err := ps.store([]byte{1, 2}, "image/png", time.Now()); err == nil {
t.Fatal("upload exceeded capacity after known stale bytes were accounted")
}
ps.mu.RLock()
pendingBytes := ps.pendingDebt
accountedBytes := ps.accountedLocked()
ps.mu.RUnlock()
if pendingBytes != 4 || accountedBytes != 4 {
t.Fatalf("known debt accounting: pending=%d accounted=%d, want 4", pendingBytes, accountedBytes)
}
if calls := remover.callCount(stalePath); calls != 2 {
t.Fatalf("known debt remove calls=%d, want startup plus upload retry", calls)
}
remover.recover(stalePath)
if _, entry, err := ps.store([]byte{1, 2}, "image/png", time.Now()); err != nil {
t.Fatalf("store after known debt recovery: %v", err)
} else if entry.Size != 2 {
t.Fatalf("stored entry size=%d, want 2", entry.Size)
}
ps.mu.RLock()
pendingBytes = ps.pendingDebt
ps.mu.RUnlock()
if pendingBytes != 0 {
t.Fatalf("known debt remained after successful retry: %d bytes", pendingBytes)
}
}
func TestPosterStoreUnknownCleanupDebtDoesNotBlockUploadAndRecovers(t *testing.T) {
dir := t.TempDir()
unknownPath := filepath.Join(dir, "unknown-dir")
if err := os.Mkdir(unknownPath, 0755); err != nil {
t.Fatalf("seed unknown artifact: %v", err)
}
remover := newDeterministicRemover()
remover.fail(unknownPath, fs.ErrPermission)
ps := newPosterStoreWithRemover(dir, 5, time.Hour, remover.remove)
if ps.startupErr == nil {
t.Fatal("startup removal failure was not reported")
}
if calls := remover.callCount(unknownPath); calls != 1 {
t.Fatalf("startup remove calls=%d, want 1", calls)
}
if _, entry, err := ps.store([]byte{1, 2, 3}, "image/png", time.Now()); err != nil {
t.Fatalf("capacity-safe upload blocked by unknown artifact: %v", err)
} else if entry.Size != 3 {
t.Fatalf("stored entry size=%d, want 3", entry.Size)
}
if calls := remover.callCount(unknownPath); calls != 1 {
t.Fatalf("upload retried permanent unknown debt: calls=%d", calls)
}
if err := ps.cleanup(time.Now()); !errors.Is(err, fs.ErrPermission) {
t.Fatalf("failed cleanup error=%v, want permission error", err)
}
ps.mu.RLock()
pending, exists := ps.pendingRemovals[filepath.Base(unknownPath)]
pendingBytes := ps.pendingDebt
ps.mu.RUnlock()
if !exists || pending.sizeKnown {
t.Fatalf("unknown debt entry=(%+v, exists=%v), want present with unknown size", pending, exists)
}
if pendingBytes != 0 {
t.Fatalf("unknown debt consumed %d known pending bytes", pendingBytes)
}
remover.recover(unknownPath)
if err := ps.cleanup(time.Now()); err != nil {
t.Fatalf("cleanup after recovery: %v", err)
}
ps.mu.RLock()
pendingCount := len(ps.pendingRemovals)
pendingBytes = ps.pendingDebt
ps.mu.RUnlock()
if pendingCount != 0 || pendingBytes != 0 {
t.Fatalf("recovered debt remains: pending=%d bytes=%d", pendingCount, pendingBytes)
}
if _, err := os.Stat(unknownPath); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("unknown artifact remains after recovery: %v", err)
}
}
func TestStorageHandlersReturnGenericErrorsForRemovalFailures(t *testing.T) {
remover := newDeterministicRemover()
logs := newLogStoreWithRemover(t.TempDir(), remover.remove)
logs.generateID = func() string { return strings.Repeat("a", logIDLength) }
posters := newPosterStoreWithRemover(t.TempDir(), 1024, time.Hour, remover.remove)
h := newStorageHarness(t, logs, posters)
now := time.Now()
logID, _, err := logs.store([]byte("expired log"), now)
if err != nil {
t.Fatalf("store log: %v", err)
}
posterID, poster, err := posters.store([]byte{1, 2, 3}, "image/png", now)
if err != nil {
t.Fatalf("store poster: %v", err)
}
logs.mu.Lock()
logEntry := logs.entries[logID]
logEntry.ExpiresAt = now.Add(-time.Minute)
logs.entries[logID] = logEntry
logs.mu.Unlock()
posters.mu.Lock()
posterEntry := posters.entries[posterID]
posterEntry.ExpiresAt = now.Add(-time.Minute)
posters.entries[posterID] = posterEntry
posters.mu.Unlock()
logPath := logs.filePath(logID)
posterPath := posters.filePath(poster.Filename)
remover.fail(logPath, fs.ErrPermission)
remover.fail(posterPath, fs.ErrPermission)
for name, target := range map[string]string{
"log": h.baseURL + "/logs/" + logID,
"poster": h.baseURL + "/posters/" + poster.Filename,
} {
resp, err := http.Get(target)
if err != nil {
t.Fatalf("%s get: %v", name, err)
}
body, readErr := io.ReadAll(resp.Body)
resp.Body.Close()
if readErr != nil {
t.Fatalf("%s response body: %v", name, readErr)
}
if resp.StatusCode != http.StatusInternalServerError {
t.Fatalf("%s status=%d want 500", name, resp.StatusCode)
}
want := "Failed to retrieve " + name + "\n"
if string(body) != want {
t.Fatalf("%s response=%q want %q", name, body, want)
}
}
remover.recover(logPath)
remover.recover(posterPath)
for name, target := range map[string]string{
"log": h.baseURL + "/logs/" + logID,
"poster": h.baseURL + "/posters/" + poster.Filename,
} {
resp, err := http.Get(target)
if err != nil {
t.Fatalf("%s recovery get: %v", name, err)
}
resp.Body.Close()
if resp.StatusCode != http.StatusNotFound {
t.Fatalf("%s recovery status=%d want 404", name, resp.StatusCode)
}
}
}
func TestPosterHandlerRejectsUploadWhenQuotaRemovalFails(t *testing.T) {
remover := newDeterministicRemover()
logs := newLogStoreWithRemover(t.TempDir(), remover.remove)
payload := []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a, 1, 2, 3}
posters := newPosterStoreWithRemover(t.TempDir(), int64(len(payload)+1), time.Hour, remover.remove)
now := time.Now()
oldID, oldEntry, err := posters.store(payload, "image/png", now)
if err != nil {
t.Fatalf("store old poster: %v", err)
}
oldPath := posters.filePath(oldEntry.Filename)
remover.fail(oldPath, fs.ErrPermission)
h := newStorageHarness(t, logs, posters)
resp := postPoster(t, h.baseURL, "9.9.9.9", payload)
body, readErr := io.ReadAll(resp.Body)
resp.Body.Close()
if readErr != nil {
t.Fatalf("read failed upload response: %v", readErr)
}
if resp.StatusCode != http.StatusInternalServerError || string(body) != "Failed to store poster\n" {
t.Fatalf("failed upload status=%d body=%q", resp.StatusCode, body)
}
posters.mu.RLock()
_, retained := posters.entries[oldID]
total := posters.used
posters.mu.RUnlock()
if !retained || total != int64(len(payload)) {
t.Fatalf("failed upload changed old poster: retained=%v total=%d", retained, total)
}
get, err := http.Get(h.baseURL + "/posters/" + oldEntry.Filename)
if err != nil {
t.Fatalf("get retained poster: %v", err)
}
get.Body.Close()
if get.StatusCode != http.StatusOK {
t.Fatalf("retained poster status=%d want 200", get.StatusCode)
}
}
func TestCleanupStepContinuesAfterRemovalFailureAndThrottlesLogging(t *testing.T) {
remover := newDeterministicRemover()
logs := newLogStoreWithRemover(t.TempDir(), remover.remove)
logs.generateID = func() string { return strings.Repeat("a", logIDLength) }
posters := newPosterStoreWithRemover(t.TempDir(), 1024, time.Hour, remover.remove)
now := time.Now()
logID, _, err := logs.store([]byte("expired"), now)
if err != nil {
t.Fatalf("store log: %v", err)
}
posterID, poster, err := posters.store([]byte{1, 2, 3}, "image/png", now)
if err != nil {
t.Fatalf("store poster: %v", err)
}
logs.mu.Lock()
logEntry := logs.entries[logID]
logEntry.ExpiresAt = now.Add(-time.Minute)
logs.entries[logID] = logEntry
logs.mu.Unlock()
posters.mu.Lock()
posterEntry := posters.entries[posterID]
posterEntry.ExpiresAt = now.Add(-time.Minute)
posters.entries[posterID] = posterEntry
posters.mu.Unlock()
logPath := logs.filePath(logID)
remover.fail(logPath, fs.ErrPermission)
srv := &Server{
rooms: make(map[string]*Room),
logs: logs,
posters: posters,
posterUploads: newPosterUploadLimiter(posterPerIPRateBurst, posterPerIPRateSustained, posterGlobalRateBurst, posterGlobalRateSustained, maxConcurrentPosterUploads, maxConcurrentPosterUploadsPerIP, now),
posterFetches: newPosterUploadLimiter(posterFetchPerIPRateBurst, posterFetchPerIPRateSustained, posterFetchGlobalRateBurst, posterFetchGlobalRateSustained, maxConcurrentPosterFetches, maxConcurrentPosterFetchesPerIP, now),
conns: newConnTracker(),
}
srv.runCleanupStep(now)
logs.mu.RLock()
_, logRetained := logs.entries[logID]
logs.mu.RUnlock()
posters.mu.RLock()
_, posterRetained := posters.entries[posterID]
posters.mu.RUnlock()
if !logRetained || posterRetained {
t.Fatalf("cleanup continuation: log retained=%v poster retained=%v", logRetained, posterRetained)
}
if _, err := os.Stat(posters.filePath(poster.Filename)); !errors.Is(err, fs.ErrNotExist) {
t.Fatalf("poster cleanup did not continue: %v", err)
}
srv.removalErrors.mu.Lock()
firstLog := srv.removalErrors.lastLog["logs:cleanup"]
srv.removalErrors.mu.Unlock()
if firstLog.IsZero() {
t.Fatal("cleanup removal failure was not made operationally visible")
}
srv.runCleanupStep(now.Add(time.Minute))
srv.removalErrors.mu.Lock()
secondLog := srv.removalErrors.lastLog["logs:cleanup"]
srv.removalErrors.mu.Unlock()
if !secondLog.Equal(firstLog) {
t.Fatalf("persistent cleanup failure was not throttled: first=%v second=%v", firstLog, secondLog)
}
if remover.callCount(logPath) != 2 {
t.Fatalf("cleanup retry calls=%d want 2", remover.callCount(logPath))
}
}
func TestRemovalFailureLogDoesNotExposeCapabilityPath(t *testing.T) {
dir := t.TempDir()
remover := newDeterministicRemover()
store := newLogStoreWithRemover(dir, remover.remove)
id := strings.Repeat("c", logIDLength)
store.generateID = func() string { return id }
now := time.Now()
if _, _, err := store.store([]byte("sensitive"), now); err != nil {
t.Fatalf("store: %v", err)
}
path := store.filePath(id)
store.mu.Lock()
entry := store.entries[id]
entry.ExpiresAt = now.Add(-time.Minute)
store.entries[id] = entry
store.mu.Unlock()
remover.fail(path, &os.PathError{Op: "remove", Path: path, Err: syscall.EACCES})
removalErr := store.cleanup(now)
if removalErr == nil {
t.Fatal("cleanup unexpectedly succeeded")
}
var output bytes.Buffer
previousOutput := log.Writer()
previousFlags := log.Flags()
previousPrefix := log.Prefix()
log.SetOutput(&output)
log.SetFlags(0)
log.SetPrefix("")
t.Cleanup(func() {
log.SetOutput(previousOutput)
log.SetFlags(previousFlags)
log.SetPrefix(previousPrefix)
})
srv := &Server{}
srv.logRemovalError("logs", "cleanup", removalErr)
message := output.String()
if strings.Contains(message, id) || strings.Contains(message, path) {
t.Fatalf("removal log exposed capability path: %q", message)
}
want := fmt.Sprintf("logs: cleanup removal failed: category=permission errno=%d", syscall.EACCES)
if !strings.Contains(message, want) {
t.Fatalf("removal log=%q, want sanitized context %q", message, want)
}
}
func TestSnapshotSurvivesRestartWithHostAuthority(t *testing.T) {
stateFile := filepath.Join(t.TempDir(), "rooms.json")
hA := newRelayHarnessAt(t, t.TempDir(), stateFile)
host := hA.dial(t, "8.0.0.1")
host.send(clientMsg{Type: relayTypeCreate, SessionID: "RESUM", PeerID: "H"})
created := host.expectAuthority(relayTypeCreated, "H")
if err := hA.srv.snap.flushAndStop(2 * time.Second); err != nil {
t.Fatalf("flushAndStop: %v", err)
}
snapshotBytes, err := os.ReadFile(stateFile)
if err != nil {
t.Fatalf("read snapshot: %v", err)
}
if bytes.Contains(snapshotBytes, []byte(created.ReconnectToken)) {
t.Fatal("snapshot persisted the raw reconnect capability")
}
if !bytes.Contains(snapshotBytes, []byte(`"hostReconnectVerifier"`)) {
t.Fatalf("snapshot omitted host verifier: %s", snapshotBytes)
}
hB := newRelayHarnessAt(t, t.TempDir(), stateFile)
hB.srv.mu.RLock()
_, reloaded := hB.srv.rooms["RESUM"]
hB.srv.mu.RUnlock()
if !reloaded {
t.Fatal("room RESUM was not reloaded from snapshot")
}
unproved := hB.dial(t, "8.0.0.2")
unproved.send(clientMsg{Type: relayTypeJoin, SessionID: "RESUM", PeerID: "H"})
unproved.expectError(relayErrorPeerIdUnavailable)
reconnected := hB.dial(t, "8.0.0.3")
reconnected.send(clientMsg{
Type: relayTypeJoin,
SessionID: "RESUM",
PeerID: "H",
ReconnectToken: created.ReconnectToken,
})
joined := reconnected.expectAuthority(relayTypeJoined, "H")
if joined.ReconnectToken != created.ReconnectToken {
t.Fatal("restored host capability changed")
}
duplicateCreate := hB.dial(t, "8.0.0.4")
duplicateCreate.send(clientMsg{Type: relayTypeCreate, SessionID: "RESUM", PeerID: "OTHER"})
duplicateCreate.expectError(relayErrorRoomExists)
}
func TestSnapshotV4RetainsModernHostAndGuestReservationsAcrossRestart(t *testing.T) {
stateFile := filepath.Join(t.TempDir(), "rooms.json")
hA := newRelayHarnessAt(t, t.TempDir(), stateFile)
hostToken, _ := mustReconnectToken(t)
host := hA.dial(t, "8.0.1.1")
host.send(clientMsg{
Type: relayTypeCreate,
SessionID: "V3_RESTART",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
host.expectAuthority(relayTypeCreated, "H")
guestToken, _ := mustReconnectToken(t)
guest := hA.dial(t, "8.0.1.2")
guest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "V3_RESTART",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guest.expectAuthority(relayTypeJoined, "H")
host.expect(relayTypePeerJoined)
if err := guest.conn.Close(); err != nil {
t.Fatalf("close guest before snapshot: %v", err)
}
left := host.expect(relayTypePeerLeft)
if left.PeerID != "G" {
t.Fatalf("pre-snapshot disconnect peerId=%q, want G", left.PeerID)
}
if err := hA.srv.snap.flushAndStop(2 * time.Second); err != nil {
t.Fatalf("flush snapshot v4: %v", err)
}
snapshotBytes, err := os.ReadFile(stateFile)
if err != nil {
t.Fatalf("read snapshot v4: %v", err)
}
if !bytes.Contains(snapshotBytes, []byte(`"version":4`)) ||
!bytes.Contains(snapshotBytes, []byte(`"peerReservations"`)) {
t.Fatalf("snapshot omitted v4 guest reservation state: %s", snapshotBytes)
}
if bytes.Contains(snapshotBytes, []byte(hostToken)) || bytes.Contains(snapshotBytes, []byte(guestToken)) {
t.Fatal("snapshot persisted a raw reconnect capability")
}
hB := newRelayHarnessAt(t, t.TempDir(), stateFile)
wrongHostToken, _ := mustReconnectToken(t)
hostThief := hB.dial(t, "8.0.1.3")
hostThief.send(clientMsg{
Type: relayTypeJoin,
SessionID: "V3_RESTART",
PeerID: "H",
ReconnectToken: wrongHostToken,
ProtocolVersion: relayProtocolVersion,
})
hostThief.expectError(relayErrorPeerIdUnavailable)
restartedHost := hB.dial(t, "8.0.1.4")
restartedHost.send(clientMsg{
Type: relayTypeJoin,
SessionID: "V3_RESTART",
PeerID: "H",
ReconnectToken: hostToken,
ProtocolVersion: relayProtocolVersion,
})
hostJoined := restartedHost.expectAuthority(relayTypeJoined, "H")
if hostJoined.ReconnectToken != hostToken || hostJoined.ProtocolVersion != relayProtocolVersion {
t.Fatalf("restored host authority changed: %+v", hostJoined)
}
wrongGuestToken, _ := mustReconnectToken(t)
guestThief := hB.dial(t, "8.0.1.5")
guestThief.send(clientMsg{
Type: relayTypeJoin,
SessionID: "V3_RESTART",
PeerID: "G",
ReconnectToken: wrongGuestToken,
ProtocolVersion: relayProtocolVersion,
})
guestThief.expectError(relayErrorPeerIdUnavailable)
restartedGuest := hB.dial(t, "8.0.1.6")
restartedGuest.send(clientMsg{
Type: relayTypeJoin,
SessionID: "V3_RESTART",
PeerID: "G",
ReconnectToken: guestToken,
ProtocolVersion: relayProtocolVersion,
})
guestJoined := restartedGuest.expectAuthority(relayTypeJoined, "H")
if guestJoined.ReconnectToken != guestToken || guestJoined.ProtocolVersion != relayProtocolVersion {
t.Fatalf("restored guest authority changed: %+v", guestJoined)
}
rejoined := restartedHost.expect(relayTypePeerJoined)
if rejoined.PeerID != "G" {
t.Fatalf("restored guest event peerId=%q, want G", rejoined.PeerID)
}
}
func TestLoadedRoomsConsumeGlobalCapacityWithoutRestoringSourceQuota(t *testing.T) {
stateFile := filepath.Join(t.TempDir(), "rooms.json")
now := time.Now().UTC()
snapshot := stateSnapshot{
Version: snapshotFormatVersion,
SavedAt: now,
Rooms: makeRoomSnapshots(maxRetainedRooms, false, now),
}
data, err := json.Marshal(snapshot)
if err != nil {
t.Fatalf("marshal full snapshot: %v", err)
}
if err := os.WriteFile(stateFile, data, 0644); err != nil {
t.Fatalf("write full snapshot: %v", err)
}
h := newRelayHarnessAt(t, t.TempDir(), stateFile)
h.srv.conns.mu.Lock()
restoredQuotaEntries := len(h.srv.conns.roomsPerIP)
h.srv.conns.mu.Unlock()
if restoredQuotaEntries != 0 {
t.Fatalf("restart restored %d process-local quota entries", restoredQuotaEntries)
}
client := h.dial(t, "8.0.0.4")
client.send(clientMsg{Type: relayTypeCreate, SessionID: "RESTARTOVER", PeerID: "H"})
client.expectError(relayErrorRateLimited)
client.send(clientMsg{Type: relayTypeJoin, SessionID: "S0000", PeerID: "G"})
client.expect(relayTypeJoined)
}
Reference in New Issue View Git Blame Copy Permalink