Files
silo-server/internal/api/handlers/admin_playback_control.go
T
CoffeeKnyte 0217cce2df feat(playback): stream kill switch + async over-cap enforcer
Add the enforcement layer on top of server-observed monitoring: a revocation
kill switch that stops any stream within ~120s and keeps it dead, plus an async
over-cap enforcer that drives kills off the live monitoring picture — entirely
off the per-segment hot path and with no client-protocol change.

- internal/streamrevoke: the central kill list. IsRevoked is a pure in-memory
  lookup safe on the request hot path; a Redis pub/sub + poll mirror keeps edge
  caches current, and a Postgres durable mirror lets kills survive a server
  restart AND a Redis flush so a restart-resilient stream cannot be reconstructed
  and re-served after being killed. A user revocation is a cutoff (kills tokens
  minted before it, spares post-reauth tokens), not a 24h ban.
- internal/streamenforcer: async over-cap brain — reads the monitoring snapshot
  and per-user limits, selects victims, and collapses every reason (exceeded
  limit, admin terminate, abuse) to the same action: write a revocation.
- Edge + native + jellycompat enforcement: proxy refuses revoked sessions on
  every request and cuts long direct-play/remux pours mid-stream; the transcode
  node guards both serve and the reconstruct path so a killed session is never
  re-spawned after a node restart; jellycompat serve surfaces close their
  kill-switch coverage holes.
- streamtoken.IssuedTime exposes the token iat the user-kill cutoff compares
  against; token IssuedTime + revocation guards wire through router, downloads,
  and admin terminate-by-id (with admin-list dedupe).
- Restore sendfile zero-copy on direct-play/remux byte counting so the monitor's
  served-byte accounting does not cost the sendfile fast path.
- migrations/sql: stream_revocations durable table.

Part of the stream monitoring & kill-switch epic.
2026-07-29 12:26:03 +00:00

271 lines
8.6 KiB
Go

package handlers
import (
"context"
"encoding/json"
"errors"
"io"
"log/slog"
"net/http"
"time"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/Silo-Server/silo-server/internal/playback"
"github.com/Silo-Server/silo-server/internal/streamrevoke"
)
const (
defaultPlaybackControlDeadline = 3 * time.Second
maxPlaybackControlDeadline = 10 * time.Second
)
type AdminPlaybackControlHandler struct {
playback *PlaybackHandler
revocation *streamrevoke.Store
}
// SetRevocationStore wires the kill switch so an admin terminate revokes the
// stream credential (stops it at the edge and refuses reconnects), not just
// dispatches a cooperative realtime command. Optional; nil keeps prior behavior.
func (h *AdminPlaybackControlHandler) SetRevocationStore(store *streamrevoke.Store) {
if h != nil {
h.revocation = store
}
}
type playbackControlRequest struct {
Reason string `json:"reason"`
Title string `json:"title"`
Message string `json:"message"`
DeadlineMS int `json:"deadline_ms"`
}
type playbackControlResponse struct {
CommandID string `json:"command_id"`
Status string `json:"status"`
}
func requiresLivePlaybackControl(name playback.CommandName) bool {
switch name {
case playback.CommandPause, playback.CommandUnpause:
return true
default:
return false
}
}
func NewAdminPlaybackControlHandler(playbackHandler *PlaybackHandler) *AdminPlaybackControlHandler {
return &AdminPlaybackControlHandler{playback: playbackHandler}
}
func (h *AdminPlaybackControlHandler) HandleStopSession(w http.ResponseWriter, r *http.Request) {
h.handleSessionCommand(w, r, playback.CommandStop)
}
func (h *AdminPlaybackControlHandler) HandlePauseSession(w http.ResponseWriter, r *http.Request) {
h.handleSessionCommand(w, r, playback.CommandPause)
}
func (h *AdminPlaybackControlHandler) HandleResumeSession(w http.ResponseWriter, r *http.Request) {
h.handleSessionCommand(w, r, playback.CommandUnpause)
}
func (h *AdminPlaybackControlHandler) HandleTerminateSession(w http.ResponseWriter, r *http.Request) {
h.handleSessionCommand(w, r, playback.CommandTerminate)
}
func (h *AdminPlaybackControlHandler) HandleMessageSession(w http.ResponseWriter, r *http.Request) {
if h == nil || h.playback == nil || h.playback.CommandDispatcher == nil {
writeError(w, http.StatusServiceUnavailable, "service_unavailable", "Playback control is unavailable")
return
}
sessionID := chi.URLParam(r, "session_id")
if sessionID == "" {
writeError(w, http.StatusBadRequest, "bad_request", "Session ID is required")
return
}
if _, err := h.playback.sessionMgr.GetSession(sessionID); err != nil {
if errors.Is(err, playback.ErrSessionNotFound) {
writeError(w, http.StatusNotFound, "not_found", "Playback session not found")
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load playback session")
return
}
var req playbackControlRequest
if err := decodeOptionalJSONBody(r, &req); err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
return
}
if req.Message == "" {
writeError(w, http.StatusBadRequest, "bad_request", "Message is required")
return
}
payload, err := json.Marshal(map[string]string{
"title": req.Title,
"message": req.Message,
})
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to build command payload")
return
}
commandID := uuid.NewString()
command, err := playback.NewCommandEnvelope(sessionID, commandID, playback.CommandDisplayMessage, payload)
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to build command")
return
}
command.Reason = req.Reason
command.IssuedBy = &playback.CommandIssuedBy{Kind: "admin"}
result := h.playback.CommandDispatcher.DispatchToSession(command, 0, nil)
if result.DispatchErr != nil {
status := http.StatusInternalServerError
code := "internal_error"
message := "Failed to dispatch command"
if errors.Is(result.DispatchErr, playback.ErrRealtimeConnectionNotFound) {
status = http.StatusConflict
code = "realtime_unavailable"
message = "Realtime connection unavailable for playback session"
}
writeError(w, status, code, message)
return
}
writeJSON(w, http.StatusAccepted, playbackControlResponse{
CommandID: commandID,
Status: "dispatched",
})
}
func (h *AdminPlaybackControlHandler) handleSessionCommand(w http.ResponseWriter, r *http.Request, name playback.CommandName) {
if h == nil || h.playback == nil || h.playback.CommandDispatcher == nil {
writeError(w, http.StatusServiceUnavailable, "service_unavailable", "Playback control is unavailable")
return
}
sessionID := chi.URLParam(r, "session_id")
if sessionID == "" {
writeError(w, http.StatusBadRequest, "bad_request", "Session ID is required")
return
}
var req playbackControlRequest
if err := decodeOptionalJSONBody(r, &req); err != nil {
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
return
}
// A terminate must stick even if the client ignores the realtime command:
// revoke the stream credential so the edge refuses further segments and any
// reconnect within one propagation/poll interval. Stop stays cooperative.
// The revocation is written BEFORE the local session lookup: the kill needs
// only the id, and the stream may exist only as an edge Redis record — e.g.
// after a central restart, or when a client that withholds progress let the
// in-memory session be reaped — which is precisely the stream an operator
// most needs to kill. Revoking an unknown id is harmless (idempotent, TTL'd).
if name == playback.CommandTerminate && h.revocation != nil {
if err := h.revocation.RevokeSession(r.Context(), sessionID, "admin_terminate"); err != nil {
slog.Warn("admin terminate: revoke stream failed", "session_id", sessionID, "error", err)
}
}
session, err := h.playback.sessionMgr.GetSession(sessionID)
if err != nil {
if errors.Is(err, playback.ErrSessionNotFound) {
// The revocation above still cut the stream at every serve surface;
// only the cooperative realtime command has no local session to go to.
if name == playback.CommandTerminate && h.revocation != nil {
writeJSON(w, http.StatusAccepted, playbackControlResponse{
CommandID: uuid.NewString(),
Status: "revoked",
})
return
}
writeError(w, http.StatusNotFound, "not_found", "Playback session not found")
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load playback session")
return
}
if requiresLivePlaybackControl(name) && (session == nil || !session.HasRealtimeConnection) {
writeError(w, http.StatusConflict, "realtime_unavailable", "Realtime connection unavailable for playback session")
return
}
commandID := uuid.NewString()
command, err := playback.NewCommandEnvelope(sessionID, commandID, name, nil)
if err != nil {
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to build command")
return
}
command.Reason = req.Reason
command.IssuedBy = &playback.CommandIssuedBy{Kind: "admin"}
deadline := boundedPlaybackControlDeadline(req.DeadlineMS)
command.DeadlineMS = int(deadline / time.Millisecond)
fallback := func() {
h.playback.forgetRealtimeCommand(commandID)
_ = h.playback.stopPlaybackSessionByID(context.Background(), sessionID, true)
}
h.playback.rememberRealtimeCommand(commandID, sessionID, name)
result := h.playback.CommandDispatcher.DispatchToSession(command, deadline, fallback)
if result.DispatchErr == nil {
writeJSON(w, http.StatusAccepted, playbackControlResponse{
CommandID: commandID,
Status: "dispatched",
})
return
}
h.playback.forgetRealtimeCommand(commandID)
if errors.Is(result.DispatchErr, playback.ErrRealtimeConnectionNotFound) {
time.AfterFunc(deadline, fallback)
writeJSON(w, http.StatusAccepted, playbackControlResponse{
CommandID: commandID,
Status: "fallback_scheduled",
})
return
}
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to dispatch command")
}
func decodeOptionalJSONBody(r *http.Request, target any) error {
if r.Body == nil {
return nil
}
defer r.Body.Close()
decoder := json.NewDecoder(r.Body)
if err := decoder.Decode(target); err != nil {
if errors.Is(err, io.EOF) {
return nil
}
if errors.Is(err, http.ErrBodyNotAllowed) {
return nil
}
return err
}
return nil
}
func boundedPlaybackControlDeadline(deadlineMS int) time.Duration {
if deadlineMS <= 0 {
return defaultPlaybackControlDeadline
}
deadline := time.Duration(deadlineMS) * time.Millisecond
if deadline > maxPlaybackControlDeadline {
return maxPlaybackControlDeadline
}
return deadline
}