Add the enforcement layer on top of server-observed monitoring: a revocation kill switch that stops any stream within ~120s and keeps it dead, plus an async over-cap enforcer that drives kills off the live monitoring picture — entirely off the per-segment hot path and with no client-protocol change. - internal/streamrevoke: the central kill list. IsRevoked is a pure in-memory lookup safe on the request hot path; a Redis pub/sub + poll mirror keeps edge caches current, and a Postgres durable mirror lets kills survive a server restart AND a Redis flush so a restart-resilient stream cannot be reconstructed and re-served after being killed. A user revocation is a cutoff (kills tokens minted before it, spares post-reauth tokens), not a 24h ban. - internal/streamenforcer: async over-cap brain — reads the monitoring snapshot and per-user limits, selects victims, and collapses every reason (exceeded limit, admin terminate, abuse) to the same action: write a revocation. - Edge + native + jellycompat enforcement: proxy refuses revoked sessions on every request and cuts long direct-play/remux pours mid-stream; the transcode node guards both serve and the reconstruct path so a killed session is never re-spawned after a node restart; jellycompat serve surfaces close their kill-switch coverage holes. - streamtoken.IssuedTime exposes the token iat the user-kill cutoff compares against; token IssuedTime + revocation guards wire through router, downloads, and admin terminate-by-id (with admin-list dedupe). - Restore sendfile zero-copy on direct-play/remux byte counting so the monitor's served-byte accounting does not cost the sendfile fast path. - migrations/sql: stream_revocations durable table. Part of the stream monitoring & kill-switch epic.
271 lines
8.6 KiB
Go
271 lines
8.6 KiB
Go
package handlers
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"time"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
"github.com/google/uuid"
|
|
|
|
"github.com/Silo-Server/silo-server/internal/playback"
|
|
"github.com/Silo-Server/silo-server/internal/streamrevoke"
|
|
)
|
|
|
|
const (
|
|
defaultPlaybackControlDeadline = 3 * time.Second
|
|
maxPlaybackControlDeadline = 10 * time.Second
|
|
)
|
|
|
|
type AdminPlaybackControlHandler struct {
|
|
playback *PlaybackHandler
|
|
revocation *streamrevoke.Store
|
|
}
|
|
|
|
// SetRevocationStore wires the kill switch so an admin terminate revokes the
|
|
// stream credential (stops it at the edge and refuses reconnects), not just
|
|
// dispatches a cooperative realtime command. Optional; nil keeps prior behavior.
|
|
func (h *AdminPlaybackControlHandler) SetRevocationStore(store *streamrevoke.Store) {
|
|
if h != nil {
|
|
h.revocation = store
|
|
}
|
|
}
|
|
|
|
type playbackControlRequest struct {
|
|
Reason string `json:"reason"`
|
|
Title string `json:"title"`
|
|
Message string `json:"message"`
|
|
DeadlineMS int `json:"deadline_ms"`
|
|
}
|
|
|
|
type playbackControlResponse struct {
|
|
CommandID string `json:"command_id"`
|
|
Status string `json:"status"`
|
|
}
|
|
|
|
func requiresLivePlaybackControl(name playback.CommandName) bool {
|
|
switch name {
|
|
case playback.CommandPause, playback.CommandUnpause:
|
|
return true
|
|
default:
|
|
return false
|
|
}
|
|
}
|
|
|
|
func NewAdminPlaybackControlHandler(playbackHandler *PlaybackHandler) *AdminPlaybackControlHandler {
|
|
return &AdminPlaybackControlHandler{playback: playbackHandler}
|
|
}
|
|
|
|
func (h *AdminPlaybackControlHandler) HandleStopSession(w http.ResponseWriter, r *http.Request) {
|
|
h.handleSessionCommand(w, r, playback.CommandStop)
|
|
}
|
|
|
|
func (h *AdminPlaybackControlHandler) HandlePauseSession(w http.ResponseWriter, r *http.Request) {
|
|
h.handleSessionCommand(w, r, playback.CommandPause)
|
|
}
|
|
|
|
func (h *AdminPlaybackControlHandler) HandleResumeSession(w http.ResponseWriter, r *http.Request) {
|
|
h.handleSessionCommand(w, r, playback.CommandUnpause)
|
|
}
|
|
|
|
func (h *AdminPlaybackControlHandler) HandleTerminateSession(w http.ResponseWriter, r *http.Request) {
|
|
h.handleSessionCommand(w, r, playback.CommandTerminate)
|
|
}
|
|
|
|
func (h *AdminPlaybackControlHandler) HandleMessageSession(w http.ResponseWriter, r *http.Request) {
|
|
if h == nil || h.playback == nil || h.playback.CommandDispatcher == nil {
|
|
writeError(w, http.StatusServiceUnavailable, "service_unavailable", "Playback control is unavailable")
|
|
return
|
|
}
|
|
|
|
sessionID := chi.URLParam(r, "session_id")
|
|
if sessionID == "" {
|
|
writeError(w, http.StatusBadRequest, "bad_request", "Session ID is required")
|
|
return
|
|
}
|
|
|
|
if _, err := h.playback.sessionMgr.GetSession(sessionID); err != nil {
|
|
if errors.Is(err, playback.ErrSessionNotFound) {
|
|
writeError(w, http.StatusNotFound, "not_found", "Playback session not found")
|
|
return
|
|
}
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load playback session")
|
|
return
|
|
}
|
|
|
|
var req playbackControlRequest
|
|
if err := decodeOptionalJSONBody(r, &req); err != nil {
|
|
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
|
return
|
|
}
|
|
if req.Message == "" {
|
|
writeError(w, http.StatusBadRequest, "bad_request", "Message is required")
|
|
return
|
|
}
|
|
|
|
payload, err := json.Marshal(map[string]string{
|
|
"title": req.Title,
|
|
"message": req.Message,
|
|
})
|
|
if err != nil {
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to build command payload")
|
|
return
|
|
}
|
|
|
|
commandID := uuid.NewString()
|
|
command, err := playback.NewCommandEnvelope(sessionID, commandID, playback.CommandDisplayMessage, payload)
|
|
if err != nil {
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to build command")
|
|
return
|
|
}
|
|
command.Reason = req.Reason
|
|
command.IssuedBy = &playback.CommandIssuedBy{Kind: "admin"}
|
|
|
|
result := h.playback.CommandDispatcher.DispatchToSession(command, 0, nil)
|
|
if result.DispatchErr != nil {
|
|
status := http.StatusInternalServerError
|
|
code := "internal_error"
|
|
message := "Failed to dispatch command"
|
|
if errors.Is(result.DispatchErr, playback.ErrRealtimeConnectionNotFound) {
|
|
status = http.StatusConflict
|
|
code = "realtime_unavailable"
|
|
message = "Realtime connection unavailable for playback session"
|
|
}
|
|
writeError(w, status, code, message)
|
|
return
|
|
}
|
|
|
|
writeJSON(w, http.StatusAccepted, playbackControlResponse{
|
|
CommandID: commandID,
|
|
Status: "dispatched",
|
|
})
|
|
}
|
|
|
|
func (h *AdminPlaybackControlHandler) handleSessionCommand(w http.ResponseWriter, r *http.Request, name playback.CommandName) {
|
|
if h == nil || h.playback == nil || h.playback.CommandDispatcher == nil {
|
|
writeError(w, http.StatusServiceUnavailable, "service_unavailable", "Playback control is unavailable")
|
|
return
|
|
}
|
|
|
|
sessionID := chi.URLParam(r, "session_id")
|
|
if sessionID == "" {
|
|
writeError(w, http.StatusBadRequest, "bad_request", "Session ID is required")
|
|
return
|
|
}
|
|
|
|
var req playbackControlRequest
|
|
if err := decodeOptionalJSONBody(r, &req); err != nil {
|
|
writeError(w, http.StatusBadRequest, "bad_request", "Invalid request body")
|
|
return
|
|
}
|
|
|
|
// A terminate must stick even if the client ignores the realtime command:
|
|
// revoke the stream credential so the edge refuses further segments and any
|
|
// reconnect within one propagation/poll interval. Stop stays cooperative.
|
|
// The revocation is written BEFORE the local session lookup: the kill needs
|
|
// only the id, and the stream may exist only as an edge Redis record — e.g.
|
|
// after a central restart, or when a client that withholds progress let the
|
|
// in-memory session be reaped — which is precisely the stream an operator
|
|
// most needs to kill. Revoking an unknown id is harmless (idempotent, TTL'd).
|
|
if name == playback.CommandTerminate && h.revocation != nil {
|
|
if err := h.revocation.RevokeSession(r.Context(), sessionID, "admin_terminate"); err != nil {
|
|
slog.Warn("admin terminate: revoke stream failed", "session_id", sessionID, "error", err)
|
|
}
|
|
}
|
|
|
|
session, err := h.playback.sessionMgr.GetSession(sessionID)
|
|
if err != nil {
|
|
if errors.Is(err, playback.ErrSessionNotFound) {
|
|
// The revocation above still cut the stream at every serve surface;
|
|
// only the cooperative realtime command has no local session to go to.
|
|
if name == playback.CommandTerminate && h.revocation != nil {
|
|
writeJSON(w, http.StatusAccepted, playbackControlResponse{
|
|
CommandID: uuid.NewString(),
|
|
Status: "revoked",
|
|
})
|
|
return
|
|
}
|
|
writeError(w, http.StatusNotFound, "not_found", "Playback session not found")
|
|
return
|
|
}
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to load playback session")
|
|
return
|
|
}
|
|
|
|
if requiresLivePlaybackControl(name) && (session == nil || !session.HasRealtimeConnection) {
|
|
writeError(w, http.StatusConflict, "realtime_unavailable", "Realtime connection unavailable for playback session")
|
|
return
|
|
}
|
|
|
|
commandID := uuid.NewString()
|
|
command, err := playback.NewCommandEnvelope(sessionID, commandID, name, nil)
|
|
if err != nil {
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to build command")
|
|
return
|
|
}
|
|
command.Reason = req.Reason
|
|
command.IssuedBy = &playback.CommandIssuedBy{Kind: "admin"}
|
|
deadline := boundedPlaybackControlDeadline(req.DeadlineMS)
|
|
command.DeadlineMS = int(deadline / time.Millisecond)
|
|
|
|
fallback := func() {
|
|
h.playback.forgetRealtimeCommand(commandID)
|
|
_ = h.playback.stopPlaybackSessionByID(context.Background(), sessionID, true)
|
|
}
|
|
|
|
h.playback.rememberRealtimeCommand(commandID, sessionID, name)
|
|
result := h.playback.CommandDispatcher.DispatchToSession(command, deadline, fallback)
|
|
if result.DispatchErr == nil {
|
|
writeJSON(w, http.StatusAccepted, playbackControlResponse{
|
|
CommandID: commandID,
|
|
Status: "dispatched",
|
|
})
|
|
return
|
|
}
|
|
|
|
h.playback.forgetRealtimeCommand(commandID)
|
|
if errors.Is(result.DispatchErr, playback.ErrRealtimeConnectionNotFound) {
|
|
time.AfterFunc(deadline, fallback)
|
|
writeJSON(w, http.StatusAccepted, playbackControlResponse{
|
|
CommandID: commandID,
|
|
Status: "fallback_scheduled",
|
|
})
|
|
return
|
|
}
|
|
|
|
writeError(w, http.StatusInternalServerError, "internal_error", "Failed to dispatch command")
|
|
}
|
|
|
|
func decodeOptionalJSONBody(r *http.Request, target any) error {
|
|
if r.Body == nil {
|
|
return nil
|
|
}
|
|
defer r.Body.Close()
|
|
decoder := json.NewDecoder(r.Body)
|
|
if err := decoder.Decode(target); err != nil {
|
|
if errors.Is(err, io.EOF) {
|
|
return nil
|
|
}
|
|
if errors.Is(err, http.ErrBodyNotAllowed) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func boundedPlaybackControlDeadline(deadlineMS int) time.Duration {
|
|
if deadlineMS <= 0 {
|
|
return defaultPlaybackControlDeadline
|
|
}
|
|
deadline := time.Duration(deadlineMS) * time.Millisecond
|
|
if deadline > maxPlaybackControlDeadline {
|
|
return maxPlaybackControlDeadline
|
|
}
|
|
return deadline
|
|
}
|