* feat(nodepool): node groups, per-node caps, and local transcode fallback control Group co-located transcode and proxy nodes so transcoded streams are served by a proxy on the same host/LAN instead of bouncing across the internal network (fixes #93): - New nodepool.Planner is the single selection entry point: it picks the transcode node and its group's proxy together (round-robin within the group), replacing the independent ProxyPool.Pick/TranscodePool.Acquire calls scattered across the native and jellycompat handlers, and absorbs the duplicated soft-affinity pick logic. - A group is only eligible while all of its enabled members are healthy; ungrouped nodes keep the historical behavior. - New per-node max_jobs cap (transcodes for transcode nodes, streams for proxies; NULL = unlimited), enforced via health-reported job counts plus short-lived reservations that expire once fresher health data arrives. Proxy health now reports real stream counts, including HLS sessions via idle-expiry tracking. - New playback.local_transcode_fallback setting (default on) lets admins refuse API-server transcoding when no eligible node exists. - Health checks now publish updated node copies under the pool lock instead of mutating shared structs in place, fixing a data race. - Admin UI: group + cap fields on the node form, group/cap columns, and the new fallback toggle in playback settings. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * feat(nodepool): proxy bandwidth measurement and egress caps Proxy nodes now measure their stream egress (rolling 60s average over everything under /stream) and report it via the health endpoint. A new per-proxy max_bandwidth_kbps cap lets the planner route new streams away from saturated proxies: - Admission combines the measured egress with the estimated bitrate of the new stream (transcode target bitrate, or source bitrate for direct play/remux) so a stream is only admitted where it fits. - Recently admitted streams are bridged as bandwidth reservations for the meter window, since the rolling average only converges on a new stream's rate gradually. - A group whose proxies lack bandwidth headroom is treated as full: its transcode nodes are skipped, same as the job cap. - Admin UI: per-proxy "Max Egress Bandwidth (Mbps)" field and a live egress column; manual health checks return the measured rate. Active streams are never interrupted - the cap only gates new admissions. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * perf(playback): trim node-mode time-to-stream-start Distributed playback paid several avoidable costs before the first frame that integrated mode doesn't have. This trims the safe ones: - Web player preconnects to the stream origin (the proxy node) as soon as /playback/start returns, overlapping DNS/TCP/TLS handshakes with the transcode dispatch instead of paying them at the first manifest fetch. - The transcode node no longer blocks its 202 on monitoring work: the Redis session-track write moves off the request path, and a replaced session's segment directory is renamed aside and deleted in the background instead of synchronously (RemoveAll of a long session can take seconds on slow disks during quality switches). - The proxy's node-facing HTTP client gets a tuned transport: a larger idle-connection pool (Go's default of 2 per host causes connection churn and TLS re-handshakes when many viewers stream through one proxy->node pair) and a response-header timeout so a hung transcode node can no longer hang client requests indefinitely. - jellycompat's remote transcode dispatch gains the same 10s timeout the native path has had; an unreachable node previously hung the compat manifest request until the OS gave up. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
257 lines
7.8 KiB
Go
257 lines
7.8 KiB
Go
package nodepool
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5"
|
|
"github.com/jackc/pgx/v5/pgxpool"
|
|
)
|
|
|
|
const (
|
|
// NodeTypeProxy identifies a proxy stream node.
|
|
NodeTypeProxy = "proxy"
|
|
// NodeTypeTranscode identifies a transcode stream node.
|
|
NodeTypeTranscode = "transcode"
|
|
)
|
|
|
|
// Node represents a stream node in the database.
|
|
type Node struct {
|
|
ID int `json:"id"`
|
|
Name string `json:"name"`
|
|
Type string `json:"type"`
|
|
URL string `json:"url"`
|
|
Enabled bool `json:"enabled"`
|
|
Healthy bool `json:"healthy"`
|
|
ActiveJobs int `json:"active_jobs"`
|
|
Group *string `json:"group"` // co-location group; nil = ungrouped
|
|
MaxJobs *int `json:"max_jobs"` // concurrent job cap; nil = unlimited
|
|
MaxBandwidthKbps *int `json:"max_bandwidth_kbps"` // egress cap in kilobits/s; nil = unlimited
|
|
EgressKbps int `json:"egress_kbps"` // health-reported rolling egress average
|
|
LastHealthCheck *time.Time `json:"last_health_check"`
|
|
CreatedAt time.Time `json:"created_at"`
|
|
}
|
|
|
|
// CreateNodeInput holds the fields for creating a new node.
|
|
type CreateNodeInput struct {
|
|
Name string `json:"name"`
|
|
Type string `json:"type"`
|
|
URL string `json:"url"`
|
|
Group string `json:"group"` // empty = ungrouped
|
|
MaxJobs *int `json:"max_jobs"` // nil or <= 0 = unlimited
|
|
MaxBandwidthKbps *int `json:"max_bandwidth_kbps"` // nil or <= 0 = unlimited
|
|
}
|
|
|
|
// Validate checks required fields and allowed values.
|
|
func (i CreateNodeInput) Validate() error {
|
|
if i.Name == "" {
|
|
return errors.New("name is required")
|
|
}
|
|
if i.Type != NodeTypeProxy && i.Type != NodeTypeTranscode {
|
|
return fmt.Errorf("type must be %q or %q", NodeTypeProxy, NodeTypeTranscode)
|
|
}
|
|
if i.URL == "" {
|
|
return errors.New("url is required")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// UpdateNodeInput holds the fields for updating a node.
|
|
// The optional fields distinguish "leave unchanged" (nil) from "clear":
|
|
// an empty-string Group clears the group, and a non-positive MaxJobs or
|
|
// MaxBandwidthKbps clears that cap.
|
|
type UpdateNodeInput struct {
|
|
Name *string `json:"name,omitempty"`
|
|
URL *string `json:"url,omitempty"`
|
|
Enabled *bool `json:"enabled,omitempty"`
|
|
Group *string `json:"group,omitempty"`
|
|
MaxJobs *int `json:"max_jobs,omitempty"`
|
|
MaxBandwidthKbps *int `json:"max_bandwidth_kbps,omitempty"`
|
|
}
|
|
|
|
// normalizeGroup trims a group label and converts empty to NULL.
|
|
func normalizeGroup(group string) *string {
|
|
g := strings.TrimSpace(group)
|
|
if g == "" {
|
|
return nil
|
|
}
|
|
return &g
|
|
}
|
|
|
|
// normalizeCap converts non-positive capacity values to NULL (unlimited).
|
|
func normalizeCap(v *int) *int {
|
|
if v == nil || *v <= 0 {
|
|
return nil
|
|
}
|
|
return v
|
|
}
|
|
|
|
// Repository provides CRUD operations for stream nodes.
|
|
type Repository struct {
|
|
pool *pgxpool.Pool
|
|
}
|
|
|
|
// NewRepository creates a new node repository.
|
|
func NewRepository(pool *pgxpool.Pool) *Repository {
|
|
return &Repository{pool: pool}
|
|
}
|
|
|
|
const nodeColumns = `id, name, type, url, enabled, healthy, active_jobs, node_group, max_jobs, max_bandwidth_kbps, egress_kbps, last_health_check, created_at`
|
|
|
|
func scanNode(row pgx.Row) (*Node, error) {
|
|
var n Node
|
|
err := row.Scan(
|
|
&n.ID, &n.Name, &n.Type, &n.URL,
|
|
&n.Enabled, &n.Healthy, &n.ActiveJobs,
|
|
&n.Group, &n.MaxJobs,
|
|
&n.MaxBandwidthKbps, &n.EgressKbps,
|
|
&n.LastHealthCheck, &n.CreatedAt,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &n, nil
|
|
}
|
|
|
|
func scanNodes(rows pgx.Rows) ([]*Node, error) {
|
|
var nodes []*Node
|
|
for rows.Next() {
|
|
var n Node
|
|
if err := rows.Scan(
|
|
&n.ID, &n.Name, &n.Type, &n.URL,
|
|
&n.Enabled, &n.Healthy, &n.ActiveJobs,
|
|
&n.Group, &n.MaxJobs,
|
|
&n.MaxBandwidthKbps, &n.EgressKbps,
|
|
&n.LastHealthCheck, &n.CreatedAt,
|
|
); err != nil {
|
|
return nil, err
|
|
}
|
|
nodes = append(nodes, &n)
|
|
}
|
|
return nodes, rows.Err()
|
|
}
|
|
|
|
// List returns all nodes ordered by type then name.
|
|
func (r *Repository) List(ctx context.Context) ([]*Node, error) {
|
|
rows, err := r.pool.Query(ctx,
|
|
`SELECT `+nodeColumns+` FROM stream_nodes ORDER BY type, name`)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list nodes: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
return scanNodes(rows)
|
|
}
|
|
|
|
// ListEnabled returns all enabled nodes of a given type.
|
|
func (r *Repository) ListEnabled(ctx context.Context, nodeType string) ([]*Node, error) {
|
|
rows, err := r.pool.Query(ctx,
|
|
`SELECT `+nodeColumns+` FROM stream_nodes WHERE type = $1 AND enabled = true ORDER BY name`,
|
|
nodeType)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("list enabled nodes: %w", err)
|
|
}
|
|
defer rows.Close()
|
|
return scanNodes(rows)
|
|
}
|
|
|
|
// GetByID returns a single node by ID.
|
|
func (r *Repository) GetByID(ctx context.Context, id int) (*Node, error) {
|
|
row := r.pool.QueryRow(ctx,
|
|
`SELECT `+nodeColumns+` FROM stream_nodes WHERE id = $1`, id)
|
|
n, err := scanNode(row)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, ErrNodeNotFound
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("get node: %w", err)
|
|
}
|
|
return n, nil
|
|
}
|
|
|
|
// Create inserts a new node and returns it.
|
|
func (r *Repository) Create(ctx context.Context, input CreateNodeInput) (*Node, error) {
|
|
if err := input.Validate(); err != nil {
|
|
return nil, err
|
|
}
|
|
row := r.pool.QueryRow(ctx,
|
|
`INSERT INTO stream_nodes (name, type, url, node_group, max_jobs, max_bandwidth_kbps)
|
|
VALUES ($1, $2, $3, $4, $5, $6)
|
|
RETURNING `+nodeColumns,
|
|
input.Name, input.Type, input.URL, normalizeGroup(input.Group),
|
|
normalizeCap(input.MaxJobs), normalizeCap(input.MaxBandwidthKbps))
|
|
return scanNode(row)
|
|
}
|
|
|
|
// Update modifies a node's mutable fields. The optional fields use sentinel
|
|
// values to clear: an empty-string group and non-positive caps set the
|
|
// column to NULL (see UpdateNodeInput).
|
|
func (r *Repository) Update(ctx context.Context, id int, input UpdateNodeInput) (*Node, error) {
|
|
var group *string
|
|
if input.Group != nil {
|
|
group = normalizeGroup(*input.Group)
|
|
}
|
|
var maxJobs, maxBandwidth *int
|
|
if input.MaxJobs != nil {
|
|
maxJobs = normalizeCap(input.MaxJobs)
|
|
}
|
|
if input.MaxBandwidthKbps != nil {
|
|
maxBandwidth = normalizeCap(input.MaxBandwidthKbps)
|
|
}
|
|
row := r.pool.QueryRow(ctx,
|
|
`UPDATE stream_nodes SET
|
|
name = COALESCE($2, name),
|
|
url = COALESCE($3, url),
|
|
enabled = COALESCE($4, enabled),
|
|
node_group = CASE WHEN $5::boolean THEN $6::text ELSE node_group END,
|
|
max_jobs = CASE WHEN $7::boolean THEN $8::integer ELSE max_jobs END,
|
|
max_bandwidth_kbps = CASE WHEN $9::boolean THEN $10::integer ELSE max_bandwidth_kbps END
|
|
WHERE id = $1
|
|
RETURNING `+nodeColumns,
|
|
id, input.Name, input.URL, input.Enabled,
|
|
input.Group != nil, group,
|
|
input.MaxJobs != nil, maxJobs,
|
|
input.MaxBandwidthKbps != nil, maxBandwidth)
|
|
n, err := scanNode(row)
|
|
if errors.Is(err, pgx.ErrNoRows) {
|
|
return nil, ErrNodeNotFound
|
|
}
|
|
if err != nil {
|
|
return nil, fmt.Errorf("update node: %w", err)
|
|
}
|
|
return n, nil
|
|
}
|
|
|
|
// Delete removes a node by ID.
|
|
func (r *Repository) Delete(ctx context.Context, id int) error {
|
|
tag, err := r.pool.Exec(ctx, `DELETE FROM stream_nodes WHERE id = $1`, id)
|
|
if err != nil {
|
|
return fmt.Errorf("delete node: %w", err)
|
|
}
|
|
if tag.RowsAffected() == 0 {
|
|
return ErrNodeNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// UpdateHealth updates a node's health status, active job count, and
|
|
// reported egress bandwidth.
|
|
func (r *Repository) UpdateHealth(ctx context.Context, id int, healthy bool, activeJobs, egressKbps int) error {
|
|
tag, err := r.pool.Exec(ctx,
|
|
`UPDATE stream_nodes SET healthy = $2, active_jobs = $3, egress_kbps = $4, last_health_check = NOW()
|
|
WHERE id = $1`,
|
|
id, healthy, activeJobs, egressKbps)
|
|
if err != nil {
|
|
return fmt.Errorf("update node health: %w", err)
|
|
}
|
|
if tag.RowsAffected() == 0 {
|
|
return ErrNodeNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Sentinel errors.
|
|
var ErrNodeNotFound = errors.New("stream node not found")
|