diff --git a/src/Cli/Commands/MonitorCommand.php b/src/Cli/Commands/MonitorCommand.php index 2baa6b8e..2c852b5a 100644 --- a/src/Cli/Commands/MonitorCommand.php +++ b/src/Cli/Commands/MonitorCommand.php @@ -12,6 +12,7 @@ use XcVm\Domain\Server\ServerRepository; use XcVm\Domain\Stream\StreamProcess; use XcVm\Domain\Stream\StreamSorter; use XcVm\Streaming\Codec\FFprobeRunner; +use XcVm\Streaming\Fanout\FanoutClient; /** * `monitor [restart]` — the per-stream watchdog. @@ -68,6 +69,15 @@ class MonitorCommand implements CommandInterface { $rStreamID = intval($rArgs[0]); $rRestart = !empty($rArgs[1]); + // A stream the fanout daemon supervises already has a monitor. This one + // can only have been started by a path that has not learned that (or in + // a race with a hand-over), and two watchdogs would fight over one + // producer. Ask the daemon rather than assume: unreachable is not "no". + if (FanoutClient::isSupervised($rStreamID) === true) { + echo "Stream is supervised by the fanout daemon; monitor standing down.\n"; + return 0; + } + global $db; $this->checkRunning($rStreamID); diff --git a/src/Cli/Commands/OndemandCommand.php b/src/Cli/Commands/OndemandCommand.php index aecbded5..ccd4a51b 100644 --- a/src/Cli/Commands/OndemandCommand.php +++ b/src/Cli/Commands/OndemandCommand.php @@ -9,6 +9,7 @@ use XcVm\Core\Process\ProcessManager; use XcVm\Domain\Stream\ConnectionTracker; use XcVm\Domain\Stream\StreamProcess; use XcVm\Infrastructure\Redis\RedisManager; +use XcVm\Streaming\Fanout\FanoutClient; /** * OndemandCommand — ondemand command @@ -144,6 +145,11 @@ class OndemandCommand implements CommandInterface { echo "Killing a stream without viewers: ID $rStreamID\n"; + // Release a supervised stream before touching its producer: killing + // the producer first is what the fanout supervisor restarts. + FanoutClient::release($rStreamID); + FanoutClient::unregister($rStreamID); + if ($rMonitorPID > 0) @posix_kill($rMonitorPID, 9); if ($rPID > 0) diff --git a/src/Cli/Commands/SignalsCommand.php b/src/Cli/Commands/SignalsCommand.php index b8db8fbb..14fe4e59 100644 --- a/src/Cli/Commands/SignalsCommand.php +++ b/src/Cli/Commands/SignalsCommand.php @@ -6,6 +6,7 @@ use XcVm\Cli\CommandInterface; use XcVm\Cli\DaemonTrait; use XcVm\Core\Config\SettingsManager; use XcVm\Domain\Server\ServerRepository; +use XcVm\Domain\Stream\StreamProcess; use XcVm\Infrastructure\Redis\RedisManager; /** @@ -21,6 +22,9 @@ use XcVm\Infrastructure\Redis\RedisManager; class SignalsCommand implements CommandInterface { use DaemonTrait; + /** Seconds between syncs of supervised streams' rows from the fanout daemon. */ + const RECONCILE_INTERVAL = 5; + public function getName(): string { return 'signals'; } @@ -46,6 +50,7 @@ class SignalsCommand implements CommandInterface { $this->initRedisIfEnabled(); $rServers = ServerRepository::getAll(); + $rLastReconcile = 0; while ($db && $db->ping()) { if (!$this->refreshOrBreak()) { @@ -62,6 +67,15 @@ class SignalsCommand implements CommandInterface { break; } + // Keep supervised streams' rows current: the fanout daemon runs their + // producers but cannot write the database, and the minute cron alone + // would leave a start showing "starting" (or a failure showing "up") + // for up to a minute. One control-socket call when nothing changed. + if (time() - $rLastReconcile >= self::RECONCILE_INTERVAL) { + $rLastReconcile = time(); + StreamProcess::reconcileSupervised(); + } + // ── Kill-сигналы из БД ────────────────────────────── if ($db->query('SELECT `signal_id`, `pid`, `rtmp` FROM `signals` WHERE `server_id` = ? AND `pid` IS NOT NULL ORDER BY `signal_id` ASC LIMIT 100', SERVER_ID)) { if ($db->num_rows() > 0) { diff --git a/src/Cli/CronJobs/StreamsCronJob.php b/src/Cli/CronJobs/StreamsCronJob.php index efbd1f33..42734204 100644 --- a/src/Cli/CronJobs/StreamsCronJob.php +++ b/src/Cli/CronJobs/StreamsCronJob.php @@ -47,6 +47,25 @@ class StreamsCronJob implements CommandInterface { return 0; } + /** + * Whether handing a stream (whose watchdog is gone) to the fanout supervisor + * must restart its producer rather than adopt it. + * + * An ffmpeg producer's feed into the daemon is a tee slave that, once broken + * by a daemon restart, stays broken for the life of the process — adopting it + * would leave the daemon's viewers on dead air until a stall restart. The + * native remuxer redials the ingest socket by itself, so it is adopted and + * the channel does not blink. + */ + private static function handOverNeedsRestart(array $rStream): int { + $rPID = file_exists(STREAMS_PATH . $rStream['stream_id'] . '_.pid') ? intval(@file_get_contents(STREAMS_PATH . $rStream['stream_id'] . '_.pid')) : intval($rStream['pid']); + if (!ProcessManager::isStreamRunning($rPID, $rStream['stream_id'])) { + return 0; // nothing running: a plain start + } + $rExe = basename((string) @readlink('/proc/' . $rPID . '/exe')); + return (strpos($rExe, 'ffmpeg') === 0 && FanoutClient::daemonStreamMissing(intval($rStream['stream_id']))) ? 1 : 0; + } + private function loadCron(): void { $rRedis = SettingsManager::getBool('redis_handler'); global $db; @@ -62,6 +81,18 @@ class StreamsCronJob implements CommandInterface { $rActivePIDs = array(); $rStreamIDs = array(); + // Bring streams_servers in step with the fanout supervisor first, so the + // pass below reads what is actually running. $rSupervised is the set it + // is supervising, or null when it cannot be asked — unknown, which the + // checks below treat as "not supervised" exactly as before this existed. + $rStates = FanoutClient::monitorStates(); + $rSupervised = StreamProcess::reconcileSupervised($rStates); + $rSupervisedSet = array_flip($rSupervised ?? array()); + // While the daemon takes hand-overs, streams still under a PHP monitor + // (started before supervision was on, or while the daemon was down) are + // moved to it — adopting their running encoder, so they do not restart. + $rMigrate = $rStates !== null && !empty($rStates['accepting']) && StreamProcess::supervisionEnabled(); + if ($rRedis) { $db->query('SELECT t2.stream_display_name, t1.stream_started, t1.stream_info, t2.fps_restart, t1.stream_status, t1.progress_info, t1.stream_id, t1.monitor_pid, t1.on_demand, t1.server_stream_id, t1.pid, servers_attached.attached, t2.vframes_server_id, t2.vframes_pid, t2.tv_archive_server_id, t2.tv_archive_pid FROM `streams_servers` t1 INNER JOIN `streams` t2 ON t2.id = t1.stream_id AND t2.direct_source = 0 INNER JOIN `streams_types` t3 ON t3.type_id = t2.type LEFT JOIN (SELECT `stream_id`, COUNT(*) AS `attached` FROM `streams_servers` WHERE `parent_id` = ? AND `pid` IS NOT NULL AND `pid` > 0 AND `monitor_pid` IS NOT NULL AND `monitor_pid` > 0) AS `servers_attached` ON `servers_attached`.`stream_id` = t1.`stream_id` WHERE (t1.pid IS NOT NULL OR t1.stream_status <> 0 OR t1.to_analyze = 1) AND t1.server_id = ? AND t3.live = 1', SERVER_ID, SERVER_ID); } else { @@ -73,7 +104,17 @@ class StreamsCronJob implements CommandInterface { echo 'Stream ID: ' . $rStream['stream_id'] . "\n"; $rStreamIDs[] = $rStream['stream_id']; - if (ProcessManager::isMonitorAlive($rStream['monitor_pid'], $rStream['stream_id']) || $rStream['on_demand']) { + $rIsSupervised = isset($rSupervisedSet[intval($rStream['stream_id'])]); + // superviseStream, not startMonitor: a stream the daemon will not + // take (delay, created channels) must keep the PHP monitor it has, + // not have a second one spawned beside it every pass. + if ($rMigrate && !$rIsSupervised && ProcessManager::isMonitorAlive($rStream['monitor_pid'], $rStream['stream_id'])) { + if (StreamProcess::superviseStream(intval($rStream['stream_id']), false)) { + echo 'Handed over to the fanout supervisor.' . "\n\n"; + continue; + } + } + if ($rIsSupervised || ProcessManager::isMonitorAlive($rStream['monitor_pid'], $rStream['stream_id']) || $rStream['on_demand']) { if ($rStream['on_demand'] == 1 && $rStream['attached'] == 0) { if ($rRedis) { $rCount = 0; @@ -209,6 +250,18 @@ class StreamsCronJob implements CommandInterface { } else { $rProgress = $rStream['progress_info']; } + // A supervised stream's codecs, resolution and bitrate come + // from the daemon, measured off the bytes (reconcileSupervised + // wrote them above); recomputing them here from a stream_info + // that nothing probes any more would blank them. Only the + // producer's progress report is this pass's to record. + if ($rIsSupervised) { + if ($rProgress !== $rStream['progress_info']) { + $db->query('UPDATE `streams_servers` SET `progress_info` = ? WHERE `server_stream_id` = ?', $rProgress, $rStream['server_stream_id']); + } + echo "\n"; + continue; + } if (file_exists(STREAMS_PATH . $rStream['stream_id'] . '_.stream_info')) { $rStreamInfo = file_get_contents(STREAMS_PATH . $rStream['stream_id'] . '_.stream_info'); unlink(STREAMS_PATH . $rStream['stream_id'] . '_.stream_info'); @@ -238,8 +291,9 @@ class StreamsCronJob implements CommandInterface { echo "\n"; } else { echo 'Start monitor...' . "\n\n"; - StreamProcess::startMonitor($rStream['stream_id']); - usleep(50000); + if (StreamProcess::startMonitor($rStream['stream_id'], self::handOverNeedsRestart($rStream)) === StreamProcess::MONITOR_PHP) { + usleep(50000); // stagger PHP monitor spawns + } } } } @@ -310,6 +364,15 @@ class StreamsCronJob implements CommandInterface { } if (SettingsManager::getBool('kill_rogue_ffmpeg')) { + // The supervisor restarts producers on its own schedule: a pid read from + // _.pid at the top of this pass may already have been replaced, and the + // replacement is not rogue. Ask the daemon for what it runs NOW. + $rStates = FanoutClient::monitorStates(); + foreach (($rStates['streams'] ?? array()) as $rState) { + if (intval($rState['pid'] ?? 0) > 0) { + $rActivePIDs[] = intval($rState['pid']); + } + } exec("ps aux | grep -v grep | grep '/*_.m3u8' | awk '{print \$2}'", $rRoguePIDs); foreach ($rRoguePIDs as $rPID) { if (is_numeric($rPID) && intval($rPID) > 0 && !in_array($rPID, $rActivePIDs)) { diff --git a/src/Core/Process/ProcessManager.php b/src/Core/Process/ProcessManager.php index d94c891d..f9541df1 100644 --- a/src/Core/Process/ProcessManager.php +++ b/src/Core/Process/ProcessManager.php @@ -157,6 +157,15 @@ class ProcessManager { ); } + // The fanout daemon's native remuxer (`xc_fanout remux … /_.m3u8`), + // which produces a copy-only stream in ffmpeg's place. Its own subcommand + // and this stream's playlist must both be there: the daemon process shares + // the executable but names no stream playlist. + if (strpos($exe, 'xc_fanout') === 0) { + $cmdline = (string) @file_get_contents('/proc/' . $pid . '/cmdline'); + return strpos($cmdline, "\0remux\0") !== false && strpos($cmdline, '/' . $streamId . '_.m3u8') !== false; + } + if (strpos($exe, 'php') === 0) { return true; } @@ -415,7 +424,9 @@ class ProcessManager { /** * Start a stream monitor process in background. * - * Extracted from ProcessManager::startMonitor(). + * Always the PHP watchdog. Stream code calls StreamProcess::startMonitor() + * instead, which hands the stream to the fanout daemon's supervisor when this + * server supervises and only falls back to this. * * @param int $streamID * @param int $restart diff --git a/src/Core/Util/StreamUtils.php b/src/Core/Util/StreamUtils.php index 5dea3fb1..e7b90fac 100644 --- a/src/Core/Util/StreamUtils.php +++ b/src/Core/Util/StreamUtils.php @@ -150,18 +150,32 @@ class StreamUtils { } $rURL .= ' live=1 timeout=10'; } else { - if ($rProtocol == 'http') { - $rPlatforms = array('livestream.com', 'ustream.tv', 'twitch.tv', 'vimeo.com', 'facebook.com', 'dailymotion.com', 'cnn.com', 'edition.cnn.com', 'youtube.com', 'youtu.be'); - $rHost = str_ireplace('www.', '', parse_url($rURL, PHP_URL_HOST)); - if (in_array($rHost, $rPlatforms)) { - $rURLs = trim(shell_exec(YOUTUBE_BIN . ' ' . escapeshellarg($rURL) . ' -q --get-url --skip-download -f best')); - list($rURL) = explode("\n", $rURLs); - } + if (self::needsResolver($rURL)) { + $rURLs = trim(shell_exec(YOUTUBE_BIN . ' ' . escapeshellarg($rURL) . ' -q --get-url --skip-download -f best')); + list($rURL) = explode("\n", $rURLs); } } return $rURL; } + /** Video platforms whose page URLs parseStreamURL() resolves through yt-dlp. */ + const RESOLVED_PLATFORMS = array('livestream.com', 'ustream.tv', 'twitch.tv', 'vimeo.com', 'facebook.com', 'dailymotion.com', 'cnn.com', 'edition.cnn.com', 'youtube.com', 'youtu.be'); + + /** + * Whether a source URL is a platform page parseStreamURL() has to resolve + * (through yt-dlp) into a playable — and short-lived — media URL. + * + * @param string $rURL Source URL. + * @return bool + */ + public static function needsResolver($rURL) { + if (strtolower(substr((string) $rURL, 0, 4)) !== 'http') { + return false; + } + $rHost = str_ireplace('www.', '', (string) parse_url($rURL, PHP_URL_HOST)); + return in_array($rHost, self::RESOLVED_PLATFORMS, true); + } + /** * Heuristically detect whether a URL is an XC_VM stream endpoint. * diff --git a/src/Domain/Stream/StreamProcess.php b/src/Domain/Stream/StreamProcess.php index 8a98b5b7..27e4ac7e 100644 --- a/src/Domain/Stream/StreamProcess.php +++ b/src/Domain/Stream/StreamProcess.php @@ -5,6 +5,7 @@ namespace XcVm\Domain\Stream; use XcVm\Core\Config\SettingsManager; use XcVm\Core\Diagnostics\DiagnosticsService; use XcVm\Core\Http\CurlClient; +use XcVm\Core\Process\ProcessManager; use XcVm\Core\Util\StreamUtils; use XcVm\Streaming\Fanout\FanoutClient; @@ -84,15 +85,49 @@ class StreamProcess { } /** - * Start the monitor process for a stream. + * Whether anything is watching this stream on this server: its PHP monitor + * (verified by command line, as ever), or the fanout supervisor. For a + * supervised stream `monitor_pid` names the daemon, which the PHP check + * rightly rejects — so "no PHP monitor" must not be read as "unwatched", or + * the caller starts a second watchdog for a stream that already has one. + * + * @param int $rStreamID Stream id. + * @param mixed $rMonitorPID The stream's recorded monitor pid. + * @return bool + */ + public static function isWatched($rStreamID, $rMonitorPID): bool { + if (ProcessManager::isMonitorAlive($rMonitorPID, $rStreamID)) { + return true; + } + return FanoutClient::isSupervised(intval($rStreamID)) === true; + } + + /** startMonitor(): the stream was handed to the fanout daemon's supervisor. */ + const MONITOR_FANOUT = 'fanout'; + /** startMonitor(): a PHP watchdog (`console.php monitor`) was started for it. */ + const MONITOR_PHP = 'php'; + + /** + * Start watching a live stream: hand it to the fanout daemon's supervisor + * when this server supervises (see superviseStream()), otherwise start the + * PHP watchdog for it. * * @param int $rStreamID Stream id. - * @param int $rRestart Restart flag/counter. - * @return mixed Start result. + * @param int $rRestart Truthy to restart what is running rather than take it as it is. + * @return string MONITOR_FANOUT or MONITOR_PHP — which one now watches it. */ public static function startMonitor($rStreamID, $rRestart = 0) { + if (self::superviseStream(intval($rStreamID), (bool) $rRestart)) { + return self::MONITOR_FANOUT; + } + // The PHP monitor takes it. A stream the daemon still supervises — its + // supervision since turned off, or the daemon refusing it now — is taken + // back first: otherwise the new monitor would find it supervised and stand + // down, and a restart would do nothing. This ends its producer; a running + // encoder cannot be handed back to PHP without a restart. + FanoutClient::release(intval($rStreamID)); shell_exec(PHP_BIN . ' ' . MAIN_HOME . 'console.php monitor ' . intval($rStreamID) . ' ' . intval($rRestart) . ' >/dev/null 2>/dev/null &'); - return true; + return self::MONITOR_PHP; } @@ -838,10 +873,13 @@ class StreamProcess { // Capped at seg_time so a small seg_time never produces a longer first segment. $rInitTime = min(2, intval($rSegmentSettings['seg_time'])); // When the xc_fanout daemon accepted an ingest registration (reachable), - // tee the HLS output to it too (ADR 0003, A2). Standard live only — not - // loopback/delay. If the daemon was unreachable ($data['ingestSock'] is - // null) the original on-disk-only HLS output runs, unchanged. - if (!$rLoopback && !$rDelayActive && !empty($data['ingestSock'])) { + // tee the HLS output to it too (ADR 0003, A2). Never for delay, whose HLS + // goes to its own directory. startStream() registers no ingest for a + // loopback stream (the PHP relay feeds the daemon for those); a supervised + // one does, because the supervisor confirms and judges a stream by the + // bytes the daemon receives. If the daemon was unreachable + // ($data['ingestSock'] is null) the original on-disk-only HLS runs. + if (!$rDelayActive && !empty($data['ingestSock'])) { // The tee muxer needs an EXPLICIT -map — plain single outputs use // ffmpeg's automatic stream selection, but tee does not ("Output file // does not contain any stream" otherwise). Reuse the stream's own map, @@ -894,7 +932,13 @@ class StreamProcess { $rFFMPEG .= '{MAP} -individual_header_trailer 0 -f hls -hls_time ' . intval($rSegmentSettings['seg_time']) . ' -hls_list_size ' . intval($rStream['stream_info']['delay_minutes']) * 6 . ' -hls_delete_threshold 4 -start_number ' . $rSegmentStart . ' -hls_flags delete_segments+discont_start+omit_endlist -hls_segment_type mpegts -hls_segment_filename "' . DELAY_PATH . intval($rStreamID) . '_%d.ts" "' . DELAY_PATH . intval($rStreamID) . '_.m3u8" '; } - $rFFMPEG .= ' >/dev/null 2>>' . STREAMS_PATH . intval($rStreamID) . '.errors & echo $! > ' . STREAMS_PATH . intval($rStreamID) . '_.pid'; + // Launched by the shell here: redirect, background, record the pid. A + // supervised command is launched by the fanout daemon instead, which has + // to be its parent to reap it and redirects stderr / writes the pid file + // itself — so it must arrive without this tail. + if (empty($data['supervised'])) { + $rFFMPEG .= ' >/dev/null 2>>' . STREAMS_PATH . intval($rStreamID) . '.errors & echo $! > ' . STREAMS_PATH . intval($rStreamID) . '_.pid'; + } $ffprobeContainer = (isset($rFFProbeOutput['container']) && is_string($rFFProbeOutput['container'])) ? $rFFProbeOutput['container'] : ''; @@ -921,6 +965,612 @@ class StreamProcess { return $rFFMPEG; } + // ── Fanout supervision + native remuxer ───────────────────────────────── + // + // With `fanout_supervise` on, a live stream is not given a PHP watchdog + // (`console.php monitor`, MonitorCommand). startMonitor() builds the stream's + // commands here and hands them to the xc_fanout daemon's supervisor + // (XC_VM_Fanout ADR 0002), which runs, watches and restarts them. The PHP + // monitor remains the fallback for a daemon that cannot be reached, and for + // the stream kinds the supervisor does not take (delay, created channels, + // sources that need a URL resolver). + // + // With `fanout_source_backend` native or auto, a copy-only stream's command is + // the daemon's native remuxer — `xc_fanout remux`, built by buildNativeLive() + // exactly as buildLive() builds an ffmpeg one — instead of ffmpeg. native runs + // only the remuxer; auto gives each source the ffmpeg command as an explicit + // fallback, which the supervisor switches to when the remuxer reports it + // cannot read that source. + + /** Exit status of `xc_fanout remux` for "cannot be served natively" (supervisor.ExitUnsupported). */ + const REMUX_EXIT_UNSUPPORTED = 3; + + /** + * Assemble the native remuxer command — `xc_fanout remux` — for one source of + * a copy-only live stream: the same source, fetch arguments, segment settings + * and ingest socket buildLive() turns into an ffmpeg `-c copy -f tee` line, + * producing the same two outputs (the on-disk HLS under this stream's names, + * the MPEG-TS feed into the daemon) with no ffmpeg. PURE: no I/O. + * + * Like a supervised buildLive() line it carries no redirect/background tail: + * the daemon runs it, redirects its stderr to .errors and writes the pid. + * + * @param array $data streamID, source (resolved URL), arguments (rows keyed by + * argument_key), segmentSettings, ingestSock, settings, binary. + * @return string The shell command line. + */ + private static function buildNativeLive(array $data): string { + $rStreamID = intval($data['streamID']); + $rSeg = $data['segmentSettings']; + $rArgs = $data['arguments']; + $rSettings = $data['settings']; + $rSegTime = max(1, intval($rSeg['seg_time'])); + + // The fetch identity the daemon's own puller uses for this stream + // (user_agent / proxy / cookie resolution is shared, not re-derived). + $rSource = FanoutClient::buildSource(array('stream_source' => json_encode(array($data['source']))), $rArgs); + + $rCmd = array( + $data['binary'], 'remux', + '-loglevel', (!empty($rSettings['ffmpeg_warnings']) ? 'warning' : 'error'), + '-i', escapeshellarg($data['source']), + ); + if ($rSource['ua'] !== '') { + $rCmd[] = '-user_agent ' . escapeshellarg($rSource['ua']); + } + if ($rSource['cookie'] !== '') { + $rCmd[] = '-cookies ' . escapeshellarg(StreamUtils::fixCookie($rSource['cookie'])); + } + if ($rSource['proxy'] !== '') { + $rCmd[] = '-http_proxy ' . escapeshellarg($rSource['proxy']); + } + if (!empty($rArgs['headers']['value'])) { + $rCmd[] = '-headers ' . escapeshellarg($rArgs['headers']['value']); + } + if (!isset($rSettings['fanout_source_insecure']) || !empty($rSettings['fanout_source_insecure'])) { + $rCmd[] = '-insecure'; + } + if (!empty($data['ingestSock'])) { + $rCmd[] = '-ingest ' . escapeshellarg('unix:' . $data['ingestSock']); + } + $rCmd[] = '-hls_time ' . $rSegTime; + $rCmd[] = '-hls_init_time ' . min(2, $rSegTime); // buildLive's fast first segment + $rCmd[] = '-hls_list_size ' . intval($rSeg['seg_list_size']); + $rCmd[] = '-hls_delete_threshold ' . intval($rSeg['seg_delete_threshold']); + $rCmd[] = '-progress ' . escapeshellarg(STREAMS_PATH . $rStreamID . '_.progress'); + $rCmd[] = '-hls_segment_filename ' . escapeshellarg(STREAMS_PATH . $rStreamID . '_%d.ts'); + $rCmd[] = escapeshellarg(STREAMS_PATH . $rStreamID . '_.m3u8'); + + return implode(' ', $rCmd); + } + + /** + * Whether a live stream's configuration is a pure copy the native remuxer + * reproduces: one MPEG-TS source passed through, with nothing configured that + * needs ffmpeg in the path. PURE. Anything unrecognised falls to ffmpeg — a + * refusal costs an ffmpeg process, a wrong acceptance a channel served wrong. + * + * @param array $rStreamInfo streams ⨝ streams_types row. + * @param array $rArgs Stream arguments keyed by argument_key. + * @return bool + */ + private static function isNativeEligible(array $rStreamInfo, array $rArgs): bool { + if (($rStreamInfo['type_key'] ?? '') !== 'live_streams') { + return false; // radio and created channels are ffmpeg's + } + if (intval($rStreamInfo['enable_transcode'] ?? 0) === 1 || !empty($rStreamInfo['custom_ffmpeg'])) { + return false; // re-encoding + } + if (!empty($rStreamInfo['custom_map'])) { + return false; // a stream selection the remuxer (which copies every PID) cannot honour + } + if (intval($rStreamInfo['rtmp_output'] ?? 0) === 1) { + return false; // FLV output + } + $rPush = json_decode((string) ($rStreamInfo['external_push'] ?? ''), true); + if (is_array($rPush) && !empty($rPush[SERVER_ID])) { + return false; // external RTMP pushes + } + // Asked for timestamp repair or realtime pacing: the source is not a + // clean live feed, and passing its bytes through unchanged is not enough. + if (intval($rStreamInfo['gen_timestamps'] ?? 0) === 1 || intval($rStreamInfo['read_native'] ?? 0) === 1) { + return false; + } + if (!empty($rArgs['force_input_acodec']['value'])) { + return false; // re-interprets the audio: an ffmpeg input option + } + return true; + } + + /** + * Whether one source URL is one the native remuxer reads (xc_fanout's + * nativesrc: MPEG-TS over http(s) — plain or as HLS with TS segments — and + * udp/rtp). What it can only discover by connecting (fMP4 or encrypted HLS) + * it reports at run time, which is what the auto-mode fallback is for. + */ + private static function isNativeSource(string $rURL): bool { + $rScheme = strtolower((string) parse_url($rURL, PHP_URL_SCHEME)); + return in_array($rScheme, array('http', 'https', 'udp', 'rtp'), true) && !StreamUtils::needsResolver($rURL); + } + + /** + * The supervisor policy for a stream, from the panel settings the PHP monitor + * obeyed. PURE. + * + * @return array The spec's `policy` object. + */ + private static function supervisorPolicy(array $rServerInfo, array $rSettings, int $rSourceCount, int $rProbeSeconds): array { + $rSegTime = max(1, intval($rSettings['seg_time'] ?? 10)); + // The PHP monitor probed (up to the analyse window plus slack) and then + // waited for the playlist; the daemon confirms a start by bytes arriving, + // which for ffmpeg comes after its own probe. Same budget. + $rStartTimeout = $rProbeSeconds + max(20, min($rSegTime * 3, 30)); + return array( + 'stop_failures' => max(0, intval($rSettings['stop_failures'] ?? 0)), + 'stream_fail_sleep' => max(1, intval($rSettings['stream_fail_sleep'] ?? 10)), + 'on_demand' => !empty($rServerInfo['on_demand']), + 'on_demand_failure_exit' => !empty($rSettings['on_demand_failure_exit']), + 'start_timeout_sec' => $rStartTimeout, + 'priority_backup_sec' => (!empty($rSettings['priority_backup']) && $rSourceCount > 1 && empty($rServerInfo['parent_id'])) ? 300 : 0, + ); + } + + /** + * The supervisor health policy for a stream, from the checks the PHP monitor + * made. PURE. Each check is off when its panel setting is. + * + * @return array The spec's `health` object. + */ + private static function supervisorHealth(array $rStreamInfo, array $rSettings): array { + $rSegTime = max(1, intval($rSettings['seg_time'] ?? 10)); + $rHealth = array( + 'stall_sec' => $rSegTime * 6, // the monitor's "playlist unchanged for seg_time × 6" + 'audio_loss_sec' => !empty($rSettings['audio_restart_loss']) ? 30 : 0, + 'fps_threshold' => 0, + 'fps_grace_sec' => max(0, intval($rSettings['fps_delay'] ?? 0)), + ); + if (intval($rStreamInfo['fps_restart'] ?? 0) === 1) { + // "FPS Threshold %": restart below this share of the stream's own rate. + $rPercent = intval($rStreamInfo['fps_threshold'] ?? 0) ?: 90; + $rHealth['fps_threshold'] = min(100, max(1, $rPercent)) / 100; + } + $rAuto = json_decode((string) ($rStreamInfo['auto_restart'] ?? ''), true); + if (is_array($rAuto) && !empty($rAuto['days']) && !empty($rAuto['at'])) { + $rHealth['auto_restart'] = array('days' => array_values((array) $rAuto['days']), 'at' => (string) $rAuto['at']); + } + return $rHealth; + } + + /** + * Whether live streams on this server are handed to the fanout supervisor. + */ + public static function supervisionEnabled(): bool { + return !empty(SettingsManager::get('fanout_supervise')) && defined('FANOUT_CTL_SOCK') && file_exists(FANOUT_CTL_SOCK); + } + + /** + * Build the supervisor spec for a live stream on this server, or null when it + * has to run under the PHP monitor (not found, a kind the supervisor does not + * take, or no daemon ingest to feed). + * + * Registers the stream's ingest with the daemon and writes its HLS key/iv, + * because both are baked into the commands. + * + * @param int $rStreamID Stream id. + * @return array|null The spec for FanoutClient::supervise(), or null. + */ + public static function buildSupervisorSpec(int $rStreamID): ?array { + global $rSettings, $rServers, $rFFMPEG_CPU, $rFFMPEG_GPU, $rFFPROBE; + $db = self::db(); + $rFFMPEGCpu = $rFFMPEG_CPU ?: \XcVm\Streaming\Codec\FfmpegPaths::cpu(); + $rFFMPEGGpu = $rFFMPEG_GPU ?: \XcVm\Streaming\Codec\FfmpegPaths::gpu(); + $rFFProbeBin = $rFFPROBE ?: \XcVm\Streaming\Codec\FfmpegPaths::probe(); + + $db->query('SELECT * FROM `streams` t1 INNER JOIN `streams_types` t2 ON t2.type_id = t1.type AND t2.live = 1 LEFT JOIN `profiles` t4 ON t1.transcode_profile_id = t4.profile_id WHERE t1.direct_source = 0 AND t1.id = ?', $rStreamID); + if ($db->num_rows() <= 0) { + return null; + } + $rStream = array('stream_info' => $db->get_row()); + $db->query('SELECT * FROM `streams_servers` WHERE stream_id = ? AND `server_id` = ?', $rStreamID, SERVER_ID); + if ($db->num_rows() <= 0) { + return null; + } + $rStream['server_info'] = $db->get_row(); + $db->query('SELECT t1.*, t2.* FROM `streams_options` t1, `streams_arguments` t2 WHERE t1.stream_id = ? AND t1.argument_id = t2.id', $rStreamID); + $rStream['stream_arguments'] = $db->get_rows(); + + $rInfo = $rStream['stream_info']; + $rParentID = intval($rStream['server_info']['parent_id']); + // Kinds the PHP monitor keeps: a delayed stream runs its own playlist + // worker off the encoder's, and a created channel resumes at an offset + // computed at each start. + if ((intval($rInfo['delay_minutes']) > 0 && $rParentID === 0) || $rInfo['type_key'] === 'created_live') { + return null; + } + + if ($rParentID > 0) { + $rLoopURL = (!is_null($rServers[SERVER_ID]['private_url_ip']) && !is_null($rServers[$rParentID]['private_url_ip']) ? $rServers[$rParentID]['private_url_ip'] : $rServers[$rParentID]['public_url_ip']); + $rSources = array($rLoopURL . 'admin/live?stream=' . intval($rStreamID) . '&password=' . urlencode($rSettings['live_streaming_pass']) . '&extension=ts'); + $rLabels = array('Loopback: #' . $rParentID); + } else { + $rSources = array_values(array_filter(array_map('trim', (array) json_decode((string) $rInfo['stream_source'], true)), 'strlen')); + $rLabels = $rSources; + } + if (count($rSources) === 0) { + return null; + } + foreach ($rSources as $rSource) { + // A platform URL is resolved to a short-lived one at each start; a + // command built now would carry a URL that expires under it. + if (StreamUtils::needsResolver($rSource)) { + return null; + } + } + + $rLoopback = $rParentID > 0; + $rLLOD = !empty($rStream['server_info']['on_demand']) && ($rLoopback || intval($rInfo['llod']) > 0); + $rSegmentSettings = array('seg_time' => intval($rSettings['seg_time']), 'seg_list_size' => intval($rSettings['seg_list_size']), 'seg_delete_threshold' => intval($rSettings['seg_delete_threshold'])); + list($rProbesize, $rAnalyseDuration, $rTimeout) = self::resolveProbeSettings($rStream['server_info']['on_demand'], $rInfo['probesize_ondemand'], $rLLOD, $rSettings); + + self::writeStreamKeyIv($rStreamID); + $rEncKey = $rEncIV = null; + if (!empty($rSettings['encrypt_hls']) && !$rLoopback) { + $rEncKey = @bin2hex((string) @file_get_contents(STREAMS_PATH . $rStreamID . '_.key')); + $rEncIV = @bin2hex((string) @file_get_contents(STREAMS_PATH . $rStreamID . '_.iv')); + } + $rIngestSock = FanoutClient::registerIngest($rStreamID, $rEncKey, $rEncIV); + if ($rIngestSock === null) { + return null; // no daemon to feed: the stream runs the legacy way + } + + $rArgsByKey = array(); + foreach ($rStream['stream_arguments'] as $rArg) { + $rArgsByKey[$rArg['argument_key']] = $rArg; + } + $rBackend = (string) ($rSettings['fanout_source_backend'] ?? 'auto'); + $rNativeStream = $rBackend !== 'ffmpeg' && self::isNativeEligible($rInfo, $rArgsByKey); + $rPriority = !empty($rSettings['priority_backup']) && count($rSources) > 1 && !$rLoopback; + + $rSpecSources = array(); + foreach ($rSources as $i => $rSource) { + $rStreamSource = StreamUtils::parseStreamURL($rSource); + $rProtocol = strtolower(substr($rStreamSource, 0, (int) strpos($rStreamSource, '://'))); + $rArguments = $rStream['stream_arguments']; + $rIsXC_VM = $rLoopback || StreamUtils::detectXC_VM($rStreamSource); + if ($rIsXC_VM && !$rLoopback && !empty($rSettings['send_xc_vm_header'])) { + $rArguments = self::appendHeaderArgument($rArguments, 'X-XC_VM-Detect:1'); + } + $rProbeArguments = self::appendHeaderArgument($rArguments, 'X-XC_VM-Prebuffer:1'); + if ($rIsXC_VM && !empty($rStream['server_info']['on_demand']) && !empty($rSettings['request_prebuffer'])) { + $rArguments = self::appendHeaderArgument($rArguments, 'X-XC_VM-Prebuffer:1'); + } + $rFetchOptions = implode(' ', StreamUtils::getArguments($rArguments, $rProtocol, 'fetch')); + + $rFFMPEG = self::buildLive(array( + 'stream' => $rStream, 'settings' => $rSettings, 'servers' => $rServers, + 'streamID' => $rStreamID, 'streamSource' => $rStreamSource, + 'fetchOptions' => $rFetchOptions, 'ffprobe' => self::cachedProbe($rSource, $rStreamSource), + 'protocol' => $rProtocol, 'source' => $rSource, + 'segmentSettings' => $rSegmentSettings, 'externalPush' => array(), + 'probesize' => $rProbesize, 'analyseDuration' => $rAnalyseDuration, + 'llod' => $rLLOD, 'loopback' => $rLoopback, + 'segmentStart' => 0, 'delayActive' => false, + 'ffmpegCpu' => $rFFMPEGCpu, 'ffmpegGpu' => $rFFMPEGGpu, + 'ingestSock' => $rIngestSock, 'supervised' => true, + )); + + $rEntry = array('label' => $rLabels[$i], 'cmd' => $rFFMPEG); + if ($rNativeStream && self::isNativeSource($rStreamSource)) { + $rNativeArgs = array(); + foreach ($rArguments as $rArg) { + $rNativeArgs[$rArg['argument_key']] = $rArg; + } + $rEntry['cmd'] = self::buildNativeLive(array( + 'streamID' => $rStreamID, 'source' => $rStreamSource, 'arguments' => $rNativeArgs, + 'segmentSettings' => $rSegmentSettings, 'ingestSock' => $rIngestSock, + 'settings' => $rSettings, 'binary' => FanoutClient::binaryPath(), + )); + if ($rBackend === 'auto') { + $rEntry['fallback_cmd'] = $rFFMPEG; + } + } + if ($rPriority) { + $rProbeOptions = implode(' ', StreamUtils::getArguments($rProbeArguments, $rProtocol, 'fetch')); + $rEntry['probe_cmd'] = 'timeout ' . intval($rTimeout) . ' ' . $rFFProbeBin . ' ' . $rProbeOptions . ' -probesize ' . intval($rProbesize) . ' -analyzeduration ' . intval($rAnalyseDuration) . ' -i ' . escapeshellarg($rStreamSource) . ' -v quiet -print_format json -show_streams -show_format'; + } + $rSpecSources[] = $rEntry; + } + + return array( + 'sources' => $rSpecSources, + 'policy' => self::supervisorPolicy($rStream['server_info'], $rSettings, count($rSpecSources), intval($rTimeout)), + 'health' => self::supervisorHealth($rInfo, $rSettings), + 'pid_path' => STREAMS_PATH . $rStreamID . '_.pid', + 'errors_path' => STREAMS_PATH . $rStreamID . '.errors', + 'log_path' => (SettingsManager::get('save_restart_logs') != 0 ? LOGS_TMP_PATH . 'stream_log.log' : ''), + 'server_id' => intval(SERVER_ID), + // Both producers name this stream's playlist, and nothing else does: + // an encoder that outlived a daemon restart is recognised by it. + 'adopt_match' => STREAMS_PATH . $rStreamID . '_.m3u8', + ); + } + + /** + * The last ffprobe result cached for a source, for the codec-dependent parts + * of an ffmpeg command. A supervised spec carries a command for EVERY source, + * and probing each one on the hand-over path would cost seconds per source, + * so a source with nothing cached gets the minimum buildLive() needs. + */ + private static function cachedProbe(string $rSource, string $rStreamSource): array { + $rCache = CACHE_TMP_PATH . md5($rSource); + if (file_exists($rCache)) { + $rProbe = @igbinary_unserialize((string) @file_get_contents($rCache)); + if (is_array($rProbe) && !isset($rProbe['codecs']) && isset($rProbe['streams'])) { + $rProbe = \XcVm\Streaming\Codec\FFprobeRunner::parseFFProbe($rProbe); + } + if (is_array($rProbe) && isset($rProbe['codecs'])) { + return $rProbe; + } + } + $rPath = strtolower((string) parse_url($rStreamSource, PHP_URL_PATH)); + return array('container' => (substr($rPath, -5) === '.m3u8' ? 'hls' : 'mpegts'), 'codecs' => array()); + } + + /** + * Kill this stream's PHP watchdog, if one is running — only ever a process + * whose command line is exactly `XC_VM[]`. Leaves its encoder alone: a + * hand-over without a restart adopts it. + */ + private static function killPhpMonitor(int $rStreamID): void { + $rCandidates = array(); + if (file_exists(STREAMS_PATH . $rStreamID . '_.monitor')) { + $rCandidates[] = intval(@file_get_contents(STREAMS_PATH . $rStreamID . '_.monitor')); + } + self::db()->query('SELECT `monitor_pid` FROM `streams_servers` WHERE `stream_id` = ? AND `server_id` = ?', $rStreamID, SERVER_ID); + if (self::db()->num_rows() > 0) { + $rCandidates[] = intval(self::db()->get_row()['monitor_pid']); + } + foreach (array_unique($rCandidates) as $rPID) { + if ($rPID > 0 && ProcessManager::isMonitorAlive($rPID, $rStreamID)) { + posix_kill($rPID, 9); + } + } + @unlink(STREAMS_PATH . $rStreamID . '_.monitor'); + } + + /** + * Hand a live stream to the fanout daemon's supervisor — what startMonitor() + * does instead of spawning a PHP watchdog. + * + * Without a restart, a stream the daemon already supervises is left alone, + * and a running encoder is adopted rather than replaced (a PHP-monitored + * stream moves over without a blip); nothing is touched unless the daemon can + * take the stream. With a restart, whatever is running is ended first — the + * PHP monitor's restart semantics. + * + * @param int $rStreamID Stream id. + * @param bool $rRestart Restart it (an admin start/restart) rather than take it as it is. + * @return bool True when the daemon is now supervising it; false = run the PHP monitor. + */ + public static function superviseStream(int $rStreamID, bool $rRestart): bool { + if (!self::supervisionEnabled()) { + return false; + } + // Asked before anything is touched: a daemon that is down or not taking + // hand-overs leaves the stream entirely to the PHP monitor. + $rStates = FanoutClient::monitorStates(); + if ($rStates === null || empty($rStates['accepting'])) { + return false; + } + if (!$rRestart && isset($rStates['streams'][(string) $rStreamID])) { + return true; // already supervised + } + + if ($rRestart) { + // End everything first — the spec below writes the stream's fresh HLS + // key/iv, which the cleanup would otherwise delete. + self::killPhpMonitor($rStreamID); + FanoutClient::release($rStreamID); + self::killProducer($rStreamID, false); + shell_exec('rm -f ' . STREAMS_PATH . intval($rStreamID) . '_*'); + } + + // Built before anything of a running stream is touched: a stream the + // daemon will not take keeps its PHP monitor and its encoder as they are. + $rSpec = self::buildSupervisorSpec($rStreamID); + if ($rSpec === null) { + return false; + } + + $rAdopting = false; + if (!$rRestart) { + // One watchdog at a time: the PHP monitor goes before the daemon takes + // over, and a producer the daemon cannot adopt goes with it. + self::killPhpMonitor($rStreamID); + $rAdopting = self::killProducer($rStreamID, true); + } + + // The daemon is the monitor now: record its pid where the panel looks for + // "is anything watching this stream" BEFORE handing over. The reconcile + // releases any supervised stream whose row reads stopped, and must not + // catch this one in the moment between the hand-over and this write. A + // fresh start is marked in progress until the reconcile sees it confirmed; + // an adopted, already-running one keeps its status and start time. + $rDaemonPID = intval($rStates['daemon_pid'] ?? 0) ?: null; + if ($rAdopting) { + self::db()->query('UPDATE `streams_servers` SET `monitor_pid` = ? WHERE `stream_id` = ? AND `server_id` = ?', $rDaemonPID, $rStreamID, SERVER_ID); + } else { + self::db()->query('UPDATE `streams_servers` SET `monitor_pid` = ?, `pid` = NULL, `stream_status` = 2, `to_analyze` = 0, `stream_started` = ?, `current_source` = ? WHERE `stream_id` = ? AND `server_id` = ?', $rDaemonPID, time(), $rSpec['sources'][0]['label'], $rStreamID, SERVER_ID); + } + + if (!FanoutClient::supervise($rStreamID, $rSpec)) { + // Refused after all. Make sure the daemon holds nothing for this stream + // before a PHP monitor starts a second producer for it; the PHP + // monitor records its own pid over the daemon's. + FanoutClient::release($rStreamID); + return false; + } + self::updateStream($rStreamID); + return true; + } + + /** + * End this stream's running producer, if it has one — or, with $rKeepAdoptable, + * only if the daemon could not adopt it: adoption needs the producer's command + * line to name this stream's playlist (spec adopt_match), which ffmpeg and the + * native remuxer do and the PHP LLOD segmenter and PHP loopback relay do not. + * Left running, one of those would share the stream's files with the + * replacement the daemon starts. + * + * @return bool True when an adoptable producer was left running. + */ + private static function killProducer(int $rStreamID, bool $rKeepAdoptable): bool { + $rPID = self::pidFromFileOrColumn($rStreamID, 'pid', '_.pid'); + if ($rPID <= 0 || !\XcVm\Streaming\Health\ProcessChecker::checkPID($rPID, array($rStreamID . '_.m3u8', $rStreamID . '_%d.ts', 'LLOD[' . $rStreamID . ']', 'Loopback[' . $rStreamID . ']'))) { + return false; + } + if ($rKeepAdoptable && strpos((string) @file_get_contents('/proc/' . $rPID . '/cmdline'), STREAMS_PATH . $rStreamID . '_.m3u8') !== false) { + return true; + } + posix_kill($rPID, 9); + return false; + } + + /** + * Bring streams_servers in step with what the fanout supervisor reports for + * this server's streams — the DB writes the PHP monitor used to make itself. + * Called by cron:streams every pass and by the signals daemon every few + * seconds, so a start or a failure shows in the panel promptly. + * + * Streams the daemon supervises but the panel no longer runs here (deleted, + * or stopped in a race) are released. + * + * @param array|null $rStates FanoutClient::monitorStates(), or null to fetch it. + * @return int[]|null Ids supervised after the pass; null when the daemon is + * unreachable (unknown — never "none"). + */ + public static function reconcileSupervised(?array $rStates = null): ?array { + if ($rStates === null) { + $rStates = FanoutClient::monitorStates(); + } + if ($rStates === null) { + return null; + } + $rIDs = array_map('intval', array_keys($rStates['streams'])); + if (count($rIDs) === 0) { + return array(); + } + $db = self::db(); + $db->query('SELECT `stream_id`, `pid`, `monitor_pid`, `stream_status`, `current_source`, `stream_started`, `stream_info`, `audio_codec`, `video_codec`, `resolution`, `bitrate`, `compatible` FROM `streams_servers` WHERE `server_id` = ? AND `stream_id` IN (' . implode(',', $rIDs) . ')', SERVER_ID); + $rRows = array(); + foreach ($db->get_rows() as $rRow) { + $rRows[intval($rRow['stream_id'])] = $rRow; + } + + $rKept = array(); + $rChanged = array(); + foreach ($rStates['streams'] as $rID => $rState) { + $rID = intval($rID); + $rRow = $rRows[$rID] ?? null; + // No row, or a row the panel has marked stopped: nothing should be + // producing this stream here. + if ($rRow === null || (is_null($rRow['monitor_pid']) && is_null($rRow['pid']) && intval($rRow['stream_status']) === 0)) { + FanoutClient::release($rID); + continue; + } + $rKept[] = $rID; + $rSet = self::supervisedRowUpdate($rRow, $rState, (bool) SettingsManager::get('player_allow_hevc'), time()); + if (count($rSet) > 0) { + $rCols = array(); + $rVals = array(); + foreach ($rSet as $rCol => $rVal) { + $rCols[] = '`' . $rCol . '` = ?'; + $rVals[] = $rVal; + } + $rVals[] = $rID; + $rVals[] = SERVER_ID; + $db->query('UPDATE `streams_servers` SET ' . implode(', ', $rCols) . ' WHERE `stream_id` = ? AND `server_id` = ?', ...$rVals); + $rChanged[] = $rID; + } + } + if (count($rChanged) > 0) { + self::updateStreams($rChanged); + } + return $rKept; + } + + /** + * The streams_servers changes one supervisor state implies, as column => + * value; only columns whose value differs. PURE. + * + * Status follows the PHP monitor's meaning: 2 while a start is in progress, + * 0 once it is confirmed, 1 when the supervisor has given up or is between + * failed starts. Metadata the daemon could not determine is left as it is — + * a correct value is never overwritten with a blank. + * + * @param array $rRow Current streams_servers columns. + * @param array $rState One stream's supervisor state. + * @param bool $rAllowHevc player_allow_hevc (for `compatible`). + * @param int $rNow Current time. + * @return array Column => new value. + */ + private static function supervisedRowUpdate(array $rRow, array $rState, bool $rAllowHevc, int $rNow): array { + $rRunning = !empty($rState['running']); + $rConfirmed = $rRunning && !empty($rState['confirmed']); + if ($rConfirmed) { + $rStatus = 0; + } elseif (!empty($rState['gave_up']) || (!$rRunning && intval($rState['failures'] ?? 0) > 0)) { + $rStatus = 1; + } else { + $rStatus = 2; + } + $rWant = array( + 'stream_status' => $rStatus, + 'pid' => ($rRunning && intval($rState['pid'] ?? 0) > 0) ? intval($rState['pid']) : null, + ); + if (intval($rState['daemon_pid'] ?? 0) > 0) { + $rWant['monitor_pid'] = intval($rState['daemon_pid']); + } + if (($rState['source'] ?? '') !== '') { + $rWant['current_source'] = (string) $rState['source']; + } + // stream_started is when the running producer came up. + if ($rConfirmed && intval($rRow['stream_status']) !== 0) { + $rWant['stream_started'] = $rNow - intdiv(intval($rState['uptime_ms'] ?? 0), 1000); + } + + $rMeta = (isset($rState['meta']) && is_array($rState['meta'])) ? $rState['meta'] : array(); + if (!empty($rMeta['video_codec']) || !empty($rMeta['audio_codec'])) { + $rVideo = (string) ($rMeta['video_codec'] ?? '') ?: $rRow['video_codec']; + $rAudio = (string) ($rMeta['audio_codec'] ?? '') ?: $rRow['audio_codec']; + $rWant['video_codec'] = $rVideo; + $rWant['audio_codec'] = $rAudio; + $rCodecs = array(); + if ($rVideo) { + $rCodecs['video'] = array('codec_name' => $rVideo, 'codec_type' => 'video'); + } + if ($rAudio) { + $rCodecs['audio'] = array('codec_name' => $rAudio, 'codec_type' => 'audio'); + } + $rWant['compatible'] = intval(DiagnosticsService::checkCompatibility(array('codecs' => $rCodecs), $rAllowHevc)); + } + if (intval($rMeta['height'] ?? 0) > 0) { + $rWant['resolution'] = StreamSorter::getNearest(array(240, 360, 480, 576, 720, 1080, 1440, 2160), intval($rMeta['height'])); + } + if (intval($rMeta['bitrate_kbps'] ?? 0) > 0) { + $rWant['bitrate'] = intval($rMeta['bitrate_kbps']); + } + + $rSet = array(); + foreach ($rWant as $rCol => $rVal) { + $rHave = $rRow[$rCol] ?? null; + if ((is_null($rVal) !== is_null($rHave)) || (!is_null($rVal) && (string) $rVal !== (string) $rHave)) { + $rSet[$rCol] = $rVal; + } + } + return $rSet; + } + public static function createChannelItem($rStreamID, $rSource) { global $rSettings, $rServers, $rFFMPEG_CPU, $rFFMPEG_GPU; $db = self::db(); @@ -980,6 +1630,13 @@ class StreamProcess { * @return mixed Stop result. */ public static function stopStream($rStreamID, $rStop = false) { + // A supervised stream is released FIRST: its producer dying is exactly + // what the fanout supervisor restarts, so killing it before the release + // would have the daemon start a replacement and the stream refuse to stop. + // The release kills the producer itself. A no-op for a stream the daemon + // does not supervise (or a daemon that is not there). + FanoutClient::release(intval($rStreamID)); + $rMonitor = self::pidFromFileOrColumn($rStreamID, 'monitor_pid', '_.monitor'); if (0 < $rMonitor && \XcVm\Streaming\Health\ProcessChecker::checkPID($rMonitor, array('XC_VM[' . $rStreamID . ']')) && is_numeric($rMonitor)) { diff --git a/src/Public/Controllers/Api/InternalApiController.php b/src/Public/Controllers/Api/InternalApiController.php index 63cd73f1..f328e9f5 100644 --- a/src/Public/Controllers/Api/InternalApiController.php +++ b/src/Public/Controllers/Api/InternalApiController.php @@ -189,12 +189,11 @@ class InternalApiController { switch ($rFunction) { case 'start': foreach ($rStreamIDs as $rStreamID) { - if (StreamProcess::startMonitor($rStreamID, true)) { + // Handed to the fanout supervisor synchronously, or a PHP + // monitor spawned — those are staggered so a bulk start does + // not fork hundreds of PHP processes in the same instant. + if (StreamProcess::startMonitor($rStreamID, true) === StreamProcess::MONITOR_PHP) { usleep(50000); - } else { - echo json_encode(array('result' => false)); - - exit(); } } @@ -227,7 +226,12 @@ class InternalApiController { $rForceID = intval($rRequest['force_id']); if ($rStreamID > 0) { - file_put_contents(SIGNALS_TMP_PATH . $rStreamID . '.force', $rForceID); + // A supervised stream switches through the daemon. The .force file + // is only ever read by the PHP monitor, which a supervised stream + // does not have, so writing it there would silently do nothing. + if (!FanoutClient::forceSource($rStreamID, $rForceID)) { + file_put_contents(SIGNALS_TMP_PATH . $rStreamID . '.force', $rForceID); + } } exit(json_encode(array('result' => true))); diff --git a/src/Public/admin/live.php b/src/Public/admin/live.php index 81ad97c2..edaeb9e2 100644 --- a/src/Public/admin/live.php +++ b/src/Public/admin/live.php @@ -86,13 +86,17 @@ if (0 < $db->num_rows()) { $rChannelInfo['pid'] = null; if ($rChannelInfo['on_demand'] == 1) { - if (!ProcessManager::isMonitorAlive($rChannelInfo['monitor_pid'], $rStreamID)) { - StreamProcess::startMonitor($rStreamID); - - for ($rRetries = 0; !file_exists(STREAMS_PATH . intval($rStreamID) . '_.monitor') && $rRetries < 300; $rRetries++) { - usleep(10000); + if (!StreamProcess::isWatched($rStreamID, $rChannelInfo['monitor_pid'])) { + DatabaseFactory::connect(); // closed above; the hand-over reads the stream's config + if (StreamProcess::startMonitor($rStreamID) === StreamProcess::MONITOR_FANOUT) { + // The daemon is the monitor, and writes no _.monitor file. + $rChannelInfo['monitor_pid'] = -1; + } else { + for ($rRetries = 0; !file_exists(STREAMS_PATH . intval($rStreamID) . '_.monitor') && $rRetries < 300; $rRetries++) { + usleep(10000); + } + $rChannelInfo['monitor_pid'] = intval(@file_get_contents(STREAMS_PATH . $rStreamID . '_.monitor')); } - $rChannelInfo['monitor_pid'] = intval(file_get_contents(STREAMS_PATH . $rStreamID . '_.monitor')); } } else { generate404(); diff --git a/src/Public/stream/live.php b/src/Public/stream/live.php index 98264b82..e98d4499 100644 --- a/src/Public/stream/live.php +++ b/src/Public/stream/live.php @@ -4,6 +4,7 @@ use XcVm\Core\Logging\DatabaseLogger; use XcVm\Core\Process\ProcessManager; use XcVm\Core\Util\NetworkUtils; use XcVm\Domain\Stream\ConnectionTracker; +use XcVm\Domain\Stream\StreamProcess; use XcVm\Infrastructure\Database\DatabaseFactory; use XcVm\Infrastructure\Redis\RedisManager; use XcVm\Streaming\AsyncFileOperations; @@ -149,16 +150,24 @@ if ($rChannelInfo) { $rChannelInfo["pid"] = NULL; if ($rChannelInfo["on_demand"] == 1) { - if (!ProcessManager::isMonitorAlive($rChannelInfo["monitor_pid"], $rStreamID)) { + // Watched = a live PHP monitor, or the fanout supervisor (whose pid + // the PHP check rightly rejects). Either way, do not start another. + if (!StreamProcess::isWatched($rStreamID, $rChannelInfo["monitor_pid"])) { if (($rActivityStart + $rCreateExpiration) - intval($rServers[SERVER_ID]["time_offset"]) < time()) { generateError("TOKEN_EXPIRED"); } - ProcessManager::startMonitor($rStreamID); - - if (AsyncFileOperations::awaitFileExists(STREAMS_PATH . $rStreamID . "_.monitor", 300, 10)) { + DatabaseFactory::connect(); // the hand-over reads the stream's config + if (StreamProcess::startMonitor($rStreamID) === StreamProcess::MONITOR_FANOUT) { + // The daemon is the monitor: there is no _.monitor file to wait + // for, and waiting its full three seconds would be pure latency + // on the viewer's connect. It writes _.pid as it launches. + $rChannelInfo["monitor_pid"] = -1; + } elseif (AsyncFileOperations::awaitFileExists(STREAMS_PATH . $rStreamID . "_.monitor", 300, 10)) { $rChannelInfo["monitor_pid"] = (intval(AsyncFileOperations::readFile(STREAMS_PATH . $rStreamID . "_.monitor")) ?: NULL); } + } elseif (!$rChannelInfo["monitor_pid"]) { + $rChannelInfo["monitor_pid"] = -1; // supervised; its pid is the daemon's } if (!$rChannelInfo["monitor_pid"]) { @@ -199,7 +208,7 @@ if ($rChannelInfo) { generateError("WAIT_TIME_EXPIRED"); } else { // Verify stream is still running - if (!(ProcessManager::isMonitorAlive($rChannelInfo["monitor_pid"], $rStreamID) && ProcessManager::isStreamAlive($rChannelInfo["pid"], $rStreamID))) { + if (!(StreamProcess::isWatched($rStreamID, $rChannelInfo["monitor_pid"]) && ProcessManager::isStreamAlive($rChannelInfo["pid"], $rStreamID))) { OffAirHandler::showNotOnAir($rExtension, $rUserInfo, $rIP, $rCountryCode, $rServerID, $rProxyID); } } diff --git a/src/Public/stream/rtmp.php b/src/Public/stream/rtmp.php index de230ac6..20cacc45 100644 --- a/src/Public/stream/rtmp.php +++ b/src/Public/stream/rtmp.php @@ -7,6 +7,7 @@ use XcVm\Core\Process\ProcessManager; use XcVm\Core\Util\Encryption; use XcVm\Domain\Security\BlocklistService; use XcVm\Domain\Stream\ConnectionTracker; +use XcVm\Domain\Stream\StreamProcess; use XcVm\Domain\User\UserRepository; use XcVm\Infrastructure\Redis\RedisManager; use XcVm\Streaming\Auth\StreamAuth; @@ -110,9 +111,9 @@ if (!($_GET['addr'] == '127.0.0.1' && $_GET['call'] == 'publish')) { if (ProcessManager::isStreamAlive($rChannelInfo['pid'], $rStreamID)) { } else { if ($rChannelInfo['on_demand'] == 1) { - if (ProcessManager::isMonitorAlive($rChannelInfo['monitor_pid'], $rStreamID)) { + if (StreamProcess::isWatched($rStreamID, $rChannelInfo['monitor_pid'])) { } else { - ProcessManager::startMonitor($rStreamID); + StreamProcess::startMonitor($rStreamID); sleep(5); } } else { @@ -239,9 +240,9 @@ if (!($_GET['addr'] == '127.0.0.1' && $_GET['call'] == 'publish')) { if (ProcessManager::isStreamAlive($rChannelInfo['pid'], $rStreamID)) { } else { if ($rChannelInfo['on_demand'] == 1) { - if (ProcessManager::isMonitorAlive($rChannelInfo['monitor_pid'], $rStreamID)) { + if (StreamProcess::isWatched($rStreamID, $rChannelInfo['monitor_pid'])) { } else { - ProcessManager::startMonitor($rStreamID); + StreamProcess::startMonitor($rStreamID); sleep(5); } } else { diff --git a/src/Streaming/Fanout/FanoutClient.php b/src/Streaming/Fanout/FanoutClient.php index 83c4678b..906513e1 100644 --- a/src/Streaming/Fanout/FanoutClient.php +++ b/src/Streaming/Fanout/FanoutClient.php @@ -422,6 +422,152 @@ class FanoutClient { return self::call('DELETE', $rStreamID, null); } + // ── Encoder supervision (XC_VM_Fanout ADR 0002) ───────────────────────── + // + // The daemon's supervisor runs and watches a stream's producer — the ffmpeg + // command StreamProcess::buildLive composes, or the native remuxer + // (`xc_fanout remux`) StreamProcess::buildNativeLive composes — in place of the + // per-stream PHP watchdog (`console.php monitor`, MonitorCommand). PHP still + // builds every command and still owns every database write; the daemon runs + // what it is handed and reports back. + + /** Path of the daemon binary, which is also the native remuxer (`xc_fanout remux`). */ + public static function binaryPath(): string { + return BIN_PATH . 'xc_fanout/xc_fanout'; + } + + /** + * Hand a stream to the daemon's supervisor (PUT /monitor/). Re-handing a + * supervised stream replaces its spec and restarts its producer. + * + * @param int $rStreamID Stream id. + * @param array $rSpec Spec: sources (label/cmd/fallback_cmd/probe_cmd), + * policy, health, pid/errors/log paths, server_id, + * adopt_match — see StreamProcess::buildSupervisorSpec(). + * @return bool True when the daemon took it (204); false when it is + * unreachable, not accepting hand-overs (501) or refused the spec. + */ + public static function supervise(int $rStreamID, array $rSpec): bool { + $rBody = json_encode($rSpec, JSON_UNESCAPED_SLASHES); + if ($rBody === false) { + return false; + } + return self::request('PUT', '/monitor/' . $rStreamID, $rBody, 2, 5)['code'] === 204; + } + + /** + * Stop supervising a stream and kill its producer (DELETE /monitor/). + * Idempotent: a stream the daemon is not supervising answers 204 too. + * + * This MUST come before anything kills the producer by pid — killing it + * first is precisely what the supervisor exists to react to, so it would + * start a replacement and the stream would refuse to stop. + * + * @param int $rStreamID Stream id. + * @return bool True when the daemon answered (whether or not it was supervising). + */ + public static function release(int $rStreamID): bool { + $rCode = self::request('DELETE', '/monitor/' . $rStreamID, null, 2, 5)['code']; + return $rCode >= 200 && $rCode < 300; + } + + /** + * Every supervised stream's state in one call (GET /monitors/state), for the + * streams_servers reconcile. + * + * Returns null — never an empty list — when the daemon cannot be reached, so + * a dead socket is never mistaken for "supervising nothing" (which would have + * the caller start a PHP monitor for every stream on the node). + * + * @return array{accepting:bool,daemon_pid:int,streams:array}|null + * accepting — whether a new hand-over would be taken now; + * daemon_pid — the daemon's pid, a supervised stream's monitor_pid; + * streams — id => state (running, pid, source, source_idx, restarts, + * failures, uptime_ms, gave_up, adopted, fallback, last_error, + * daemon_pid, meta{…}). + */ + public static function monitorStates(): ?array { + $rRes = self::request('GET', '/monitors/state', null, 1, 3); + if ($rRes['code'] !== 200 || !is_string($rRes['body'])) { + return null; + } + $rData = json_decode($rRes['body'], true); + if (!is_array($rData) || !isset($rData['streams']) || !is_array($rData['streams'])) { + return null; + } + return array('accepting' => !empty($rData['accepting']), 'daemon_pid' => intval($rData['daemon_pid'] ?? 0), 'streams' => $rData['streams']); + } + + /** + * Whether the daemon is supervising this stream. + * + * @param int $rStreamID Stream id. + * @return bool|null True/false from a reachable daemon; null when it cannot + * be asked, which callers must treat as "unknown", not "no". + */ + public static function isSupervised(int $rStreamID): ?bool { + $rRes = self::request('GET', '/monitor/' . $rStreamID, null, 1, 2); + if ($rRes['errno'] !== 0 || $rRes['code'] === 0) { + return null; + } + if ($rRes['code'] === 404) { + return false; + } + if ($rRes['code'] !== 200 || !is_string($rRes['body'])) { + return null; + } + $rData = json_decode($rRes['body'], true); + return is_array($rData) && !empty($rData['supervised']); + } + + /** + * Force a supervised stream onto one of its sources (POST + * /monitor//source) — the supervised form of the `.force` signal file, + * which only the PHP monitor ever read. + * + * @param int $rStreamID Stream id. + * @param int $rIndex Index into the stream's source list. + * @return bool True when the daemon queued the switch. + */ + public static function forceSource(int $rStreamID, int $rIndex): bool { + $rCode = self::request('POST', '/monitor/' . $rStreamID . '/source', json_encode(array('index' => $rIndex)), 1, 3)['code']; + return $rCode >= 200 && $rCode < 300; + } + + /** + * One control request over the daemon's unix socket. + * + * @param string $rMethod HTTP method. + * @param string $rPath Request path. + * @param string|null $rBody JSON body, or null. + * @param int $rConnect Connect timeout, seconds. + * @param int $rTimeout Whole-request timeout, seconds. + * @return array{code:int,body:string|null,errno:int} code 0 / errno set when unreachable. + */ + private static function request(string $rMethod, string $rPath, ?string $rBody, int $rConnect, int $rTimeout): array { + if (!function_exists('curl_init') || !defined('FANOUT_CTL_SOCK') || !file_exists(FANOUT_CTL_SOCK)) { + return array('code' => 0, 'body' => null, 'errno' => -1); + } + $rCurl = curl_init(); + curl_setopt_array($rCurl, [ + CURLOPT_UNIX_SOCKET_PATH => FANOUT_CTL_SOCK, + CURLOPT_URL => 'http://localhost' . $rPath, + CURLOPT_CUSTOMREQUEST => $rMethod, + CURLOPT_RETURNTRANSFER => true, + CURLOPT_CONNECTTIMEOUT => $rConnect, + CURLOPT_TIMEOUT => $rTimeout, + ]); + if ($rBody !== null) { + curl_setopt($rCurl, CURLOPT_POSTFIELDS, $rBody); + curl_setopt($rCurl, CURLOPT_HTTPHEADER, ['Content-Type: application/json']); + } + $rResponse = curl_exec($rCurl); + $rCode = (int) curl_getinfo($rCurl, CURLINFO_HTTP_CODE); + $rErrno = curl_errno($rCurl); + curl_close($rCurl); + return array('code' => $rCode, 'body' => is_string($rResponse) ? $rResponse : null, 'errno' => $rErrno); + } + /** * Issue one control request over the daemon's unix socket via cURL. * diff --git a/tests/Unit/StreamProcessBuildLiveTest.php b/tests/Unit/StreamProcessBuildLiveTest.php index e86e02d2..60e0afb9 100644 --- a/tests/Unit/StreamProcessBuildLiveTest.php +++ b/tests/Unit/StreamProcessBuildLiveTest.php @@ -120,6 +120,30 @@ final class StreamProcessBuildLiveTest extends TestCase { $this->assertStringContainsString('echo $! > ' . STREAMS_PATH . '42_.pid', $out); } + // ── supervised (launched by the fanout daemon) ───────────── + + /** The daemon is the parent: the shell's redirect/background/pid tail must go. */ + public function testSupervisedOmitsTheLaunchTail(): void { + $out = $this->build(['supervised' => true]); + $this->assertStringNotContainsString('>/dev/null', $out); + $this->assertStringNotContainsString('echo $!', $out); + $this->assertStringNotContainsString(' & ', $out); + $this->assertStringContainsString(STREAMS_PATH . '42_.m3u8', $out, 'still names its playlist (adopt_match)'); + } + + /** A supervised loopback feeds the daemon, which confirms and judges it by those bytes. */ + public function testSupervisedLoopbackTeesIntoTheDaemon(): void { + $out = $this->build(['loopback' => true, 'ingestSock' => '/run/ingest/42.sock', 'supervised' => true]); + $this->assertStringContainsString('-f tee', $out); + $this->assertStringContainsString('unix:/run/ingest/42.sock', $out); + $this->assertStringContainsString('-map 0 -copy_unknown', $out); + } + + /** The legacy path registers no ingest for loopback and must stay on-disk only. */ + public function testLegacyLoopbackStaysOnDiskOnly(): void { + $this->assertStringNotContainsString('-f tee', $this->build(['loopback' => true])); + } + // ── custom_ffmpeg branch ─────────────────────────────────── public function testCustomFfmpegBypassesTemplate(): void { diff --git a/tests/Unit/StreamProcessSupervisionTest.php b/tests/Unit/StreamProcessSupervisionTest.php new file mode 100644 index 00000000..10a4e346 --- /dev/null +++ b/tests/Unit/StreamProcessSupervisionTest.php @@ -0,0 +1,229 @@ + 1, + 'STREAMS_PATH' => '/tmp/xcvm-test-streams/', + 'DELAY_PATH' => '/tmp/xcvm-test-delay/', + 'FFMPEG_BIN_40' => '/bin/ffmpeg40', + 'FFPROBE_BIN_40' => '/bin/ffprobe40', + ] as $k => $v) { + if (!defined($k)) { + define($k, $v); + } + } + } + + private static function call(string $rMethod, ...$rArgs) { + $m = new ReflectionMethod(StreamProcess::class, $rMethod); + $m->setAccessible(true); + return $m->invoke(null, ...$rArgs); + } + + private function native(array $rOverrides = []): string { + return self::call('buildNativeLive', array_merge([ + 'streamID' => 42, + 'source' => 'http://src.example/live/u/p/9.ts', + 'arguments' => [], + 'segmentSettings' => ['seg_time' => 6, 'seg_list_size' => 8, 'seg_delete_threshold' => 4], + 'ingestSock' => '/run/fanout/ingest/42.sock', + 'settings' => ['ffmpeg_warnings' => 0, 'fanout_source_insecure' => 1], + 'binary' => '/home/xc_vm/bin/xc_fanout/xc_fanout', + ], $rOverrides)); + } + + // ── buildNativeLive ─────────────────────────────────────────── + + public function testNativeCommandShape(): void { + $c = $this->native(); + $this->assertStringStartsWith('/home/xc_vm/bin/xc_fanout/xc_fanout remux ', $c); + $this->assertStringContainsString("-i 'http://src.example/live/u/p/9.ts'", $c); + $this->assertStringContainsString("-ingest 'unix:/run/fanout/ingest/42.sock'", $c); + $this->assertStringContainsString('-hls_time 6 ', $c); + $this->assertStringContainsString('-hls_init_time 2 ', $c, "buildLive's fast first segment"); + $this->assertStringContainsString('-hls_list_size 8 ', $c); + $this->assertStringContainsString('-hls_delete_threshold 4 ', $c); + $this->assertStringContainsString("-progress '" . STREAMS_PATH . "42_.progress'", $c); + $this->assertStringContainsString("-hls_segment_filename '" . STREAMS_PATH . "42_%d.ts'", $c); + $this->assertStringEndsWith("'" . STREAMS_PATH . "42_.m3u8'", $c, 'the playlist is the one positional argument, last'); + } + + /** The daemon launches it: no redirect, background or pid tail may ride along. */ + public function testNativeCommandHasNoLaunchTail(): void { + $c = $this->native(); + $this->assertStringNotContainsString('>', $c); + $this->assertStringNotContainsString('&', $c); + $this->assertStringNotContainsString('echo', $c); + } + + /** The same fetch identity the daemon's own puller uses for the stream. */ + public function testNativeCommandCarriesFetchArguments(): void { + $c = $this->native(['arguments' => [ + 'user_agent' => ['value' => 'VLC/3.0', 'argument_default_value' => 'Mozilla/5.0'], + 'proxy' => ['value' => '10.0.0.1:3128'], + 'cookie' => ['value' => 'a=b'], + 'headers' => ['value' => "X-A: 1\r\nX-B: 2\r\n"], + ]]); + $this->assertStringContainsString("-user_agent 'VLC/3.0'", $c); + $this->assertStringContainsString("-http_proxy '10.0.0.1:3128'", $c); + $this->assertStringContainsString('-cookies ', $c); + $this->assertStringContainsString("-headers 'X-A: 1\r\nX-B: 2\r\n'", $c); + + $this->assertStringContainsString("-user_agent 'Mozilla/5.0'", $this->native(), 'no UA set: the puller default'); + } + + public function testNativeInsecureFollowsTheSetting(): void { + $this->assertStringContainsString(' -insecure ', $this->native()); + $this->assertStringNotContainsString('-insecure', $this->native(['settings' => ['fanout_source_insecure' => 0]])); + } + + /** A source URL is shell data: quotes in it must not escape the argument. */ + public function testNativeSourceIsShellEscaped(): void { + $c = $this->native(['source' => "http://x/a'; rm -rf /;'.ts"]); + $this->assertStringContainsString("-i 'http://x/a'\\''; rm -rf /;'\\''.ts'", $c); + } + + // ── eligibility ─────────────────────────────────────────────── + + private function plainStream(array $rOverrides = []): array { + return array_merge([ + 'type_key' => 'live_streams', + 'enable_transcode' => 0, + 'custom_ffmpeg' => '', + 'custom_map' => '', + 'rtmp_output' => 0, + 'external_push' => '', + 'gen_timestamps' => 0, + 'read_native' => 0, + ], $rOverrides); + } + + public function testPlainCopyStreamIsNativeEligible(): void { + $this->assertTrue(self::call('isNativeEligible', $this->plainStream(), [])); + } + + public function testAnythingNeedingFfmpegIsNotNativeEligible(): void { + foreach ([ + 'transcode' => ['enable_transcode' => 1], + 'custom ffmpeg' => ['custom_ffmpeg' => '-i x -c:v libx264 y'], + 'custom map' => ['custom_map' => '-map 0:0'], + 'rtmp output' => ['rtmp_output' => 1], + 'external push here' => ['external_push' => json_encode([1 => ['rtmp://push/x']])], + 'timestamp repair' => ['gen_timestamps' => 1], + 'realtime pacing' => ['read_native' => 1], + 'radio' => ['type_key' => 'radio_streams'], + 'created channel' => ['type_key' => 'created_live'], + ] as $rWhy => $rOverride) { + $this->assertFalse(self::call('isNativeEligible', $this->plainStream($rOverride), []), $rWhy); + } + $this->assertTrue(self::call('isNativeEligible', $this->plainStream(['external_push' => json_encode([7 => ['rtmp://push/x']])]), []), "another server's push is not this one's"); + $this->assertFalse(self::call('isNativeEligible', $this->plainStream(), ['force_input_acodec' => ['value' => 'ac3']]), 'forced input codec'); + } + + public function testNativeSources(): void { + foreach (['http://h/x.ts', 'https://h/x.m3u8', 'udp://239.0.0.1:1234', 'rtp://239.0.0.1:5000'] as $rURL) { + $this->assertTrue(self::call('isNativeSource', $rURL), $rURL); + } + foreach (['rtmp://h/app/s', 'srt://h:9000', '/srv/film.mp4', 'https://www.youtube.com/watch?v=x'] as $rURL) { + $this->assertFalse(self::call('isNativeSource', $rURL), $rURL); + } + } + + // ── policy / health ─────────────────────────────────────────── + + public function testHealthMirrorsThePhpMonitor(): void { + $h = self::call('supervisorHealth', ['fps_restart' => 1, 'fps_threshold' => 90, 'auto_restart' => json_encode(['days' => ['Monday'], 'at' => '04:30'])], ['seg_time' => 10, 'audio_restart_loss' => 1, 'fps_delay' => 60]); + $this->assertSame(60, $h['stall_sec'], 'seg_time × 6'); + $this->assertSame(30, $h['audio_loss_sec']); + $this->assertEqualsWithDelta(0.9, $h['fps_threshold'], 1e-9, '"FPS Threshold %" is a percentage; the daemon takes a fraction'); + $this->assertSame(60, $h['fps_grace_sec']); + $this->assertSame(['days' => ['Monday'], 'at' => '04:30'], $h['auto_restart']); + + $off = self::call('supervisorHealth', ['fps_restart' => 0, 'fps_threshold' => 90, 'auto_restart' => ''], ['seg_time' => 10]); + $this->assertSame(0, $off['fps_threshold'], 'fps_restart off'); + $this->assertSame(0, $off['audio_loss_sec']); + $this->assertArrayNotHasKey('auto_restart', $off); + } + + public function testPolicyMirrorsThePhpMonitor(): void { + $p = self::call('supervisorPolicy', ['on_demand' => 1, 'parent_id' => 0], ['stop_failures' => 3, 'stream_fail_sleep' => 7, 'on_demand_failure_exit' => 1, 'priority_backup' => 1, 'seg_time' => 10], 2, 5); + $this->assertSame(3, $p['stop_failures']); + $this->assertSame(7, $p['stream_fail_sleep']); + $this->assertTrue($p['on_demand']); + $this->assertTrue($p['on_demand_failure_exit']); + $this->assertSame(300, $p['priority_backup_sec']); + $this->assertSame(5 + 30, $p['start_timeout_sec'], 'probe window + the playlist wait'); + + $one = self::call('supervisorPolicy', ['on_demand' => 0, 'parent_id' => 0], ['priority_backup' => 1, 'seg_time' => 10], 1, 5); + $this->assertSame(0, $one['priority_backup_sec'], 'nothing to switch back to with one source'); + $loop = self::call('supervisorPolicy', ['on_demand' => 0, 'parent_id' => 3], ['priority_backup' => 1, 'seg_time' => 10], 2, 5); + $this->assertSame(0, $loop['priority_backup_sec'], 'a loopback has no sources of its own'); + } + + // ── reconcile ───────────────────────────────────────────────── + + private function row(array $rOverrides = []): array { + return array_merge([ + 'pid' => null, 'monitor_pid' => 900, 'stream_status' => 2, 'current_source' => 'http://a/1.ts', + 'stream_started' => 1000, 'stream_info' => null, 'audio_codec' => null, 'video_codec' => null, + 'resolution' => null, 'bitrate' => null, 'compatible' => 0, + ], $rOverrides); + } + + private function update(array $rRow, array $rState): array { + return self::call('supervisedRowUpdate', $rRow, array_merge(['daemon_pid' => 900], $rState), false, 2000); + } + + public function testConfirmedStartIsUpAndDated(): void { + $set = $this->update($this->row(), ['running' => true, 'confirmed' => true, 'pid' => 4321, 'uptime_ms' => 5000, 'source' => 'http://a/1.ts']); + $this->assertSame(0, $set['stream_status']); + $this->assertSame(4321, $set['pid']); + $this->assertSame(1995, $set['stream_started'], 'when the running producer came up'); + $this->assertArrayNotHasKey('current_source', $set, 'unchanged columns are not rewritten'); + } + + public function testStatusFollowsThePhpMonitorsMeaning(): void { + $this->assertSame(2, $this->update($this->row(['stream_status' => 0]), ['running' => true, 'confirmed' => false, 'pid' => 5])['stream_status'], 'launched, not confirmed'); + $this->assertSame(1, $this->update($this->row(), ['running' => false, 'failures' => 2])['stream_status'], 'between failed starts'); + $this->assertSame(1, $this->update($this->row(), ['running' => false, 'gave_up' => true])['stream_status'], 'gave up'); + $this->assertArrayNotHasKey('stream_status', $this->update($this->row(), ['running' => false, 'failures' => 0]), 'a first start still pending stays "starting"'); + $this->assertNull($this->update($this->row(['pid' => 77, 'stream_status' => 0]), ['running' => false, 'failures' => 1])['pid'], 'nothing running, no pid'); + } + + public function testMetadataFromTheBytes(): void { + $set = $this->update($this->row(), ['running' => true, 'confirmed' => true, 'pid' => 1, 'meta' => ['video_codec' => 'h264', 'audio_codec' => 'aac', 'height' => 1088, 'bitrate_kbps' => 4500]]); + $this->assertSame('h264', $set['video_codec']); + $this->assertSame('aac', $set['audio_codec']); + $this->assertSame(1, $set['compatible']); + $this->assertSame(1080, $set['resolution'], 'snapped to the nearest standard height'); + $this->assertSame(4500, $set['bitrate']); + } + + /** Unknown is left unknown: a correct value is never overwritten with a blank. */ + public function testUnknownMetadataKeepsWhatThePanelHas(): void { + $rRow = $this->row(['video_codec' => 'hevc', 'audio_codec' => 'ac3', 'resolution' => 2160, 'bitrate' => 9000, 'stream_status' => 0, 'pid' => 1]); + $set = $this->update($rRow, ['running' => true, 'confirmed' => true, 'pid' => 1, 'meta' => []]); + foreach (['video_codec', 'audio_codec', 'resolution', 'bitrate', 'compatible'] as $rCol) { + $this->assertArrayNotHasKey($rCol, $set, $rCol); + } + $audioOnly = $this->update($rRow, ['running' => true, 'confirmed' => true, 'pid' => 1, 'meta' => ['audio_codec' => 'aac']]); + $this->assertArrayNotHasKey('video_codec', $audioOnly, 'a known video codec is not blanked by an audio-only reading'); + } + + public function testNothingChangedWritesNothing(): void { + $rRow = $this->row(['pid' => 4321, 'stream_status' => 0]); + $this->assertSame([], $this->update($rRow, ['running' => true, 'confirmed' => true, 'pid' => 4321, 'source' => 'http://a/1.ts'])); + } +}