XC_VM/tests/Unit/StreamProcessSupervisionTest.php

<?php

use PHPUnit\Framework\TestCase;
use XcVm\Domain\Stream\StreamProcess;

/**
 * The pure parts of handing a live stream to the xc_fanout supervisor: the
 * native remuxer command (buildNativeLive), which streams may use it
 * (nativeRefusal / isNativeSource), the policy and health the PHP monitor
 * obeyed (supervisorPolicy / supervisorHealth), and what a supervisor state
 * means for streams_servers (supervisedRowUpdate).
 */
final class StreamProcessSupervisionTest extends TestCase {

	public static function setUpBeforeClass(): void {
		foreach ([
			'SERVER_ID' => 1,
			'STREAMS_PATH' => '/tmp/xcvm-test-streams/',
			'DELAY_PATH' => '/tmp/xcvm-test-delay/',
			'FFMPEG_BIN_40' => '/bin/ffmpeg40',
			'FFPROBE_BIN_40' => '/bin/ffprobe40',
		] as $k => $v) {
			if (!defined($k)) {
				define($k, $v);
			}
		}
	}

	private static function call(string $rMethod, ...$rArgs) {
		$m = new ReflectionMethod(StreamProcess::class, $rMethod);
		$m->setAccessible(true);
		return $m->invoke(null, ...$rArgs);
	}

	private function native(array $rOverrides = []): string {
		return self::call('buildNativeLive', array_merge([
			'streamID' => 42,
			'source' => 'http://src.example/live/u/p/9.ts',
			'arguments' => [],
			'segmentSettings' => ['seg_time' => 6, 'seg_list_size' => 8, 'seg_delete_threshold' => 4],
			'ingestSock' => '/run/fanout/ingest/42.sock',
			'settings' => ['ffmpeg_warnings' => 0, 'fanout_source_insecure' => 1],
			'binary' => '/home/xc_vm/bin/xc_fanout/xc_fanout',
		], $rOverrides));
	}

	// ── buildNativeLive ───────────────────────────────────────────

	public function testNativeCommandShape(): void {
		$c = $this->native();
		$this->assertStringStartsWith('/home/xc_vm/bin/xc_fanout/xc_fanout remux ', $c);
		$this->assertStringContainsString("-i 'http://src.example/live/u/p/9.ts'", $c);
		$this->assertStringContainsString("-ingest 'unix:/run/fanout/ingest/42.sock'", $c);
		$this->assertStringContainsString('-hls_time 6 ', $c);
		$this->assertStringContainsString('-hls_init_time 2 ', $c, "buildLive's fast first segment");
		$this->assertStringContainsString('-hls_list_size 8 ', $c);
		$this->assertStringContainsString('-hls_delete_threshold 4 ', $c);
		$this->assertStringContainsString("-progress '" . STREAMS_PATH . "42_.progress'", $c);
		$this->assertStringContainsString("-hls_segment_filename '" . STREAMS_PATH . "42_%d.ts'", $c);
		$this->assertStringEndsWith("'" . STREAMS_PATH . "42_.m3u8'", $c, 'the playlist is the one positional argument, last');
	}

	/** The daemon launches it: no redirect, background or pid tail may ride along. */
	public function testNativeCommandHasNoLaunchTail(): void {
		$c = $this->native();
		$this->assertStringNotContainsString('>', $c);
		$this->assertStringNotContainsString('&', $c);
		$this->assertStringNotContainsString('echo', $c);
	}

	/** standalone (the PHP monitor runs it, fanout off): it carries buildLive's launch tail. */
	public function testNativeCommandStandaloneCarriesTheLaunchTail(): void {
		$c = $this->native(['standalone' => true]);
		$this->assertStringContainsString('>/dev/null 2>>' . STREAMS_PATH . '42.errors', $c);
		$this->assertStringContainsString('& echo $! > ' . STREAMS_PATH . '42_.pid', $c);
		$this->assertStringContainsString("'" . STREAMS_PATH . "42_.m3u8' >/dev/null", $c, 'the playlist is still the last positional, before the tail');
		$this->assertStringNotContainsString('echo', $this->native(), 'the supervised form still has no tail');
	}

	/** The same fetch identity the daemon's own puller uses for the stream. */
	public function testNativeCommandCarriesFetchArguments(): void {
		$c = $this->native(['arguments' => [
			'user_agent' => ['value' => 'VLC/3.0', 'argument_default_value' => 'Mozilla/5.0'],
			'proxy' => ['value' => '10.0.0.1:3128'],
			'cookie' => ['value' => 'a=b'],
			'headers' => ['value' => "X-A: 1\r\nX-B: 2\r\n"],
		]]);
		$this->assertStringContainsString("-user_agent 'VLC/3.0'", $c);
		// remux takes host:port or http://host:port; the panel normalises to the URL ffmpeg needs.
		$this->assertStringContainsString("-http_proxy 'http://10.0.0.1:3128'", $c);
		$this->assertStringContainsString('-cookies ', $c);
		$this->assertStringContainsString("-headers 'X-A: 1\r\nX-B: 2\r\n'", $c);

		$this->assertStringContainsString("-user_agent 'Mozilla/5.0'", $this->native(), 'no UA set: the puller default');
	}

	public function testNativeInsecureFollowsTheSetting(): void {
		$this->assertStringContainsString(' -insecure ', $this->native());
		$this->assertStringNotContainsString('-insecure', $this->native(['settings' => ['fanout_source_insecure' => 0]]));
	}

	/** A source URL is shell data: quotes in it must not escape the argument. */
	public function testNativeSourceIsShellEscaped(): void {
		$c = $this->native(['source' => "http://x/a'; rm -rf /;'.ts"]);
		$this->assertStringContainsString("-i 'http://x/a'\\''; rm -rf /;'\\''.ts'", $c);
	}

	private const CREDENTIAL_ARGUMENTS = [
		'user_agent' => ['value' => 'VLC/3.0'],
		'proxy' => ['value' => 'pu:pp@10.0.0.1:3128'],
		'cookie' => ['value' => 'session=s3cret'],
		'headers' => ['value' => "Authorization: Bearer t0ken\r\n"],
	];

	/** With a source file, nothing that can carry an account is left in argv (/proc/<pid>/cmdline). */
	public function testNativeCommandWithSourceFileCarriesNoCredentials(): void {
		$c = $this->native(['sourceFile' => STREAMS_PATH . '42_.source_0', 'arguments' => self::CREDENTIAL_ARGUMENTS]);
		$this->assertStringContainsString("-source_file '" . STREAMS_PATH . "42_.source_0'", $c);
		foreach (['-i ', '/live/u/p/', 'VLC', 'pp@', 's3cret', 't0ken', '-cookies', '-headers', '-http_proxy', '-user_agent'] as $rLeak) {
			$this->assertStringNotContainsString($rLeak, $c);
		}
		$this->assertStringContainsString("-ingest 'unix:/run/fanout/ingest/42.sock'", $c);
		$this->assertStringEndsWith("'" . STREAMS_PATH . "42_.m3u8'", $c);
	}

	/** The file holds exactly what the argv line would, under the remux flag names. */
	public function testNativeSourceHoldsTheArgvValues(): void {
		$rSource = self::call('nativeSource', ['source' => 'http://src.example/live/u/p/9.ts', 'arguments' => self::CREDENTIAL_ARGUMENTS]);
		$this->assertSame('http://src.example/live/u/p/9.ts', $rSource['i']);
		$this->assertSame('VLC/3.0', $rSource['user_agent']);
		$this->assertSame('http://pu:pp@10.0.0.1:3128', $rSource['http_proxy']);
		$this->assertStringContainsString('session=s3cret', $rSource['cookies']);
		$this->assertSame("Authorization: Bearer t0ken\r\n", $rSource['headers']);
		$this->assertSame(['i', 'user_agent', 'cookies', 'http_proxy', 'headers'], array_keys($rSource));
	}

	/** Written 0600, holding exactly nativeSource(), at the path the command names. */
	public function testWriteNativeSourceIsPrivate(): void {
		@mkdir(STREAMS_PATH, 0777, true);
		$rData = ['streamID' => 42, 'source' => 'http://src.example/live/u/p/9.ts', 'arguments' => self::CREDENTIAL_ARGUMENTS];
		try {
			$rPath = self::call('writeNativeSource', $rData, 1);
			$this->assertSame(STREAMS_PATH . '42_.source_1', $rPath);
			clearstatcache();
			$this->assertSame(0600, fileperms($rPath) & 0777);
			$this->assertSame(self::call('nativeSource', $rData), json_decode((string) file_get_contents($rPath), true));
		} finally {
			@unlink(STREAMS_PATH . '42_.source_1');
		}
	}

	/** The source file is used only on a daemon that says it reads one, as this process last asked it; an older one keeps argv. */
	public function testSourceFilesNeedTheRunningDaemonsFeature(): void {
		$rFeatures = new ReflectionProperty(\XcVm\Streaming\Fanout\FanoutClient::class, 'features');
		$rFeatures->setAccessible(true);
		$rWas = $rFeatures->getValue();
		try {
			$rFeatures->setValue(null, ['remux']);
			$this->assertFalse(\XcVm\Streaming\Fanout\FanoutClient::supportsLive('remux_source_file'));
			$rFeatures->setValue(null, ['remux', 'remux_source_file']);
			$this->assertTrue(\XcVm\Streaming\Fanout\FanoutClient::supportsLive('remux_source_file'));
			$this->assertTrue(\XcVm\Streaming\Fanout\FanoutClient::supportsRemux());
		} finally {
			$rFeatures->setValue(null, $rWas);
		}
	}

	// ── eligibility ───────────────────────────────────────────────

	private function plainStream(array $rOverrides = []): array {
		return array_merge([
			// `streams_types`: (1, 'Live Streams', 'live'), (3, 'created_live'),
			// (4, 'radio_streams') — the key, not the table name.
			'type_key' => 'live',
			'enable_transcode' => 0,
			'custom_ffmpeg' => '',
			'custom_map' => '',
			'rtmp_output' => 0,
			'external_push' => '',
			'gen_timestamps' => 0,
			'read_native' => 0,
		], $rOverrides);
	}

	/**
	 * The stream's Always Use ffmpeg switch refuses the remuxer, but not a source
	 * driver (nativeRefusal): ffmpeg cannot read a driver's URL, so refusing it
	 * would leave the stream with no producer.
	 */
	public function testAlwaysUseFfmpegRefusesOnlyTheRemuxer(): void {
		$rForced = $this->plainStream(['force_ffmpeg' => 1]);
		$this->assertSame('the stream is set to always use ffmpeg', self::call('remuxRefusal', $rForced, []));
		$this->assertNull(self::call('nativeRefusal', $rForced, []), 'a source driver still runs');
		$this->assertNull(self::call('remuxRefusal', $this->plainStream(), []));
		$this->assertSame('transcoding is enabled', self::call('remuxRefusal', $this->plainStream(['enable_transcode' => 1]), []));
	}

	public function testPlainCopyStreamIsNativeEligible(): void {
		$this->assertNull(self::call('nativeRefusal', $this->plainStream(), []));
	}

	/**
	 * Both default to 1 in `streams`, so they say nothing about this channel —
	 * refusing them is refusing every channel, which is how the native backend
	 * came to never run.
	 */
	public function testSchemaDefaultsDoNotRefuseTheRemuxer(): void {
		$this->assertNull(self::call('nativeRefusal', $this->plainStream(['gen_timestamps' => 1, 'read_native' => 1]), []));
	}

	public function testAnythingNeedingFfmpegIsNotNativeEligible(): void {
		foreach ([
			'transcode' => ['enable_transcode' => 1],
			'custom ffmpeg' => ['custom_ffmpeg' => '-i x -c:v libx264 y'],
			'custom map' => ['custom_map' => '-map 0:0'],
			'rtmp output' => ['rtmp_output' => 1],
			'external push here' => ['external_push' => json_encode([1 => ['rtmp://push/x']])],
			'radio' => ['type_key' => 'radio_streams'],
			'created channel' => ['type_key' => 'created_live'],
			'a type key that is not the schema\'s' => ['type_key' => 'live_streams'],
		] as $rWhy => $rOverride) {
			// The refusal is written to the stream's log, so every one of them
			// must say something an operator can act on.
			$this->assertIsString(self::call('nativeRefusal', $this->plainStream($rOverride), []), $rWhy);
		}
		$this->assertNull(self::call('nativeRefusal', $this->plainStream(['external_push' => json_encode([7 => ['rtmp://push/x']])]), []), "another server's push is not this one's");
		$this->assertIsString(self::call('nativeRefusal', $this->plainStream(), ['force_input_acodec' => ['value' => 'ac3']]), 'forced input codec');
	}

	public function testRemuxCommandIsRecognisedForTheCommandRecord(): void {
		$rCmd = $this->native(['binary' => \XcVm\Streaming\Fanout\FanoutClient::binaryPath()]);
		$this->assertTrue(self::call('isRemuxCommand', $rCmd), 'the remuxer command is recorded as <id>_.fanout');
		$this->assertFalse(self::call('isRemuxCommand', '/bin/ffmpeg40 -i x -c copy -f tee y'), 'an ffmpeg command is not');
	}

	public function testNativeSources(): void {
		foreach (['http://h/x.ts', 'https://h/x.m3u8', 'udp://239.0.0.1:1234', 'rtp://239.0.0.1:5000'] as $rURL) {
			$this->assertTrue(self::call('isNativeSource', $rURL), $rURL);
		}
		foreach (['rtmp://h/app/s', 'srt://h:9000', '/srv/film.mp4', 'https://www.youtube.com/watch?v=x'] as $rURL) {
			$this->assertFalse(self::call('isNativeSource', $rURL), $rURL);
		}
	}

	// ── policy / health ───────────────────────────────────────────

	public function testHealthMirrorsThePhpMonitor(): void {
		$h = self::call('supervisorHealth', ['fps_restart' => 1, 'fps_threshold' => 90, 'auto_restart' => json_encode(['days' => ['Monday'], 'at' => '04:30'])], ['seg_time' => 10, 'audio_restart_loss' => 1, 'fps_delay' => 60]);
		$this->assertSame(60, $h['stall_sec'], 'seg_time × 6');
		$this->assertSame(30, $h['audio_loss_sec']);
		$this->assertEqualsWithDelta(0.9, $h['fps_threshold'], 1e-9, '"FPS Threshold %" is a percentage; the daemon takes a fraction');
		$this->assertSame(60, $h['fps_grace_sec']);
		$this->assertSame(['days' => ['Monday'], 'at' => '04:30'], $h['auto_restart']);

		$off = self::call('supervisorHealth', ['fps_restart' => 0, 'fps_threshold' => 90, 'auto_restart' => ''], ['seg_time' => 10]);
		$this->assertSame(0, $off['fps_threshold'], 'fps_restart off');
		$this->assertSame(0, $off['audio_loss_sec']);
		$this->assertArrayNotHasKey('auto_restart', $off);
	}

	public function testPolicyMirrorsThePhpMonitor(): void {
		$p = self::call('supervisorPolicy', ['on_demand' => 1, 'parent_id' => 0], ['stop_failures' => 3, 'stream_fail_sleep' => 7, 'on_demand_failure_exit' => 1, 'priority_backup' => 1, 'seg_time' => 10], 2, 5);
		$this->assertSame(3, $p['stop_failures']);
		$this->assertSame(7, $p['stream_fail_sleep']);
		$this->assertTrue($p['on_demand']);
		$this->assertTrue($p['on_demand_failure_exit']);
		$this->assertSame(300, $p['priority_backup_sec']);
		$this->assertSame(5 + 30, $p['start_timeout_sec'], 'probe window + the playlist wait');

		$one = self::call('supervisorPolicy', ['on_demand' => 0, 'parent_id' => 0], ['priority_backup' => 1, 'seg_time' => 10], 1, 5);
		$this->assertSame(0, $one['priority_backup_sec'], 'nothing to switch back to with one source');
		$loop = self::call('supervisorPolicy', ['on_demand' => 0, 'parent_id' => 3], ['priority_backup' => 1, 'seg_time' => 10], 2, 5);
		$this->assertSame(0, $loop['priority_backup_sec'], 'a loopback has no sources of its own');
	}

	// ── reconcile ─────────────────────────────────────────────────

	private function row(array $rOverrides = []): array {
		return array_merge([
			'pid' => null, 'monitor_pid' => 900, 'stream_status' => 2, 'current_source' => 'http://a/1.ts',
			'stream_started' => 1000, 'stream_info' => null, 'audio_codec' => null, 'video_codec' => null,
			'resolution' => null, 'bitrate' => null, 'compatible' => 0,
		], $rOverrides);
	}

	private function update(array $rRow, array $rState): array {
		return self::call('supervisedRowUpdate', $rRow, array_merge(['daemon_pid' => 900], $rState), false, 2000);
	}

	public function testConfirmedStartIsUpAndDated(): void {
		$set = $this->update($this->row(), ['running' => true, 'confirmed' => true, 'pid' => 4321, 'uptime_ms' => 5000, 'source' => 'http://a/1.ts']);
		$this->assertSame(0, $set['stream_status']);
		$this->assertSame(4321, $set['pid']);
		$this->assertSame(1995, $set['stream_started'], 'when the running producer came up');
		$this->assertArrayNotHasKey('current_source', $set, 'unchanged columns are not rewritten');
	}

	public function testStatusFollowsThePhpMonitorsMeaning(): void {
		$this->assertSame(2, $this->update($this->row(['stream_status' => 0]), ['running' => true, 'confirmed' => false, 'pid' => 5])['stream_status'], 'launched, not confirmed');
		$this->assertSame(1, $this->update($this->row(), ['running' => false, 'failures' => 2])['stream_status'], 'between failed starts');
		$this->assertSame(1, $this->update($this->row(), ['running' => false, 'gave_up' => true])['stream_status'], 'gave up');
		$this->assertArrayNotHasKey('stream_status', $this->update($this->row(), ['running' => false, 'failures' => 0]), 'a first start still pending stays "starting"');
		$this->assertNull($this->update($this->row(['pid' => 77, 'stream_status' => 0]), ['running' => false, 'failures' => 1])['pid'], 'nothing running, no pid');
	}

	public function testMetadataFromTheBytes(): void {
		$set = $this->update($this->row(), ['running' => true, 'confirmed' => true, 'pid' => 1, 'meta' => ['video_codec' => 'h264', 'audio_codec' => 'aac', 'width' => 1920, 'height' => 1088, 'bitrate_kbps' => 4500]]);
		$this->assertSame('h264', $set['video_codec']);
		$this->assertSame('aac', $set['audio_codec']);
		$this->assertSame(1, $set['compatible']);
		$this->assertSame(1080, $set['resolution'], 'snapped to the nearest standard height');
		$this->assertSame(4500, $set['bitrate']);
	}

	/**
	 * The streams list, the adaptive master playlist and the viewer's codec all
	 * read the `stream_info` JSON, not the flat columns — a supervised stream
	 * that never runs ffprobe has to have it written from the daemon's reading.
	 */
	public function testMetadataAlsoFillsTheStreamInfoJson(): void {
		$set = $this->update($this->row(), ['running' => true, 'confirmed' => true, 'pid' => 1, 'meta' => ['video_codec' => 'hevc', 'audio_codec' => 'ac3', 'width' => 1920, 'height' => 1080, 'bitrate_kbps' => 4500]]);
		$rInfo = json_decode($set['stream_info'], true);
		$this->assertSame('hevc', $rInfo['codecs']['video']['codec_name']);
		$this->assertSame('video', $rInfo['codecs']['video']['codec_type']);
		$this->assertSame(1920, $rInfo['codecs']['video']['width']);
		$this->assertSame(1080, $rInfo['codecs']['video']['height']);
		$this->assertSame('ac3', $rInfo['codecs']['audio']['codec_name']);
		$this->assertSame(4500000, $rInfo['bitrate'], 'bit/s here, kbit/s in the column');
	}

	/** Whatever ffprobe once found is kept; only what the daemon read is replaced. */
	public function testStreamInfoIsMergedNotReplaced(): void {
		$rRow = $this->row(['stream_info' => json_encode([
			'container' => 'mpegts',
			'codecs' => ['video' => ['codec_name' => 'h264', 'codec_type' => 'video', 'width' => 1280, 'height' => 720, 'r_frame_rate' => 25]],
		])]);
		$set = $this->update($rRow, ['running' => true, 'confirmed' => true, 'pid' => 1, 'meta' => ['video_codec' => 'h264', 'width' => 1920, 'height' => 1080]]);
		$rInfo = json_decode($set['stream_info'], true);
		$this->assertSame('mpegts', $rInfo['container']);
		$this->assertSame(25, $rInfo['codecs']['video']['r_frame_rate'], 'a field the daemon does not read survives');
		$this->assertSame(1920, $rInfo['codecs']['video']['width']);
		$this->assertSame(1080, $rInfo['codecs']['video']['height']);
	}

	/** Nothing new read means no write: the JSON is not rewritten every pass. */
	public function testStreamInfoIsNotRewrittenWhenNothingChanged(): void {
		$rRow = $this->row(['pid' => 1, 'stream_status' => 0, 'video_codec' => 'h264', 'audio_codec' => 'aac', 'stream_info' => json_encode([
			'codecs' => [
				'video' => ['codec_name' => 'h264', 'codec_type' => 'video', 'width' => 1920, 'height' => 1080],
				'audio' => ['codec_name' => 'aac', 'codec_type' => 'audio'],
			],
		])]);
		$set = $this->update($rRow, ['running' => true, 'confirmed' => true, 'pid' => 1, 'meta' => ['video_codec' => 'h264', 'audio_codec' => 'aac', 'width' => 1920, 'height' => 1080]]);
		$this->assertArrayNotHasKey('stream_info', $set);
	}

	/** Unknown is left unknown: a correct value is never overwritten with a blank. */
	public function testUnknownMetadataKeepsWhatThePanelHas(): void {
		$rRow = $this->row(['video_codec' => 'hevc', 'audio_codec' => 'ac3', 'resolution' => 2160, 'bitrate' => 9000, 'stream_status' => 0, 'pid' => 1]);
		$set = $this->update($rRow, ['running' => true, 'confirmed' => true, 'pid' => 1, 'meta' => []]);
		foreach (['video_codec', 'audio_codec', 'resolution', 'bitrate', 'compatible'] as $rCol) {
			$this->assertArrayNotHasKey($rCol, $set, $rCol);
		}
		$audioOnly = $this->update($rRow, ['running' => true, 'confirmed' => true, 'pid' => 1, 'meta' => ['audio_codec' => 'aac']]);
		$this->assertArrayNotHasKey('video_codec', $audioOnly, 'a known video codec is not blanked by an audio-only reading');
	}

	public function testNothingChangedWritesNothing(): void {
		$rRow = $this->row(['pid' => 4321, 'stream_status' => 0]);
		$this->assertSame([], $this->update($rRow, ['running' => true, 'confirmed' => true, 'pid' => 4321, 'source' => 'http://a/1.ts']));
	}
}