mirror of
https://github.com/Vateron-Media/XC_VM.git
synced 2026-11-07 11:03:41 +01:00
obscuremind
7aae87d7dd
MAIN stamps a stream's version before it sends the command that starts it, so the record is in the node's next delta; the start could still run before the agent's sync and find no entry. StreamSource::streamRow (the stream, movie and supervisor starts, the relays' thumbnails) now asks its agent for a sync on such a miss (POST /v1/streams_sync) and reads again, where the entry is built from the record the agent stored. Only with an agent that says `streams_sync` in flows.json, never for a stream with an entry or whose record did not read, once per stream a minute in a process. This is the plan's `stream_bundle` on a miss, done with the delta MAIN already serves rather than a per-stream op and record type. ADR 0004 records it, the blocklist's wake-up on change, and why the node section's nginx settings need no applier of their own.
632 lines
34 KiB
PHP
632 lines
34 KiB
PHP
<?php
|
|
|
|
use PHPUnit\Framework\TestCase;
|
|
use XcVm\Core\Cache\FileCache;
|
|
use XcVm\Core\Cluster\AgentClient;
|
|
use XcVm\Core\Cluster\NodeFlows;
|
|
use XcVm\Core\Cluster\ReplicaApply;
|
|
use XcVm\Core\Cluster\ReplicaSections;
|
|
use XcVm\Core\Cluster\ReplicaStreamCache;
|
|
use XcVm\Core\Cluster\StreamRecords;
|
|
use XcVm\Core\Cluster\StreamRuntime;
|
|
use XcVm\Domain\Stream\StreamSource;
|
|
use XcVm\Infrastructure\Database\DatabaseFactory;
|
|
use XcVm\Tests\Support\InstallSchema;
|
|
use XcVm\Tests\Support\ReplicaFixture;
|
|
|
|
/**
|
|
* The node's half of the R2 `streams` section (cluster plan, section 9,
|
|
* "Storage and boot"; section 10): cluster:apply turns the records the agent
|
|
* stored into the node's stream caches, in the shapes its readers took from
|
|
* MAIN's database (StreamSource: the stream starts, the monitor, the proxy
|
|
* producer, live.php, the scanner, the recorder). With the STREAMS flow on
|
|
* they are authoritative: the readers stop reading MAIN's database once an
|
|
* apply built them. With it off the apply only reports how the section
|
|
* differs from MAIN's database, ids and names only. A record that does not
|
|
* read never deletes its stream's entry; a stream the agent removed loses
|
|
* it. `--from-disk` verifies each record. Mode 0 is unchanged.
|
|
*/
|
|
final class ReplicaStreamCacheTest extends TestCase {
|
|
private string $rDir;
|
|
|
|
private ReplicaFixture $rFixture;
|
|
|
|
private TestDb $rDb;
|
|
|
|
/** This node (SERVER_ID, whichever test defined it first) and another server. */
|
|
private int $rSid;
|
|
|
|
private int $rOther;
|
|
|
|
/** The store's directory before this test (the suite's own, tests/bootstrap.php). */
|
|
private string $rRuntimeDir;
|
|
|
|
protected function setUp(): void {
|
|
if (!defined('SERVER_ID')) {
|
|
define('SERVER_ID', 5);
|
|
}
|
|
$this->rSid = (int) SERVER_ID;
|
|
$this->rOther = $this->rSid + 1;
|
|
$this->rDir = sys_get_temp_dir() . '/xcvm-stream-cache-' . bin2hex(random_bytes(4)) . '/';
|
|
mkdir($this->rDir . 'cluster', 0777, true);
|
|
mkdir($this->rDir . 'cache', 0777, true);
|
|
$this->rFixture = new ReplicaFixture($this->rDir . 'cluster/');
|
|
ReplicaApply::useDir($this->rFixture->dir());
|
|
ReplicaApply::useConfigDir($this->rDir);
|
|
// A recording status another test kept would win over the record's.
|
|
$this->rRuntimeDir = StreamRuntime::dir();
|
|
StreamRuntime::useDir($this->rDir . 'cluster/runtime/');
|
|
(new \ReflectionProperty(FileCache::class, 'defaultInstance'))->setValue(null, new FileCache($this->rDir . 'cache/'));
|
|
$this->flows(NodeFlows::STREAMS);
|
|
$this->rDb = new TestDb();
|
|
foreach (['streams', 'streams_servers', 'recordings', 'profiles', 'streams_types', 'streams_arguments', 'streams_options'] as $rTable) {
|
|
$this->rDb->exec(InstallSchema::table($rTable));
|
|
}
|
|
$this->rDb->exec("INSERT INTO `streams_types` VALUES (1, 'Live Streams', 'live', 'live', 1), (2, 'Movies', 'movie', 'movie', 0), (3, 'Created Live', 'created_live', 'live', 1)");
|
|
$this->rDb->exec("INSERT INTO `profiles` VALUES (7, 'hd', '{\"3\":{\"cmd\":\"-b:v 4M\"}}')");
|
|
$this->rDb->exec("INSERT INTO `streams_arguments` VALUES (1, 'fetch', 'User Agent', 'shown in the form', 'http', 'user_agent', '-user_agent \"%s\"', 'text', 'VLC'), (2, 'fetch', 'Proxy', 'a proxy', 'http', 'proxy', '-http_proxy \"%s\"', 'text', NULL)");
|
|
// 10 live, assigned here and relayed on by the other server; 11 a movie
|
|
// assigned here; 12 recorded here (its TV archive) only; 13 with a
|
|
// recording scheduled here; 14 on the other server alone; 15 a direct
|
|
// source assigned here.
|
|
$this->rDb->exec("INSERT INTO `streams` (`id`, `type`, `stream_display_name`, `stream_source`, `notes`, `order`, `transcode_profile_id`, `enable_transcode`, `tv_archive_server_id`, `tv_archive_duration`, `vframes_server_id`, `direct_source`, `probesize_ondemand`, `target_container`, `tmdb_id`) VALUES
|
|
(10, 1, 'News', '[\"http://user:pass@src.example/a\"]', 'secret-notes', 3, 7, 1, 0, 0, 0, 0, 512000, NULL, 0),
|
|
(11, 2, 'Film', '[\"http://src.example/film.mkv\"]', NULL, 4, 0, 0, 0, 0, 0, 0, 0, 'mkv', 42),
|
|
(12, 1, 'Archive', '[\"http://src.example/c\"]', NULL, 5, 0, 0, {$this->rSid}, 24, 0, 0, 0, NULL, 0),
|
|
(13, 1, 'Recorded', '[\"http://src.example/d\"]', NULL, 6, 0, 0, 0, 0, 0, 0, 0, NULL, 0),
|
|
(14, 1, 'Elsewhere', '[\"http://src.example/e\"]', NULL, 7, 0, 0, 0, 0, 0, 0, 0, NULL, 0),
|
|
(15, 1, 'Direct', '[\"http://src.example/f\"]', NULL, 8, 0, 0, 0, 0, 0, 1, 0, NULL, 0)");
|
|
$this->rDb->exec("INSERT INTO `streams_servers` (`server_stream_id`, `stream_id`, `server_id`, `parent_id`, `on_demand`, `pid`, `stream_status`, `current_source`, `bitrate`) VALUES
|
|
(1, 10, {$this->rSid}, NULL, 1, 4242, 0, 'http://user:pass@src.example/a', 3000),
|
|
(2, 10, {$this->rOther}, {$this->rSid}, 0, 0, 0, NULL, NULL),
|
|
(3, 11, {$this->rSid}, NULL, 0, 99, 0, NULL, NULL),
|
|
(4, 14, {$this->rOther}, NULL, 0, 0, 0, NULL, NULL),
|
|
(5, 15, {$this->rSid}, NULL, 0, 0, 0, NULL, NULL)");
|
|
$this->rDb->exec("INSERT INTO `streams_options` (`id`, `stream_id`, `argument_id`, `value`) VALUES (1, 10, 1, 'curl/8'), (2, 10, 2, 'http://proxy.example:3128')");
|
|
$this->rDb->exec("INSERT INTO `recordings` (`id`, `stream_id`, `created_id`, `source_id`, `title`, `start`, `end`, `archive`, `status`) VALUES (1, 13, 0, {$this->rSid}, 'Match', 100, 200, 0, 0), (2, 13, 0, {$this->rOther}, 'Match elsewhere', 100, 200, 0, 0)");
|
|
DatabaseFactory::set($this->rDb);
|
|
}
|
|
|
|
/** @var resource|null the stand-in agent (streamsSyncAgent) */
|
|
private $rAgent = null;
|
|
|
|
protected function tearDown(): void {
|
|
if ($this->rAgent !== null) {
|
|
proc_terminate($this->rAgent);
|
|
proc_close($this->rAgent);
|
|
}
|
|
AgentClient::useSocket(null);
|
|
DatabaseFactory::reset();
|
|
StreamSource::useLoader(null);
|
|
ReplicaApply::useDir(null);
|
|
ReplicaApply::useConfigDir(null);
|
|
StreamRuntime::useDir($this->rRuntimeDir);
|
|
NodeFlows::usePath(null);
|
|
(new \ReflectionProperty(FileCache::class, 'defaultInstance'))->setValue(null, null);
|
|
exec('rm -rf ' . escapeshellarg($this->rDir));
|
|
}
|
|
|
|
private function flows(?int $rFlows, int $rMode = 1): void {
|
|
if ($rFlows === null) {
|
|
@unlink($this->rDir . 'flows.json');
|
|
} else {
|
|
file_put_contents($this->rDir . 'flows.json', json_encode(['mode' => $rMode, 'flows' => $rFlows, 'state' => 'active']));
|
|
}
|
|
NodeFlows::usePath($this->rDir . 'flows.json');
|
|
}
|
|
|
|
/** The section as the agent stores it: MAIN's records of the streams this node holds, and a completed pass. */
|
|
private function storeSection(int $rSince = 7): void {
|
|
foreach (StreamRecords::data($this->rSid, StreamRecords::held($this->rSid, null)) as $rID => $rData) {
|
|
$this->rFixture->stream($rID, $rData, 3);
|
|
}
|
|
$this->rFixture->streamsSince($rSince);
|
|
}
|
|
|
|
/** @return array<string, mixed> */
|
|
private function apply(bool $rFromDisk = false): array {
|
|
$rReport = ReplicaApply::run(false, 1800000000, $this->rSid, $rFromDisk);
|
|
$this->assertIsArray($rReport);
|
|
$this->assertArrayHasKey(ReplicaSections::STREAMS, $rReport);
|
|
return $rReport;
|
|
}
|
|
|
|
/** MAIN's database gone: whatever reads it now fails. */
|
|
private function mainUnreachable(): void {
|
|
DatabaseFactory::reset();
|
|
}
|
|
|
|
/**
|
|
* What every reader of a stream's definition gets.
|
|
*
|
|
* @return array<string, mixed>
|
|
*/
|
|
private function answers(): array {
|
|
$rOut = [];
|
|
foreach ([10, 11, 12, 13, 14, 15, 99] as $rID) {
|
|
$rOut[$rID] = [
|
|
'live' => StreamSource::streamRow($rID, true),
|
|
'vod' => StreamSource::streamRow($rID, false),
|
|
'server' => StreamSource::serverRow($rID),
|
|
'arguments' => StreamSource::arguments($rID),
|
|
'keyed' => StreamSource::arguments($rID, true),
|
|
'source' => StreamSource::sourceRow($rID),
|
|
];
|
|
}
|
|
$rOut['recordings'] = [1 => StreamSource::recording(1), 2 => StreamSource::recording(2)];
|
|
return $rOut;
|
|
}
|
|
|
|
/** Scalars as strings, as a driver reads them, at any depth. */
|
|
private static function loose(mixed $rValue): mixed {
|
|
if (is_array($rValue)) {
|
|
return array_map([self::class, 'loose'], $rValue);
|
|
}
|
|
return $rValue === null ? null : (string) $rValue;
|
|
}
|
|
|
|
/**
|
|
* The same shape: every key the SQL answer has, the same values but for
|
|
* the columns no record carries, which are null.
|
|
*/
|
|
private function assertSameShape(?array $rSql, ?array $rReplica, string $rWhat): void {
|
|
if ($rSql === null || $rReplica === null) {
|
|
$this->assertSame($rSql, $rReplica, $rWhat);
|
|
return;
|
|
}
|
|
$this->assertEqualsCanonicalizing(array_keys($rSql), array_keys($rReplica), $rWhat . ': the same columns');
|
|
$rLocal = array_merge(ReplicaSections::STREAM_LOCAL, ReplicaSections::STREAM_SERVER_LOCAL, ['argument_description']);
|
|
foreach ($rSql as $rKey => $rValue) {
|
|
if (is_array($rValue)) {
|
|
$this->assertSameShape($rValue, $rReplica[$rKey], $rWhat . '.' . $rKey);
|
|
} elseif (in_array($rKey, $rLocal, true)) {
|
|
$this->assertNull($rReplica[$rKey], $rWhat . '.' . $rKey . ': no record carries it');
|
|
} else {
|
|
$this->assertSame(self::loose($rValue), self::loose($rReplica[$rKey]), $rWhat . '.' . $rKey);
|
|
}
|
|
}
|
|
}
|
|
|
|
public function testTheCachesAnswerInTheShapesMainsDatabaseDid(): void {
|
|
$this->flows(0);
|
|
$rSql = $this->answers();
|
|
$this->assertSame('curl/8', $rSql[10]['keyed']['user_agent']['value'], 'the SQL backend, as before');
|
|
$this->assertSame('Match', $rSql['recordings'][1]['title']);
|
|
|
|
$this->storeSection();
|
|
$this->flows(NodeFlows::STREAMS);
|
|
$rReport = $this->apply();
|
|
$this->assertSame(['since' => 7, 'streams' => 5, 'mode' => 'applied', 'written' => 5, 'removed' => 0, 'unreadable' => []], $rReport[ReplicaSections::STREAMS]);
|
|
$this->assertTrue(ReplicaStreamCache::owned());
|
|
|
|
// MAIN's database is gone: every answer comes from the caches.
|
|
$this->mainUnreachable();
|
|
$rReplica = $this->answers();
|
|
foreach ([10, 11, 12, 13, 15] as $rID) {
|
|
foreach ($rSql[$rID] as $rWhat => $rAnswer) {
|
|
$this->assertSameShape($rAnswer, $rReplica[$rID][$rWhat], $rID . '.' . $rWhat);
|
|
}
|
|
}
|
|
$this->assertSameShape($rSql['recordings'][1], $rReplica['recordings'][1], 'recordings.1');
|
|
// A stream, or a recording, the node does not hold is none of its business.
|
|
$this->assertSame(['live' => null, 'vod' => null, 'server' => null, 'arguments' => [], 'keyed' => [], 'source' => []], $rReplica[14]);
|
|
$this->assertNull($rReplica['recordings'][2]);
|
|
$this->assertNotNull($rSql[14]['live'], 'MAIN\'s database answered for any stream');
|
|
|
|
// The answers that matter, spelled out.
|
|
$this->assertSame(['http://user:pass@src.example/a'], json_decode((string) $rReplica[10]['live']['stream_source'], true));
|
|
$this->assertSame(['live', 7, 'hd'], [$rReplica[10]['live']['type_key'], $rReplica[10]['live']['profile_id'], $rReplica[10]['live']['profile_name']]);
|
|
$this->assertNull($rReplica[10]['live']['notes'], 'MAIN\'s catalogue metadata stays MAIN\'s');
|
|
$this->assertSame([1, null, null], [$rReplica[10]['server']['on_demand'], $rReplica[10]['server']['pid'], $rReplica[10]['server']['current_source']], 'the runtime state is the node\'s, no record carries it');
|
|
$this->assertSame(['user_agent', 'proxy'], array_keys($rReplica[10]['keyed']));
|
|
$this->assertSame(2, $rReplica[10]['keyed']['proxy']['id'], 'the argument\'s id wins the join, as in SQL');
|
|
$this->assertNull($rReplica[11]['live'], 'a movie is not a live stream');
|
|
$this->assertSame('mkv', $rReplica[11]['vod']['target_container']);
|
|
$this->assertNull($rReplica[12]['server'], 'held for its archive, not run here');
|
|
$this->assertNull($rReplica[15]['live'], 'a direct source is not run by a node');
|
|
$this->assertSame(['stream_source' => '["http://src.example/f"]'], $rReplica[15]['source']);
|
|
$this->assertSame([$this->rOther], ReplicaStreamCache::get(10)['children']);
|
|
|
|
// Another server's row is not the replica's to answer.
|
|
DatabaseFactory::set($this->rDb);
|
|
$this->assertSame($this->rOther, (int) StreamSource::serverRow(10, $this->rOther)['server_id']);
|
|
}
|
|
|
|
public function testWithStreamsOffTheApplyOnlyReportsHowTheSectionDiffers(): void {
|
|
$this->storeSection();
|
|
// MAIN changed since the agent's last sync: a source, an unassignment, an assignment.
|
|
$this->rDb->exec("UPDATE `streams` SET `stream_source` = '[\"http://user:pass@src.example/a2\"]' WHERE `id` = 10");
|
|
$this->rDb->exec('DELETE FROM `streams_servers` WHERE `stream_id` = 11');
|
|
$this->rDb->exec("INSERT INTO `streams_servers` (`server_stream_id`, `stream_id`, `server_id`, `parent_id`, `on_demand`) VALUES (6, 14, {$this->rSid}, NULL, 0)");
|
|
$this->rDb->exec("UPDATE `recordings` SET `title` = 'Final' WHERE `id` = 1");
|
|
$this->flows(NodeFlows::CONFIG);
|
|
$rReport = $this->apply()[ReplicaSections::STREAMS];
|
|
$this->assertSame(['since' => 7, 'streams' => 5, 'mode' => 'shadow', 'missing' => [14], 'extra' => [11], 'unreadable' => [], 'differ' => ['10.stream.stream_source', '13.recordings']], $rReport);
|
|
$rJson = (string) json_encode($rReport);
|
|
foreach (['src.example', 'pass', 'Final', 'Match', 'tickets'] as $rValue) {
|
|
$this->assertStringNotContainsString($rValue, $rJson, 'names only, never a value or the tickets');
|
|
}
|
|
$this->assertSame([], ReplicaStreamCache::cached(), 'nothing written');
|
|
$this->assertFalse(ReplicaStreamCache::owned());
|
|
$this->assertSame('["http://user:pass@src.example/a2"]', StreamSource::streamRow(10, true)['stream_source'], 'the readers keep MAIN\'s database');
|
|
|
|
// MAIN's database does not answer: nothing to compare with, and still nothing written.
|
|
$this->mainUnreachable();
|
|
$this->assertSame(['since' => 7, 'streams' => 5, 'mode' => 'shadow', 'compared' => false, 'unreadable' => []], $this->apply()[ReplicaSections::STREAMS]);
|
|
}
|
|
|
|
public function testTheShadowComparisonWalksTheStreamsInSteps(): void {
|
|
// Past two steps of 1000 held streams, whatever holds them.
|
|
$rRows = [];
|
|
foreach (range(100, 2199) as $rID) {
|
|
$rRows[] = "({$rID}, 1, 'S{$rID}', '[\"http://src.example/{$rID}\"]', 0)";
|
|
}
|
|
$this->rDb->exec('INSERT INTO `streams` (`id`, `type`, `stream_display_name`, `stream_source`, `direct_source`) VALUES ' . implode(', ', $rRows));
|
|
$this->rDb->exec("INSERT INTO `streams_servers` (`stream_id`, `server_id`, `on_demand`) SELECT `id`, {$this->rSid}, 0 FROM `streams` WHERE `id` >= 100");
|
|
$this->storeSection();
|
|
$this->rFixture->stream(5000, ReplicaFixture::streamData(5000, $this->rSid));
|
|
// Since the agent's last sync: one stream renamed at each end, one
|
|
// taken off the node, and two the section lacks, in the second and third steps.
|
|
$this->rDb->exec("UPDATE `streams` SET `stream_display_name` = 'Renamed' WHERE `id` IN (100, 2199)");
|
|
$this->rDb->exec('DELETE FROM `streams_servers` WHERE `stream_id` = 1500');
|
|
foreach ([1105, 2150] as $rID) {
|
|
unlink($this->rFixture->dir() . 'streams/' . $rID . '.json');
|
|
}
|
|
$this->flows(0);
|
|
$rReport = $this->apply()[ReplicaSections::STREAMS];
|
|
$this->assertSame(['mode' => 'shadow', 'missing' => [1105, 2150], 'extra' => [1500, 5000], 'unreadable' => [], 'differ' => ['100.stream.stream_display_name', '2199.stream.stream_display_name']], array_diff_key($rReport, ['since' => 0, 'streams' => 0]));
|
|
$this->assertSame(2104, $rReport['streams']);
|
|
}
|
|
|
|
public function testTheFlowDecidesAndHandsTheStreamsBack(): void {
|
|
$this->storeSection();
|
|
$this->apply();
|
|
$this->assertTrue(ReplicaStreamCache::owned());
|
|
$this->rDb->exec("UPDATE `streams` SET `stream_display_name` = 'Renamed' WHERE `id` = 10");
|
|
$this->assertSame('News', StreamSource::streamRow(10, true)['stream_display_name'], 'the replica\'s, however MAIN\'s database changed');
|
|
|
|
// CONFIG going off leaves the streams to their own flow.
|
|
ReplicaApply::disown();
|
|
$this->assertTrue(ReplicaStreamCache::owned());
|
|
|
|
// STREAMS off: cron:cache's minute hands them back at once, and they are MAIN's database's.
|
|
$this->flows(NodeFlows::CONFIG);
|
|
$this->assertFalse(ReplicaStreamCache::owned(), 'the flow is read at each call');
|
|
$this->assertNull(ReplicaApply::streamsMinute($this->rSid));
|
|
$this->assertFalse(ReplicaApply::built(ReplicaSections::STREAMS));
|
|
$this->assertSame('Renamed', StreamSource::streamRow(10, true)['stream_display_name']);
|
|
|
|
// On again: nothing is the replica's until an apply built it; the minute does.
|
|
$this->flows(NodeFlows::STREAMS);
|
|
$this->assertFalse(ReplicaStreamCache::owned());
|
|
$this->assertSame('applied', ReplicaApply::streamsMinute($this->rSid)['mode']);
|
|
$this->assertSame(0, ReplicaApply::streamsMinute($this->rSid)['written'], 'an unchanged record is not written again');
|
|
$this->assertSame('News', StreamSource::streamRow(10, true)['stream_display_name']);
|
|
|
|
// STREAMS off with CONFIG on: cron:cache's apply never compares a shadow
|
|
// section with MAIN's database, and keeps the agent's last comparison.
|
|
$this->flows(0);
|
|
$rShadow = $this->apply()[ReplicaSections::STREAMS];
|
|
$this->assertSame(['shadow', ['10.stream.stream_display_name']], [$rShadow['mode'], $rShadow['differ']]);
|
|
$this->flows(NodeFlows::CONFIG);
|
|
$this->mainUnreachable();
|
|
$rReport = ReplicaApply::run(true, 1800000000, $this->rSid, false, true);
|
|
$this->assertSame($rShadow, $rReport[ReplicaSections::STREAMS]);
|
|
$this->assertFalse(ReplicaStreamCache::owned());
|
|
}
|
|
|
|
public function testCronCachesMinuteFollowsBothFlows(): void {
|
|
$this->storeSection();
|
|
// CONFIG off, STREAMS on (the rollout's order): the minute applies the
|
|
// section and owns it, and writes no report (the agent's apply keeps it).
|
|
$this->assertSame('applied', ReplicaApply::minute($this->rSid)[ReplicaSections::STREAMS]['mode']);
|
|
$this->assertTrue(ReplicaStreamCache::owned());
|
|
$this->assertFileDoesNotExist($this->rFixture->dir() . 'apply.json');
|
|
$this->rDb->exec("UPDATE `streams` SET `stream_display_name` = 'Renamed' WHERE `id` = 10");
|
|
$this->assertSame('News', StreamSource::streamRow(10, true)['stream_display_name']);
|
|
|
|
// STREAMS off, CONFIG off: handed back at the minute, and turning it on
|
|
// again waits for the next minute instead of reusing the entries.
|
|
$this->flows(0);
|
|
$this->assertNull(ReplicaApply::minute($this->rSid));
|
|
$this->assertFalse(ReplicaApply::built(ReplicaSections::STREAMS));
|
|
$this->flows(NodeFlows::STREAMS);
|
|
$this->assertFalse(ReplicaStreamCache::owned());
|
|
$this->assertSame('Renamed', StreamSource::streamRow(10, true)['stream_display_name'], 'MAIN\'s database until then');
|
|
ReplicaApply::minute($this->rSid);
|
|
$this->assertTrue(ReplicaStreamCache::owned());
|
|
|
|
// CONFIG on, STREAMS off: the agent's apply compares with MAIN's
|
|
// database; the minute never does, and keeps the agent's comparison.
|
|
$this->flows(NodeFlows::CONFIG);
|
|
$rAgent = ReplicaApply::run(true, 1800000000, $this->rSid)[ReplicaSections::STREAMS];
|
|
$this->assertSame(['shadow', ['10.stream.stream_display_name']], [$rAgent['mode'], $rAgent['differ']]);
|
|
$this->mainUnreachable();
|
|
$rMinute = ReplicaApply::minute($this->rSid);
|
|
$this->assertSame($rAgent, $rMinute[ReplicaSections::STREAMS], 'no comparison: MAIN\'s database was never read');
|
|
$this->assertSame($rAgent, json_decode((string) file_get_contents($this->rFixture->dir() . 'apply.json'), true)[ReplicaSections::STREAMS]);
|
|
$this->assertFalse(ReplicaApply::built(ReplicaSections::STREAMS));
|
|
}
|
|
|
|
public function testAStreamTheAgentRemovedLosesAnEntryItsFileBuilt(): void {
|
|
$this->storeSection();
|
|
$this->apply();
|
|
// Stored after the apply: built from the agent's file at the first read, not indexed.
|
|
$this->rFixture->stream(20, ReplicaFixture::streamData(20, $this->rSid));
|
|
$this->assertSame('S20', StreamSource::streamRow(20, true)['stream_display_name']);
|
|
$this->assertContains(20, ReplicaStreamCache::cached());
|
|
$this->assertArrayNotHasKey(20, ReplicaStreamCache::index());
|
|
unlink($this->rFixture->dir() . 'streams/20.json');
|
|
unlink($this->rFixture->dir() . 'streams/20.rep');
|
|
$this->assertSame(1, $this->apply()[ReplicaSections::STREAMS]['removed']);
|
|
$this->mainUnreachable();
|
|
$this->assertNull(StreamSource::streamRow(20, true));
|
|
$this->assertNotContains(20, ReplicaStreamCache::cached());
|
|
}
|
|
|
|
/**
|
|
* A stand-in agent on its socket: each POST /v1/streams_sync moves what is
|
|
* staged in $rStaged into the replica's streams (the records a sync would
|
|
* store) and answers 200; each request is logged.
|
|
*/
|
|
private function streamsSyncAgent(string $rStaged): string {
|
|
$rSock = $this->rDir . 'agent.sock';
|
|
$rScript = $this->rDir . 'agent.php';
|
|
file_put_contents($rScript, <<<'PHP'
|
|
<?php
|
|
[, $rSock, $rLog, $rStaged, $rStreams] = $argv;
|
|
$rServer = stream_socket_server('unix://' . $rSock, $rErrNo, $rErr);
|
|
while (($rConn = @stream_socket_accept($rServer, 10)) !== false) {
|
|
$rHead = '';
|
|
while (!str_contains($rHead, "\r\n\r\n") && ($rChunk = fread($rConn, 8192)) !== false && $rChunk !== '') {
|
|
$rHead .= $rChunk;
|
|
}
|
|
file_put_contents($rLog, strtok($rHead, "\r\n") . "\n", FILE_APPEND);
|
|
foreach (glob($rStaged . '/*') ?: [] as $rFile) {
|
|
rename($rFile, $rStreams . '/' . basename($rFile));
|
|
}
|
|
$rBody = '{"synced":true}';
|
|
fwrite($rConn, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: " . strlen($rBody) . "\r\nConnection: close\r\n\r\n" . $rBody);
|
|
fclose($rConn);
|
|
}
|
|
PHP);
|
|
$rNull = ['file', '/dev/null', 'w'];
|
|
$this->rAgent = proc_open([PHP_BINARY, $rScript, $rSock, $this->rDir . 'requests.log', $rStaged, $this->rFixture->dir() . 'streams'], [0 => ['file', '/dev/null', 'r'], 1 => $rNull, 2 => $rNull], $rPipes) ?: null;
|
|
for ($i = 0; $i < 100 && !file_exists($rSock); $i++) {
|
|
usleep(20000);
|
|
}
|
|
$this->assertFileExists($rSock, 'the stand-in agent listens');
|
|
AgentClient::useSocket($rSock);
|
|
return $this->rDir . 'requests.log';
|
|
}
|
|
|
|
/**
|
|
* A start that finds no entry for a stream MAIN assigned a moment ago (its
|
|
* record not yet synced) has the agent sync the section and reads again:
|
|
* the plan's `stream_bundle` on a miss. Once per stream a minute, and only
|
|
* with an agent that says it can.
|
|
*/
|
|
public function testAStartThatMissesHasTheAgentSyncAndReadsAgain(): void {
|
|
$this->storeSection();
|
|
$this->apply();
|
|
// Stream 20, assigned since: its record is what the next sync stores.
|
|
$this->rFixture->stream(20, ReplicaFixture::streamData(20, $this->rSid));
|
|
$rStaged = $this->rDir . 'staged';
|
|
mkdir($rStaged);
|
|
foreach (['20.json', '20.rep'] as $rFile) {
|
|
rename($this->rFixture->dir() . 'streams/' . $rFile, $rStaged . '/' . $rFile);
|
|
}
|
|
$rLog = $this->streamsSyncAgent($rStaged);
|
|
$this->mainUnreachable();
|
|
|
|
// An agent that does not say it: no request, no stream.
|
|
$this->assertNull(StreamSource::streamRow(20, true));
|
|
$this->assertFileDoesNotExist($rLog);
|
|
|
|
file_put_contents($this->rDir . 'flows.json', json_encode(['mode' => 1, 'flows' => NodeFlows::STREAMS, 'state' => 'active', 'features' => [ReplicaStreamCache::FEATURE_SYNC]]));
|
|
NodeFlows::usePath($this->rDir . 'flows.json');
|
|
$this->assertSame('S20', StreamSource::streamRow(20, true)['stream_display_name'] ?? null, 'the start finds it after the sync');
|
|
$this->assertSame(['POST /v1/streams_sync HTTP/1.0'], file($rLog, FILE_IGNORE_NEW_LINES));
|
|
|
|
// A stream the sync does not bring (not the node's): asked once a minute, not at every start.
|
|
$this->assertNull(StreamSource::streamRow(30, true));
|
|
$this->assertNull(StreamSource::streamRow(30, true));
|
|
$this->assertCount(2, file($rLog, FILE_IGNORE_NEW_LINES));
|
|
// One the node has an entry for never asks.
|
|
$this->assertNotNull(StreamSource::streamRow(10, true));
|
|
$this->assertCount(2, file($rLog, FILE_IGNORE_NEW_LINES));
|
|
}
|
|
|
|
public function testARecordingStoredSinceTheLastApplyIsFound(): void {
|
|
$this->storeSection();
|
|
$this->apply();
|
|
$rData = ReplicaFixture::streamData(21, $this->rSid);
|
|
$rData['recordings'] = [ReplicaSections::typed(['id' => 777, 'stream_id' => 21, 'source_id' => $this->rSid, 'title' => 'Late', 'start' => 100, 'end' => 200], ReplicaSections::RECORDING_FIELDS)];
|
|
$this->rFixture->stream(21, $rData);
|
|
$this->mainUnreachable();
|
|
$this->assertSame('Late', StreamSource::recording(777)['title'], 'the recorder starts it');
|
|
$this->assertSame('Match', StreamSource::recording(1)['title'], 'the index first');
|
|
$this->assertNull(StreamSource::recording(778));
|
|
}
|
|
|
|
public function testFromDiskTheBlocklistIsAppliedBeforeTheStreams(): void {
|
|
$this->storeSection();
|
|
$this->rFixture->blocklist(['ip' => ['203.0.113.1'], 'asn' => [], 'ua' => [], 'isp' => [], 'rtmp' => []], 7);
|
|
$this->flows(NodeFlows::CONFIG | NodeFlows::STREAMS);
|
|
// The streams part fails (as `service`'s timeout or a full tmpfs would stop it).
|
|
$rPath = FileCache::defaultPath() . ReplicaStreamCache::DIR;
|
|
$rStores = new \ReflectionProperty(ReplicaStreamCache::class, 'rStores');
|
|
$rStores->setValue(null, [$rPath => new class ($rPath) extends FileCache {
|
|
public function set($key, $data, $ttl = 0) {
|
|
throw new \RuntimeException('the streams part stopped');
|
|
}
|
|
}]);
|
|
try {
|
|
ReplicaApply::run(true, 1800000000, $this->rSid, true);
|
|
$this->fail('the streams part ran');
|
|
} catch (\RuntimeException $rE) {
|
|
$this->assertSame('the streams part stopped', $rE->getMessage());
|
|
} finally {
|
|
$rStores->setValue(null, []);
|
|
}
|
|
$this->assertSame(['203.0.113.1'], FileCache::getCache('blocked_ips'), 'the bans are in place');
|
|
$this->assertFalse(ReplicaApply::built(ReplicaSections::STREAMS), 'the streams are not the replica\'s');
|
|
$rReport = ReplicaApply::run(true, 1800000000, $this->rSid, true);
|
|
$this->assertSame(['verified' => ['blocklist', 'streams'], 'unverified' => []], $rReport['from_disk']);
|
|
$this->assertTrue(ReplicaStreamCache::owned());
|
|
}
|
|
|
|
public function testARecordThatDoesNotReadNeverDeletesItsStreamsEntry(): void {
|
|
$rData = StreamRecords::data($this->rSid, [10, 11]);
|
|
$this->storeSection();
|
|
$this->apply();
|
|
file_put_contents($this->rFixture->dir() . 'streams/10.json', '{"etag": "torn');
|
|
$rReport = $this->apply()[ReplicaSections::STREAMS];
|
|
$this->assertSame(['applied', 0, 0, [10]], [$rReport['mode'], $rReport['written'], $rReport['removed'], $rReport['unreadable']]);
|
|
$this->mainUnreachable();
|
|
$this->assertSame('News', StreamSource::streamRow(10, true)['stream_display_name'], 'kept as it was');
|
|
// One that is another stream's, or names another server's row, is not this one's either.
|
|
$this->rFixture->stream(10, $rData[11]);
|
|
$this->assertSame([10], $this->apply()[ReplicaSections::STREAMS]['unreadable']);
|
|
$rRecording = ReplicaSections::typed(['id' => 9, 'stream_id' => 10, 'source_id' => $this->rOther, 'title' => 'Elsewhere'], ReplicaSections::RECORDING_FIELDS);
|
|
foreach ([
|
|
'a server row naming another server' => ['server', 'server_id', $this->rOther],
|
|
'a server row of another stream' => ['server', 'stream_id', 11],
|
|
'a recording scheduled on another server' => ['recordings', null, [$rRecording]],
|
|
'a type that is not MAIN\'s' => ['type', null, ['type_id' => 'x']],
|
|
'a profile that is not MAIN\'s' => ['profile', null, 'x'],
|
|
'children that are not server ids' => ['children', null, ['x']],
|
|
] as $rWhy => [$rPart, $rField, $rValue]) {
|
|
$rTen = $rData[10];
|
|
if ($rField === null) {
|
|
$rTen[$rPart] = $rValue;
|
|
} else {
|
|
$rTen[$rPart][$rField] = $rValue;
|
|
}
|
|
$this->rFixture->stream(10, $rTen);
|
|
$this->assertSame([10], $this->apply()[ReplicaSections::STREAMS]['unreadable'], $rWhy);
|
|
$this->assertSame('News', StreamSource::streamRow(10, true)['stream_display_name'], $rWhy . ': kept as it was');
|
|
}
|
|
|
|
// From disk, a record missing behind its .json is not a removal either.
|
|
DatabaseFactory::set($this->rDb);
|
|
$this->storeSection();
|
|
unlink($this->rFixture->dir() . 'streams/10.rep');
|
|
$rReport = $this->apply(true);
|
|
$this->assertSame([10], $rReport[ReplicaSections::STREAMS]['unreadable']);
|
|
$this->assertSame(['streams'], $rReport['from_disk']['unverified']);
|
|
$this->assertNotNull(ReplicaStreamCache::get(10));
|
|
|
|
// The agent removed it (both files): the node no longer holds it, and never asks MAIN.
|
|
unlink($this->rFixture->dir() . 'streams/10.json');
|
|
$rReport = $this->apply()[ReplicaSections::STREAMS];
|
|
$this->assertSame([1, 4], [$rReport['removed'], $rReport['streams']]);
|
|
$this->mainUnreachable();
|
|
$this->assertNull(StreamSource::streamRow(10, true));
|
|
$this->assertSame([11, 12, 13, 15], ReplicaStreamCache::cached());
|
|
$this->assertSame([11, 12, 13, 15], array_keys(ReplicaStreamCache::index()));
|
|
}
|
|
|
|
public function testWithoutAWholeSectionTheReadersKeepMainsDatabase(): void {
|
|
$this->storeSection(0);
|
|
$this->assertSame(['since' => 0, 'mode' => 'incomplete'], $this->apply()[ReplicaSections::STREAMS], 'the agent has not completed a pass');
|
|
$this->assertFalse(ReplicaStreamCache::owned());
|
|
$this->assertSame([], ReplicaStreamCache::cached());
|
|
|
|
$this->rFixture->streamsSince(7);
|
|
$this->apply();
|
|
$this->assertTrue(ReplicaStreamCache::owned());
|
|
// A file that names no stream: the section is not the agent's, nothing is touched, MAIN's database again.
|
|
file_put_contents($this->rFixture->dir() . 'streams/notes.json', '{}');
|
|
$this->assertSame(['since' => 7, 'mode' => 'refused'], $this->apply()[ReplicaSections::STREAMS]);
|
|
$this->assertFalse(ReplicaStreamCache::owned());
|
|
$this->assertSame([10, 11, 12, 13, 15], ReplicaStreamCache::cached(), 'no entry deleted');
|
|
$this->rDb->exec("UPDATE `streams` SET `stream_display_name` = 'Renamed' WHERE `id` = 10");
|
|
$this->assertSame('Renamed', StreamSource::streamRow(10, true)['stream_display_name']);
|
|
unlink($this->rFixture->dir() . 'streams/notes.json');
|
|
|
|
// The directory lost: a section lost, never one that holds nothing.
|
|
exec('rm -rf ' . escapeshellarg($this->rFixture->dir() . 'streams'));
|
|
$this->assertSame(['since' => 7, 'mode' => 'incomplete'], $this->apply()[ReplicaSections::STREAMS]);
|
|
$this->assertSame([10, 11, 12, 13, 15], ReplicaStreamCache::cached());
|
|
|
|
// No section at all: nothing to report.
|
|
unlink($this->rFixture->dir() . 'streams.json');
|
|
$this->assertNull(ReplicaApply::run(true, 1800000000, $this->rSid), 'nothing the agent wrote');
|
|
}
|
|
|
|
public function testFromDiskEachRecordIsVerifiedAndWins(): void {
|
|
$this->storeSection();
|
|
// The .json says something else (planted or torn): the signed record wins.
|
|
$rTen = StreamRecords::data($this->rSid, [10])[10];
|
|
$rPlanted = $rTen;
|
|
$rPlanted['stream']['stream_source'] = '["http://evil.example/"]';
|
|
$this->rFixture->stream(10, $rTen, 3, $rPlanted);
|
|
$rReport = $this->apply(true);
|
|
$this->assertSame(['verified' => ['streams'], 'unverified' => []], $rReport['from_disk']);
|
|
$this->assertSame(5, $rReport[ReplicaSections::STREAMS]['written']);
|
|
$this->mainUnreachable();
|
|
$this->assertSame('["http://user:pass@src.example/a"]', StreamSource::sourceRow(10)['stream_source']);
|
|
// An entry written from a planted .json is written again from its record.
|
|
ReplicaApply::run(false, 1800000000, $this->rSid);
|
|
$rPlanted['stream']['stream_display_name'] = 'Planted';
|
|
$this->rFixture->stream(10, $rTen, 3, $rPlanted);
|
|
ReplicaStreamCache::store()->set('10', ReplicaStreamCache::entry(10, $rPlanted, $this->rSid, ReplicaStreamCache::index()[10]['etag'], 3));
|
|
$this->assertSame('Planted', StreamSource::streamRow(10, true)['stream_display_name']);
|
|
$this->assertSame(5, $this->apply(true)[ReplicaSections::STREAMS]['written']);
|
|
$this->assertSame('News', StreamSource::streamRow(10, true)['stream_display_name']);
|
|
|
|
// Signed by another panel key, for another node, another stream's, corrupt: none is taken.
|
|
DatabaseFactory::set($this->rDb);
|
|
$this->storeSection();
|
|
$rData = StreamRecords::data($this->rSid, [11, 12, 13]);
|
|
file_put_contents($this->rFixture->dir() . 'streams/11.rep', $this->rFixture->record('rep', ['section' => 'stream', 'node' => $this->rFixture->rUuid, 'stream_id' => 11, 'ver' => 3, 'etag' => str_repeat('c', 64), 'data' => $rData[11]], ReplicaFixture::otherPanel()));
|
|
file_put_contents($this->rFixture->dir() . 'streams/12.rep', $this->rFixture->record('rep', ['section' => 'stream', 'node' => '00000000-0000-4000-a000-000000000000', 'stream_id' => 12, 'ver' => 3, 'etag' => str_repeat('c', 64), 'data' => $rData[12]]));
|
|
copy($this->rFixture->dir() . 'streams/10.rep', $this->rFixture->dir() . 'streams/13.rep');
|
|
$this->rFixture->corrupt('streams/15.rep');
|
|
ReplicaStreamCache::store()->flush();
|
|
$rReport = $this->apply(true);
|
|
$this->assertSame([11, 12, 13, 15], $rReport[ReplicaSections::STREAMS]['unreadable']);
|
|
$this->assertSame(['streams'], $rReport['from_disk']['unverified']);
|
|
$this->assertSame([10], ReplicaStreamCache::cached());
|
|
$this->assertStringNotContainsString('src.example', (string) json_encode($rReport), 'the report names streams, never their content');
|
|
// Nor is a stream whose record did not verify built from its unsigned
|
|
// .json at a read until the next apply.
|
|
$this->assertNull(StreamSource::streamRow(11, false));
|
|
$this->assertNull(StreamSource::streamRow(12, true));
|
|
$this->assertSame([10], ReplicaStreamCache::cached());
|
|
// That apply (the agent's, or cron:cache's minute) trusts the agent's .json, as for every section.
|
|
ReplicaApply::minute($this->rSid);
|
|
$this->assertSame('Film', StreamSource::streamRow(11, false)['stream_display_name']);
|
|
}
|
|
|
|
public function testAnEntryGoneIsBuiltAgainFromTheAgentsFile(): void {
|
|
$this->storeSection();
|
|
$this->apply();
|
|
// The next apply writes it again, unchanged record or not.
|
|
ReplicaStreamCache::store()->delete('10');
|
|
$this->assertSame(1, $this->apply()[ReplicaSections::STREAMS]['written']);
|
|
$this->assertContains(10, ReplicaStreamCache::cached());
|
|
$this->assertSame('0700', substr(sprintf('%o', fileperms($this->rDir . 'cache/' . ReplicaStreamCache::DIR)), -4), 'the entries hold the sources');
|
|
ReplicaStreamCache::store()->delete('10');
|
|
$this->mainUnreachable();
|
|
$this->assertSame('News', StreamSource::streamRow(10, true)['stream_display_name'], 'the agent\'s file, never MAIN\'s database');
|
|
$this->assertContains(10, ReplicaStreamCache::cached(), 'and cached again');
|
|
}
|
|
|
|
public function testModeZeroIsUnchanged(): void {
|
|
$this->storeSection();
|
|
$this->flows(null);
|
|
$rReport = $this->apply()[ReplicaSections::STREAMS];
|
|
$this->assertSame('shadow', $rReport['mode'], 'a legacy node only compares');
|
|
$this->assertSame([], ReplicaStreamCache::cached());
|
|
// Even a stale record of ownership changes nothing without the flow.
|
|
FileCache::setCache(ReplicaApply::OWNED_CACHE, [ReplicaSections::STREAMS => '7']);
|
|
$this->assertFalse(ReplicaStreamCache::owned());
|
|
$this->rDb->exec("UPDATE `streams` SET `stream_display_name` = 'Renamed' WHERE `id` = 10");
|
|
$this->assertSame('Renamed', StreamSource::streamRow(10, true)['stream_display_name']);
|
|
$this->assertNull(ReplicaApply::streamsMinute($this->rSid));
|
|
$this->assertFalse(ReplicaApply::built(ReplicaSections::STREAMS));
|
|
}
|
|
}
|