diff --git a/docs/adr/0004-cluster-api.md b/docs/adr/0004-cluster-api.md new file mode 100644 index 00000000..d9c41b13 --- /dev/null +++ b/docs/adr/0004-cluster-api.md @@ -0,0 +1,85 @@ +# ADR 0004 — Cluster API between MAIN and load balancers: the panel's contract + +- **Status:** Accepted. Phase 0 (seams) and Phase 1 (crypto contract, schema, settings) are implemented; Phases 2–11 are not. +- **Date:** 2026-09-25 +- **Plan:** `docs/superpowers/specs/2026-09-21-main-lb-api-communication-design.md` (MAIN ↔ LB API communication, revision 3 plus corrections). +- **Extension side:** `xcvm_core` ADR-002, "Cluster API: the extension's half of MAIN ↔ LB communication", cluster API version 1. + +## Context + +Load balancers today reach MAIN's MariaDB and Redis directly (the `db_grant` model) and MAIN reaches them through `/api` with the live-streaming password. The plan replaces both with a signed, encrypted HTTP API started by each LB's Go `xc_agent`. The cryptographic root lives in `xcvm_core`. PHP runs the protocol, and the agent is its other end. Three codebases must therefore produce the same bytes. + +## Decisions + +### Who owns which part of the wire + +| Part | Owner | Where it is defined | +| --- | --- | --- | +| XCVM-SEAL-v1, XCVM-BOX-v1, token derivation, panel signature domain, tag registry and record classes, lease, pin, DR bundle | `xcvm_core` | ADR-002; vectors `tests/Support/cluster_vectors.json` (copied verbatim from the extension, generated from its Rust) | +| Canonical request/response serialisation and the request/response MAC | panel | `Core\Cluster\Crypto\Canonical`; vectors `tests/Support/cluster_canonical_vectors.json` | +| Node signatures (`X-XCVM-Node-Sig`, relay auth, file digests from an LB) | panel | `Core\Cluster\Crypto\NodeSig` (domain `xcvm-node-sig-v1`, never the panel's `xcvm-sig-v1`) | +| Relay and file tickets, `X-XCVM-Relay-Auth`, `X-XCVM-File-Digest` | panel | `Ticket`, `RelayAuth`, `FileDigest` | + +The Go agent (`internal/clustercrypto` in XC_VM_Fanout) must pass both vector files. That package is not written yet. + +### Canonical form + +```text +req_ctx = "xcvm-req-v1" ‖ u32(proto) ‖ lp(agent) ‖ lp(METHOD) ‖ lp(path) ‖ lp(query) + ‖ lp(content_type) ‖ lp(content_encoding) ‖ lp(node) ‖ u64(epoch) ‖ u64(ts_ms) ‖ nonce[16] +res_ctx = "xcvm-res-v1" ‖ SHA-256(req_ctx) ‖ u32(status) ‖ lp(content_type) ‖ u64(ts_ms) ‖ nonce[16] +mac = HMAC-SHA256(K_mac_up | K_mac_down, "xcvm-mac-v1" ‖ lp(ctx) ‖ SHA-256(body)) +``` + +The rules that make the form canonical: + +- The query string is percent-decoded (`+` becomes a space), re-encoded per RFC 3986, and sorted by key then value. +- The method is upper-cased. The content type and encoding are lower-cased and trimmed. +- `node` is a uuid, or `sid:` before enrolment finishes. +- The request context is the BOX context in both directions, and the response context hashes the request in, so a reply cannot be moved onto another request. +- The window is ±90 s. + +### Fail closed + +`ClusterCryptoFactory::create()` throws `ClusterUnavailableException` when: + +- `xcvm_core` or `XC_VM::cluster_session` is missing; +- `cluster_info()['api']` is outside `API_MIN..API_MAX` (currently 1..1); +- sodium or AES-GCM is unavailable. + +The node then stays legacy and `db_grant` remains the gate. No PHP implementation of the extension's secret parts ships: the binding, token MAC, session keys and panel signing with a seed live in `tests/Support/ClusterReference.php`. `ClusterCryptoFailClosedTest` checks that nothing under `Core/Cluster` signs as the panel or computes tokens. + +### Schema numbering + +The plan numbers its Phase 1 migrations 026–032. 026 and 027 were already taken (`ssh_hostkey_sha1`, `fanout_enabled`), so they ship as 028–034. The contents are unchanged. The plan's Phase 10 `033_drop_connection_sync_timer` will get the next free number. + +| Migration | Contents | +| --- | --- | +| 028 | the 19 settings | +| 029 | `cluster_nodes`, `cluster_node_epochs`, `cluster_meta` | +| 030 | `cluster_commands` | +| 031 | enrolment codes and requests | +| 032 | audit, nonces, reservations | +| 033 | `crontab.role` plus a disabled `cluster` row | +| 034 | `cluster_changes`, `cluster_stream_ver` | + +### Settings + +`Core\Cluster\ClusterSettings` holds the defaults and bounds. It lives in `Core/Cluster`, which ships to LBs, not in `Domain/Cluster`, which the plan strips from LB builds, so nodes can apply the same bounds. It clamps silently and refuses only where clamping would change intent: + +- a taken or out-of-range port; +- a host name that is an IP; +- `https_required` without working HTTPS; +- enabling the API without the extension; +- `api` mode before the cutover. + +### Extension updates + +`console.php xcvm_core` rolls back an update whose cluster API falls outside the panel's range when the installed one was inside it. `console.php xcvm_core status` reports what is loaded. Installing an exact pinned version needs versioned paths in the binaries repo; that prerequisite is still open. + +## Consequences + +- Changing any formula in Canonical changes `cluster_canonical_vectors.json`. That is a protocol change: raise `proto` and keep accepting N−1, per the plan's mixed-version rules. +- A new extension API version needs `API_MAX` raised, and new vectors copied in, in the same panel release. +- The crypto pipeline measures about 0.25 ms p99 for a 64 KB request against the plan's 1 ms budget. It measures about 41 ms for 8 MB in the CI container against the plan's 40 ms target, because the body is hashed twice and encrypted twice. `ClusterCryptoBenchTest` guards 8 MB at 2× the target; the target itself needs a check on bundled PHP and production hardware. +- `ClusterExtensionIntegrationTest` runs the panel against a real test-hooks build of `xcvm_core` (opt-in, throwaway `XCVM_CONFIG_DIR`). It passed against the 2.2.2 build at the time of writing. diff --git a/src/Cli/Commands/XcvmCoreCommand.php b/src/Cli/Commands/XcvmCoreCommand.php index 6b4a2deb..34e0e420 100644 --- a/src/Cli/Commands/XcvmCoreCommand.php +++ b/src/Cli/Commands/XcvmCoreCommand.php @@ -3,6 +3,7 @@ namespace XcVm\Cli\Commands; use XcVm\Cli\CommandInterface; +use XcVm\Core\Cluster\Crypto\ClusterCryptoFactory; use XcVm\Core\Updates\UpdateChannels; /** @@ -30,7 +31,14 @@ use XcVm\Core\Updates\UpdateChannels; * Unlike the daemon (release assets), the extension lives in the repo *tree*, so * it is fetched raw from the default branch (rolling latest at a fixed path). * - * Usage: `console.php xcvm_core` (add `force` to reinstall the same version). + * Cluster API pin: an update never replaces an extension whose cluster API + * this panel speaks (ClusterCryptoFactory::API_MIN..API_MAX) with one whose API + * it does not. The new `.so` is load-tested for that too, and rolled back. + * Versioned download paths, to install exactly a pinned version, are a + * prerequisite still open on the binaries repo. + * + * Usage: `console.php xcvm_core` (add `force` to reinstall the same version), + * `console.php xcvm_core status` (what is loaded, and its cluster API state). * * @package XC_VM_CLI_Commands * @author Divarion_D @@ -54,6 +62,9 @@ class XcvmCoreCommand implements CommandInterface { } public function execute(array $rArgs): int { + if (in_array('status', $rArgs, true)) { + return $this->status(); + } if (posix_getpwuid(posix_geteuid())['name'] !== 'root') { echo "Please run as root!\n"; return 1; @@ -202,8 +213,10 @@ class XcvmCoreCommand implements CommandInterface { // Load-test the freshly-installed .so in a fresh php process (it loads the // extension from php.ini). A wrong-ABI/broken .so fails HERE, before we - // touch the running php-fpm workers — so we can roll back cleanly. - if (!$this->loads()) { + // touch the running php-fpm workers — so we can roll back cleanly. So does + // one that would drop the cluster API this panel speaks. + $rApiBefore = $this->clusterApi(); + if (!$this->loads() || !self::clusterApiKept($rApiBefore, $this->clusterApi())) { if ($rHadOld && is_file($rBackup)) { @rename($rBackup, $rSo); // rollback } else { @@ -220,6 +233,37 @@ class XcvmCoreCommand implements CommandInterface { } /** True if a fresh php (as xc_vm) loads xcvm_core and it is the new API build. */ + /** + * Would swapping an extension with cluster API $rBefore for one with $rAfter + * lose the API this panel speaks? (0 = no cluster API.) Keeping an + * out-of-range or absent API as it was is allowed: nothing worked before. + */ + public static function clusterApiKept(int $rBefore, int $rAfter): bool { + $rInRange = static fn(int $rApi) => $rApi >= ClusterCryptoFactory::API_MIN && $rApi <= ClusterCryptoFactory::API_MAX; + return !$rInRange($rBefore) || $rInRange($rAfter); + } + + /** The cluster API version of the extension a fresh php loads (0 = none). */ + private function clusterApi(): int { + $rBin = defined('PHP_BIN') ? PHP_BIN : (BIN_PATH . 'php/bin/php'); + $rCheck = 'echo (class_exists("XC_VM") && method_exists("XC_VM","cluster_info")) ? intval(XC_VM::cluster_info()["api"] ?? 0) : 0;'; + return intval(trim((string) shell_exec('sudo -u xc_vm ' . escapeshellarg($rBin) . ' -r ' . escapeshellarg($rCheck) . ' 2>/dev/null'))); + } + + /** `console.php xcvm_core status`: the loaded extension and its cluster API. */ + private function status(): int { + $rStatus = ClusterCryptoFactory::status(); + echo 'xcvm_core: ' . (phpversion('xcvm_core') ?: 'not loaded') . "\n"; + echo 'cluster API: ' . ($rStatus['api'] ?? 'none') . ' (panel speaks ' . $rStatus['range'] . ') -> ' . ($rStatus['available'] ? 'usable' : 'unavailable: ' . $rStatus['reason']) . "\n"; + if ($rStatus['available']) { + $rInfo = \XC_VM::cluster_info(); + echo 'root: ' . (!empty($rInfo['initialised']) ? 'initialised, fingerprint ' . bin2hex((string) ($rInfo['panel_fp'] ?? '')) : 'not initialised (' . ($rInfo['root_error'] ?? '') . ')') . "\n"; + echo 'licence binding: ' . (!empty($rInfo['licensed']) ? 'kid ' . $rInfo['kid'] : 'none (token issue refused)') . "\n"; + echo 'clock: ' . (!empty($rInfo['clock_ok']) ? 'ok' : 'ROLLED BACK - cluster calls refuse') . "\n"; + } + return $rStatus['available'] ? 0 : 2; + } + private function loads(): bool { $rBin = defined('PHP_BIN') ? PHP_BIN : (BIN_PATH . 'php/bin/php'); $rCheck = 'echo (extension_loaded("xcvm_core") && method_exists("XC_VM","config_set_redis")) ? "OK" : "NO";'; diff --git a/tests/Unit/ClusterExtensionIntegrationTest.php b/tests/Unit/ClusterExtensionIntegrationTest.php new file mode 100644 index 00000000..03acea8f --- /dev/null +++ b/tests/Unit/ClusterExtensionIntegrationTest.php @@ -0,0 +1,109 @@ +markTestSkipped('xcvm_core with the cluster API is not loaded'); + } + if ($rDir === '' || !str_starts_with(realpath($rDir) ?: '', realpath(sys_get_temp_dir()) ?: '/tmp') || file_exists($rDir . '/config.enc')) { + $this->markTestSkipped('XCVM_CONFIG_DIR must be a throwaway directory under the temp dir'); + } + $this->rDir = $rDir; + $rPair = sodium_crypto_sign_keypair(); + $this->rVerdictSk = sodium_crypto_sign_secretkey($rPair); + putenv('XCVM_TEST_VERDICT_PK_HEX=' . bin2hex(sodium_crypto_sign_publickey($rPair))); + putenv('XCVM_TEST_CLUSTER_LIC_TTL=0'); + $rPayload = json_encode(['v' => 1, 'jti' => bin2hex(random_bytes(16)), 'hwid' => \XC_VM::install_id(), 'exp' => null, 'kid' => 1, 'iat' => time()], JSON_UNESCAPED_SLASHES); + $rB64 = static fn(string $b) => rtrim(strtr(base64_encode($b), '+/', '-_'), '='); + file_put_contents($this->rDir . '/activation_key', 'XCVM1.' . $rB64($rPayload) . '.' . $rB64(sodium_crypto_sign_detached($rPayload, $this->rVerdictSk))); + if (!\XC_VM::license_valid()) { + $this->markTestSkipped('not a test-hooks build (the fake licence was not accepted)'); + } + } + + protected function tearDown(): void { + putenv('XCVM_TEST_VERDICT_PK_HEX'); + putenv('XCVM_TEST_CLUSTER_LIC_TTL'); + if (isset($this->rDir)) { + @unlink($this->rDir . '/activation_key'); + } + } + + public function testPanelAndExtensionAgreeEndToEnd(): void { + $rCrypto = ClusterCryptoFactory::create(); + $rRoot = $rCrypto->init(); + $this->assertSame(32, strlen($rRoot['panel_sign_pub'])); + $this->assertSame(1, $rCrypto->info()['api']); + + // The agent: static node key, per-epoch ephemeral key. + $rNodePair = sodium_crypto_sign_keypair(); + $rEphSk = random_bytes(32); + $rUuid = '0f8fad5b-d9cb-469f-a165-70867728950e'; + $rIssued = $rCrypto->tokenIssue([ + 'node_uuid' => $rUuid, 'server_id' => 7, 'gen' => 1, 'epoch' => 1, 'rotation_min' => 60, + 'node_sign_pub' => sodium_crypto_sign_publickey($rNodePair), 'agent_eph_pub' => sodium_crypto_scalarmult_base($rEphSk), + ]); + $this->assertSame(15, $rIssued['grace_min']); + + // Agent side: open with the ephemeral key, verify the panel signature, derive. + $rBody = Seal::open($rEphSk, 'token', $rUuid, $rIssued['token_sealed']); + $this->assertNotNull($rBody); + $rLen = unpack('N', substr($rBody, 0, 4))[1]; + $rDoc = substr($rBody, 4, $rLen); + $this->assertTrue(PanelSig::verify($rRoot['panel_sign_pub'], 'tok', $rDoc, substr($rBody, 4 + $rLen))); + $rAgentKeys = Ref::sessionKeys(hex2bin(json_decode($rDoc, true)['token'])); + + // MAIN side: the session of the stored epoch record. + $rSession = $rCrypto->session($rIssued['epoch_record'], $rUuid); + $this->assertSame($rAgentKeys['mac_up'], $rSession->rMacUp); + $this->assertSame($rAgentKeys['enc_down'], $rSession->rEncDown); + + // A request up and a reply down, with the panel's canonical MAC. + $rCtx = Canonical::request(['proto' => 1, 'agent' => 'test', 'method' => 'POST', 'path' => '/cluster/v1/heartbeat', 'query' => '', + 'content_type' => 'application/octet-stream', 'content_encoding' => '', 'node' => $rUuid, 'epoch' => 1, 'ts_ms' => time() * 1000, 'nonce' => random_bytes(16)]); + $rUp = Box::box($rAgentKeys['enc_up'], $rCtx, '{"hb":1}'); + $this->assertTrue(Canonical::verifyMac($rSession->rMacUp, $rCtx, $rUp, Canonical::mac($rAgentKeys['mac_up'], $rCtx, $rUp))); + $this->assertSame('{"hb":1}', Box::open($rSession->rEncUp, $rCtx, $rUp)); + $rDown = Box::box($rSession->rEncDown, $rCtx, '{"ok":true}'); + $this->assertSame('{"ok":true}', Box::open($rAgentKeys['enc_down'], $rCtx, $rDown)); + + // A record sealed for this node does not open as another node. + $this->expectException(ClusterRefusedException::class); + $rCrypto->session($rIssued['epoch_record'], '11111111-2222-4333-8444-555555555555'); + } + + public function testSignedRecordsVerifyWithThePanelClass(): void { + $rCrypto = ClusterCryptoFactory::create(); + $rPub = $rCrypto->init()['panel_sign_pub']; + $rCmd = json_encode(['v' => 1, 'type' => 'conn.drop', 'exp' => time() + 300, 'iat' => time(), 'cmd_id' => str_repeat('a', 32), 'seq' => 1, + 'node_uuid' => '0f8fad5b-d9cb-469f-a165-70867728950e', 'gen' => 1, 'dedupe_key' => null, 'args' => ['uuid' => 'x']]); + $this->assertSame('R', $rCrypto->recordClass('cmd', $rCmd)); + $this->assertTrue(PanelSig::verify($rPub, 'cmd', $rCmd, $rCrypto->sign('cmd', $rCmd))); + $this->assertFalse(PanelSig::verify($rPub, 'blk', $rCmd, $rCrypto->sign('cmd', $rCmd))); + } +} diff --git a/tests/Unit/XcvmCoreClusterPinTest.php b/tests/Unit/XcvmCoreClusterPinTest.php new file mode 100644 index 00000000..8871ef46 --- /dev/null +++ b/tests/Unit/XcvmCoreClusterPinTest.php @@ -0,0 +1,23 @@ +assertTrue(XcvmCoreCommand::clusterApiKept(1, 1)); + $this->assertFalse(XcvmCoreCommand::clusterApiKept(1, 0), 'an extension without the cluster API'); + $this->assertFalse(XcvmCoreCommand::clusterApiKept(1, 2), 'an API this panel does not speak yet'); + } + + public function testNothingToLoseWhenNothingWorkedBefore(): void { + $this->assertTrue(XcvmCoreCommand::clusterApiKept(0, 0)); + $this->assertTrue(XcvmCoreCommand::clusterApiKept(0, 1)); + $this->assertTrue(XcvmCoreCommand::clusterApiKept(2, 0)); + } +}