Fix Core update health checks and rollback

core/updates/tests/health.test.js

@@ -0,0 +1,35 @@
+'use strict';
+const test = require('node:test'), assert = require('node:assert/strict');
+const fs = require('node:fs'), path = require('node:path'), os = require('node:os'), http = require('node:http');
+const { coreHooks } = require('../../../host/releases/core');
+const { requestHealth } = require('../../../host/releases/health');
+const { atomic } = require('../../installations/src/release-delivery-files');
+async function server(t, handler) {
+  const server = http.createServer(handler);
+  await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
+  t.after(() => { server.closeAllConnections(); return new Promise(resolve => server.close(resolve)); });
+  return server.address().port;
+}
+test('Core loopback health preserves public Host, HTTPS forwarding and recovery nonce', async t => {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-core-health-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  fs.mkdirSync(path.join(root, 'config'), { mode: 0o700 });
+  atomic(path.join(root, 'config/dashboard.json'), { version: 1, port: 4310, publicOrigin: 'https://dispatch.example.test' });
+  const digest = 'a'.repeat(64), nonce = 'b'.repeat(64); let requests = 0;
+  const port = await server(t, (request, response) => {
+    requests++;
+    const allowed = request.headers.host === 'dispatch.example.test' && request.headers['cf-visitor'] === '{"scheme":"https"}'
+      && request.headers['x-dispatch-recovery-probe'] === nonce;
+    response.writeHead(allowed ? 200 : 403, { 'Content-Type': 'application/json' });
+    response.end(JSON.stringify({ ok: allowed, data: { digest, version: '0.0.2', recoveryProbe: 'passed' } }));
+  });
+  const hooks = coreHooks({ paths: { local: root, platformRoot: root }, configuration: { apiPort: port }, releases: () => ({}), healthTimeoutMs: 1000 });
+  assert.equal(await hooks.verify({ digest, manifest: { version: '0.0.2' }, preparation: { nonce } }), true);
+  assert.equal(requests, 1);
+});
+test('Core health rejects an oversized response and respects cancellation', async t => {
+  const oversized = await server(t, (_request, response) => response.end('x'.repeat(4097)));
+  await assert.rejects(requestHealth(`http://127.0.0.1:${oversized}/`, { signal: AbortSignal.timeout(1000) }), /release_health_response_invalid/);
+  const hung = await server(t, () => {});
+  await assert.rejects(requestHealth(`http://127.0.0.1:${hung}/`, { signal: AbortSignal.timeout(25) }), { name: 'AbortError' });
+});

core/updates/tests/lifecycle.test.js

@@ -32,14 +32,20 @@ function fixture(t) {
   }
   return { root, paths, database, artifact };
 }
-async function coreFixture(t) {
+async function coreFixture(t, { missingBackend = false, stopFailure = false } = {}) {
   const f = fixture(t), before = f.artifact('core', '1.0.0'), next = f.artifact('core', '1.1.0');
   fs.rmdirSync(f.paths.live); secureCopy(path.join(before.directory, 'code'), f.paths.live);
   let releases, fail = false;
   const events = [];
   const hooks = coreHooks({ paths: f.paths, configuration: { apiPort: 4999 }, releases: () => releases.state(), healthTimeoutMs: 5,
     systemctl: async args => {
       events.push(args);
+      if (args[0] === 'stop' && stopFailure) throw new Error('service_stop_failed');
+      if (missingBackend && args[1].startsWith('dispatch-backend-')) {
+        if (args[0] === 'stop') throw new Error('unit_not_loaded');
+        if (args.includes('LoadState')) return 'LoadState=not-found\n';
+      }
+      if (args.includes('LoadState')) return 'LoadState=loaded\n';
       if (args[0] === 'show') return 'ActiveState=inactive\nMainPID=0\nControlPID=0\n';
       if (args[0] === 'start' && args[1] === 'dispatch-api.service') {
         const current = JSON.parse(fs.readFileSync(receiptFile(f.paths)));
@@ -78,6 +84,19 @@ test('Core failed health restores the previous code and compatible database sche
   assert.equal(fs.existsSync(path.join(f.paths.local, 'state/new-file.json')), false);
   assert.equal(f.releases.state().active.core, f.before.digest); assert.equal(f.releases.state().operation, null);
 });
+test('Core rollback accepts an already collected backend service', async t => {
+  const f = await coreFixture(t, { missingBackend: true }); f.fail();
+  await assert.rejects(f.releases.updateCore(f.next.digest), /release_health_failed/);
+  assert.equal(f.releases.state().active.core, f.before.digest);
+  assert.equal(f.releases.state().operation, null);
+  assert.match(fs.readFileSync(path.join(f.paths.live, 'value.js'), 'utf8'), /1.0.0/);
+});
+test('Core stop failures for a loaded service still prevent the swap', async t => {
+  const f = await coreFixture(t, { stopFailure: true });
+  await assert.rejects(f.releases.updateCore(f.next.digest), /release_recovery_required/);
+  assert.match(fs.readFileSync(path.join(f.paths.live, 'value.js'), 'utf8'), /1.0.0/);
+  assert.equal(f.releases.state().operation.phase, 'failed');
+});
 test('Core recovery repairs a crash between the two live-directory renames', async t => {
   const f = await coreFixture(t);
   const c = { product: 'core', digest: f.next.digest, previousDigest: f.before.digest, directory: f.next.directory, manifest: f.next.manifest };

host/releases/core.js

@@ -12,7 +12,7 @@ function verifyLive(paths, manifest) {
   const expected = manifest.files.filter(item => item.path.startsWith('code/')).map(item => ({ ...item, path: item.path.slice(5) }));
   if (JSON.stringify(inventory(paths.live)) !== JSON.stringify(expected)) throw new Error('release_core_baseline_changed');
 }
-function coreHooks({ paths, configuration, releases, systemctl = async args => privileged(['/usr/bin/systemctl', ...args], { timeout: 300000 }), fetchImpl = fetch, healthTimeoutMs = 90000 }) {
+function coreHooks({ paths, configuration, releases, systemctl = async args => privileged(['/usr/bin/systemctl', ...args], { timeout: 300000 }), fetchImpl = require('./health').requestHealth, healthTimeoutMs = 90000 }) {
   const root = privateDirectory(path.join(paths.local, 'backups/updates/core'));
   let controllerLock, operationLock;
   const unlock = () => {
@@ -32,7 +32,15 @@ function coreHooks({ paths, configuration, releases, systemctl = async args => p
     return path.join(root, token.id);
   };
   const stop = async () => {
-    for (const unit of units) await systemctl(['stop', unit]);
+    for (const unit of units) {
+      try { await systemctl(['stop', unit]); }
+      catch (error) {
+        // A collected backend unit may already be gone after a failed API
+        // startup. Other stop failures must still block the code/state swap.
+        const status = await systemctl(['show', unit, '-p', 'LoadState']);
+        if (status.trim() !== 'LoadState=not-found') throw error;
+      }
+    }
     for (const unit of units) {
       const status = await systemctl(['show', unit, '-p', 'ActiveState', '-p', 'MainPID', '-p', 'ControlPID']);
       const fields = Object.fromEntries(status.trim().split('\n').map(line => line.split('=')));

host/releases/health.js

@@ -0,0 +1,27 @@
+'use strict';
+const http = require('node:http');
+
+// Core's loopback listener validates the public Host header. Node fetch can
+// replace that header with the loopback address, so use the HTTP transport.
+function requestHealth(url, { headers, signal }) {
+  return new Promise((resolve, reject) => {
+    const request = http.get(url, { headers, signal }, response => {
+      const chunks = []; let bytes = 0;
+      response.on('data', chunk => {
+        bytes += chunk.length;
+        if (bytes > 4096) {
+          const error = new Error('release_health_response_invalid');
+          reject(error); response.destroy(); request.destroy();
+        }
+        else chunks.push(chunk);
+      });
+      response.on('error', reject);
+      response.on('end', () => resolve({
+        ok: response.statusCode === 200,
+        json: async () => JSON.parse(Buffer.concat(chunks).toString('utf8')),
+      }));
+    });
+    request.on('error', reject);
+  });
+}
+module.exports = { requestHealth };