diff --git a/core/updates/tests/health.test.js b/core/updates/tests/health.test.js new file mode 100644 index 0000000..572122e --- /dev/null +++ b/core/updates/tests/health.test.js @@ -0,0 +1,35 @@ +'use strict'; +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'), http = require('node:http'); +const { coreHooks } = require('../../../host/releases/core'); +const { requestHealth } = require('../../../host/releases/health'); +const { atomic } = require('../../installations/src/release-delivery-files'); +async function server(t, handler) { + const server = http.createServer(handler); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => { server.closeAllConnections(); return new Promise(resolve => server.close(resolve)); }); + return server.address().port; +} +test('Core loopback health preserves public Host, HTTPS forwarding and recovery nonce', async t => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'dispatch-core-health-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + fs.mkdirSync(path.join(root, 'config'), { mode: 0o700 }); + atomic(path.join(root, 'config/dashboard.json'), { version: 1, port: 4310, publicOrigin: 'https://dispatch.example.test' }); + const digest = 'a'.repeat(64), nonce = 'b'.repeat(64); let requests = 0; + const port = await server(t, (request, response) => { + requests++; + const allowed = request.headers.host === 'dispatch.example.test' && request.headers['cf-visitor'] === '{"scheme":"https"}' + && request.headers['x-dispatch-recovery-probe'] === nonce; + response.writeHead(allowed ? 200 : 403, { 'Content-Type': 'application/json' }); + response.end(JSON.stringify({ ok: allowed, data: { digest, version: '0.0.2', recoveryProbe: 'passed' } })); + }); + const hooks = coreHooks({ paths: { local: root, platformRoot: root }, configuration: { apiPort: port }, releases: () => ({}), healthTimeoutMs: 1000 }); + assert.equal(await hooks.verify({ digest, manifest: { version: '0.0.2' }, preparation: { nonce } }), true); + assert.equal(requests, 1); +}); +test('Core health rejects an oversized response and respects cancellation', async t => { + const oversized = await server(t, (_request, response) => response.end('x'.repeat(4097))); + await assert.rejects(requestHealth(`http://127.0.0.1:${oversized}/`, { signal: AbortSignal.timeout(1000) }), /release_health_response_invalid/); + const hung = await server(t, () => {}); + await assert.rejects(requestHealth(`http://127.0.0.1:${hung}/`, { signal: AbortSignal.timeout(25) }), { name: 'AbortError' }); +}); diff --git a/core/updates/tests/lifecycle.test.js b/core/updates/tests/lifecycle.test.js index d410388..5cdba9a 100644 --- a/core/updates/tests/lifecycle.test.js +++ b/core/updates/tests/lifecycle.test.js @@ -32,7 +32,7 @@ function fixture(t) { } return { root, paths, database, artifact }; } -async function coreFixture(t) { +async function coreFixture(t, { missingBackend = false, stopFailure = false } = {}) { const f = fixture(t), before = f.artifact('core', '1.0.0'), next = f.artifact('core', '1.1.0'); fs.rmdirSync(f.paths.live); secureCopy(path.join(before.directory, 'code'), f.paths.live); let releases, fail = false; @@ -40,6 +40,12 @@ async function coreFixture(t) { const hooks = coreHooks({ paths: f.paths, configuration: { apiPort: 4999 }, releases: () => releases.state(), healthTimeoutMs: 5, systemctl: async args => { events.push(args); + if (args[0] === 'stop' && stopFailure) throw new Error('service_stop_failed'); + if (missingBackend && args[1].startsWith('dispatch-backend-')) { + if (args[0] === 'stop') throw new Error('unit_not_loaded'); + if (args.includes('LoadState')) return 'LoadState=not-found\n'; + } + if (args.includes('LoadState')) return 'LoadState=loaded\n'; if (args[0] === 'show') return 'ActiveState=inactive\nMainPID=0\nControlPID=0\n'; if (args[0] === 'start' && args[1] === 'dispatch-api.service') { const current = JSON.parse(fs.readFileSync(receiptFile(f.paths))); @@ -78,6 +84,19 @@ test('Core failed health restores the previous code and compatible database sche assert.equal(fs.existsSync(path.join(f.paths.local, 'state/new-file.json')), false); assert.equal(f.releases.state().active.core, f.before.digest); assert.equal(f.releases.state().operation, null); }); +test('Core rollback accepts an already collected backend service', async t => { + const f = await coreFixture(t, { missingBackend: true }); f.fail(); + await assert.rejects(f.releases.updateCore(f.next.digest), /release_health_failed/); + assert.equal(f.releases.state().active.core, f.before.digest); + assert.equal(f.releases.state().operation, null); + assert.match(fs.readFileSync(path.join(f.paths.live, 'value.js'), 'utf8'), /1.0.0/); +}); +test('Core stop failures for a loaded service still prevent the swap', async t => { + const f = await coreFixture(t, { stopFailure: true }); + await assert.rejects(f.releases.updateCore(f.next.digest), /release_recovery_required/); + assert.match(fs.readFileSync(path.join(f.paths.live, 'value.js'), 'utf8'), /1.0.0/); + assert.equal(f.releases.state().operation.phase, 'failed'); +}); test('Core recovery repairs a crash between the two live-directory renames', async t => { const f = await coreFixture(t); const c = { product: 'core', digest: f.next.digest, previousDigest: f.before.digest, directory: f.next.directory, manifest: f.next.manifest }; diff --git a/host/releases/core.js b/host/releases/core.js index 4f579a2..c983660 100644 --- a/host/releases/core.js +++ b/host/releases/core.js @@ -12,7 +12,7 @@ function verifyLive(paths, manifest) { const expected = manifest.files.filter(item => item.path.startsWith('code/')).map(item => ({ ...item, path: item.path.slice(5) })); if (JSON.stringify(inventory(paths.live)) !== JSON.stringify(expected)) throw new Error('release_core_baseline_changed'); } -function coreHooks({ paths, configuration, releases, systemctl = async args => privileged(['/usr/bin/systemctl', ...args], { timeout: 300000 }), fetchImpl = fetch, healthTimeoutMs = 90000 }) { +function coreHooks({ paths, configuration, releases, systemctl = async args => privileged(['/usr/bin/systemctl', ...args], { timeout: 300000 }), fetchImpl = require('./health').requestHealth, healthTimeoutMs = 90000 }) { const root = privateDirectory(path.join(paths.local, 'backups/updates/core')); let controllerLock, operationLock; const unlock = () => { @@ -32,7 +32,15 @@ function coreHooks({ paths, configuration, releases, systemctl = async args => p return path.join(root, token.id); }; const stop = async () => { - for (const unit of units) await systemctl(['stop', unit]); + for (const unit of units) { + try { await systemctl(['stop', unit]); } + catch (error) { + // A collected backend unit may already be gone after a failed API + // startup. Other stop failures must still block the code/state swap. + const status = await systemctl(['show', unit, '-p', 'LoadState']); + if (status.trim() !== 'LoadState=not-found') throw error; + } + } for (const unit of units) { const status = await systemctl(['show', unit, '-p', 'ActiveState', '-p', 'MainPID', '-p', 'ControlPID']); const fields = Object.fromEntries(status.trim().split('\n').map(line => line.split('='))); diff --git a/host/releases/health.js b/host/releases/health.js new file mode 100644 index 0000000..dbf3a35 --- /dev/null +++ b/host/releases/health.js @@ -0,0 +1,27 @@ +'use strict'; +const http = require('node:http'); + +// Core's loopback listener validates the public Host header. Node fetch can +// replace that header with the loopback address, so use the HTTP transport. +function requestHealth(url, { headers, signal }) { + return new Promise((resolve, reject) => { + const request = http.get(url, { headers, signal }, response => { + const chunks = []; let bytes = 0; + response.on('data', chunk => { + bytes += chunk.length; + if (bytes > 4096) { + const error = new Error('release_health_response_invalid'); + reject(error); response.destroy(); request.destroy(); + } + else chunks.push(chunk); + }); + response.on('error', reject); + response.on('end', () => resolve({ + ok: response.statusCode === 200, + json: async () => JSON.parse(Buffer.concat(chunks).toString('utf8')), + })); + }); + request.on('error', reject); + }); +} +module.exports = { requestHealth };