From b6e51caca11a94ce26a8ee07b397c057ebc212e4 Mon Sep 17 00:00:00 2001 From: Dillon <260170482+dillonlille@users.noreply.github.com> Date: Sun, 13 Sep 2026 04:36:42 +0000 Subject: [PATCH] Recover verified SQLite sidecars during DSP backup erasure --- dashboard/tests/directory-platform.test.js | 45 +++++++++ host/storage/backup-erasure-verification.js | 55 +++++++++++ host/storage/erase-backup-job.js | 16 +++- host/storage/erase-backups.js | 3 +- .../tests/backup-erasure-verification.test.js | 94 +++++++++++++++++++ tooling/tests.json | 1 + 6 files changed, 208 insertions(+), 6 deletions(-) create mode 100644 host/storage/backup-erasure-verification.js create mode 100644 host/storage/tests/backup-erasure-verification.test.js diff --git a/dashboard/tests/directory-platform.test.js b/dashboard/tests/directory-platform.test.js index 2d8494b..bec798c 100644 --- a/dashboard/tests/directory-platform.test.js +++ b/dashboard/tests/directory-platform.test.js @@ -123,12 +123,29 @@ test('deletion scrubs shared platform backups, removes DSP backups, and blocks o fs.writeFileSync(path.join(c.paths.local, 'config/platform.json'), JSON.stringify({ version: 1, platformRoot: c.paths.platformRoot }), { mode: 0o600 }); fs.writeFileSync(path.join(c.paths.dsps, c.target.runtime_key, 'data/target.txt'), 'synthetic target contents', { mode: 0o600 }); fs.writeFileSync(path.join(c.paths.dsps, c.neighbor.runtime_key, 'data/neighbor.txt'), 'synthetic neighbor contents', { mode: 0o600 }); + const { DatabaseSync } = require('node:sqlite'); + for (const row of [c.target, c.neighbor]) { + const file = path.join(c.paths.dsps, row.runtime_key, 'data/snapshot.sqlite3'); + const db = new DatabaseSync(file); + db.exec("PRAGMA journal_mode=WAL; CREATE TABLE entries(value TEXT); INSERT INTO entries VALUES('synthetic retained data')"); + db.close(); fs.chmodSync(file, 0o600); + } backups.volumes = { ensure: async () => ({ limited: false }) }; const records = c.app.runtime.manager.journal.all(); const platformId = 'mbk_' + '1'.repeat(32), dspId = 'mbk_' + '2'.repeat(32); await backups.create({ scope: 'platform', organizationId: null, records, backupId: platformId }, null); await backups.create({ scope: 'dsp', organizationId: c.target.organization_id, records: [records.find(r => r.id === c.target.runtime_key)], backupId: dspId }, null); const original = fs.readFileSync(path.join(backups.root, platformId, 'manifest.json')); + for (const row of [c.target, c.neighbor]) { + const file = path.join(backups.root, platformId, 'payload', row.runtime_key + '_data/snapshot.sqlite3'); + const reader = new DatabaseSync(file, { readOnly: true }); + reader.prepare('SELECT value FROM entries').get(); reader.close(); + for (const suffix of ['-wal', '-shm']) { + fs.chmodSync(file + suffix, 0o600); + const later = (JSON.parse(original).createdAt + 1000) / 1000; + fs.utimesSync(file + suffix, later, later); + } + } await c.app.access.requestPlatformRemoval(c.login.session, c.command(), 'destroy'); assert.throws(() => backups.inspect(platformId), /directory_deletion_in_progress/); await c.app.deletions.runPending(); @@ -137,6 +154,7 @@ test('deletion scrubs shared platform backups, removes DSP backups, and blocks o const retained = backups.inspect(platformId); assert.deepEqual(retained.dsps.map(dsp => dsp.id), [c.neighbor.runtime_key]); assert.equal(fs.readFileSync(path.join(backups.root, platformId, 'payload', c.neighbor.runtime_key + '_data/neighbor.txt'), 'utf8'), 'synthetic neighbor contents'); + assert.equal(fs.existsSync(path.join(backups.root, platformId, 'payload', c.neighbor.runtime_key + '_data/snapshot.sqlite3-shm')), false); const saved = new (require('node:sqlite').DatabaseSync)(path.join(backups.root, platformId, 'payload/core/access-control.sqlite3'), { readOnly: true }); assert.equal(saved.prepare('SELECT 1 FROM organizations WHERE id=?').get(c.target.organization_id), undefined); assert.ok(saved.prepare('SELECT 1 FROM organizations WHERE id=?').get(c.neighbor.organization_id)); saved.close(); @@ -144,6 +162,33 @@ test('deletion scrubs shared platform backups, removes DSP backups, and blocks o assert.throws(() => backups.inspect(platformId), /directory_dsp_deleted/); }); +test('backup integrity failures retain their code and deletion retries after the original data is restored', async t => { + const c = await deletionFixture(t), backups = c.app.backups; + await c.remove(c.target); + const data = path.join(c.paths.dsps, c.target.runtime_key, 'data/target.txt'); + fs.writeFileSync(data, 'synthetic original data', { mode: 0o600 }); + backups.volumes = { ensure: async () => ({ limited: false }) }; + const backupId = 'mbk_' + '3'.repeat(32); + await backups.create({ scope: 'dsp', organizationId: c.target.organization_id, + records: [c.app.runtime.manager.journal.record(c.target.runtime_key)], backupId }, null); + const saved = path.join(backups.root, backupId, 'payload', c.target.runtime_key + '_data/target.txt'); + fs.writeFileSync(saved, 'changed data'); + const input = c.command(); + await c.app.access.requestPlatformRemoval(c.login.session, input, 'destroy'); + await c.app.deletions.runPending(); + assert.equal(c.app.deletions.get(c.target.organization_id).phase, 'backups'); + assert.equal(c.app.deletions.get(c.target.organization_id).failureCode, 'directory_backup_changed'); + assert.equal(c.errors.at(-1).code, 'directory_backup_changed'); + assert.equal(fs.existsSync(data), true); + assert.ok(c.app.store.organization(c.target.organization_id)); + fs.writeFileSync(saved, 'synthetic original data'); + await c.app.access.requestPlatformRemoval(c.login.session, input, 'destroy'); + await c.app.deletions.runPending(); + assert.equal(c.app.deletions.get(c.target.organization_id).status, 'complete'); + assert.equal(fs.existsSync(path.join(backups.root, backupId)), false); + assert.ok(c.app.store.organization(c.neighbor.organization_id)); +}); + test('an interrupted deletion remains blocked from restore and resumes its recorded phase', async t => { const c = await deletionFixture(t); await c.remove(c.target); const erase = c.app.deletions.eraseFiles; let calls = 0; diff --git a/host/storage/backup-erasure-verification.js b/host/storage/backup-erasure-verification.js new file mode 100644 index 0000000..afa5f80 --- /dev/null +++ b/host/storage/backup-erasure-verification.js @@ -0,0 +1,55 @@ +'use strict'; + +const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto'); +const files = require('./backup-files'); +const { fail, syncDirectory } = require('../controller/operations'); + +function matches(scan, expected) { + return scan.treeDigest === expected.treeDigest && scan.totalBytes === expected.totalBytes; +} + +function walDatabase(file) { + const before = files.checked(file, false), fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + try { + const info = fs.fstatSync(fd), header = Buffer.alloc(20); + if (info.dev !== before.dev || info.ino !== before.ino) fail('directory_backup_changed'); + return fs.readSync(fd, header, 0, header.length, 0) === header.length + && header.subarray(0, 16).equals(Buffer.from('SQLite format 3\0')) && header[18] === 2 && header[19] === 2; + } finally { fs.closeSync(fd); } +} + +// The caller holds the host operation lock. Even a read-only SQLite connection +// can leave an empty WAL and shared-memory index in a sealed backup. Recover only +// additions whose removal reproduces the complete original manifest checksum. +// Restore validation remains strict; this recovery is specific to erasure. +function verifyForErasure(root, expected, createdAt) { + const scanned = files.scan(root); + if (matches(scanned, expected)) return; + if (!Number.isSafeInteger(createdAt) || createdAt < 0) fail('directory_backup_changed'); + const entries = new Map(scanned.entries.map(entry => [entry.path, entry])), additions = []; + for (const entry of scanned.entries) { + if (entry.type !== 'file' || !/\.(?:sqlite3?|db)-(?:wal|shm)$/.test(entry.path)) continue; + const database = entry.path.slice(0, -4), wal = entries.get(database + '-wal'); + if (entries.get(database)?.type !== 'file' || wal && (wal.type !== 'file' || wal.bytes !== 0)) continue; + if (entry.path.endsWith('-wal') ? entry.bytes !== 0 : entry.bytes !== 32768) continue; + // Exclude only files clearly newer than the snapshot. Timestamps select + // candidates; the original hash, including every retained file, is proof. + if (files.checked(path.join(root, entry.path), false).mtimeMs <= createdAt + 1) continue; + if (walDatabase(path.join(root, database))) additions.push(entry); + } + const removed = new Set(additions.map(entry => entry.path)); + const retained = scanned.entries.filter(entry => !removed.has(entry.path)); + const candidate = { + treeDigest: crypto.createHash('sha256').update(JSON.stringify(retained)).digest('hex'), + totalBytes: scanned.totalBytes - additions.reduce((bytes, entry) => bytes + entry.bytes, 0), + }; + if (!additions.length || !matches(candidate, expected) || !matches(files.scan(root), scanned)) fail('directory_backup_changed'); + for (const entry of additions) { + const file = path.join(root, entry.path), current = files.digest(file); + if (current.sha256 !== entry.sha256 || current.bytes !== entry.bytes) fail('directory_backup_changed'); + fs.unlinkSync(file); syncDirectory(path.dirname(file)); + } + if (!matches(files.scan(root), expected)) fail('directory_backup_changed'); +} + +module.exports = { verifyForErasure }; diff --git a/host/storage/erase-backup-job.js b/host/storage/erase-backup-job.js index 4cb9e9f..3721709 100644 --- a/host/storage/erase-backup-job.js +++ b/host/storage/erase-backup-job.js @@ -2,6 +2,11 @@ const path = require('node:path'); const { spawn } = require('node:child_process'); +function failure(code) { + const safe = /^directory_[a-z_]{1,80}$/.test(code || '') ? code : 'directory_backup_erasure_failed'; + return Object.assign(new Error(safe), { code: safe }); +} + // Hashing large retained snapshots runs in a finite process, keeping Core's // request loop responsive and releasing the scan's memory when it finishes. function eraseBackupJob(paths, job, lockFd) { @@ -14,11 +19,14 @@ function eraseBackupJob(paths, job, lockFd) { child.stdout.on('data', chunk => { bytes += chunk.length; if (bytes > 4096) child.kill('SIGKILL'); else output += chunk; }); child.stderr.on('data', chunk => { bytes += chunk.length; if (bytes > 4096) child.kill('SIGKILL'); }); child.stdin.on('error', () => {}); - child.on('error', () => { clearTimeout(timer); reject(Error('directory_backup_erasure_failed')); }); + child.on('error', () => { clearTimeout(timer); reject(failure()); }); child.on('close', code => { clearTimeout(timer); - try { if (code !== 0 || JSON.parse(output).ok !== true) throw Error(); resolve(); } - catch { reject(Error('directory_backup_erasure_failed')); } + try { + const result = JSON.parse(output); + if (code !== 0 || result.ok !== true) reject(failure(result.code)); + else resolve(); + } catch { reject(failure()); } }); child.stdin.end(JSON.stringify({ platformRoot: paths.platformRoot, id: job.id, organizationId: job.organizationId, runtimeKey: job.runtimeKey })); @@ -38,7 +46,7 @@ if (require.main === module) { const backups = new (require('./manual-backups').ManualBackups)({ paths, store }); require('./erase-backups').eraseBackups(backups, job); process.stdout.write('{"ok":true}\n'); - } catch { process.stdout.write('{"ok":false}\n'); process.exitCode = 1; } + } catch (error) { process.stdout.write(JSON.stringify({ ok: false, code: failure(error.code).code }) + '\n'); process.exitCode = 1; } finally { store?.close(); } })(); } diff --git a/host/storage/erase-backups.js b/host/storage/erase-backups.js index bb0b964..837e567 100644 --- a/host/storage/erase-backups.js +++ b/host/storage/erase-backups.js @@ -49,8 +49,7 @@ function eraseBackups(backups, job) { } if (!pending) { for (const entry of manifest.roots) { - const scan = files.scan(path.join(root, 'payload', entry.label)); - if (scan.treeDigest !== entry.treeDigest || scan.totalBytes !== entry.totalBytes) fail('directory_backup_changed'); + require('./backup-erasure-verification').verifyForErasure(path.join(root, 'payload', entry.label), entry, manifest.createdAt); } atomic(marker, { version: 1, jobId: job.id, manifest }); } diff --git a/host/storage/tests/backup-erasure-verification.test.js b/host/storage/tests/backup-erasure-verification.test.js new file mode 100644 index 0000000..02d0b94 --- /dev/null +++ b/host/storage/tests/backup-erasure-verification.test.js @@ -0,0 +1,94 @@ +'use strict'; + +const test = require('node:test'), assert = require('node:assert/strict'); +const fs = require('node:fs'), path = require('node:path'), os = require('node:os'); +const { DatabaseSync } = require('node:sqlite'); +const files = require('../backup-files'); +const { verifyForErasure } = require('../backup-erasure-verification'); + +function fixture(t, { retainedSidecars = false } = {}) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'backup-erasure-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const source = path.join(root, 'source.sqlite3'), payload = path.join(root, 'payload'); + fs.mkdirSync(payload, { mode: 0o700 }); + const db = new DatabaseSync(source); + db.exec("PRAGMA journal_mode=WAL; CREATE TABLE entries(value TEXT); INSERT INTO entries VALUES('synthetic contents')"); + db.close(); + const databases = ['first.sqlite3', 'second.sqlite3'].map(name => path.join(payload, name)); + for (const file of databases) { fs.copyFileSync(source, file); fs.chmodSync(file, 0o600); } + const open = file => { + const reader = new DatabaseSync(file, { readOnly: true }); + try { assert.equal(reader.prepare('SELECT value FROM entries').get().value, 'synthetic contents'); } + finally { reader.close(); } + for (const suffix of ['-wal', '-shm']) fs.chmodSync(file + suffix, 0o600); + }; + if (retainedSidecars) open(databases[1]); + const expected = files.scan(payload), createdAt = Date.now(); + if (retainedSidecars) for (const suffix of ['-wal', '-shm']) fs.utimesSync(databases[1] + suffix, (createdAt - 1000) / 1000, (createdAt - 1000) / 1000); + const add = file => { + open(file); + for (const suffix of ['-wal', '-shm']) fs.utimesSync(file + suffix, (createdAt + 1000) / 1000, (createdAt + 1000) / 1000); + }; + return { root, payload, databases, expected, createdAt, add, verify: () => verifyForErasure(payload, expected, createdAt) }; +} + +test('erasure recovers read-only SQLite additions while preserving originally backed-up sidecars', t => { + const c = fixture(t, { retainedSidecars: true }); c.add(c.databases[0]); + assert.notEqual(files.scan(c.payload).treeDigest, c.expected.treeDigest); + c.verify(); + assert.deepEqual(files.scan(c.payload), c.expected); + assert.equal(fs.existsSync(c.databases[0] + '-shm'), false); + assert.equal(fs.existsSync(c.databases[1] + '-shm'), true); + c.verify(); +}); + +test('erasure recovers multiple additions and resumes after a sidecar was already removed', t => { + const c = fixture(t); c.databases.forEach(c.add); + fs.unlinkSync(c.databases[0] + '-shm'); + fs.unlinkSync(c.databases[1] + '-wal'); + c.verify(); assert.deepEqual(files.scan(c.payload), c.expected); +}); + +test('changed database bytes or unrelated added files cannot be accepted as SQLite additions', t => { + for (const kind of ['database', 'extra']) { + const c = fixture(t); c.add(c.databases[0]); + if (kind === 'database') fs.appendFileSync(c.databases[1], 'unexpected data'); + else fs.writeFileSync(path.join(c.payload, 'unexpected.txt'), 'unexpected data', { mode: 0o600 }); + const before = files.scan(c.payload); + assert.throws(c.verify, { code: 'directory_backup_changed' }); + assert.deepEqual(files.scan(c.payload), before); + } +}); + +test('erasure never discards a nonempty WAL, an old sidecar, or an unrecognized database', t => { + for (const kind of ['nonempty', 'old', 'not-sqlite']) { + const c = fixture(t); c.add(c.databases[0]); + let expected = c.expected; + if (kind === 'nonempty') fs.writeFileSync(c.databases[0] + '-wal', 'uncheckpointed data'); + if (kind === 'old') for (const suffix of ['-shm', '-wal']) fs.utimesSync(c.databases[0] + suffix, 1, 1); + if (kind === 'not-sqlite') { + fs.writeFileSync(c.databases[0], 'ordinary file'); + const entries = files.scan(c.payload).entries.filter(entry => !entry.path.endsWith('-shm') && !entry.path.endsWith('-wal')); + expected = { treeDigest: require('node:crypto').createHash('sha256').update(JSON.stringify(entries)).digest('hex'), + totalBytes: entries.reduce((sum, entry) => sum + (entry.bytes || 0), 0) }; + } + const before = files.scan(c.payload); + assert.throws(() => verifyForErasure(c.payload, expected, c.createdAt), { code: 'directory_backup_changed' }); + assert.deepEqual(files.scan(c.payload), before); + } +}); + +test('erasure fails closed on sidecar links and files changed after the recovery scan', t => { + const linked = fixture(t); linked.add(linked.databases[0]); + fs.unlinkSync(linked.databases[0] + '-wal'); fs.symlinkSync(linked.databases[1], linked.databases[0] + '-wal'); + assert.throws(linked.verify, { code: 'directory_backup_unsafe' }); + const c = fixture(t); c.add(c.databases[0]); + const digest = files.digest; + t.mock.method(files, 'digest', file => { + if (file === c.databases[0] + '-shm') fs.appendFileSync(file, 'changed'); + return digest(file); + }); + assert.throws(c.verify, { code: 'directory_backup_changed' }); + assert.equal(fs.existsSync(c.databases[0] + '-shm'), true); + assert.equal(fs.existsSync(c.databases[0] + '-wal'), true); +}); diff --git a/tooling/tests.json b/tooling/tests.json index d2c23cf..e427712 100644 --- a/tooling/tests.json +++ b/tooling/tests.json @@ -14,6 +14,7 @@ "host/plugins/tests/lifecycle.test.js", "host/plugins/tests/sdk-socket.test.js", "host/plugins/tests/worker-layout.test.js", + "host/storage/tests/backup-erasure-verification.test.js", "core/accounts/tests/plugins.test.js", "core/agents/tests/execution.test.js" ],