Recover DSP deletion blocked by SQLite backup sidecars

dashboard/tests/directory-platform.test.js

@@ -123,12 +123,29 @@ test('deletion scrubs shared platform backups, removes DSP backups, and blocks o
   fs.writeFileSync(path.join(c.paths.local, 'config/platform.json'), JSON.stringify({ version: 1, platformRoot: c.paths.platformRoot }), { mode: 0o600 });
   fs.writeFileSync(path.join(c.paths.dsps, c.target.runtime_key, 'data/target.txt'), 'synthetic target contents', { mode: 0o600 });
   fs.writeFileSync(path.join(c.paths.dsps, c.neighbor.runtime_key, 'data/neighbor.txt'), 'synthetic neighbor contents', { mode: 0o600 });
+  const { DatabaseSync } = require('node:sqlite');
+  for (const row of [c.target, c.neighbor]) {
+    const file = path.join(c.paths.dsps, row.runtime_key, 'data/snapshot.sqlite3');
+    const db = new DatabaseSync(file);
+    db.exec("PRAGMA journal_mode=WAL; CREATE TABLE entries(value TEXT); INSERT INTO entries VALUES('synthetic retained data')");
+    db.close(); fs.chmodSync(file, 0o600);
+  }
   backups.volumes = { ensure: async () => ({ limited: false }) };
   const records = c.app.runtime.manager.journal.all();
   const platformId = 'mbk_' + '1'.repeat(32), dspId = 'mbk_' + '2'.repeat(32);
   await backups.create({ scope: 'platform', organizationId: null, records, backupId: platformId }, null);
   await backups.create({ scope: 'dsp', organizationId: c.target.organization_id, records: [records.find(r => r.id === c.target.runtime_key)], backupId: dspId }, null);
   const original = fs.readFileSync(path.join(backups.root, platformId, 'manifest.json'));
+  for (const row of [c.target, c.neighbor]) {
+    const file = path.join(backups.root, platformId, 'payload', row.runtime_key + '_data/snapshot.sqlite3');
+    const reader = new DatabaseSync(file, { readOnly: true });
+    reader.prepare('SELECT value FROM entries').get(); reader.close();
+    for (const suffix of ['-wal', '-shm']) {
+      fs.chmodSync(file + suffix, 0o600);
+      const later = (JSON.parse(original).createdAt + 1000) / 1000;
+      fs.utimesSync(file + suffix, later, later);
+    }
+  }
   await c.app.access.requestPlatformRemoval(c.login.session, c.command(), 'destroy');
   assert.throws(() => backups.inspect(platformId), /directory_deletion_in_progress/);
   await c.app.deletions.runPending();
@@ -137,13 +154,41 @@ test('deletion scrubs shared platform backups, removes DSP backups, and blocks o
   const retained = backups.inspect(platformId);
   assert.deepEqual(retained.dsps.map(dsp => dsp.id), [c.neighbor.runtime_key]);
   assert.equal(fs.readFileSync(path.join(backups.root, platformId, 'payload', c.neighbor.runtime_key + '_data/neighbor.txt'), 'utf8'), 'synthetic neighbor contents');
+  assert.equal(fs.existsSync(path.join(backups.root, platformId, 'payload', c.neighbor.runtime_key + '_data/snapshot.sqlite3-shm')), false);
   const saved = new (require('node:sqlite').DatabaseSync)(path.join(backups.root, platformId, 'payload/core/access-control.sqlite3'), { readOnly: true });
   assert.equal(saved.prepare('SELECT 1 FROM organizations WHERE id=?').get(c.target.organization_id), undefined);
   assert.ok(saved.prepare('SELECT 1 FROM organizations WHERE id=?').get(c.neighbor.organization_id)); saved.close();
   fs.writeFileSync(path.join(backups.root, platformId, 'manifest.json'), original);
   assert.throws(() => backups.inspect(platformId), /directory_dsp_deleted/);
 });
 
+test('backup integrity failures retain their code and deletion retries after the original data is restored', async t => {
+  const c = await deletionFixture(t), backups = c.app.backups;
+  await c.remove(c.target);
+  const data = path.join(c.paths.dsps, c.target.runtime_key, 'data/target.txt');
+  fs.writeFileSync(data, 'synthetic original data', { mode: 0o600 });
+  backups.volumes = { ensure: async () => ({ limited: false }) };
+  const backupId = 'mbk_' + '3'.repeat(32);
+  await backups.create({ scope: 'dsp', organizationId: c.target.organization_id,
+    records: [c.app.runtime.manager.journal.record(c.target.runtime_key)], backupId }, null);
+  const saved = path.join(backups.root, backupId, 'payload', c.target.runtime_key + '_data/target.txt');
+  fs.writeFileSync(saved, 'changed data');
+  const input = c.command();
+  await c.app.access.requestPlatformRemoval(c.login.session, input, 'destroy');
+  await c.app.deletions.runPending();
+  assert.equal(c.app.deletions.get(c.target.organization_id).phase, 'backups');
+  assert.equal(c.app.deletions.get(c.target.organization_id).failureCode, 'directory_backup_changed');
+  assert.equal(c.errors.at(-1).code, 'directory_backup_changed');
+  assert.equal(fs.existsSync(data), true);
+  assert.ok(c.app.store.organization(c.target.organization_id));
+  fs.writeFileSync(saved, 'synthetic original data');
+  await c.app.access.requestPlatformRemoval(c.login.session, input, 'destroy');
+  await c.app.deletions.runPending();
+  assert.equal(c.app.deletions.get(c.target.organization_id).status, 'complete');
+  assert.equal(fs.existsSync(path.join(backups.root, backupId)), false);
+  assert.ok(c.app.store.organization(c.neighbor.organization_id));
+});
+
 test('an interrupted deletion remains blocked from restore and resumes its recorded phase', async t => {
   const c = await deletionFixture(t); await c.remove(c.target);
   const erase = c.app.deletions.eraseFiles; let calls = 0;

host/storage/backup-erasure-verification.js

@@ -0,0 +1,55 @@
+'use strict';
+
+const fs = require('node:fs'), path = require('node:path'), crypto = require('node:crypto');
+const files = require('./backup-files');
+const { fail, syncDirectory } = require('../controller/operations');
+
+function matches(scan, expected) {
+  return scan.treeDigest === expected.treeDigest && scan.totalBytes === expected.totalBytes;
+}
+
+function walDatabase(file) {
+  const before = files.checked(file, false), fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
+  try {
+    const info = fs.fstatSync(fd), header = Buffer.alloc(20);
+    if (info.dev !== before.dev || info.ino !== before.ino) fail('directory_backup_changed');
+    return fs.readSync(fd, header, 0, header.length, 0) === header.length
+      && header.subarray(0, 16).equals(Buffer.from('SQLite format 3\0')) && header[18] === 2 && header[19] === 2;
+  } finally { fs.closeSync(fd); }
+}
+
+// The caller holds the host operation lock. Even a read-only SQLite connection
+// can leave an empty WAL and shared-memory index in a sealed backup. Recover only
+// additions whose removal reproduces the complete original manifest checksum.
+// Restore validation remains strict; this recovery is specific to erasure.
+function verifyForErasure(root, expected, createdAt) {
+  const scanned = files.scan(root);
+  if (matches(scanned, expected)) return;
+  if (!Number.isSafeInteger(createdAt) || createdAt < 0) fail('directory_backup_changed');
+  const entries = new Map(scanned.entries.map(entry => [entry.path, entry])), additions = [];
+  for (const entry of scanned.entries) {
+    if (entry.type !== 'file' || !/\.(?:sqlite3?|db)-(?:wal|shm)$/.test(entry.path)) continue;
+    const database = entry.path.slice(0, -4), wal = entries.get(database + '-wal');
+    if (entries.get(database)?.type !== 'file' || wal && (wal.type !== 'file' || wal.bytes !== 0)) continue;
+    if (entry.path.endsWith('-wal') ? entry.bytes !== 0 : entry.bytes !== 32768) continue;
+    // Exclude only files clearly newer than the snapshot. Timestamps select
+    // candidates; the original hash, including every retained file, is proof.
+    if (files.checked(path.join(root, entry.path), false).mtimeMs <= createdAt + 1) continue;
+    if (walDatabase(path.join(root, database))) additions.push(entry);
+  }
+  const removed = new Set(additions.map(entry => entry.path));
+  const retained = scanned.entries.filter(entry => !removed.has(entry.path));
+  const candidate = {
+    treeDigest: crypto.createHash('sha256').update(JSON.stringify(retained)).digest('hex'),
+    totalBytes: scanned.totalBytes - additions.reduce((bytes, entry) => bytes + entry.bytes, 0),
+  };
+  if (!additions.length || !matches(candidate, expected) || !matches(files.scan(root), scanned)) fail('directory_backup_changed');
+  for (const entry of additions) {
+    const file = path.join(root, entry.path), current = files.digest(file);
+    if (current.sha256 !== entry.sha256 || current.bytes !== entry.bytes) fail('directory_backup_changed');
+    fs.unlinkSync(file); syncDirectory(path.dirname(file));
+  }
+  if (!matches(files.scan(root), expected)) fail('directory_backup_changed');
+}
+
+module.exports = { verifyForErasure };

host/storage/erase-backup-job.js

@@ -2,6 +2,11 @@
 const path = require('node:path');
 const { spawn } = require('node:child_process');
 
+function failure(code) {
+  const safe = /^directory_[a-z_]{1,80}$/.test(code || '') ? code : 'directory_backup_erasure_failed';
+  return Object.assign(new Error(safe), { code: safe });
+}
+
 // Hashing large retained snapshots runs in a finite process, keeping Core's
 // request loop responsive and releasing the scan's memory when it finishes.
 function eraseBackupJob(paths, job, lockFd) {
@@ -14,11 +19,14 @@ function eraseBackupJob(paths, job, lockFd) {
     child.stdout.on('data', chunk => { bytes += chunk.length; if (bytes > 4096) child.kill('SIGKILL'); else output += chunk; });
     child.stderr.on('data', chunk => { bytes += chunk.length; if (bytes > 4096) child.kill('SIGKILL'); });
     child.stdin.on('error', () => {});
-    child.on('error', () => { clearTimeout(timer); reject(Error('directory_backup_erasure_failed')); });
+    child.on('error', () => { clearTimeout(timer); reject(failure()); });
     child.on('close', code => {
       clearTimeout(timer);
-      try { if (code !== 0 || JSON.parse(output).ok !== true) throw Error(); resolve(); }
-      catch { reject(Error('directory_backup_erasure_failed')); }
+      try {
+        const result = JSON.parse(output);
+        if (code !== 0 || result.ok !== true) reject(failure(result.code));
+        else resolve();
+      } catch { reject(failure()); }
     });
     child.stdin.end(JSON.stringify({ platformRoot: paths.platformRoot, id: job.id,
       organizationId: job.organizationId, runtimeKey: job.runtimeKey }));
@@ -38,7 +46,7 @@ if (require.main === module) {
       const backups = new (require('./manual-backups').ManualBackups)({ paths, store });
       require('./erase-backups').eraseBackups(backups, job);
       process.stdout.write('{"ok":true}\n');
-    } catch { process.stdout.write('{"ok":false}\n'); process.exitCode = 1; }
+    } catch (error) { process.stdout.write(JSON.stringify({ ok: false, code: failure(error.code).code }) + '\n'); process.exitCode = 1; }
     finally { store?.close(); }
   })();
 }

host/storage/erase-backups.js

@@ -49,8 +49,7 @@ function eraseBackups(backups, job) {
     }
     if (!pending) {
       for (const entry of manifest.roots) {
-        const scan = files.scan(path.join(root, 'payload', entry.label));
-        if (scan.treeDigest !== entry.treeDigest || scan.totalBytes !== entry.totalBytes) fail('directory_backup_changed');
+        require('./backup-erasure-verification').verifyForErasure(path.join(root, 'payload', entry.label), entry, manifest.createdAt);
       }
       atomic(marker, { version: 1, jobId: job.id, manifest });
     }

host/storage/tests/backup-erasure-verification.test.js

@@ -0,0 +1,94 @@
+'use strict';
+
+const test = require('node:test'), assert = require('node:assert/strict');
+const fs = require('node:fs'), path = require('node:path'), os = require('node:os');
+const { DatabaseSync } = require('node:sqlite');
+const files = require('../backup-files');
+const { verifyForErasure } = require('../backup-erasure-verification');
+
+function fixture(t, { retainedSidecars = false } = {}) {
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), 'backup-erasure-'));
+  t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+  const source = path.join(root, 'source.sqlite3'), payload = path.join(root, 'payload');
+  fs.mkdirSync(payload, { mode: 0o700 });
+  const db = new DatabaseSync(source);
+  db.exec("PRAGMA journal_mode=WAL; CREATE TABLE entries(value TEXT); INSERT INTO entries VALUES('synthetic contents')");
+  db.close();
+  const databases = ['first.sqlite3', 'second.sqlite3'].map(name => path.join(payload, name));
+  for (const file of databases) { fs.copyFileSync(source, file); fs.chmodSync(file, 0o600); }
+  const open = file => {
+    const reader = new DatabaseSync(file, { readOnly: true });
+    try { assert.equal(reader.prepare('SELECT value FROM entries').get().value, 'synthetic contents'); }
+    finally { reader.close(); }
+    for (const suffix of ['-wal', '-shm']) fs.chmodSync(file + suffix, 0o600);
+  };
+  if (retainedSidecars) open(databases[1]);
+  const expected = files.scan(payload), createdAt = Date.now();
+  if (retainedSidecars) for (const suffix of ['-wal', '-shm']) fs.utimesSync(databases[1] + suffix, (createdAt - 1000) / 1000, (createdAt - 1000) / 1000);
+  const add = file => {
+    open(file);
+    for (const suffix of ['-wal', '-shm']) fs.utimesSync(file + suffix, (createdAt + 1000) / 1000, (createdAt + 1000) / 1000);
+  };
+  return { root, payload, databases, expected, createdAt, add, verify: () => verifyForErasure(payload, expected, createdAt) };
+}
+
+test('erasure recovers read-only SQLite additions while preserving originally backed-up sidecars', t => {
+  const c = fixture(t, { retainedSidecars: true }); c.add(c.databases[0]);
+  assert.notEqual(files.scan(c.payload).treeDigest, c.expected.treeDigest);
+  c.verify();
+  assert.deepEqual(files.scan(c.payload), c.expected);
+  assert.equal(fs.existsSync(c.databases[0] + '-shm'), false);
+  assert.equal(fs.existsSync(c.databases[1] + '-shm'), true);
+  c.verify();
+});
+
+test('erasure recovers multiple additions and resumes after a sidecar was already removed', t => {
+  const c = fixture(t); c.databases.forEach(c.add);
+  fs.unlinkSync(c.databases[0] + '-shm');
+  fs.unlinkSync(c.databases[1] + '-wal');
+  c.verify(); assert.deepEqual(files.scan(c.payload), c.expected);
+});
+
+test('changed database bytes or unrelated added files cannot be accepted as SQLite additions', t => {
+  for (const kind of ['database', 'extra']) {
+    const c = fixture(t); c.add(c.databases[0]);
+    if (kind === 'database') fs.appendFileSync(c.databases[1], 'unexpected data');
+    else fs.writeFileSync(path.join(c.payload, 'unexpected.txt'), 'unexpected data', { mode: 0o600 });
+    const before = files.scan(c.payload);
+    assert.throws(c.verify, { code: 'directory_backup_changed' });
+    assert.deepEqual(files.scan(c.payload), before);
+  }
+});
+
+test('erasure never discards a nonempty WAL, an old sidecar, or an unrecognized database', t => {
+  for (const kind of ['nonempty', 'old', 'not-sqlite']) {
+    const c = fixture(t); c.add(c.databases[0]);
+    let expected = c.expected;
+    if (kind === 'nonempty') fs.writeFileSync(c.databases[0] + '-wal', 'uncheckpointed data');
+    if (kind === 'old') for (const suffix of ['-shm', '-wal']) fs.utimesSync(c.databases[0] + suffix, 1, 1);
+    if (kind === 'not-sqlite') {
+      fs.writeFileSync(c.databases[0], 'ordinary file');
+      const entries = files.scan(c.payload).entries.filter(entry => !entry.path.endsWith('-shm') && !entry.path.endsWith('-wal'));
+      expected = { treeDigest: require('node:crypto').createHash('sha256').update(JSON.stringify(entries)).digest('hex'),
+        totalBytes: entries.reduce((sum, entry) => sum + (entry.bytes || 0), 0) };
+    }
+    const before = files.scan(c.payload);
+    assert.throws(() => verifyForErasure(c.payload, expected, c.createdAt), { code: 'directory_backup_changed' });
+    assert.deepEqual(files.scan(c.payload), before);
+  }
+});
+
+test('erasure fails closed on sidecar links and files changed after the recovery scan', t => {
+  const linked = fixture(t); linked.add(linked.databases[0]);
+  fs.unlinkSync(linked.databases[0] + '-wal'); fs.symlinkSync(linked.databases[1], linked.databases[0] + '-wal');
+  assert.throws(linked.verify, { code: 'directory_backup_unsafe' });
+  const c = fixture(t); c.add(c.databases[0]);
+  const digest = files.digest;
+  t.mock.method(files, 'digest', file => {
+    if (file === c.databases[0] + '-shm') fs.appendFileSync(file, 'changed');
+    return digest(file);
+  });
+  assert.throws(c.verify, { code: 'directory_backup_changed' });
+  assert.equal(fs.existsSync(c.databases[0] + '-shm'), true);
+  assert.equal(fs.existsSync(c.databases[0] + '-wal'), true);
+});

tooling/tests.json

@@ -14,6 +14,7 @@
     "host/plugins/tests/lifecycle.test.js",
     "host/plugins/tests/sdk-socket.test.js",
     "host/plugins/tests/worker-layout.test.js",
+    "host/storage/tests/backup-erasure-verification.test.js",
     "core/accounts/tests/plugins.test.js",
     "core/agents/tests/execution.test.js"
   ],