From ce1afa66f69e9724d31dd6870aa47982a639775d Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 04:57:13 +0000 Subject: [PATCH 1/4] fix(cli,runtime): os migrate resume, recorded-by and value-shapes answer a fresh project with empty work The read-only boot of these three commands defers schema DDL, and the driver already measures which tables the deferred sync would create (`create_table`, decided by `hasTable`). The commands now consult that measurement (`SchemaStack.tableAbsent`) and do not read a table their boot found absent: no runs to list, no sentinel rows, no stored values. The boot's migration journal scan reads a missing journal table as "no runs" through `isMissingTableError`, for that table only. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_016GiHYRmLSNWTfbX9gVQkpz --- .../cli/src/commands/migrate/recorded-by.ts | 16 +++++++-- packages/cli/src/commands/migrate/resume.ts | 15 ++++++-- .../cli/src/commands/migrate/value-shapes.ts | 36 ++++++++++++++++--- .../cli/src/utils/data-migration-plugins.ts | 8 +++-- packages/cli/src/utils/schema-migrate.ts | 26 ++++++++++++++ .../runtime/src/migration-recovery-plugin.ts | 21 ++++++++++- 6 files changed, 110 insertions(+), 12 deletions(-) diff --git a/packages/cli/src/commands/migrate/recorded-by.ts b/packages/cli/src/commands/migrate/recorded-by.ts index e74172d2ed2..4fc3e783c48 100644 --- a/packages/cli/src/commands/migrate/recorded-by.ts +++ b/packages/cli/src/commands/migrate/recorded-by.ts @@ -11,6 +11,7 @@ import { import { createRecordedBySentinelPlan, findSentinelHistoryRows, + METADATA_HISTORY_OBJECT, RECORDED_BY_SENTINEL, RECORDED_BY_SENTINEL_PLAN_ID, } from '@objectstack/metadata-protocol'; @@ -130,7 +131,13 @@ export default class MigrateRecordedBy extends Command { plans?.register?.(plan); } catch { /* no registry composed — resume reports it, this run still works */ } - const pending = await findSentinelHistoryRows(engine); + // [#21529] Not asked: the dry run's read-only boot measured whether the + // history table exists, and a table that does not exist holds no + // sentinel rows. Reading it anyway answered a fresh project's "nothing to + // convert" with a query fault and exit 1. `--apply` booted plain, so its + // table exists by now. + const historyAbsent = !flags.apply && stack.tableAbsent(METADATA_HISTORY_OBJECT); + const pending = historyAbsent ? [] : await findSentinelHistoryRows(engine); // ── dry run (default): read-only ───────────────────────────────── if (!flags.apply) { @@ -139,7 +146,12 @@ export default class MigrateRecordedBy extends Command { return; } if (pending.length === 0) { - printSuccess(`No sys_metadata_history row holds the '${RECORDED_BY_SENTINEL}' sentinel — nothing to convert.`); + printSuccess( + historyAbsent + ? `No ${METADATA_HISTORY_OBJECT} row holds the '${RECORDED_BY_SENTINEL}' sentinel — this database ` + + 'has no such table yet, so there is nothing to convert.' + : `No sys_metadata_history row holds the '${RECORDED_BY_SENTINEL}' sentinel — nothing to convert.`, + ); return; } printWarning(`${pending.length} sys_metadata_history row(s) hold recorded_by = '${RECORDED_BY_SENTINEL}'.`); diff --git a/packages/cli/src/commands/migrate/resume.ts b/packages/cli/src/commands/migrate/resume.ts index 6e2c7de017a..06a8af42843 100644 --- a/packages/cli/src/commands/migrate/resume.ts +++ b/packages/cli/src/commands/migrate/resume.ts @@ -13,6 +13,7 @@ import { } from '@objectstack/core'; import { describeInterruptedRun } from '@objectstack/runtime'; import type { IObjectQLEngine } from '@objectstack/spec/contracts'; +import { MIGRATION_JOURNAL_OBJECT } from '@objectstack/spec/system'; import { printHeader, printSuccess, @@ -140,7 +141,12 @@ export default class MigrateResume extends Command { // which is the truthful answer for this process. } - const interrupted = await findInterruptedRuns(engine); + // [#21529] Not asked: the list mode's read-only boot measured whether the + // journal table exists, and a table that does not exist holds no runs. + // Reading it anyway answered a fresh project's empty list with a query + // fault and exit 1. `--run` booted plain, so its table exists by now. + const journalAbsent = !flags.run && stack.tableAbsent(MIGRATION_JOURNAL_OBJECT); + const interrupted = journalAbsent ? [] : await findInterruptedRuns(engine); // ── list mode (no --run): read-only ────────────────────────────── if (!flags.run) { @@ -153,7 +159,12 @@ export default class MigrateResume extends Command { return; } if (interrupted.length === 0) { - printSuccess('No interrupted migration runs — every run in the journal concluded.'); + printSuccess( + journalAbsent + ? `No interrupted migration runs — this database has no ${MIGRATION_JOURNAL_OBJECT} table yet, ` + + 'so no run was ever journalled here.' + : 'No interrupted migration runs — every run in the journal concluded.', + ); return; } printWarning(`${interrupted.length} interrupted migration run(s):`); diff --git a/packages/cli/src/commands/migrate/value-shapes.ts b/packages/cli/src/commands/migrate/value-shapes.ts index 70e62148148..4ec27cedf0c 100644 --- a/packages/cli/src/commands/migrate/value-shapes.ts +++ b/packages/cli/src/commands/migrate/value-shapes.ts @@ -178,11 +178,37 @@ export default class MigrateValueShapes extends Command { ? { info: (m: string) => console.error(m), warn: (m: string) => console.error(m) } : { info: (m: string) => printInfo(m), warn: (m: string) => printWarning(m) }; - const report = await scanValueShapes(engine, logger, { + // [#21529] Not asked: the scan's read-only boot measured which tables + // exist, and a table that does not exist stores no value to check. The + // scan reads through this view, which answers such a table with its true + // contents (no rows) without issuing the read. Read anyway, every covered + // object on a fresh project was "unreadable", the gate closed, and the + // scan exited 1 over data that does not exist. `--apply` booted plain, so + // there every table exists and every read is real. + // ⛔ Only a table the boot measured absent: any other refused read still + // lands in `unreadableObjects` and still closes the gate. + const notStored = new Set(); + const scanView = { + getObject: (name: string) => engine.getObject(name), + ...(typeof engine.getConfigs === 'function' ? { getConfigs: () => engine.getConfigs() } : {}), + find: (object: string, options: Record) => { + if (!apply && stack.tableAbsent(object)) { + notStored.add(object); + return Promise.resolve([]); + } + return engine.find(object, options); + }, + }; + + const report = await scanValueShapes(scanView, logger, { objects: flags.object, maxRecordsPerObject: flags['max-records'], }); const passed = valueShapeScanPassed(report); + const notStoredLine = notStored.size > 0 + ? `${notStored.size} scanned object(s) have no table in this database yet, so nothing is stored in ` + + `them and they were not read: ${[...notStored].sort().join(', ')}.` + : null; // The flag write is the ONLY write this command makes, and only on an // apply run that passed. A failing apply run still records — deliberately: @@ -213,6 +239,7 @@ export default class MigrateValueShapes extends Command { } if (flags.json) { + if (notStoredLine) logger.info(notStoredLine); await emitJson({ database: stack.dbLabel, apply, @@ -229,6 +256,7 @@ export default class MigrateValueShapes extends Command { console.log(''); console.log(formatValueShapeScanReport(report).join('\n')); console.log(''); + if (notStoredLine) printInfo(notStoredLine); if (passed && apply) { printSuccess( @@ -250,9 +278,9 @@ export default class MigrateValueShapes extends Command { // throws oclif's ExitError: rethrown, never re-reported. Caught here, it // printed a second `--json` document (`{"error":"EEXIT: 1"}`) after the // scan's own — the shape `summary-nulls` and `files-to-references` - // already guard against. A scan with unreadable objects fails the gate, - // and the read-only scan of a database without the app's tables has - // nothing else. + // already guard against. A scan with unreadable objects fails the gate. + // (A table the read-only boot measured absent is not one of them since + // #21529: it is answered from that measurement, above, never read.) if (isExitSignal(error)) throw error; if (flags.json) { await emitJson({ error: error.message, ...errorCodeFields(error) }, 0, { compact: true }); this.exit(1); } printError(error.message || String(error)); diff --git a/packages/cli/src/utils/data-migration-plugins.ts b/packages/cli/src/utils/data-migration-plugins.ts index bab15d8279d..ba7a256a04c 100644 --- a/packages/cli/src/utils/data-migration-plugins.ts +++ b/packages/cli/src/utils/data-migration-plugins.ts @@ -48,9 +48,11 @@ export async function buildDataMigrationPlugins( // // The plugin's journal scan rides along. Its measured effects here: a data // command booted over an interrupted run warns about that run on stderr - // before it does anything else, and a read-only boot of a database that has no - // journal table yet warns that it could not check. That database is one these - // commands already refuse, because the tables they read are not there either. + // before it does anything else. [#21529] A read-only boot of a database that + // has no journal table yet is silent: the scan reads the missing table as + // "no runs" (`isMissingTableError`, for that table only), and `resume`, + // `recorded-by` and `value-shapes` answer such a database with empty work + // instead of reading the tables their boot measured absent. const { MigrationRecoveryPlugin } = await import('@objectstack/runtime'); plugins.push(new MigrationRecoveryPlugin()); if (opts.audit === true) { diff --git a/packages/cli/src/utils/schema-migrate.ts b/packages/cli/src/utils/schema-migrate.ts index 5a6895a7b4b..e981dfe615c 100644 --- a/packages/cli/src/utils/schema-migrate.ts +++ b/packages/cli/src/utils/schema-migrate.ts @@ -25,6 +25,7 @@ import type { PendingSchemaWork, } from '@objectstack/driver-sql'; import type { IObjectQLEngine } from '@objectstack/spec/contracts'; +import { StorageNameMapping } from '@objectstack/spec/system'; import { describeDriverConnection } from './connection-display.js'; import { reserveStdoutForJson } from './json-stdout.js'; import { @@ -70,6 +71,21 @@ export interface SchemaStack { * (#3917). Always `[]` unless the stack was booted with `deferSchemaDdl`. */ pendingSchemaWork: PendingSchemaWork[]; + /** + * [#21529] Did this boot MEASURE `objectName`'s table as absent? `true` only + * when the held-back sync above would CREATE it (`create_table`), which the + * driver decides with `hasTable` — a fact about the database, not a guess + * from a failed read. Always `false` unless the stack was booted with + * `deferSchemaDdl`, and `false` again once {@link flushSchemaDdl} has run. + * + * What it is for: a read-only command does not read a table its own boot + * deferred. A table that does not exist holds no rows, so the command + * answers its empty work from this ("not asked") instead of issuing the read + * and turning the refusal into a query fault. ⛔ Only `create_table` counts: + * a table that exists but lacks a column (`add_columns`) is still read, and + * whatever that read refuses is still reported. + */ + tableAbsent: (objectName: string) => boolean; /** * Every object the booted stack knows about — the set the plan is computed * against, including objects that arrived from installed packages rather than @@ -546,6 +562,10 @@ export async function bootSchemaStack( if (d.previewDeferredSchemaWork) pendingSchemaWork.push(...(await d.previewDeferredSchemaWork())); } sortPendingSchemaWork(pendingSchemaWork); + // [#21529] The tables the held-back sync would create: measured absent. + const absentTables = new Set( + pendingSchemaWork.filter((w) => w.kind === 'create_table').map((w) => w.table), + ); return { driver, @@ -553,6 +573,10 @@ export async function bootSchemaStack( managedTableCount, kernel, pendingSchemaWork, + // The driver keys its deferred work by physical table, so ask in the same + // name (`StorageNameMapping`, the mapping the driver itself applies). + tableAbsent: (objectName: string): boolean => + absentTables.has(StorageNameMapping.resolveTableName({ name: objectName })), /** * Every object this booted stack knows about — the same set the plan is * computed against. @@ -583,6 +607,8 @@ export async function bootSchemaStack( for (const d of deferral?.drivers ?? []) { if (d.flushDeferredSchemaDdl) performed.push(...(await d.flushDeferredSchemaDdl())); } + // [#21529] The tables exist now; nothing is measured absent any more. + absentTables.clear(); deferral?.release(); return sortPendingSchemaWork(performed); }, diff --git a/packages/runtime/src/migration-recovery-plugin.ts b/packages/runtime/src/migration-recovery-plugin.ts index dcff42a3882..ef5909dc958 100644 --- a/packages/runtime/src/migration-recovery-plugin.ts +++ b/packages/runtime/src/migration-recovery-plugin.ts @@ -9,6 +9,7 @@ import { } from '@objectstack/core'; import type { IObjectQLEngine } from '@objectstack/spec/contracts'; import { MIGRATION_JOURNAL_OBJECT } from '@objectstack/spec/system'; +import { isMissingTableError } from '@objectstack/types'; /** * MigrationRecoveryPlugin — boot reconciliation for the ADR-0119 D2 migration @@ -53,7 +54,9 @@ import { MIGRATION_JOURNAL_OBJECT } from '@objectstack/spec/system'; * journal has no interrupted runs to find, and a warning there would train * operators to ignore this plugin's output, which is the one thing it cannot * afford. A scan that FAILS, by contrast, is reported: "I could not check" and - * "there is nothing to find" are different answers. + * "there is nothing to find" are different answers. A journal object that is + * registered but whose TABLE does not exist yet (a read-only boot of a fresh + * database) is the second answer, not the first: no table, no runs (#21529). */ export class MigrationRecoveryPlugin implements Plugin { readonly name = 'com.objectstack.migration-recovery'; @@ -88,6 +91,22 @@ export class MigrationRecoveryPlugin implements Plugin { try { interrupted = await findInterruptedRuns(engine); } catch (err) { + // [#21529] A journal table that does not exist holds no runs: no run + // was ever journalled on this database. That is "there is nothing to + // find", answered by the one shared predicate and only for the journal + // itself. A read-only CLI boot of a fresh project (`os migrate resume` + // and the other data commands) defers the table's DDL and lands here + // on every run, and a warning there trains operators to skim this + // plugin's output — the one thing the header says it cannot afford. + // ⛔ Any other refusal, a missing table it did not ask about included, + // is still "I could not check", below. + if (isMissingTableError(err, MIGRATION_JOURNAL_OBJECT)) { + logger?.debug?.( + `Migration journal scan: ${MIGRATION_JOURNAL_OBJECT} does not exist on this database yet, ` + + 'so no migration run was ever journalled here — nothing to report.', + ); + return; + } // "I could not check" is not "there is nothing to find". logger?.warn?.( `Migration journal scan failed; interrupted migrations (if any) were NOT detected this boot: ${ From b3f7cf99e4e7a5572a338b3ab507ab7a9d6df089 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 05:06:50 +0000 Subject: [PATCH 2/4] test(cli,runtime): pin empty work on an absent database and the silent journal scan Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_016GiHYRmLSNWTfbX9gVQkpz --- ...mmands.absent-database.integration.test.ts | 316 ++++++++++++++++++ ...on-recovery-plugin.missing-journal.test.ts | 158 +++++++++ 2 files changed, 474 insertions(+) create mode 100644 packages/cli/src/commands/migrate/data-commands.absent-database.integration.test.ts create mode 100644 packages/runtime/src/migration-recovery-plugin.missing-journal.test.ts diff --git a/packages/cli/src/commands/migrate/data-commands.absent-database.integration.test.ts b/packages/cli/src/commands/migrate/data-commands.absent-database.integration.test.ts new file mode 100644 index 00000000000..7a2808dc225 --- /dev/null +++ b/packages/cli/src/commands/migrate/data-commands.absent-database.integration.test.ts @@ -0,0 +1,316 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21529] `os migrate resume`, `os migrate recorded-by` and `os migrate + * value-shapes` answer a project whose database does not exist yet with empty + * work and exit 0, the answer each gives a booted database with nothing to do. + * + * ## The measured defect + * + * Each command's default mode boots read-only: the SQL driver defers every + * table's DDL, and a missing sqlite file is opened as an empty in-memory + * stand-in. Each then read the very tables its boot had just deferred, so on a + * fresh project (measured at the public door, base `49161683fb`): + * + * - `resume --json` exited 1 with "The database refused to run this query for + * object 'sys_migration_journal'"; + * - `recorded-by --json` exited 1 the same way for `sys_metadata_history`; + * - `value-shapes --json` exited 1 with every covered object "unreadable" and + * the gate closed, over data that does not exist; + * - and every one of those boots also logged "Migration journal scan failed" + * from `MigrationRecoveryPlugin`'s scan of the same absent journal. + * + * ## The answer pinned + * + * "Not asked": the read-only boot already measured which tables are absent + * (the deferred sync lists them as `create_table`, decided by `hasTable`), so + * a command does not read them. The documented exit for empty work is 0 — an + * `os migrate` subcommand's `--json` success exits 0 (the platform checklist's + * migrate item), and for `value-shapes` a clean scan exits 0 + * (`content/docs/deployment/cli.mdx`) — and it is the exit each command gives + * the booted control below. + * + * Per command, on the absent database: exit 0, the empty-work document, no + * read of its own tables refused, no journal-scan warning, no file created. + * Human mode says the table is not there yet rather than implying it looked. + * + * ## The control + * + * A booted database holding one row of work for each command: a journalled + * run that never concluded, a history row still carrying the sentinel, and an + * app record with a covered field. Each command READS it and reports it. A + * fix that answered empty work without looking would fail here. + * + * Every spawn runs in the hook; a case only reads what a run printed. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawn, spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CLI_ROOT = resolve(HERE, '..', '..', '..'); +/** The source entry, as `test/helpers/serve-process.ts` spawns it; `src/` cannot import that helper. */ +const CLI = resolve(HERE, '../../../bin/run-dev.js'); + +const HOOK_TIMEOUT_MS = 480_000; +const RUN_BUDGET_MS = 120_000; + +const ARTIFACT = { + manifest: { id: 'com.example.os21529', name: 'Absent database', version: '0.0.0', type: 'app' }, + objects: [ + { name: 'os21529_account', fields: { name: { type: 'text' } } }, + // A lookup is a covered value class, so `value-shapes` walks this object. + { + name: 'os21529_contact', + fields: { name: { type: 'text' }, account: { type: 'lookup', reference: 'os21529_account' } }, + }, + ], +}; + +/** The interrupted run the control's journal holds. */ +const RUN_ID = 'run_21529_control'; + +/** Env that would point a boot somewhere other than the fixture. */ +const OVERRIDING_ENV = ['OS_DATABASE_URL', 'DATABASE_URL', 'TURSO_DATABASE_URL', 'OS_DATABASE_DRIVER', 'OS_HOME'] as const; + +/** + * This process's environment for a child, minus the families + * `test/helpers/serve-process.ts` `childEnv()` strips (its header says why). + * An `undefined` override unsets a variable. + */ +function childEnv(overrides: Record): Record { + const env: Record = {}; + for (const [key, value] of Object.entries(process.env)) { + if (key === 'TEST' || key === 'VITEST' || key.startsWith('VITEST_') || key === 'NODE_PATH') continue; + env[key] = value; + } + const unset: Record = {}; + for (const key of OVERRIDING_ENV) unset[key] = undefined; + return { ...env, ...unset, ...overrides }; +} + +/** + * The control's database: a served-shape boot (DDL performed) of the same data + * stack the commands compose, then one row of work for each command. + */ +const SEED_CHILD = ` +const rt = await import('@objectstack/runtime'); +const { PlatformObjectsPlugin } = await import('@objectstack/platform-objects/plugin'); +const stack = await rt.createStandaloneStack({ + projectRoot: process.env.FIXTURE_PROJECT, + databaseUrl: 'file:' + process.env.FIXTURE_DB, + skipSeedData: true, + armLifecycleSweep: false, +}); +const runtime = new rt.Runtime({ cluster: false }); +const kernel = runtime.getKernel(); +for (const plugin of stack.plugins) await kernel.use(plugin); +await kernel.use(new PlatformObjectsPlugin()); +await runtime.start(); +const ql = kernel.getService('objectql'); +const SYSTEM = { context: { isSystem: true } }; +await ql.insert('sys_metadata_history', { + id: 'h_21529', event_seq: 1, name: 'os21529_note', type: 'object', version: 1, + operation_type: 'create', recorded_by: 'system', recorded_at: new Date().toISOString(), +}, SYSTEM); +await ql.insert('sys_migration_journal', { + run_id: process.env.FIXTURE_RUN, seq: 0, kind: 'run_started', plan_hash: 'h', + detail: JSON.stringify({ planId: 'plan_21529' }), +}, SYSTEM); +await ql.insert('sys_migration_journal', { + run_id: process.env.FIXTURE_RUN, seq: 1, kind: 'chunk_started', chunk_index: 0, +}, SYSTEM); +const account = await ql.insert('os21529_account', { name: 'Acme' }, SYSTEM); +await ql.insert('os21529_contact', { name: 'Ann', account: account.id }, SYSTEM); +await kernel.shutdown(); +process.stderr.write('[fixture] seeded\\n'); +process.exit(0); +`; + +interface Run { + code: number | null; + stdout: string; + stderr: string; +} + +let dir: string; +let absentDb: string; +let bootedDb: string; + +function runCommand(command: string, extra: string[], dbFile: string): Promise { + return new Promise((resolveRun, rejectRun) => { + const child = spawn( + process.execPath, + [CLI, 'migrate', command, ...extra, '--database-url', `file:${dbFile}`], + { + cwd: dir, + env: childEnv({ + NO_COLOR: '1', + OS_ARTIFACT_PATH: join(dir, 'dist', 'objectstack.json'), + // The fixed key `serve-process.ts` hands its children, so no boot + // mints and persists a crypto key into this runner's home directory. + OS_SECRET_KEY: '0e2e'.repeat(16), + }), + stdio: ['ignore', 'pipe', 'pipe'], + }, + ); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (c) => { stdout += String(c); }); + child.stderr.on('data', (c) => { stderr += String(c); }); + const timer = setTimeout(() => { + child.kill('SIGKILL'); + rejectRun(new Error(`os migrate ${command} did not finish within ${RUN_BUDGET_MS}ms\n${stderr}`)); + }, RUN_BUDGET_MS); + child.on('error', (err) => { clearTimeout(timer); rejectRun(err); }); + child.on('close', (code) => { + clearTimeout(timer); + resolveRun({ code, stdout, stderr }); + }); + }); +} + +const COMMANDS = ['resume', 'recorded-by', 'value-shapes'] as const; +type CommandName = (typeof COMMANDS)[number]; + +/** The tables each command reads for its answer — the ones its boot defers on a fresh project. */ +const OWN_TABLES: Record = { + resume: ['sys_migration_journal'], + 'recorded-by': ['sys_metadata_history'], + // The objects with a covered field: the scan walks each. + 'value-shapes': ['os21529_contact', 'sys_metadata', 'sys_view_definition'], +}; + +const absentJson = {} as Record; +const absentHuman = {} as Record; +const bootedJson = {} as Record; + +beforeAll(async () => { + dir = mkdtempSync(join(tmpdir(), 'os-21529-')); + mkdirSync(join(dir, 'dist'), { recursive: true }); + mkdirSync(join(dir, 'data'), { recursive: true }); + writeFileSync(join(dir, 'dist', 'objectstack.json'), JSON.stringify(ARTIFACT)); + absentDb = join(dir, 'data', 'absent.db'); + bootedDb = join(dir, 'data', 'booted.db'); + + const seed = spawnSync(process.execPath, ['--input-type=module', '-e', SEED_CHILD], { + cwd: CLI_ROOT, + env: childEnv({ + OS_ARTIFACT_PATH: join(dir, 'dist', 'objectstack.json'), + OS_SECRET_KEY: '0e2e'.repeat(16), + FIXTURE_PROJECT: dir, + FIXTURE_DB: bootedDb, + FIXTURE_RUN: RUN_ID, + }), + encoding: 'utf8', + timeout: RUN_BUDGET_MS, + }); + if (seed.status !== 0 || !String(seed.stderr).includes('[fixture] seeded')) { + throw new Error(`the control database was not seeded (status ${seed.status})\n${seed.stderr}`); + } + + for (const command of COMMANDS) { + absentJson[command] = await runCommand(command, ['--json'], absentDb); + absentHuman[command] = await runCommand(command, [], absentDb); + bootedJson[command] = await runCommand(command, ['--json'], bootedDb); + } +}, HOOK_TIMEOUT_MS); + +afterAll(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); +}); + +/** Lines, on either stream, where a read of one of `tables` was refused. */ +function refusedReads(run: Run, tables: readonly string[]): string[] { + return `${run.stdout}\n${run.stderr}` + .split('\n') + .filter((line) => /refused to run this query|cannot read/.test(line)) + .filter((line) => tables.some((t) => line.includes(`'${t}'`) || line.includes(`read ${t} `))); +} + +function journalScanWarnings(run: Run): string[] { + return `${run.stdout}\n${run.stderr}`.split('\n').filter((line) => /journal scan failed/i.test(line)); +} + +describe('[#21529] on a database that does not exist yet: empty work, exit 0', () => { + it.each(COMMANDS)('%s: reads none of its own tables, and the boot scan does not warn', (command) => { + for (const run of [absentJson[command], absentHuman[command]]) { + expect(refusedReads(run, OWN_TABLES[command]), run.stderr).toEqual([]); + expect(journalScanWarnings(run)).toEqual([]); + } + // Still a read-only boot: no database file was brought into existence. + expect(existsSync(absentDb)).toBe(false); + }); + + it('resume --json: no interrupted runs, exit 0', () => { + const run = absentJson.resume; + expect(run.code, run.stderr).toBe(0); + expect(JSON.parse(run.stdout)).toMatchObject({ interrupted: [], count: 0 }); + }); + + it('recorded-by --json: nothing to convert, exit 0', () => { + const run = absentJson['recorded-by']; + expect(run.code, run.stderr).toBe(0); + expect(JSON.parse(run.stdout)).toMatchObject({ + planId: 'metadata.recorded-by-sentinel-to-null', + sentinel: 'system', + pending: 0, + applied: false, + }); + }); + + it('value-shapes --json: a clean, complete scan of zero records, exit 0', () => { + const run = absentJson['value-shapes']; + expect(run.code, run.stderr).toBe(0); + const payload = JSON.parse(run.stdout); + expect(payload).toMatchObject({ apply: false, gatePassed: true, flag: null }); + expect(payload.scan).toMatchObject({ scannedRecords: 0, blocking: 0, truncated: false, unreadableObjects: [] }); + // The objects were in scope, and answered from the measurement rather than a read. + expect(payload.scan.scannedObjects).toContain('os21529_contact'); + expect(run.stderr).toMatch(/have no table in this database yet[^\n]*os21529_contact/); + }); + + it('human mode says the table is not there yet, exit 0', () => { + expect(absentHuman.resume.code, absentHuman.resume.stderr).toBe(0); + expect(absentHuman.resume.stdout).toContain('No interrupted migration runs — this database has no sys_migration_journal table yet'); + expect(absentHuman['recorded-by'].code, absentHuman['recorded-by'].stderr).toBe(0); + expect(absentHuman['recorded-by'].stdout).toContain('this database has no such table yet, so there is nothing to convert'); + expect(absentHuman['value-shapes'].code, absentHuman['value-shapes'].stderr).toBe(0); + expect(absentHuman['value-shapes'].stdout).toMatch(/have no table in this database yet[^\n]*os21529_contact/); + expect(absentHuman['value-shapes'].stdout).toContain('Scan clean'); + }); +}); + +describe('[#21529] the control: a booted database is read, and its work is reported', () => { + it('resume --json lists the interrupted run, exit 0', () => { + const run = bootedJson.resume; + expect(run.code, run.stderr).toBe(0); + const payload = JSON.parse(run.stdout); + expect(payload.count).toBe(1); + expect(payload.interrupted.map((r: { runId: string }) => r.runId)).toEqual([RUN_ID]); + }); + + it('recorded-by --json counts the sentinel row, exit 0', () => { + const run = bootedJson['recorded-by']; + expect(run.code, run.stderr).toBe(0); + expect(JSON.parse(run.stdout)).toMatchObject({ pending: 1, applied: false }); + }); + + it('value-shapes --json walks the stored record, exit 0', () => { + const run = bootedJson['value-shapes']; + expect(run.code, run.stderr).toBe(0); + const payload = JSON.parse(run.stdout); + expect(payload.gatePassed).toBe(true); + expect(payload.scan.scannedRecords).toBeGreaterThanOrEqual(1); + expect(payload.scan.unreadableObjects).toEqual([]); + expect(run.stderr).not.toMatch(/have no table in this database yet/); + }); + + it.each(COMMANDS)('%s: the boot scan does not warn here either', (command) => { + expect(journalScanWarnings(bootedJson[command])).toEqual([]); + }); +}); diff --git a/packages/runtime/src/migration-recovery-plugin.missing-journal.test.ts b/packages/runtime/src/migration-recovery-plugin.missing-journal.test.ts new file mode 100644 index 00000000000..76b7a36f40e --- /dev/null +++ b/packages/runtime/src/migration-recovery-plugin.missing-journal.test.ts @@ -0,0 +1,158 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21529] The boot's migration journal scan reads a journal TABLE that does + * not exist as "no runs", and says nothing. + * + * ## The measured defect + * + * `os migrate resume`, `recorded-by` and `value-shapes` boot read-only on a + * project whose database does not exist yet: the SQL driver defers every + * table's DDL, so `sys_migration_journal` is registered and absent. The scan + * this plugin runs at `kernel:ready` read it anyway and logged "Migration + * journal scan failed; interrupted migrations (if any) were NOT detected" on + * every such run. A database with no journal table has no journalled run, so + * that line was a false alarm on exactly the boot an operator runs first. + * + * ## What is pinned + * + * Every case drives the plugin's real `kernel:ready` scan over a real + * `ObjectQL` engine on a real SQLite `SqlDriver`, with the platform's own + * journal object registered: + * + * 1. DDL deferred, the journal table absent: no warning at all (the refusal + * the driver throws is the one the CLI boot meets). + * 2. The control: the same engine with its DDL performed and one interrupted + * run journalled. The scan reads the table and reports the run, so the + * silence in 1 is not a scan that never ran. + * 3. The predicate's other side: a missing-table refusal naming a relation + * that is not the journal, in the shape a driver gives when it declares + * no target (the dialect's own `no such table: `), still warns + * "scan failed". The plugin names the journal to the predicate, so only + * the journal's own absence is "no runs"; asked without that name, the + * predicate would read this refusal as benign too. + * + * Any other refusal still warning is pinned in `migration-recovery-plugin.test.ts` + * ("reports a scan FAILURE rather than reading it as nothing found"). + * + * Every engine is built in a hook: a case only drives the scan. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { SysMigrationJournal } from '@objectstack/platform-objects/system'; +import { MigrationRecoveryPlugin } from './migration-recovery-plugin.js'; + +const JOURNAL = 'sys_migration_journal'; +const SYSTEM = { context: { isSystem: true } }; + +const quiet = { debug: () => {}, info: () => {}, warn: () => {}, error: () => {} }; + +interface Booted { + driver: SqlDriver; + engine: ObjectQL; +} + +/** An engine over an in-memory SQLite database, the journal object registered. */ +async function engineOver(opts: { deferDdl: boolean }): Promise { + const driver = new SqlDriver({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + } as any); + (driver as unknown as { logger: typeof quiet }).logger = quiet; + // What the read-only CLI boot arms before schema sync. + if (opts.deferDdl) driver.setDeferredDdl(true); + const engine = new ObjectQL({ logger: quiet } as any); + engine.registerDriver(driver as any, true); + await engine.init(); + engine.registry.registerObject(SysMigrationJournal as any); + await engine.syncSchemas(); + return { driver, engine }; +} + +/** Run the plugin's `kernel:ready` scan over `engine`, and return what it warned. */ +async function scanWarnings(engine: unknown): Promise { + const warns: string[] = []; + const hooks: Array<() => Promise | void> = []; + const services = new Map([['objectql', engine]]); + const ctx: any = { + logger: { ...quiet, warn: (m: unknown) => { warns.push(String(m)); } }, + registerService: (name: string, service: unknown) => services.set(name, service), + getService: (name: string) => { + if (!services.has(name)) throw new Error(`service '${name}' not registered`); + return services.get(name); + }, + hook: (event: string, fn: () => Promise | void) => { + if (event === 'kernel:ready') hooks.push(fn); + }, + }; + const plugin = new MigrationRecoveryPlugin(); + await plugin.init(ctx); + await plugin.start(ctx); + for (const fn of hooks) await fn(); + return warns; +} + +let fresh: Booted; +let booted: Booted; +/** A dialect refusal for a relation that is not the journal, with no declared target. */ +const otherTableRefusal = Object.assign(new Error('SQLITE_ERROR: no such table: os21529_other'), { + code: 'SQLITE_ERROR', +}); + +let freshWarns: string[]; +let bootedWarns: string[]; +let otherTableWarns: string[]; + +beforeAll(async () => { + fresh = await engineOver({ deferDdl: true }); + booted = await engineOver({ deferDdl: false }); + await booted.engine.insert( + JOURNAL, + { run_id: 'run_21529', seq: 0, kind: 'run_started', plan_hash: 'h', detail: JSON.stringify({ planId: 'plan_21529' }) }, + SYSTEM, + ); + await booted.engine.insert(JOURNAL, { run_id: 'run_21529', seq: 1, kind: 'chunk_started', chunk_index: 0 }, SYSTEM); + + const refusing = { + getObject: (name: string) => fresh.engine.getObject(name), + find: async () => { throw otherTableRefusal; }, + }; + + freshWarns = await scanWarnings(fresh.engine); + bootedWarns = await scanWarnings(booted.engine); + otherTableWarns = await scanWarnings(refusing); +}, 60_000); + +afterAll(async () => { + await fresh?.driver.disconnect(); + await booted?.driver.disconnect(); +}); + +describe('[#21529] the boot journal scan on a database with no journal table', () => { + it('the deferred journal table really is absent: the scan read it and was refused', async () => { + // Non-vacuity: the same read the scan makes is refused on this engine. + await expect( + fresh.engine.find(JOURNAL, { where: { kind: 'run_started' } } as any, SYSTEM as any), + ).rejects.toMatchObject({ code: 'DATABASE_ERROR', status: 500 }); + }); + + it('says nothing: no journal table is no journalled run', () => { + expect(freshWarns).toEqual([]); + }); + + it('control: with the table there, the scan reads it and reports the interrupted run', () => { + const all = bootedWarns.join('\n'); + expect(all).toContain("run 'run_21529'"); + expect(all).toContain("plan 'plan_21529'"); + expect(all).not.toContain('scan failed'); + }); + + it('a missing table that is not the journal is still "I could not check"', () => { + expect(otherTableWarns).toHaveLength(1); + expect(otherTableWarns[0]).toContain('Migration journal scan failed'); + expect(otherTableWarns[0]).toContain('NOT detected'); + }); +}); From 6531dd1450195919107fe129c64959eec164c6cb Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 05:17:20 +0000 Subject: [PATCH 3/4] docs(cli): the read-only data commands' missing-table edge, and the changeset Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_016GiHYRmLSNWTfbX9gVQkpz --- .../21529-absent-database-empty-work.md | 26 +++++++++++++++++++ content/docs/deployment/cli.mdx | 12 ++++++--- 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100644 .changeset/21529-absent-database-empty-work.md diff --git a/.changeset/21529-absent-database-empty-work.md b/.changeset/21529-absent-database-empty-work.md new file mode 100644 index 00000000000..b371dc2d127 --- /dev/null +++ b/.changeset/21529-absent-database-empty-work.md @@ -0,0 +1,26 @@ +--- +"@objectstack/cli": patch +"@objectstack/runtime": patch +--- + +`os migrate resume`, `os migrate recorded-by` and `os migrate value-shapes` answer a project whose database does not exist yet with empty work and exit 0, instead of exiting 1 with "The database refused to run this query" (#21529) + +Clause-②: no + +Each of these commands boots read-only by default: the schema sync is held back, and a missing SQLite file is opened as an empty in-memory stand-in. That boot already measures which tables the database lacks, because the held-back sync lists each one as a table to create. Each command then read the very tables it had just found missing. On a never-booted database (or a `--database-url` that points at one), every default run failed: + +- `os migrate resume` exited 1, naming `sys_migration_journal`; +- `os migrate recorded-by` exited 1, naming `sys_metadata_history`; +- `os migrate value-shapes` reported every scanned object as unreadable, kept the gate closed and exited 1, over data that does not exist. + +Each command now reads only the tables its boot found present. A table that does not exist holds nothing, so: + +- `os migrate resume` lists no interrupted runs (`{"interrupted": [], "count": 0}`), exit 0; +- `os migrate recorded-by` reports `pending: 0`, nothing to convert, exit 0; +- `os migrate value-shapes` completes a clean scan of zero records, exit 0, and names the objects it did not read because they have no table yet (on stderr under `--json`). + +Human mode says the table is not there yet, instead of implying the command looked through one. `--json` documents have the same shape as on a booted database with nothing to do. The write modes (`--run`, `--apply`) are unchanged: they boot with the schema sync, so their tables exist before they read. + +`MigrationRecoveryPlugin` (`@objectstack/runtime`), which every one of these boots composes, scans the migration journal at boot. On such a database it logged "Migration journal scan failed; interrupted migrations (if any) were NOT detected" on every run. It now treats a missing journal table as "no runs" and says nothing. It recognises that case only with the shared `isMissingTableError` predicate, asked about `sys_migration_journal` itself. Any other failure of the scan still warns. + +There is nothing to migrate. diff --git a/content/docs/deployment/cli.mdx b/content/docs/deployment/cli.mdx index d2f6c080d07..f37d4c27649 100644 --- a/content/docs/deployment/cli.mdx +++ b/content/docs/deployment/cli.mdx @@ -1053,10 +1053,14 @@ your app to read its metadata. Without `--apply`, that boot is read-only, the sa loaded, and a SQLite file that does not exist is not created. With `--apply`, the boot creates missing tables and columns so the migration has somewhere to write, but it still loads no seed data: the only rows that change are the migration's own. -One edge follows from the read-only boot. A dry run pointed at a database that lacks a -table it reads (a never-booted database, or the wrong `--database-url`) can fail and exit -1, naming the table, where it used to create the table and report nothing to do. Point -`--database-url` at the deployment's database, or boot the deployment once first. +One edge follows from the read-only boot: it finds out which tables the database lacks +(a never-booted database, or the wrong `--database-url`) instead of creating them. +`os migrate value-shapes` does not read a table it found missing, since that table +holds nothing: the scan is clean over zero records, exits 0, and names the objects it +did not read. `os migrate recorded-by` and `os migrate resume` answer the same way +(nothing to convert, no interrupted runs). Another dry run that reads a missing table +can still fail and exit 1, naming the table. Point `--database-url` at the +deployment's database, or boot the deployment once first. ```bash os migrate files-to-references # Dry run: full report, writes nothing From e860ae7a212c1882f0c0d11cda0d81819c481a4b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 05:46:07 +0000 Subject: [PATCH 4/4] test(runtime): type the journal read's query options in the missing-journal pin Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_016GiHYRmLSNWTfbX9gVQkpz --- .../src/migration-recovery-plugin.missing-journal.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/runtime/src/migration-recovery-plugin.missing-journal.test.ts b/packages/runtime/src/migration-recovery-plugin.missing-journal.test.ts index 76b7a36f40e..141006c7a21 100644 --- a/packages/runtime/src/migration-recovery-plugin.missing-journal.test.ts +++ b/packages/runtime/src/migration-recovery-plugin.missing-journal.test.ts @@ -135,7 +135,7 @@ describe('[#21529] the boot journal scan on a database with no journal table', ( it('the deferred journal table really is absent: the scan read it and was refused', async () => { // Non-vacuity: the same read the scan makes is refused on this engine. await expect( - fresh.engine.find(JOURNAL, { where: { kind: 'run_started' } } as any, SYSTEM as any), + fresh.engine.find(JOURNAL, { where: { kind: 'run_started' } }, SYSTEM), ).rejects.toMatchObject({ code: 'DATABASE_ERROR', status: 500 }); });