diff --git a/.changeset/21573-migrate-unmapped-columns.md b/.changeset/21573-migrate-unmapped-columns.md new file mode 100644 index 00000000000..e3ed45d9994 --- /dev/null +++ b/.changeset/21573-migrate-unmapped-columns.md @@ -0,0 +1,16 @@ +--- +'@objectstack/cli': minor +--- + +feat(cli): `os migrate unmapped-columns --object NAME` reads the values of a retired field's columns, keyed by record id, for a conversion before `os migrate apply --allow-destructive` drops them (#21573) + +Clause-②: yes (widening) + +- **What it reads.** The columns `os migrate plan` reports as `unmapped_column` for one object's table: a column that is still in the table and that no metadata declares, typically one a retired field left behind. The column set is the plan's own findings, from the same differ on the same read-only boot, so the command never reads a column the plan does not report. Each record is emitted as `{ id, values }`. `--json` prints one document, `{ database, object, table, columns, count, records, duration }`. The text face lists the columns and each record's values. +- **Why it exists.** A read or a write through the engine now serves an object's declared fields only, and naming an undeclared column is refused. An app that moves a retired field's values into the field that replaced it reads them once with this command, writes them with its own script, and then drops the columns with `os migrate apply --allow-destructive`. That is the route the read and write narrowing in `@objectstack/objectql` names for this case. +- **Operator-only and read-only.** It runs under the database credentials you pass (`--database-url`, else `OS_DATABASE_URL`, else the project database), and it reads every organization's rows. No REST route, API flag or per-request option serves these values, and the runtime doors are unchanged. It boots the way `os migrate plan` does: no schema DDL, no seed data, and no database file created. +- **Values as stored.** An unmapped column has no declared type, so each value is emitted as the database client returns it, with no field-type decoding; a PostgreSQL `timestamp` arrives as a date and is emitted as its ISO 8601 text. A value JSON cannot carry as stored (binary bytes, a `bigint`, or a non-finite number) is refused in both faces with exit 1, naming the column and the record id, and no record is emitted: read that column with the database's own client. No column the platform creates for a field type answers with one of these, on SQLite or on PostgreSQL. +- **Answers.** An object with no unmapped column, or with no table yet: empty work, exit 0. No SQL driver: `os migrate plan`'s own `no_sql_driver` answer, exit 0. An undeclared object name: `OBJECT_NOT_FOUND`, exit 1. An object the plan does not diff (federated, or bound to another datasource): refused, exit 1. A read that cannot be complete, such as one stopped by `--max-records`: refused, exit 1, and no partial set is emitted. A value JSON cannot carry as stored: refused, exit 1, as above. +- `MigrateUnmappedColumnsCommand` is exported from `@objectstack/cli` beside the other `os migrate` commands. + +Nothing that ran before changes. This is a new command. diff --git a/content/docs/data-modeling/queries.mdx b/content/docs/data-modeling/queries.mdx index 74e3ae3dc06..13d50f04a18 100644 --- a/content/docs/data-modeling/queries.mdx +++ b/content/docs/data-modeling/queries.mdx @@ -290,7 +290,11 @@ system columns (`id`, `created_at`, `updated_at`, and the tenant, owner and audi the registry adds). A column no metadata declares is never returned — for example one a retired field left in the table until `os migrate apply --allow-destructive` drops it — and naming it in `fields` on the data API is refused with `400 INVALID_FIELD`. To read such a column's values -for a one-time conversion, run the conversion before the field is retired. +for a one-time conversion, run the conversion before the field is retired, or, once it is +retired, read them with the operator-only +[`os migrate unmapped-columns`](/docs/deployment/cli#os-migrate-unmapped-columns) and convert +them before `os migrate apply --allow-destructive` drops the column. No runtime door serves +them. ### Nested / Related Fields diff --git a/content/docs/deployment/cli.mdx b/content/docs/deployment/cli.mdx index e4d25b9f5e9..6c7a0463810 100644 --- a/content/docs/deployment/cli.mdx +++ b/content/docs/deployment/cli.mdx @@ -519,7 +519,7 @@ as the project's **default** datasource it is refused at boot the same way. **What it boots:** - Reads the artifact's `manifest`, `objects`, `views`, `flows`, … - Auto-registers the platform services declared in `requires: [...]` (e.g. `ai`, `automation`, `analytics`, `auth`, `ui`). Declaring a **service** capability (`automation`, `analytics`, `ai`, `audit`, …) is a *requirement*: if its provider package isn't installed, boot **fails fast** with a clear error instead of silently starting without a capability you asked for. (`auth` and `ui` are tier-gated with their own opt-in rules — `auth`'s secret-gated skip is described below.) -- Auto-detects the driver from the database URL scheme (`libsql://`/`https://*.turso.*` → Turso — via the optional `@objectstack/driver-turso` package, and a loud failure with the install command when it is missing rather than a fallback to sqlite —, `postgres[ql]://`/`pg://` → pg, `mongodb[+srv]://` → MongoDB, otherwise sqlite; `:memory:` is SQLite's own in-memory database). `memory://` and `mingo://` are refused — see the callout below. +- Auto-detects the driver from the database URL scheme (`libsql://`/`https://*.turso.*` → Turso — via the optional `@objectstack/driver-turso` package, and a loud failure with the install command when it is missing rather than a fallback to sqlite —, `postgres[ql]://`/`pg://` → pg, `mongodb[+srv]://` → MongoDB, otherwise sqlite; `:memory:` is SQLite's own in-memory database). `memory://` and `mingo://` are refused — see the callout above. - Runs standalone boot mode with one active environment. **Authentication:** @@ -853,6 +853,7 @@ diverges from the live schema, and the physical column wins at write time. | `os migrate plan` | Dry-run: show how the database has drifted from metadata, categorised safe / needs-confirm / destructive (no changes applied) | | `os migrate apply` | Reconcile the database to metadata. Applies loosening changes; destructive ones require `--allow-destructive` | | `os migrate multi-value-columns` | Migrate a stale `varchar`/`text` column to `json` where the field declares `multiple: true` — one of three drift ops `apply` never reconciles for you. Dry run by default; `--apply` runs the statement the finding prints | +| `os migrate unmapped-columns` | Read the values of the columns `plan` reports as `unmapped_column` for one object, keyed by record id — the conversion route for a retired field's values before `apply --allow-destructive` drops its columns. Read-only, and operator-only: no runtime door serves these values | ```bash os migrate plan # Preview drift (no changes) @@ -861,6 +862,7 @@ os migrate apply --yes # Skip the prompt (CI / scripts) os migrate apply --allow-destructive --yes # Also drop orphaned columns, tighten NOT NULL, narrow types os migrate apply --force # Migrate even though another process is using the database os migrate plan --json # Machine-readable output +os migrate unmapped-columns --object contact --json # A retired field's stored values, keyed by record id (read-only) ``` #### Nothing is written before you confirm @@ -1048,6 +1050,54 @@ hook or an integration already copied into some *other* single-value column is not something it looks for, and it is deliberately not something it will grow into: that repair is specific to what your automations did with the value. +#### `os migrate unmapped-columns` + +Retiring a field leaves its column in the table: the additive sync never drops +one, and `os migrate plan` reports it as `unmapped_column` until +`os migrate apply --allow-destructive` does. In between, the values are still +stored, and no runtime door serves them: a read or a write through the data API +returns the object's declared fields only, and naming the column in `fields` is +refused. When the values have to move into the field that replaced it, this is +the read: + +```bash +os migrate unmapped-columns --object contact # The columns, and every record's values +os migrate unmapped-columns --object contact --json > out.json # The same, for a conversion script +os migrate unmapped-columns --object contact --max-records 1000000 --json +os migrate unmapped-columns --object contact --database-url postgres://… +``` + +The conversion route is three steps: read the values with this command, write +them into the declared fields with your own script, then run +`os migrate apply --allow-destructive` to drop the columns. + +- **One column set.** It reads exactly the columns `os migrate plan` reports as + `unmapped_column` for that object's table: the same differ, the same boot. + Columns the differ never reports are never read either, such as the driver's + own `id`, `created_at` and `updated_at`. +- **Operator-only and read-only.** It runs under the database credentials you + pass, and covers every organization's rows. There is no REST route or API + flag behind it. It boots the way `plan` does, so it writes nothing, and it + drops nothing. +- **Values as stored.** An unmapped column has no declared type, so each value is + emitted as the database client returns it, with no field-type decoding: a + retired `json` field on SQLite reads as its stored text, a retired `boolean` + as `0` or `1`, and a retired `datetime` on PostgreSQL arrives as a date and is + emitted as its ISO 8601 text. A value JSON cannot carry as stored (binary + bytes, a `bigint`, or a non-finite number) is refused with exit 1, naming the + column and the record id, and no record is emitted; read that column with + the database's own client. +- **Empty work, exit 0**, for an object with no unmapped column, or with no table + in this database yet. `--json` prints one document: + `{ object, table, columns, count, records: [{ id, values }] }`. +- **Refused, exit 1**: an object name the deployment does not declare + (`OBJECT_NOT_FOUND`); an object `plan` does not diff (federated, or bound to + another datasource), whose empty answer would be unmeasured; a read that + cannot be complete, such as one stopped by `--max-records`; and a value JSON + cannot carry as stored, described above. A partial set is + never emitted, because a conversion over part of a table, followed by the + drop, loses the rest. + #### Data migrations The commands above reconcile **schema**. A *data* migration rewrites rows, and diff --git a/packages/cli/src/commands/migrate/data-commands.absent-database.integration.test.ts b/packages/cli/src/commands/migrate/data-commands.absent-database.integration.test.ts index dcb401837b7..5ce8b25946a 100644 --- a/packages/cli/src/commands/migrate/data-commands.absent-database.integration.test.ts +++ b/packages/cli/src/commands/migrate/data-commands.absent-database.integration.test.ts @@ -58,6 +58,10 @@ * (`--json` and human), a booted database holding one row of work for each * (the control: the door READS the table and reports the row), and the four * doors that already exited 0 on the absent database, which still do. + * + * [#21573] `os migrate unmapped-columns` joined the roster later, born with + * the same answer: it asks `tableAbsent` before the differ. Its control row is + * a retired field's column on `os21529_contact`, holding a value. */ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; @@ -418,6 +422,9 @@ await k('sys_secret').insert({ await k('sys_file').insert({ id: 'file_21552', key: 'attachments/os21552.txt', name: 'os21552.txt', size: 12, scope: 'attachments', status: 'committed', }); +// A retired field's column: in the table, in no metadata, holding a value. +await k.raw('ALTER TABLE os21529_contact ADD COLUMN legacy_note text'); +await k('os21529_contact').insert({ id: 'con_21573', name: 'Ann', legacy_note: 'kept-21573' }); await raw.disconnect(); process.stderr.write('[fixture] seeded\\n'); process.exit(0); @@ -513,6 +520,19 @@ const DOORS: readonly Door[] = [ tables: ['sys_file', 'sys_attachment'], work: (doc) => expect(doc).toMatchObject({ filesScanned: 1, stranded: 1 }), }, + { + // Born after #21552 with the family's answer: it asks `tableAbsent` before + // the differ, so an absent table is empty work and is never read. + name: 'migrate unmapped-columns', + argv: ['migrate', 'unmapped-columns', '--object', 'os21529_contact'], + empty: (doc) => expect(doc).toMatchObject({ object: 'os21529_contact', columns: [], count: 0, records: [] }), + humanEmpty: /No unmapped column on os21529_contact/, + tables: ['os21529_contact'], + work: (doc) => { + expect(doc.columns).toEqual([{ column: 'legacy_note', actual: 'text' }]); + expect(doc.records).toContainEqual({ id: 'con_21573', values: { legacy_note: 'kept-21573' } }); + }, + }, ]; /** The four doors that already answered the absent database with exit 0: they must still. */ diff --git a/packages/cli/src/commands/migrate/unmapped-columns.integration.test.ts b/packages/cli/src/commands/migrate/unmapped-columns.integration.test.ts new file mode 100644 index 00000000000..89559ccd4c2 --- /dev/null +++ b/packages/cli/src/commands/migrate/unmapped-columns.integration.test.ts @@ -0,0 +1,331 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `os migrate unmapped-columns` at the public door: the CLI spawned against a + * real SQLite database that a field retirement left behind. + * + * ## The fixture + * + * One boot of release ONE writes records while `um_contact` still declares + * `mailing_street` and `mailing_city`. Release TWO retires both fields, so + * their columns stay in the table (the additive sync never drops one) and no + * runtime door serves them any more. A driver-owned hash-shadow column is + * added by hand beside them: it exists in no metadata either, and the differ + * leaves it out because it carries a UNIQUE index, not a retired field's + * values. `um_account` retires nothing. + * + * ## What is pinned + * + * 1. an object with retired columns: the door emits them, keyed by record id, + * every row, a NULL as a NULL, and nothing declared; + * 2. ONE column set: the columns are exactly the ones `os migrate plan + * --json` reports as `unmapped_column` for the same table, on the same + * database, and the hash shadow is out of both; + * 3. an object with none: empty work, exit 0; + * 4. an object name the registry does not hold: `OBJECT_NOT_FOUND`, exit 1, + * one document; + * 5. a row cap the table exceeds: refused, exit 1, and no records emitted; + * 6. a value JSON cannot carry as stored (bytes in a BLOB column added by + * hand to `um_blob`; the platform creates no binary column for any field + * type): refused in both faces, exit 1, naming the column and the record + * id, no record emitted; + * 7. the door writes nothing: the schema and every row are byte-identical + * after it ran. + * + * The runtime half, that the engine's data door never serves these columns, is + * the read narrowing's own pin + * (`packages/rest/src/data-query-unprojected-declared-fields.test.ts`) and is + * deliberately not restated here. The PostgreSQL leg of the read was measured + * by hand on a live server and is recorded on the pull request: the driver + * call this door issues is one code path for both dialects. + * + * Every spawn runs in the hook; a case only reads what a run printed. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawn, spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CLI_ROOT = resolve(HERE, '..', '..', '..'); +/** The source entry, as `test/helpers/serve-process.ts` spawns it; `src/` cannot import that helper. */ +const CLI = resolve(HERE, '../../../bin/run-dev.js'); + +const HOOK_TIMEOUT_MS = 480_000; +const RUN_BUDGET_MS = 120_000; + +const MANIFEST = { id: 'com.example.os21573', name: 'Unmapped columns', version: '0.0.0', type: 'app' }; + +/** Release one: the two mailing fields are still declared. */ +const RELEASE_ONE = { + manifest: MANIFEST, + objects: [ + { + name: 'um_contact', + fields: { name: { type: 'text' }, mailing_street: { type: 'text' }, mailing_city: { type: 'text' } }, + }, + { name: 'um_account', fields: { name: { type: 'text' } } }, + { name: 'um_blob', fields: { name: { type: 'text' } } }, + ], +}; + +/** Release two: both mailing fields retired. Their columns stay in the table. */ +const RELEASE_TWO = { + manifest: MANIFEST, + objects: [ + { name: 'um_contact', fields: { name: { type: 'text' } } }, + { name: 'um_account', fields: { name: { type: 'text' } } }, + { name: 'um_blob', fields: { name: { type: 'text' } } }, + ], +}; + +/** The driver-owned hash-shadow spelling the differ skips (`HASH_SHADOW_SUFFIX`). */ +const HASH_SHADOW = 'legacy_code__hash'; + +/** Env that would point a boot somewhere other than the fixture. */ +const OVERRIDING_ENV = ['OS_DATABASE_URL', 'DATABASE_URL', 'TURSO_DATABASE_URL', 'OS_DATABASE_DRIVER', 'OS_HOME'] as const; + +function childEnv(overrides: Record): Record { + const env: Record = {}; + for (const [key, value] of Object.entries(process.env)) { + if (key === 'TEST' || key === 'VITEST' || key.startsWith('VITEST_') || key === 'NODE_PATH') continue; + env[key] = value; + } + const unset: Record = {}; + for (const key of OVERRIDING_ENV) unset[key] = undefined; + return { ...env, ...unset, ...overrides }; +} + +/** + * Release one, served: DDL performed, three contacts and an account written + * while the mailing fields are declared. Then, by hand, the hash shadow and a + * BLOB column on `um_blob` holding three bytes. + */ +const SEED_CHILD = ` +const rt = await import('@objectstack/runtime'); +const stack = await rt.createStandaloneStack({ + projectRoot: process.env.FIXTURE_PROJECT, + databaseUrl: 'file:' + process.env.FIXTURE_DB, + skipSeedData: true, + armLifecycleSweep: false, +}); +const runtime = new rt.Runtime({ cluster: false }); +const kernel = runtime.getKernel(); +for (const plugin of stack.plugins) await kernel.use(plugin); +await runtime.start(); +const ql = kernel.getService('objectql'); +const SYSTEM = { context: { isSystem: true } }; +await ql.insert('um_contact', { id: 'c1', name: 'Ann', mailing_street: '1 Retired Way', mailing_city: 'Oldtown' }, SYSTEM); +await ql.insert('um_contact', { id: 'c2', name: 'Bob', mailing_street: '2 Retired Way', mailing_city: 'Newtown' }, SYSTEM); +await ql.insert('um_contact', { id: 'c3', name: 'Cy' }, SYSTEM); +await ql.insert('um_account', { id: 'a1', name: 'Acme' }, SYSTEM); +await ql.insert('um_blob', { id: 'b1', name: 'Bin' }, SYSTEM); +await kernel.shutdown(); +const { SqlDriver } = await import('@objectstack/driver-sql'); +const raw = new SqlDriver({ client: 'better-sqlite3', connection: { filename: process.env.FIXTURE_DB }, useNullAsDefault: true }); +await raw.knex.raw('ALTER TABLE um_contact ADD COLUMN ${HASH_SHADOW} text'); +await raw.knex('um_contact').update({ ${HASH_SHADOW}: 'shadow' }); +await raw.knex.raw('ALTER TABLE um_blob ADD COLUMN legacy_bytes blob'); +await raw.knex('um_blob').update({ legacy_bytes: Buffer.from([1, 2, 255]) }); +await raw.disconnect(); +process.stderr.write('[fixture] seeded\\n'); +process.exit(0); +`; + +/** The schema and every row, read on a connection of our own. */ +const READ_STATE_CHILD = ` +const { SqlDriver } = await import('@objectstack/driver-sql'); +const raw = new SqlDriver({ client: 'better-sqlite3', connection: { filename: process.env.FIXTURE_DB }, useNullAsDefault: true }); +const k = raw.knex; +const schema = await k.raw('SELECT type, name, sql FROM sqlite_master ORDER BY type, name'); +const rows = {}; +for (const entry of schema) { + if (entry.type !== 'table' || entry.name.startsWith('sqlite_')) continue; + rows[entry.name] = await k.raw('SELECT * FROM "' + entry.name + '" ORDER BY rowid'); +} +await raw.disconnect(); +process.stdout.write(JSON.stringify({ schema, rows })); +process.exit(0); +`; + +interface Run { + code: number | null; + stdout: string; + stderr: string; +} + +let dir: string; +let db: string; + +function childNode(code: string, env: Record): { status: number | null; stdout: string; stderr: string } { + const out = spawnSync(process.execPath, ['--input-type=module', '-e', code], { + cwd: CLI_ROOT, + env: childEnv({ OS_ARTIFACT_PATH: join(dir, 'dist', 'objectstack.json'), OS_SECRET_KEY: '0e2e'.repeat(16), ...env }), + encoding: 'utf8', + timeout: RUN_BUDGET_MS, + }); + return { status: out.status, stdout: String(out.stdout), stderr: String(out.stderr) }; +} + +function readState(): string { + const out = childNode(READ_STATE_CHILD, { FIXTURE_DB: db }); + if (out.status !== 0) throw new Error(`could not read the fixture database (status ${out.status})\n${out.stderr}`); + return out.stdout; +} + +/** One `os --database-url file:` run, in the fixture project. */ +function runCli(argv: string[]): Promise { + return new Promise((resolveRun, rejectRun) => { + const child = spawn(process.execPath, [CLI, ...argv, '--database-url', `file:${db}`], { + cwd: dir, + env: childEnv({ + NO_COLOR: '1', + OS_ARTIFACT_PATH: join(dir, 'dist', 'objectstack.json'), + OS_SECRET_KEY: '0e2e'.repeat(16), + }), + stdio: ['ignore', 'pipe', 'pipe'], + }); + let stdout = ''; + let stderr = ''; + child.stdout.on('data', (c) => { stdout += String(c); }); + child.stderr.on('data', (c) => { stderr += String(c); }); + const timer = setTimeout(() => { + child.kill('SIGKILL'); + rejectRun(new Error(`os ${argv.join(' ')} did not finish within ${RUN_BUDGET_MS}ms\n${stderr}`)); + }, RUN_BUDGET_MS); + child.on('error', (err) => { clearTimeout(timer); rejectRun(err); }); + child.on('close', (code) => { + clearTimeout(timer); + resolveRun({ code, stdout, stderr }); + }); + }); +} + +const door = (object: string, ...extra: string[]): string[] => ['migrate', 'unmapped-columns', '--object', object, ...extra]; + +let before: string; +let after: string; +let retiredJson: Run; +let retiredHuman: Run; +let planJson: Run; +let noneJson: Run; +let noneHuman: Run; +let unknownJson: Run; +let cappedJson: Run; +let bytesJson: Run; +let bytesHuman: Run; + +beforeAll(async () => { + dir = mkdtempSync(join(tmpdir(), 'os-21573-')); + mkdirSync(join(dir, 'dist'), { recursive: true }); + mkdirSync(join(dir, 'data'), { recursive: true }); + db = join(dir, 'data', 'app.db'); + + writeFileSync(join(dir, 'dist', 'objectstack.json'), JSON.stringify(RELEASE_ONE)); + const seed = childNode(SEED_CHILD, { FIXTURE_PROJECT: dir, FIXTURE_DB: db }); + if (seed.status !== 0 || !seed.stderr.includes('[fixture] seeded')) { + throw new Error(`the fixture database was not seeded (status ${seed.status})\n${seed.stderr}`); + } + // The upgrade: every run below is against release two. + writeFileSync(join(dir, 'dist', 'objectstack.json'), JSON.stringify(RELEASE_TWO)); + + before = readState(); + retiredJson = await runCli(door('um_contact', '--json')); + retiredHuman = await runCli(door('um_contact')); + noneJson = await runCli(door('um_account', '--json')); + noneHuman = await runCli(door('um_account')); + unknownJson = await runCli(door('um_nope', '--json')); + cappedJson = await runCli(door('um_contact', '--max-records', '2', '--json')); + bytesJson = await runCli(door('um_blob', '--json')); + bytesHuman = await runCli(door('um_blob')); + after = readState(); + planJson = await runCli(['migrate', 'plan', '--json']); +}, HOOK_TIMEOUT_MS); + +afterAll(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); +}); + +describe('os migrate unmapped-columns: an object with retired columns', () => { + it('--json: emits the retired columns\' values keyed by record id, every row, exit 0', () => { + expect(retiredJson.code, retiredJson.stderr).toBe(0); + const doc = JSON.parse(retiredJson.stdout); + expect(doc).toMatchObject({ object: 'um_contact', table: 'um_contact', count: 3 }); + expect(doc.columns.map((c: { column: string }) => c.column)).toEqual(['mailing_city', 'mailing_street']); + expect(doc.records).toEqual([ + { id: 'c1', values: { mailing_city: 'Oldtown', mailing_street: '1 Retired Way' } }, + { id: 'c2', values: { mailing_city: 'Newtown', mailing_street: '2 Retired Way' } }, + // Written with the fields declared but left empty: a stored NULL, emitted as one. + { id: 'c3', values: { mailing_city: null, mailing_street: null } }, + ]); + }); + + it('emits nothing declared and nothing the differ leaves out: no field, no built-in column, no hash shadow', () => { + const doc = JSON.parse(retiredJson.stdout); + const emitted = new Set(doc.records.flatMap((r: { values: object }) => Object.keys(r.values))); + for (const key of ['name', 'id', 'created_at', 'updated_at', HASH_SHADOW]) expect(emitted.has(key), key).toBe(false); + expect(doc.columns.map((c: { column: string }) => c.column)).not.toContain(HASH_SHADOW); + }); + + it('is ONE column set: exactly what os migrate plan --json reports as unmapped_column for this table', () => { + expect(planJson.code, planJson.stderr).toBe(0); + const plan = JSON.parse(planJson.stdout); + const planned = (plan.changes as Array<{ kind: string; table: string; column: string; actual: string }>) + .filter((c) => c.kind === 'unmapped_column' && c.table === 'um_contact') + .map((c) => ({ column: c.column, actual: c.actual })); + // Non-vacuity: the plan does report the retired columns. + expect(planned.length).toBe(2); + expect(JSON.parse(retiredJson.stdout).columns).toEqual(planned); + // The differ's exclusion holds on both: the shadow is in the table and in neither answer. + expect(planned.map((c) => c.column)).not.toContain(HASH_SHADOW); + }); + + it('human face: names the columns and the records, exit 0', () => { + expect(retiredHuman.code, retiredHuman.stderr).toBe(0); + expect(retiredHuman.stdout).toMatch(/2 unmapped column\(s\)/); + expect(retiredHuman.stdout).toContain('mailing_street'); + expect(retiredHuman.stdout).toContain('"mailing_city":"Oldtown"'); + expect(retiredHuman.stdout).toMatch(/Read 3 record\(s\)/); + }); + + it('writes nothing: the schema and every row are byte-identical after the door ran', () => { + expect(JSON.parse(before).rows.um_contact).toHaveLength(3); + expect(after).toBe(before); + }); +}); + +describe('os migrate unmapped-columns: the other answers', () => { + it('an object with none: empty work, exit 0', () => { + expect(noneJson.code, noneJson.stderr).toBe(0); + expect(JSON.parse(noneJson.stdout)).toMatchObject({ object: 'um_account', columns: [], count: 0, records: [] }); + expect(noneHuman.code, noneHuman.stderr).toBe(0); + expect(noneHuman.stdout).toContain('No unmapped column on um_account'); + }); + + it('an object the registry does not hold: OBJECT_NOT_FOUND, exit 1, one document', () => { + expect(unknownJson.code).toBe(1); + expect(JSON.parse(unknownJson.stdout)).toMatchObject({ code: 'OBJECT_NOT_FOUND' }); + }); + + it('a row cap the table exceeds: refused, exit 1, and no records emitted', () => { + expect(cappedJson.code).toBe(1); + const doc = JSON.parse(cappedJson.stdout); + expect(doc.error).toMatch(/stopped at 2 row\(s\)[\s\S]*--max-records/); + expect(doc).not.toHaveProperty('records'); + }); + + it('a value JSON cannot carry as stored: refused in both faces, exit 1, naming the column and the record id', () => { + const says = /Record b1 of um_blob holds binary bytes in the column legacy_bytes[\s\S]*database's own client/; + expect(bytesJson.code).toBe(1); + const doc = JSON.parse(bytesJson.stdout); + expect(doc.error).toMatch(says); + expect(doc).not.toHaveProperty('records'); + expect(bytesHuman.code).toBe(1); + expect(bytesHuman.stdout).toMatch(says); + // Neither face carries the stand-in a JSON serialisation of the bytes would be. + for (const out of [bytesJson.stdout, bytesHuman.stdout]) expect(out).not.toContain('"type":"Buffer"'); + }); +}); diff --git a/packages/cli/src/commands/migrate/unmapped-columns.test.ts b/packages/cli/src/commands/migrate/unmapped-columns.test.ts new file mode 100644 index 00000000000..996ba4d3407 --- /dev/null +++ b/packages/cli/src/commands/migrate/unmapped-columns.test.ts @@ -0,0 +1,232 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * `os migrate unmapped-columns`, its two pure halves: which columns it reads, + * and how it reads them. + * + * ## The column set is the plan's, by construction + * + * {@link unmappedColumnsOf} filters `detectManagedDrift()`'s findings; it never + * diffs anything itself. So the cases here hold the FILTER: every + * `unmapped_column` finding of the named table, in the differ's order, and + * nothing else the differ reports (a type mismatch, an index op, another + * table's orphan). What the differ leaves out, a built-in column or a + * driver-owned hash shadow, never reaches this filter: that half is pinned on + * a real database by `unmapped-columns.integration.test.ts`, beside + * `os migrate plan`'s own answer for the same table. + * + * ## The read refuses rather than emit a partial set + * + * A conversion script run over part of a table, followed by the destructive + * drop, loses the rest. So a cap that stops the walk, a row missing a + * reported column (the SQL driver answers a projection naming a missing column + * with the whole row, measured on SQLite and PostgreSQL) and an answer that is + * not rows each REFUSE, and none of them returns records. + * + * ## …and rather than emit a value JSON cannot carry as stored + * + * Binary bytes, a `bigint` and a non-finite number each reach a JSON document + * as a stand-in (an object shaped like the bytes, a thrown serialisation, a + * `null`), which a conversion would write as the value. One case per class, + * each naming the column and the record id, and a control that a `Date`, a + * parsed json object and a string pass unchanged. What the drivers hand back + * for the columns the platform itself creates is measured on the pull request: + * none of them is in the refused set. + */ + +import { describe, it, expect } from 'vitest'; +import type { ManagedDriftEntry } from '@objectstack/driver-sql'; +import { + unmappedColumnsOf, + readUnmappedColumnValues, + unrepresentableKind, + type UnmappedColumnReader, +} from './unmapped-columns.js'; + +/** A drift finding in the differ's shape; only the fields the filter reads matter. */ +function finding(partial: Partial & { kind: ManagedDriftEntry['kind']; table: string }): ManagedDriftEntry { + return { + remoteName: partial.table, + expected: '(absent)', + actual: 'text', + severity: 'warning', + category: 'destructive', + op: { type: 'drop_column', table: partial.table, column: partial.column ?? '' }, + message: '', + ...partial, + } as ManagedDriftEntry; +} + +/** + * A reader over in-memory rows that honours what the keyset walk asks of a + * driver: the `id > cursor` seek, ascending order, the page limit and the + * projection. A condition is an equality or a `$gt`; any other operator is + * refused rather than read as a match. Every query it was asked is recorded. + */ +function rowsReader(rows: Array>): UnmappedColumnReader & { queries: Array> } { + const queries: Array> = []; + const holds = (row: Record, key: string, cond: unknown): boolean => { + if (cond !== null && typeof cond === 'object') { + const ops = Object.keys(cond); + if (ops.length === 1 && ops[0] === '$gt') return String(row[key]) > String((cond as { $gt: unknown }).$gt); + throw new Error(`this double implements equality and $gt only, not ${JSON.stringify(cond)}`); + } + return row[key] === cond; + }; + return { + queries, + async find(_object, query) { + queries.push(query); + const where = (query?.where ?? {}) as Record; + const matched = rows + .filter((r) => Object.entries(where).every(([key, cond]) => holds(r, key, cond))) + .sort((a, b) => String(a.id).localeCompare(String(b.id))); + // The caller's bound, by presence, after the filter: a driver applies it there too. + const page = typeof query?.limit === 'number' ? matched.slice(0, query.limit) : matched; + const fields = Array.isArray(query?.fields) ? (query.fields as string[]) : null; + return fields ? page.map((r) => Object.fromEntries(fields.filter((f) => f in r).map((f) => [f, r[f]]))) : page; + }, + }; +} + +describe('unmappedColumnsOf — the plan\'s unmapped_column findings for one table, and nothing else', () => { + const drift: ManagedDriftEntry[] = [ + finding({ kind: 'unmapped_column', table: 'contact', column: 'mailing_city', actual: 'varchar(255)' }), + finding({ kind: 'unmapped_column', table: 'contact', column: 'mailing_street', actual: 'text' }), + finding({ kind: 'type_mismatch', table: 'contact', column: 'name', actual: 'varchar(80)' }), + finding({ kind: 'unmapped_column', table: 'account', column: 'legacy_rank', actual: 'integer' }), + // An orphaned generated INDEX over the same column: unmapped, but not a column. + finding({ kind: 'unmapped_index', table: 'contact', column: 'mailing_street' }), + ]; + + it('picks every unmapped_column finding of the named table, in the differ\'s order, with its physical type', () => { + expect(unmappedColumnsOf(drift, 'contact')).toEqual([ + { column: 'mailing_city', actual: 'varchar(255)' }, + { column: 'mailing_street', actual: 'text' }, + ]); + }); + + it('leaves out another table\'s orphan and every other kind of finding on this one', () => { + expect(unmappedColumnsOf(drift, 'account')).toEqual([{ column: 'legacy_rank', actual: 'integer' }]); + expect(unmappedColumnsOf(drift, 'lead')).toEqual([]); + }); + + it('answers empty work for a table the differ reported nothing unmapped on', () => { + expect(unmappedColumnsOf([finding({ kind: 'type_mismatch', table: 'contact', column: 'name' })], 'contact')).toEqual([]); + expect(unmappedColumnsOf([], 'contact')).toEqual([]); + }); +}); + +describe('readUnmappedColumnValues — every row, keyed by record id, values as the driver returned them', () => { + it('asks the driver for id plus exactly the unmapped columns, and keys each record by its id', async () => { + const reader = rowsReader([ + { id: 'c2', name: 'Bob', mailing_street: null, mailing_city: null }, + { id: 'c1', name: 'Ann', mailing_street: '1 Retired Way', mailing_city: 'Oldtown' }, + ]); + const records = await readUnmappedColumnValues(reader, 'contact', ['mailing_city', 'mailing_street']); + + expect(reader.queries[0]?.fields).toEqual(['id', 'mailing_city', 'mailing_street']); + expect(records).toEqual([ + { id: 'c1', values: { mailing_city: 'Oldtown', mailing_street: '1 Retired Way' } }, + // A NULL is a stored value: the row is emitted with it, never dropped. + { id: 'c2', values: { mailing_city: null, mailing_street: null } }, + ]); + // A declared field the reader happened to carry is not an unmapped value. + expect(Object.keys(records[0]!.values)).not.toContain('name'); + }); + + it('decodes nothing: a JSON-looking string, a 0/1, a Date, a parsed json object and a string pass unchanged', async () => { + const at = new Date('2026-01-02T03:04:05.000Z'); + const parsed = { a: [1, 2], b: 'x' }; + const reader = rowsReader([ + { id: 'c1', legacy_flags: '{"a":[1,2]}', legacy_on: 1, legacy_at: at, legacy_json: parsed, legacy_note: 'kept' }, + ]); + const [record] = await readUnmappedColumnValues(reader, 'contact', [ + 'legacy_flags', 'legacy_on', 'legacy_at', 'legacy_json', 'legacy_note', + ]); + + expect(record!.values.legacy_flags).toBe('{"a":[1,2]}'); + expect(record!.values.legacy_on).toBe(1); + // The same instances: nothing is converted on the way through. + expect(record!.values.legacy_at).toBe(at); + expect(record!.values.legacy_json).toBe(parsed); + expect(record!.values.legacy_note).toBe('kept'); + // …and each one JSON carries unambiguously: a Date as its ISO 8601 text. + expect(JSON.parse(JSON.stringify(record!.values))).toEqual({ + legacy_flags: '{"a":[1,2]}', legacy_on: 1, legacy_at: '2026-01-02T03:04:05.000Z', legacy_json: parsed, legacy_note: 'kept', + }); + }); + + it.each([ + ['a Buffer (a SQLite blob, a PostgreSQL bytea)', Buffer.from([1, 2, 255]), /holds binary bytes/], + ['another ArrayBufferView', new Float64Array([1.5]), /holds binary bytes/], + ['a bigint', 9007199254740993n, /holds a bigint/], + ['NaN', Number.NaN, /holds a non-finite number \(NaN\)/], + ['Infinity', Number.POSITIVE_INFINITY, /holds a non-finite number \(Infinity\)/], + ['-Infinity', Number.NEGATIVE_INFINITY, /holds a non-finite number \(-Infinity\)/], + ])('REFUSES %s, naming the column and the record id, and emits no record', async (_label, value, says) => { + // The bad value sits on the SECOND row: the first is never handed back on its own. + const reader = rowsReader([ + { id: 'c1', legacy: 'fine' }, + { id: 'c2', legacy: value }, + ]); + const read = readUnmappedColumnValues(reader, 'contact', ['legacy']); + await expect(read).rejects.toThrow(says); + await expect(readUnmappedColumnValues(reader, 'contact', ['legacy'])).rejects.toThrow( + /Record c2 of contact holds [^\n]* in the column legacy, which JSON cannot carry as the database stored it/, + ); + }); + + it('the predicate answers null for every value JSON carries as stored', () => { + for (const value of [null, 'text', '', 0, -1.5, 42, true, false, new Date(0), { a: 1 }, [1, 'x']]) { + expect(unrepresentableKind(value), String(value)).toBeNull(); + } + }); + + it('walks past one page by seeking on id, and reads every row once', async () => { + const rows = Array.from({ length: 1203 }, (_, i) => ({ id: `r${String(i).padStart(5, '0')}`, legacy: i })); + const reader = rowsReader(rows); + const records = await readUnmappedColumnValues(reader, 'contact', ['legacy']); + + expect(records).toHaveLength(1203); + expect(new Set(records.map((r) => r.id)).size).toBe(1203); + expect(reader.queries.length).toBeGreaterThan(1); + expect(reader.queries[1]?.where).toEqual({ id: { $gt: 'r00499' } }); + }); + + it('answers no records for an empty table', async () => { + expect(await readUnmappedColumnValues(rowsReader([]), 'contact', ['legacy'])).toEqual([]); + }); + + it('REFUSES when the row cap stops the walk, rather than emit a partial set', async () => { + const reader = rowsReader([{ id: 'a', legacy: 1 }, { id: 'b', legacy: 2 }, { id: 'c', legacy: 3 }]); + await expect(readUnmappedColumnValues(reader, 'contact', ['legacy'], { max: 2 })).rejects.toThrow( + /stopped at 2 row\(s\) without reaching the end of the table[\s\S]*--max-records/, + ); + }); + + it('reads a table that holds exactly the cap, without refusing', async () => { + const reader = rowsReader([{ id: 'a', legacy: 1 }, { id: 'b', legacy: 2 }]); + expect(await readUnmappedColumnValues(reader, 'contact', ['legacy'], { max: 2 })).toHaveLength(2); + }); + + it('REFUSES a row missing a reported column (the whole-row answer to a projection naming a missing column)', async () => { + const reader: UnmappedColumnReader = { + async find() { + return [{ id: 'c1', name: 'Ann', created_at: '2026-01-01' }]; + }, + }; + await expect(readUnmappedColumnValues(reader, 'contact', ['mailing_street'])).rejects.toThrow( + /returned record c1 without the column mailing_street/, + ); + }); + + it('REFUSES an answer that is not an array of rows, rather than read it as an empty table', async () => { + const reader: UnmappedColumnReader = { + async find() { + return { records: [] }; + }, + }; + await expect(readUnmappedColumnValues(reader, 'contact', ['legacy'])).rejects.toThrow(/not an array of rows/); + }); +}); diff --git a/packages/cli/src/commands/migrate/unmapped-columns.ts b/packages/cli/src/commands/migrate/unmapped-columns.ts new file mode 100644 index 00000000000..35c1a6c619a --- /dev/null +++ b/packages/cli/src/commands/migrate/unmapped-columns.ts @@ -0,0 +1,440 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { Command, Flags } from '@oclif/core'; +import chalk from 'chalk'; +import { objectNotFoundError } from '@objectstack/core'; +import { keysetWalk } from '@objectstack/types'; +import { StorageNameMapping } from '@objectstack/spec/system'; +import type { ManagedDriftEntry } from '@objectstack/driver-sql'; +import { + printHeader, + printSuccess, + printError, + printInfo, + printStep, + createTimer, + emitJson, + errorCodeFields, + isExitSignal, +} from '../../utils/format.js'; +import { bootSchemaStack } from '../../utils/schema-migrate.js'; +import { exitOneShotCommand } from '../../utils/one-shot-exit.js'; +import { + refuseWhenHostConfigUnloadable, + type SchemaMigrationComposition, +} from '../../utils/schema-migration-plugins.js'; + +/** Rows per page of the read. The projection is `id` plus the unmapped columns, so a page stays narrow. */ +const PAGE_SIZE = 500; + +/** One unmapped column, as `os migrate plan` reports it: the name, and the physical type the differ read. */ +export interface UnmappedColumn { + column: string; + actual: string; +} + +/** One record's unmapped values, keyed by its id. */ +export interface UnmappedColumnRecord { + id: unknown; + values: Record; +} + +/** + * The columns `os migrate plan` reports as `unmapped_column` for one table. + * + * ⛔ A filter over the plan's own findings, never a second column diff. The + * differ (`diffManagedTable` in `@objectstack/driver-sql`, reached through + * `detectManagedDrift()`) owns what counts as unmapped, including what it + * leaves out: the columns the driver creates unconditionally (`id`, + * `created_at`, `updated_at`) and a driver-owned hash-shadow column, which + * carries a UNIQUE index rather than a retired field's values. A column this + * door emits is therefore always a column the plan reports, and a column the + * plan reports for this table is always one this door emits. + */ +export function unmappedColumnsOf(drift: readonly ManagedDriftEntry[], table: string): UnmappedColumn[] { + return drift + .filter((d) => d.kind === 'unmapped_column' && d.table === table && typeof d.column === 'string') + .map((d) => ({ column: d.column as string, actual: String(d.actual) })); +} + +/** The driver read this door issues: `find` only, read-only by construction. */ +export interface UnmappedColumnReader { + find(object: string, query: Record): Promise; +} + +/** + * The class of a value JSON cannot carry as the database stored it, or `null` + * when it can. + * + * Each of the three arrives in a JSON document as something else, with nothing + * to say so: binary bytes as an object shaped `{ type, data }` (a `Buffer`) or + * as an index-keyed object (another typed array), a `bigint` as a thrown + * serialisation error, and `NaN` / `±Infinity` as `null`. A conversion script + * would write the stand-in into the replacing field and report success, so the + * door refuses these instead of emitting them. ⛔ No codec: choosing a + * representation would make a representation part of this door's contract. + * + * A `Date` is not in the set: PostgreSQL hands a `timestamp` column back as + * one, and it serialises to its unambiguous ISO 8601 text. + */ +export function unrepresentableKind(value: unknown): string | null { + if (ArrayBuffer.isView(value)) return 'binary bytes'; + if (typeof value === 'bigint') return 'a bigint'; + if (typeof value === 'number' && !Number.isFinite(value)) return `a non-finite number (${String(value)})`; + return null; +} + +/** + * Read every row's unmapped values, keyed by record id. + * + * Through the DRIVER, never the engine: the engine's read verbs serve the + * declared fields and refuse a name no metadata declares, which is the + * runtime rule this operator door exists beside, not a gap in it. No tenant + * scope is passed, so the read covers every organization's rows, under the + * credentials the operator's database URL carries. + * + * Values are emitted as the driver hands them back. An unmapped column has no + * declared type, so the driver applies none of its field-type decoding to it, + * and this function applies none either: no parsing, no hydration, no + * decryption. + * + * ⛔ Refuses, and emits nothing, when the read cannot be complete: + * - the walk stops before the end of the table (`--max-records`, or a row it + * cannot seek past): a conversion run over part of a table, followed by the + * destructive drop, loses the rest; + * - a row comes back without a column the differ reported: the SQL driver + * answers a projection naming a missing column with the whole row instead, + * and a record emitted without its value would read as converted; + * - a value is one JSON cannot carry as stored ({@link unrepresentableKind}): + * it would be emitted as a stand-in a conversion would write as the value; + * - the driver answers something other than an array of rows. + */ +export async function readUnmappedColumnValues( + reader: UnmappedColumnReader, + object: string, + columns: readonly string[], + opts: { max?: number } = {}, +): Promise { + const walk = keysetWalk>( + async (q) => { + const page = await reader.find(object, { + where: q.where, + orderBy: q.orderBy, + limit: q.limit, + fields: ['id', ...columns], + }); + if (!Array.isArray(page)) { + throw new Error( + `Reading ${object} answered ${page === null ? 'null' : typeof page}, not an array of rows. ` + + 'Refusing rather than reading an uninterpretable answer as an empty one.', + ); + } + return page as Array>; + }, + { pageSize: PAGE_SIZE, ...(opts.max != null ? { max: opts.max } : {}) }, + ); + + const records: UnmappedColumnRecord[] = []; + for await (const page of walk.pages()) { + for (const row of page) { + const values: Record = {}; + for (const column of columns) { + if (!Object.prototype.hasOwnProperty.call(row, column)) { + throw new Error( + `Reading ${object} returned record ${String(row.id)} without the column ${column}, which ` + + '"os migrate plan" reports as unmapped. The driver answered a different projection than ' + + 'the one asked for. Refusing rather than emitting the record without that value.', + ); + } + const kind = unrepresentableKind(row[column]); + if (kind !== null) { + throw new Error( + `Record ${String(row.id)} of ${object} holds ${kind} in the column ${column}, which JSON cannot ` + + 'carry as the database stored it. Read that column with the database\'s own client. Refusing ' + + 'rather than emitting a stand-in a conversion would write as the value; no record was emitted.', + ); + } + values[column] = row[column]; + } + records.push({ id: row.id, values }); + } + } + + if (walk.truncated) { + throw new Error( + `The read of ${object} stopped at ${walk.scanned} row(s) without reaching the end of the table, ` + + 'so the rows it did not read cannot be emitted. ' + + (opts.max != null ? 'Re-run with a higher row cap (--max-records). ' : '') + + 'Refusing rather than emitting a partial set: a conversion over part of the table, followed by ' + + 'the destructive drop, loses the rest.', + ); + } + return records; +} + +/** + * `os migrate unmapped-columns` — read the values of the columns `os migrate + * plan` reports as `unmapped_column` for one object, keyed by record id. + * + * ## What it is for + * + * Retiring a field leaves its column in the table: the additive schema sync + * never drops one, and `os migrate apply --allow-destructive` is what does. + * Between the two, the column's values are still stored, and no runtime door + * serves them: a read or a write through the engine answers the object's + * declared fields only, and naming the column is refused. An app that moves a + * retired field's values into the field that replaced it needs to read them + * once, and this is that read: the operator runs it, a conversion script + * writes the values into the declared fields, and only then does the + * destructive apply drop the columns. + * + * ## Why it is a CLI read and nothing else + * + * - **Operator-only.** It runs under the operator's own database credentials + * (`--database-url`), like every `os migrate` subcommand. There is no REST + * route, API flag or per-request option behind it: a runtime door that + * served undeclared columns is exactly what the read narrowing closed. + * - **Read-only.** It boots the way `os migrate plan` does (schema DDL + * deferred, no seed, no database file created) and issues one paged + * `find` per page. Dropping the columns stays `os migrate apply + * --allow-destructive`'s job. + * - **One column set.** The columns are the plan's own `unmapped_column` + * findings for the object's table ({@link unmappedColumnsOf}). The boot is + * the plan's boot, host composition included, so the object set the differ + * runs over is the plan's too. + * + * ## The answers + * + * - columns found: their values, keyed by record id, exit 0; + * - an object with no unmapped column, or with no table in this database yet: + * empty work, exit 0; + * - no SQL driver: the plan's own answer (`no_sql_driver`, exit 0), since no + * differ runs there; + * - an object name no registry entry resolves: `OBJECT_NOT_FOUND`, exit 1; + * - an object the plan does not diff (federated, or bound to another + * datasource): refused, exit 1, because an empty answer there would be + * unmeasured rather than clean; + * - a read that cannot be complete, or a value JSON cannot carry as stored: + * refused, exit 1, no record emitted ({@link readUnmappedColumnValues}). + */ +export default class MigrateUnmappedColumns extends Command { + // No tracker id in this string: a command description reaches operators, + // who have no tracker to resolve one against (`check:doc-authoring`). + static override description = + 'Read the values of the columns "os migrate plan" reports as unmapped_column for one object, keyed by ' + + 'record id, so a conversion script can move a retired field\'s values into the field that replaced it ' + + 'before "os migrate apply --allow-destructive" drops the columns. Read-only; runs under the database ' + + 'credentials you pass, and no runtime door serves these values.'; + + static override examples = [ + '$ os migrate unmapped-columns --object contact', + '$ os migrate unmapped-columns --object contact --json > contact-unmapped.json', + '$ os migrate unmapped-columns --object contact --max-records 1000000 --json', + '$ os migrate unmapped-columns --object contact --database-url postgres://…', + ]; + + static override flags = { + 'database-url': Flags.string({ + description: 'Database URL to read (defaults to $OS_DATABASE_URL / the project DB)', + env: 'OS_DATABASE_URL', + }), + object: Flags.string({ + description: 'The object whose unmapped columns to read (one per run)', + required: true, + }), + 'max-records': Flags.integer({ + description: + 'Row cap for the read. Reaching it REFUSES rather than emitting a partial set.', + }), + json: Flags.boolean({ description: 'Output the columns and their values, keyed by record id, as JSON' }), + }; + + /** + * What {@link read} composed, read by {@link run} after it returns — the + * `os migrate plan` wrapper's shape. `null` until the stack has booted. + */ + private composition: SchemaMigrationComposition | null = null; + + /** + * The body is {@link read}; this wrapper ends the process deliberately, for + * the reason `os migrate plan`'s does: the composed host boot can leave the + * event loop alive after the kernel reports a clean shutdown. The failure + * paths exit through oclif's own signal and do not come back here. + */ + async run(): Promise { + await this.read(); + // A host config that exists but could not be loaded means the object set + // above is a fraction of the deployment's: the document stands, the run + // is refused, exactly as the plan refuses it. + if (this.composition) refuseWhenHostConfigUnloadable(this.composition); + await exitOneShotCommand(typeof process.exitCode === 'number' ? process.exitCode : 0); + } + + private async read(): Promise { + const { flags } = await this.parse(MigrateUnmappedColumns); + const timer = createTimer(); + const object = flags.object; + const table = StorageNameMapping.resolveTableName({ name: object }); + + if (!flags.json) { + printHeader('Migrate · unmapped-columns'); + printStep('Booting schema stack (read-only)…'); + } + + let stack; + try { + // The `os migrate plan` boot, so the differ runs over the plan's object + // set: schema DDL deferred and no seed (`deferSchemaDdl`), no database + // file brought into existence (`readOnlyProbe`), and the deployment's + // own composition (`composeHostStack`). + stack = await bootSchemaStack({ + jsonOutput: flags.json, + ...(flags['database-url'] ? { databaseUrl: flags['database-url'] } : {}), + deferSchemaDdl: true, + readOnlyProbe: true, + composeHostStack: true, + }); + } catch (error: any) { + if (flags.json) { + await emitJson( + { error: 'boot_failed', detail: error?.message ?? String(error), ...errorCodeFields(error) }, + 1, + { compact: true }, + ); + return; + } + printError(error?.message ?? String(error)); + this.exit(1); + return; + } + this.composition = stack.composition; + + try { + // The plan's answer where no differ runs: no SQL driver, no drift report. + if (!stack.driver) { + if (flags.json) { + await emitJson({ error: 'no_sql_driver', object, columns: [], records: [] }, 0, { compact: true }); + return; + } + printInfo( + 'Unmapped columns are reported by the SQL drivers\' schema differ (SQLite / Postgres). No SQL driver ' + + 'is active, so "os migrate plan" reports none and there is nothing to read.', + ); + return; + } + + const declared = stack + .allObjects() + .find((o) => (o as { name?: unknown } | null)?.name === object) as { external?: unknown } | undefined; + if (!declared) throw objectNotFoundError(object); + + // The coverage pass's own judgement (`measureComposedCoverage`): the plan + // diffs the objects bound to its driver and nothing else. For any other + // object it reports nothing, and that nothing is unmeasured, not clean. + const engine = (stack.kernel as { getService?: (n: string) => unknown }).getService?.call( + stack.kernel, + 'objectql', + ) as { getDriverForObject?: (name: string) => unknown } | undefined; + const bound = engine?.getDriverForObject?.(object); + const outside = + declared.external != null + ? 'it is federated (no managed table)' + : !bound + ? 'it is bound to no driver' + : bound !== stack.driver + ? 'it is bound to a different datasource' + : null; + if (outside) { + throw new Error( + `${object} is not in the set "os migrate plan" diffs: ${outside}. The plan reports no unmapped column ` + + 'for it, and that is unmeasured, not clean. Refusing rather than answering empty work.', + ); + } + + const reader = stack.driver as unknown as Partial; + if (typeof reader.find !== 'function') { + throw new Error(`The SQL driver on this stack has no find(); ${object} cannot be read from here.`); + } + + // [#21529] Not asked: the read-only boot measured this table absent, and + // a table that does not exist holds no column to read. + if (stack.tableAbsent(object)) { + const line = `${object} has no table in this database yet, so it holds no column to read and it was not read.`; + if (flags.json) { + console.error(line); + await emitJson({ + database: stack.dbLabel, + object, + table, + columns: [], + count: 0, + records: [], + duration: timer.elapsed(), + }); + return; + } + printInfo(`Database: ${chalk.white(stack.dbLabel)}`); + printInfo(line); + printSuccess(`No unmapped column on ${object} — nothing to read.`); + return; + } + + const drift = await stack.driver.detectManagedDrift(); + const columns = unmappedColumnsOf(drift, table); + + const records = + columns.length === 0 + ? [] + : await readUnmappedColumnValues( + { find: (o, q) => (reader.find as UnmappedColumnReader['find']).call(stack.driver, o, q) }, + object, + columns.map((c) => c.column), + flags['max-records'] != null ? { max: flags['max-records'] } : {}, + ); + + if (flags.json) { + await emitJson({ + database: stack.dbLabel, + object, + table, + columns, + count: records.length, + records, + duration: timer.elapsed(), + }); + return; + } + + printInfo(`Database: ${chalk.white(stack.dbLabel)}`); + printInfo(`Object: ${chalk.white(object)} (table ${table})`); + console.log(''); + if (columns.length === 0) { + printSuccess(`No unmapped column on ${object} — nothing to read.`); + console.log(chalk.dim(` ${timer.display()}`)); + return; + } + printInfo(`${columns.length} unmapped column(s), as "os migrate plan" reports them:`); + for (const c of columns) console.log(` ${chalk.bold(c.column)} ${chalk.dim(c.actual)}`); + console.log(''); + for (const r of records) console.log(` ${String(r.id)} ${JSON.stringify(r.values)}`); + console.log(''); + printSuccess( + `Read ${records.length} record(s). Convert these values into the declared fields that replaced the ` + + 'columns, then run "os migrate apply --allow-destructive" to drop them.', + ); + console.log(chalk.dim(` ${timer.display()}`)); + } catch (error: any) { + // `this.exit(1)` throws oclif's exit signal; re-reporting it would print + // a second document after the first. + if (isExitSignal(error)) throw error; + if (flags.json) { + await emitJson({ error: error?.message ?? String(error), ...errorCodeFields(error) }, 1, { compact: true }); + this.exit(1); + } + printError(error?.message ?? String(error)); + this.exit(1); + } finally { + await stack.shutdown(); + } + } +} diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 1269d214d7f..60c5fe93256 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -35,6 +35,10 @@ export { default as MigrateRecordedByCommand } from './commands/migrate/recorded // `sys_account.issuer` is dropped and account identity re-keys onto // (provider_id, account_id). export { default as MigrateAccountIssuerCommand } from './commands/migrate/account-issuer.js'; +// #21573: the operator-only read of the columns `os migrate plan` reports as +// `unmapped_column` for one object, keyed by record id, for a conversion run +// before `os migrate apply --allow-destructive` drops them. +export { default as MigrateUnmappedColumnsCommand } from './commands/migrate/unmapped-columns.js'; // ─── Environments topic subcommands ───────────────────────────────── export { default as EnvironmentsListCommand } from './commands/environments/list.js'; diff --git a/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts b/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts index 2382e6a2051..905837cdaa4 100644 --- a/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts +++ b/packages/cli/src/utils/schema-migrate.one-shot-family.integration.test.ts @@ -66,6 +66,7 @@ import MigratePlan from '../commands/migrate/plan.js'; import MigrateRecordedBy from '../commands/migrate/recorded-by.js'; import MigrateResume from '../commands/migrate/resume.js'; import MigrateSummaryNulls from '../commands/migrate/summary-nulls.js'; +import MigrateUnmappedColumns from '../commands/migrate/unmapped-columns.js'; import MigrateValueShapes from '../commands/migrate/value-shapes.js'; import SecretOrphans from '../commands/secret/orphans.js'; import SecretRewrap from '../commands/secret/rewrap.js'; @@ -211,6 +212,14 @@ const CALLERS: Record = { noWrite: [{ label: 'migrate summary-nulls', argv: ['--database-url', '@DB@', '--json'] }], write: [{ label: 'migrate summary-nulls --apply', argv: ['--apply', '--yes', '--database-url', '@DB@', '--json'] }], }, + 'commands/migrate/unmapped-columns.ts': { + run: invoke(MigrateUnmappedColumns), + // `osf_lead` is declared by both releases; the next one adds a column, + // which is pending work and not an unmapped one, so the run reads nothing. + noWrite: [{ label: 'migrate unmapped-columns', argv: ['--object', 'osf_lead', '--database-url', '@DB@', '--json'] }], + write: [], + note: 'a read-only report: it has no writing mode, and dropping the columns is `os migrate apply --allow-destructive`', + }, 'commands/migrate/value-shapes.ts': { run: invoke(MigrateValueShapes), noWrite: [{ label: 'migrate value-shapes', argv: ['--database-url', '@DB@', '--json'] }], diff --git a/packages/cli/test/json-stdout-purity.e2e.test.ts b/packages/cli/test/json-stdout-purity.e2e.test.ts index c9aeae1ec05..cc932b0671f 100644 --- a/packages/cli/test/json-stdout-purity.e2e.test.ts +++ b/packages/cli/test/json-stdout-purity.e2e.test.ts @@ -104,6 +104,10 @@ const FAMILY: Record = { 'migrate recorded-by': [], 'migrate resume': [], 'migrate summary-nulls': [], + // One object per run, so it needs a name: the fixture's own `jp_ticket`, + // which the composed host config registers. No database file exists, so the + // face driven here is its empty-work answer for a table not there yet. + 'migrate unmapped-columns': ['--object', 'jp_ticket'], 'migrate value-shapes': [], // Report-only is its DEFAULT and the only form driven here: without // `--delete` it boots, reports and writes nothing, so the family gains a