diff --git a/.changeset/21733-standalone-dev-self-heal.md b/.changeset/21733-standalone-dev-self-heal.md new file mode 100644 index 00000000000..4e57121cd22 --- /dev/null +++ b/.changeset/21733-standalone-dev-self-heal.md @@ -0,0 +1,13 @@ +--- +"@objectstack/runtime": minor +"@objectstack/cli": patch +--- + +fix(runtime,cli): a plain `os dev` now self-heals safe schema drift on restart and provisions the `telemetry` sibling database, as `content/docs/deployment/cli.mdx` already says (#21733) + +Clause-②: yes (widening) + +- **What was broken.** A config with no instantiated `plugins[]` (every fresh scaffold) boots through the standalone stack. Its `default` datasource was built without `autoMigrate: 'safe'`: only the config-load fallback that a host config or `OS_MODE=off` takes carried it. So safe drift was never applied on restart. An example is a per-organization unique index that an older release left non-NULL-safe. Meanwhile the driver's drift line and `os migrate plan` both said the change was "auto-applied at boot under dev autoMigrate: 'safe'". The same boot never provisioned the `.telemetry.` sibling either. +- **The fix.** The dev self-heal decision now lives in one place, `devAutoMigrateConfig` in `@objectstack/runtime`. That is the driver kinds whose connection contract declares `autoMigrate` (sqlite, postgres, mysql), on a dev boot. The standalone stack, the CLI's config-load fallback and the telemetry sibling all read it, so no kind gains or loses the self-heal relative to the host path. The telemetry provision is one helper (`provisionTelemetryDatasource`) that both serving paths call, under the same `resolveTelemetryDbPath` rule: dev default-on for a file-backed SQLite primary, `OS_TELEMETRY_DB=0` to opt out, `OS_TELEMETRY_DB=` to opt in anywhere. +- **Only a serving boot self-heals.** The standalone stack arms the self-heal on an explicit `dev: true`. That is what `os dev` passes. It does not arm it on the `NODE_ENV=development` default that its sqlite step-down still takes. A one-shot command (`os migrate *`, `os meta resync`, …) passes no `dev`, so it never applies drift its operator did not confirm, whatever `NODE_ENV` says. Production boots are unchanged: the definition carries no `autoMigrate`, and the SQL driver refuses it under `NODE_ENV=production` anyway. +- **Why minor.** `@objectstack/runtime` gains two exports on its only entry, `devAutoMigrateConfig` and its `DevAutoMigrateConfig` type. That is the widening: the existing decision moved out of the CLI so that the CLI reads it rather than keep a second copy. No config key, schema or accept set moves. `@objectstack/cli` is a `patch`: its fix restores documented behaviour and adds no public surface. diff --git a/packages/cli/src/commands/serve.ts b/packages/cli/src/commands/serve.ts index 121c8125eb0..3948606244a 100644 --- a/packages/cli/src/commands/serve.ts +++ b/packages/cli/src/commands/serve.ts @@ -8,6 +8,7 @@ import chalk, { chalkStderr } from 'chalk'; import { bundleRequire } from 'bundle-require'; import { loadConfig, BUNDLE_REQUIRE_EXTERNALS } from '../utils/config.js'; import { mergeBootConfig } from '../utils/merge-boot-config.js'; +import { provisionTelemetryDatasource, standaloneTelemetryPrimary } from '../utils/telemetry-datasource.js'; import { isHostConfig, shouldBootWithLibrary } from '../utils/plugin-detection.js'; import { readInternalArtifactPath, readInternalConfigOutputPath } from '../utils/internal-artifact-channel.js'; // The precedence's last rung — whether the cwd config takes part — decided by @@ -2583,6 +2584,12 @@ export default class Serve extends Command { // human and may be redacted or labelled, while this one is handed to // `stat` — see `describeDriverSqliteFile`. let servedSqliteFilePath: string | undefined; + // #21733 — the file-backed native SQLite database a composed STANDALONE + // stack declared as its `default` datasource, read BEFORE the kernel boots + // so the `telemetry` sibling can be provisioned next to it (step 2 below). + // `servedSqliteFilePath` cannot serve: on this path it is only learned by + // probing the booted kernel. Unset for every other boot and every other kind. + let standaloneSqlitePrimary: string | undefined; // Resolve the kernel logger level up front. It decides more than the // logger's own threshold: it decides whether the boot-quiet window below @@ -2741,7 +2748,7 @@ export default class Serve extends Command { // "missing artifact" error and assemble a bare kernel that // can later install marketplace apps at runtime. const { createDefaultHostConfig } = await import('@objectstack/runtime'); - const bootResult = await createDefaultHostConfig({ + const defaultHostInput = { requireArtifact: !useEmptyBoot, dev: isDev, // #8368: the already-fetched, already-verified LOCAL copy. Passing @@ -2755,7 +2762,15 @@ export default class Serve extends Command { ...(pinnedArtifact ? { artifactPath: pinnedArtifact.localPath } : supervisorArtifact ? { artifactPath: supervisorArtifact } : {}), - }); + }; + const bootResult = await createDefaultHostConfig(defaultHostInput); + // #21733 — the `default` database this stack declared, for the + // telemetry sibling (step 2). The same input the host config was + // built from: the database resolution reads the artifact only for a + // declared default datasource, and the empty boot's synthesized stub + // declares none, so the artifact rung `createDefaultHostConfig` + // settles internally cannot answer differently. + standaloneSqlitePrimary = await standaloneTelemetryPrimary(defaultHostInput); // [#4002] `api` merges per key — see mergeBootConfig. A shallow spread // let the boot builder's two scoping keys wipe the author's whole `api` // block, silently dropping `requireAuth` / `enforceProjectMembership`. @@ -2779,6 +2794,10 @@ export default class Serve extends Command { ...(supervisorArtifact ? { artifactPath: supervisorArtifact } : {}), }; const bootResult = await createStandaloneStack(standaloneInput); + // #21733 — the `default` database the stack just declared, resolved + // by the runtime's own pre-boot resolution over the SAME input, for + // the telemetry sibling (step 2). + standaloneSqlitePrimary = await standaloneTelemetryPrimary(standaloneInput); // #21501 — did the standalone stack load a compiled artifact as this // app's bundle? Its AppPlugin over the bundle is the proof (it is // pushed only when the bundle loaded, and a non-host config carries @@ -3190,6 +3209,26 @@ export default class Serve extends Command { } } + // ADR-0057 §3.6 — the `telemetry` sibling datasource, provisioned next to + // a file-backed SQLite primary by EVERY serving boot through ONE helper + // (`provisionTelemetryDatasource`, #21733): the config-load fallback in + // step 2 below and the standalone stack right after it. Dev default-on; + // `OS_TELEMETRY_DB=0` opts out, `OS_TELEMETRY_DB=` opts in anywhere + // (incl. serve) — `resolveTelemetryDbPath` is the rule, never restated here. + const provisionTelemetry = async (primaryPath: string | undefined): Promise => { + const telemetryPath = await provisionTelemetryDatasource({ + primaryPath, + env: process.env, + dev: isDev, + use: (plugin) => kernel.use(plugin as any), + warn: (m) => console.warn(chalk.yellow(m)), + }); + if (telemetryPath) { + trackPlugin('TelemetryDatasource'); + printDiagnostic(chalk.dim(` telemetry datasource: ${telemetryPath} (lifecycle-classed system data; OS_TELEMETRY_DB=0 to disable)`)); + } + }; + // 2. Auto-register storage driver // Priority: // 1. OS_DATABASE_DRIVER env var (explicit override) @@ -3234,7 +3273,7 @@ export default class Serve extends Command { // resolveStorageDefinition. The dev sqlite step-down (#2229) and // the loosen-only self-heal (#2186, via config.autoMigrate) now run // inside the factory at connect. - const { DriverPlugin, DefaultDatasourcePlugin } = await import('@objectstack/runtime'); + const { DefaultDatasourcePlugin } = await import('@objectstack/runtime'); const resolution = resolveStorageDefinition(driverType, { databaseUrl, isDev, authToken: databaseAuthToken }); if (resolution) { // #5602: libSQL/Turso is the one kind the shared open-core factory @@ -3257,52 +3296,11 @@ export default class Serve extends Command { resolvedDatabaseUrl = resolution.displayUrl; servedSqliteFilePath = resolution.sqliteFilePath; - // ADR-0057 §3.6 (#2834 ②): provision the dedicated `telemetry` - // datasource — a sibling SQLite file the engine routes every - // telemetry/event/audit-classed object to, so platform-generated - // growth can never again bloat the business DB. Dev default-on - // for file-backed primaries; `OS_TELEMETRY_DB=0` opts out, - // `OS_TELEMETRY_DB=` opts in anywhere (incl. serve). Gated on - // an explicit SQLite primary (`sqliteFilePath`, unset for the mingo - // memory driver AND the dev-default `:memory:` store). The old - // `resolution.engine !== 'memory'` refinement is unknowable now - // that the primary connects later (#3826); the telemetry - // provision's own `telemetry.engine !== 'memory'` check below - // still guards the ABI-broken step-down case. The telemetry - // driver itself stays a pre-built DriverPlugin — the documented - // escape hatch for named auxiliary drivers. - if (resolution.sqliteFilePath) { - const { resolveTelemetryDbPath } = await import('../utils/telemetry-datasource.js'); - const telemetryPath = resolveTelemetryDbPath({ primaryPath: resolution.sqliteFilePath, env: process.env, dev: isDev }); - if (telemetryPath) { - try { - const { resolveSqliteDriver } = await import('@objectstack/service-datasource'); - const telemetry = await resolveSqliteDriver({ - filename: telemetryPath, - dev: isDev, - autoMigrate: isDev ? 'safe' : undefined, - warn: (m) => console.warn(chalk.yellow(m)), - }); - if (telemetry.engine !== 'memory') { - // The engine keys datasources by driver name — the - // lifecycle router looks this exact name up. The driver - // name is the WHOLE wiring: DriverPlugin.init registers - // `driver.telemetry`, ObjectQL's discovery loop adopts - // it, and lifecycle-classed objects route to it. (An - // options bag once also asked for `datasourceName: - // 'telemetry'` metadata registration — inert since - // inception, retired in #4320.) - Object.defineProperty(telemetry.driver, 'name', { value: 'telemetry' }); - await kernel.use(new DriverPlugin(telemetry.driver)); - trackPlugin('TelemetryDatasource'); - printDiagnostic(chalk.dim(` telemetry datasource: ${telemetryPath} (lifecycle-classed system data; OS_TELEMETRY_DB=0 to disable)`)); - } - } catch { - // Best-effort: a failed telemetry provision must never block - // boot — objects simply stay on the primary datasource. - } - } - } + // ADR-0057 §3.6 (#2834 ②): the dedicated `telemetry` datasource + // next to an explicit SQLite primary (`sqliteFilePath`, unset for + // every other kind AND the dev-default `:memory:` store). The ONE + // provision, shared with the standalone boot below. + await provisionTelemetry(resolution.sqliteFilePath); } } catch (e: any) { // "declared ≠ enforced" guard (#3276-class): a selection the CLI @@ -3332,6 +3330,12 @@ export default class Serve extends Command { if (e?.code === 'MONGODB_MULTI_TENANT_UNSUPPORTED') throw e; // silent } + } else if (standaloneSqlitePrimary !== undefined) { + // #21733 — the standalone stack's `default` datasource is a file-backed + // SQLite database (resolved by the runtime's own pre-boot resolution in + // the boot-mode dispatch above), so it gets the same sibling the + // config-load fallback provisions, under the same rule. + await provisionTelemetry(standaloneSqlitePrimary); } // 3. Auto-register AppPlugin if config contains app definitions diff --git a/packages/cli/src/utils/dev-self-heal-host-parity.test.ts b/packages/cli/src/utils/dev-self-heal-host-parity.test.ts new file mode 100644 index 00000000000..eba6e0542c2 --- /dev/null +++ b/packages/cli/src/utils/dev-self-heal-host-parity.test.ts @@ -0,0 +1,97 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #21733 — both boot hosts give the `default` datasource the SAME dev + * self-heal, kind by kind. + * + * The self-heal (`autoMigrate: 'safe'`, #2186) used to be decided inline in + * the CLI host (`resolveStorageDefinition`) and not at all in the runtime host + * (`createStandaloneStack`), which every plain `os dev` composes. Both now read + * `devAutoMigrateConfig` from `@objectstack/runtime`; this file drives the two + * REAL entry points, not the helper, so a host that stops reading it — or + * starts deciding for itself — fails here by kind. Same reason the driver + * vocabulary's own cross-host pin lives in this package + * (`driver-vocabulary-parity.test.ts`): it is the one that can import both. + * + * The host path's answer is the reference: no kind gains or loses `'safe'` + * relative to it (`storage-driver.test.ts` pins that answer on its own). + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createStandaloneStack } from '@objectstack/runtime'; +import { resolveStorageDefinition } from './storage-driver.js'; + +// [#10126] Pay the first transform of these dist-resolved workspace deps at +// MODULE LOAD: `createStandaloneStack` reaches them through dynamic +// `import()`s inside clocked `it()` bodies (`scripts/check-test-source-alias.mjs`). +import '@objectstack/service-datasource'; +import '@objectstack/objectql'; +import '@objectstack/metadata'; + +const ENV_KEYS = ['OS_DATABASE_URL', 'DATABASE_URL', 'TURSO_DATABASE_URL', 'OS_DATABASE_DRIVER', 'OS_HOME'] as const; +const ORIGINAL_ENV: Record = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); +const dirs: string[] = []; + +afterEach(() => { + for (const key of ENV_KEYS) { + const original = ORIGINAL_ENV[key]; + if (original === undefined) delete process.env[key]; + else process.env[key] = original; + } + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); +}); + +/** Each canonical kind, by a URL both hosts select it from. */ +function urlsFor(dir: string): Record { + return { + sqlite: `file:${join(dir, 'p.db')}`, + 'sqlite-wasm': `wasm-sqlite://${join(dir, 'w.db')}`, + postgres: 'postgres://u:p@localhost:5432/db', + mysql: 'mysql://u:p@localhost:3306/db', + mongodb: 'mongodb://localhost:27017/db', + turso: 'libsql://my-db.turso.io', + }; +} + +async function runtimeAutoMigrate(databaseUrl: string, dev: boolean, projectRoot: string): Promise { + const stack = await createStandaloneStack({ databaseUrl, dev, projectRoot }); + const plugin = stack.plugins.find((p: any) => p?.name === 'com.objectstack.runtime.default-datasource') as any; + expect(plugin, 'the standalone stack must carry the DefaultDatasourcePlugin').toBeDefined(); + return plugin.def.config?.autoMigrate; +} + +const BOOT_TIMEOUT = 90_000; + +describe('#21733 — the dev self-heal is the same on both boot hosts', () => { + it('every kind: the standalone stack carries exactly the `autoMigrate` the CLI host does, dev and production', async () => { + for (const key of ENV_KEYS) delete process.env[key]; + const dir = mkdtempSync(join(tmpdir(), 'os-21733-parity-')); + dirs.push(dir); + const rows: string[] = []; + for (const [kind, url] of Object.entries(urlsFor(dir))) { + for (const dev of [true, false]) { + const cli = resolveStorageDefinition(kind, { databaseUrl: url, isDev: dev })?.config.autoMigrate; + const runtime = await runtimeAutoMigrate(url, dev, dir); + rows.push(`${kind} dev=${dev}: cli=${String(cli)} runtime=${String(runtime)}`); + } + } + // One assertion over the whole table, so a divergence names every kind it touches. + expect(rows).toEqual([ + 'sqlite dev=true: cli=safe runtime=safe', + 'sqlite dev=false: cli=undefined runtime=undefined', + 'sqlite-wasm dev=true: cli=undefined runtime=undefined', + 'sqlite-wasm dev=false: cli=undefined runtime=undefined', + 'postgres dev=true: cli=safe runtime=safe', + 'postgres dev=false: cli=undefined runtime=undefined', + 'mysql dev=true: cli=safe runtime=safe', + 'mysql dev=false: cli=undefined runtime=undefined', + 'mongodb dev=true: cli=undefined runtime=undefined', + 'mongodb dev=false: cli=undefined runtime=undefined', + 'turso dev=true: cli=undefined runtime=undefined', + 'turso dev=false: cli=undefined runtime=undefined', + ]); + }, BOOT_TIMEOUT); +}); diff --git a/packages/cli/src/utils/schema-migrate.dev-self-heal-fence.integration.test.ts b/packages/cli/src/utils/schema-migrate.dev-self-heal-fence.integration.test.ts new file mode 100644 index 00000000000..a935878def5 --- /dev/null +++ b/packages/cli/src/utils/schema-migrate.dev-self-heal-fence.integration.test.ts @@ -0,0 +1,231 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #21733 — the dev schema self-heal reaches the standalone stack, and a + * one-shot command boot never applies it, whatever `NODE_ENV` says. + * + * The self-heal (`autoMigrate: 'safe'`, #2186) is decided in ONE place, + * `devAutoMigrateConfig` in `@objectstack/runtime`. The standalone stack reads + * it under an EXPLICIT `dev: true` only — never under the `NODE_ENV` default + * its sqlite step-down takes — because `bootSchemaStack` passes no `dev`, and a + * one-shot command boot under `NODE_ENV=development` would otherwise inherit + * the self-heal: + * + * - a NON-deferred one-shot boot (every write mode: `os meta resync --yes`, + * the data commands' `--apply`) runs boot schema sync, so it would apply + * safe drift its operator never saw in a preview; + * - `os migrate plan` defers its DDL (#3917), so its own boot never reaches + * the reconcile — this file pins that it stays write-free on the card's + * staged database anyway, as the claim on #21733 asks. + * + * ## The fixture + * + * The card's reproduction, in-process: one org-scoped `unique` field, whose + * index a plain boot creates NULL-safe (`COALESCE(organization_id, …)`), then + * replaced by the bare `(organization_id, qa_code)` index an older release + * left behind. That is a `recreate_index` drift the duplicate probe finds + * clean, i.e. `safe`: exactly what the self-heal applies. The serving + * declaration (`createStandaloneStack({ dev: true })`, what `os dev` hands it) + * is the positive control — it heals the same staged file, so the fence cases + * cannot be green because the fixture has nothing to heal. + * + * Every case runs with `NODE_ENV=development` — the input the fence is about. + */ + +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; +import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { randomBytes } from 'node:crypto'; +import { tmpdir } from 'node:os'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { isExitSignal } from './format.js'; +import { bootSchemaStack } from './schema-migrate.js'; +import MigratePlan from '../commands/migrate/plan.js'; + +// [#10126] Pay the first transform of these dist-resolved workspace deps at +// MODULE LOAD: the boots reach them through dynamic `import()`s, which vitest +// clocks (`scripts/check-test-source-alias.mjs`). +import '@objectstack/runtime'; +import '@objectstack/objectql'; +import '@objectstack/service-datasource'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CLI_ROOT = resolve(HERE, '..', '..'); + +const CASE_TIMEOUT_MS = 120_000; + +const INDEX = 'uniq_scaf_item_organization_id_qa_code'; + +const ARTIFACT = { + manifest: { id: 'com.example.scaf', namespace: 'scaf', name: 'scaf', version: '1.0.0', type: 'app' }, + objects: [{ + name: 'scaf_item', + label: 'Item', + sharingModel: 'private', + fields: { qa_code: { type: 'text', label: 'QA code', unique: true } }, + }], +}; + +/** Env that would point a boot somewhere other than the fixture. */ +const OVERRIDING_ENV = ['OS_DATABASE_URL', 'DATABASE_URL', 'TURSO_DATABASE_URL', 'OS_DATABASE_DRIVER', 'OS_HOME'] as const; +const SAVED_KEYS = [...OVERRIDING_ENV, 'OS_ARTIFACT_PATH', 'NODE_ENV', 'OS_SECRET_KEY'] as const; +const savedEnv: Record = {}; +const savedCwd = process.cwd(); + +let dir: string; +let stagedDb: string; + +function probe(dbFile: string): SqlDriver { + return new SqlDriver({ client: 'better-sqlite3', connection: { filename: dbFile }, useNullAsDefault: true }); +} + +/** The index's DDL exactly as `sqlite_master` stores it. */ +async function indexSql(dbFile: string): Promise { + const driver = probe(dbFile); + try { + const rows = await (driver as any).knex.raw('SELECT sql FROM sqlite_master WHERE type = ? AND name = ?', ['index', INDEX]); + return (rows as Array<{ sql: string }>)[0]?.sql; + } finally { + await driver.disconnect(); + } +} + +/** A fresh copy of the staged database for one case. */ +function stagedCopy(label: string): string { + const caseDir = join(dir, 'cases', label); + mkdirSync(caseDir, { recursive: true }); + const dbFile = join(caseDir, 'app.db'); + copyFileSync(stagedDb, dbFile); + return dbFile; +} + +/** A booted serving stack: the standalone stack's own plugins, started, then shut down. */ +async function bootServing(dbFile: string, dev: boolean | undefined): Promise { + const { createStandaloneStack, Runtime } = await import('@objectstack/runtime'); + const stack = await createStandaloneStack({ + projectRoot: dir, + databaseUrl: `file:${dbFile}`, + ...(dev === undefined ? {} : { dev }), + }); + const runtime = new Runtime({ cluster: false }); + const kernel = runtime.getKernel(); + for (const plugin of stack.plugins) await kernel.use(plugin as any); + await runtime.start(); + await kernel.shutdown(); +} + +/** + * Run the real `os migrate plan --json` through oclif and return its document. + * The stdout spy goes in BEFORE the boot (a `--json` boot reserves stdout and + * keeps whatever `process.stdout.write` is at that moment); `process.exit` and + * the process-global `exitCode` `emitJson` sets are trapped and restored. + */ +async function runPlanJson(dbFile: string): Promise { + const savedExitCode = process.exitCode; + const swallow = ((_chunk: unknown, ...rest: unknown[]) => { + const cb = rest.find((a) => typeof a === 'function') as (() => void) | undefined; + if (cb) cb(); + return true; + }) as typeof process.stdout.write; + const stdout = vi.spyOn(process.stdout, 'write').mockImplementation(swallow); + vi.spyOn(process.stderr, 'write').mockImplementation(swallow); + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'warn').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + vi.spyOn(process, 'exit').mockImplementation(((code?: number) => { + throw new Error(`__PROCESS_EXIT__:${code ?? 0}`); + }) as never); + process.chdir(dirname(dbFile)); + try { + try { + await MigratePlan.run(['--database-url', `file:${dbFile}`, '--json'], { root: CLI_ROOT }); + } catch (error) { + const trapped = error instanceof Error && error.message.startsWith('__PROCESS_EXIT__'); + if (!trapped && !isExitSignal(error)) throw error; + } + const out = stdout.mock.calls.map((c) => String(c[0])).join('').trim(); + return JSON.parse(out); + } finally { + vi.restoreAllMocks(); + process.chdir(savedCwd); + process.exitCode = savedExitCode; + } +} + +const isNullSafe = (sql: string | undefined): boolean => /COALESCE\(`?organization_id`?/.test(sql ?? ''); + +beforeAll(async () => { + for (const key of SAVED_KEYS) savedEnv[key] = process.env[key]; + for (const key of OVERRIDING_ENV) delete process.env[key]; + process.env.NODE_ENV = 'development'; + // Never minted onto disk by a settings provider in this posture. + process.env.OS_SECRET_KEY = randomBytes(32).toString('hex'); + + dir = mkdtempSync(join(tmpdir(), 'os-21733-fence-')); + mkdirSync(join(dir, 'dist'), { recursive: true }); + mkdirSync(join(dir, 'data'), { recursive: true }); + process.env.OS_ARTIFACT_PATH = join(dir, 'dist', 'objectstack.json'); + writeFileSync(process.env.OS_ARTIFACT_PATH, JSON.stringify(ARTIFACT)); + + // A plain boot creates the table and its NULL-safe unique index… + stagedDb = join(dir, 'data', 'staged.db'); + await bootServing(stagedDb, false); + expect(isNullSafe(await indexSql(stagedDb)), 'the fixture boot did not create the NULL-safe index').toBe(true); + // …which the card's step 3 replaces with the bare one an older release left. + const driver = probe(stagedDb); + try { + const k = (driver as any).knex; + await k.raw(`DROP INDEX ${INDEX}`); + await k.raw(`CREATE UNIQUE INDEX ${INDEX} ON scaf_item (organization_id, qa_code)`); + } finally { + await driver.disconnect(); + } + expect(isNullSafe(await indexSql(stagedDb)), 'the staging step did not leave the bare index').toBe(false); +}, CASE_TIMEOUT_MS); + +afterAll(() => { + process.chdir(savedCwd); + for (const [key, value] of Object.entries(savedEnv)) { + if (value === undefined) delete process.env[key]; + else process.env[key] = value; + } + try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } +}); + +describe('#21733 — the dev self-heal on the standalone stack, and its one-shot fence (NODE_ENV=development)', () => { + it('the serving declaration (`dev: true`, what `os dev` passes) heals the staged index on restart', async () => { + const dbFile = stagedCopy('serving-dev'); + await bootServing(dbFile, true); + const sql = await indexSql(dbFile); + expect(isNullSafe(sql), `the dev boot left the bare index: ${sql}`).toBe(true); + }, CASE_TIMEOUT_MS); + + it('a non-deferred one-shot boot (`bootSchemaStack`, every write mode) leaves the safe drift in place', async () => { + const dbFile = stagedCopy('one-shot-write'); + const stack = await bootSchemaStack({ jsonOutput: false, databaseUrl: `file:${dbFile}`, projectRoot: dir }); + let drift: Array<{ category: string; op: { type: string } }>; + try { + drift = await stack.driver!.detectManagedDrift() as any; + } finally { + await stack.shutdown(); + } + // Still there, and still the kind the self-heal WOULD apply — so the + // boot declined it rather than having nothing to do. + expect(drift.map((d) => `${d.category}:${d.op.type}`)).toContain('safe:recreate_index'); + const sql = await indexSql(dbFile); + expect(isNullSafe(sql), `a one-shot boot auto-applied safe drift: ${sql}`).toBe(false); + }, CASE_TIMEOUT_MS); + + it('`os migrate plan` lists the drift as safe and leaves the staged database byte-identical', async () => { + const dbFile = stagedCopy('plan'); + const before = readFileSync(dbFile); + const plan = await runPlanJson(dbFile); + expect( + (plan.changes ?? []).map((d: { category: string; op: { type: string } }) => `${d.category}:${d.op.type}`), + `plan output: ${JSON.stringify(plan).slice(0, 400)}`, + ).toContain('safe:recreate_index'); + expect(Buffer.compare(readFileSync(dbFile), before), 'os migrate plan wrote to the staged database').toBe(0); + expect(isNullSafe(await indexSql(dbFile))).toBe(false); + }, CASE_TIMEOUT_MS); +}); diff --git a/packages/cli/src/utils/storage-driver.ts b/packages/cli/src/utils/storage-driver.ts index 5a7f7fd37f6..54c280a1799 100644 --- a/packages/cli/src/utils/storage-driver.ts +++ b/packages/cli/src/utils/storage-driver.ts @@ -101,6 +101,7 @@ import type { IDatasourceDriverFactory, } from '@objectstack/service-datasource'; import { + devAutoMigrateConfig, loadTursoDriverFactory as loadRuntimeTursoDriverFactory, namesRetiredMemoryEngine, retiredMemoryEngineMessage, @@ -365,9 +366,13 @@ export function resolveStorageDefinition( opts: ResolveStorageDefinitionOptions, ): StorageDefinitionResolution | null { const { databaseUrl, isDev, authToken } = opts; - // #2186: dev-only loosen-only self-heal, honored by the factory for the SQL - // kinds. Never in production, never destructive. - const autoMigrate = isDev ? ({ autoMigrate: 'safe' } as const) : {}; + // #2186's dev self-heal is NOT decided here (#21733). Every arm below spreads + // `devAutoMigrateConfig(, isDev)` — the ONE decision, in + // `@objectstack/runtime`, that the standalone stack's `default` datasource + // reads too — so which kinds carry `autoMigrate: 'safe'`, and when, is that + // module's answer and never this file's. It used to be an inline + // `isDev ? { autoMigrate: 'safe' } : {}` here, and the standalone stack (every + // plain `os dev`) had no copy at all. // ONE vocabulary since commit e2798fab7 (`@objectstack/spec`'s driver table). The arms // below therefore branch on the CANONICAL id and never on a spelling: the @@ -425,7 +430,9 @@ export function resolveStorageDefinition( const url = databaseUrl!; return { driverId: 'mongodb', - config: { url }, + // The shared self-heal decision answers `{}` for this kind: its + // contract declares no `autoMigrate`. + config: { url, ...devAutoMigrateConfig('mongodb', isDev) }, trackName: 'MongoDBDriver', label: 'MongoDBDriver', displayUrl: url, @@ -439,7 +446,7 @@ export function resolveStorageDefinition( .replace(/^sql:\/\//, ''); return { driverId: 'sqlite', - config: { filename: filePath, ...autoMigrate }, + config: { filename: filePath, ...devAutoMigrateConfig('sqlite', isDev) }, trackName: 'SqlDriver', label: 'SqlDriver(better-sqlite3)', displayUrl: databaseUrl ?? ':memory:', @@ -459,7 +466,9 @@ export function resolveStorageDefinition( driverId: 'sqlite-wasm', // `persist` passthrough: the CLI kept `on-disconnect` semantics here // (the factory's default for file-backed wasm is `on-write`). - config: { filename: filePath, persist: 'on-disconnect' }, + // No self-heal for this kind: its contract declares no `autoMigrate`, so + // the shared decision answers `{}`. + config: { filename: filePath, persist: 'on-disconnect', ...devAutoMigrateConfig('sqlite-wasm', isDev) }, trackName: 'SqliteWasmDriver', label: 'SqliteWasmDriver', displayUrl: databaseUrl ?? ':memory:', @@ -469,7 +478,7 @@ export function resolveStorageDefinition( if (kind === 'postgres') { return { driverId: 'postgres', - config: { url: databaseUrl, ...autoMigrate }, + config: { url: databaseUrl, ...devAutoMigrateConfig('postgres', isDev) }, trackName: 'PostgresDriver', label: 'SqlDriver(pg)', displayUrl: databaseUrl, @@ -479,7 +488,7 @@ export function resolveStorageDefinition( if (kind === 'mysql') { return { driverId: 'mysql', - config: { url: databaseUrl, ...autoMigrate }, + config: { url: databaseUrl, ...devAutoMigrateConfig('mysql', isDev) }, trackName: 'MySQLDriver', label: 'SqlDriver(mysql2)', displayUrl: databaseUrl, @@ -494,16 +503,17 @@ export function resolveStorageDefinition( // hatch and the retained Setup → Datasources status are therefore identical to // every other kind (#3826); only the construction differs. // - // No `autoMigrate` passthrough: unlike the postgres/mysql/sqlite branches, - // `TursoDriverConfig` declares no such key, and handing it one would be a config - // the driver silently ignores. No `sqliteFilePath` either — the telemetry sibling - // is provisioned next to an on-disk SQLite primary, which a libSQL endpoint is not. + // No `autoMigrate`: unlike the postgres/mysql/sqlite branches, `TursoDriverConfig` + // declares no such key, so the shared self-heal decision answers `{}` — handing + // it one would be a config the driver silently ignores. No `sqliteFilePath` + // either — the telemetry sibling is provisioned next to an on-disk SQLite + // primary, which a libSQL endpoint is not. if (kind === 'turso') { // The no-URL refusal is the shared one above; by here a URL is present. const url = databaseUrl!.trim(); return { driverId: 'turso', - config: { url, ...(authToken ? { authToken } : {}) }, + config: { url, ...(authToken ? { authToken } : {}), ...devAutoMigrateConfig('turso', isDev) }, trackName: 'TursoDriver', label: 'TursoDriver(libsql)', displayUrl: url, @@ -517,7 +527,7 @@ export function resolveStorageDefinition( if (isDev) { return { driverId: 'sqlite', - config: { filename: ':memory:', ...autoMigrate }, + config: { filename: ':memory:', ...devAutoMigrateConfig('sqlite', isDev) }, trackName: 'SqlDriver', label: 'SqlDriver(better-sqlite3)', displayUrl: ':memory:', diff --git a/packages/cli/src/utils/telemetry-datasource.provision.integration.test.ts b/packages/cli/src/utils/telemetry-datasource.provision.integration.test.ts new file mode 100644 index 00000000000..5e5ac9580dc --- /dev/null +++ b/packages/cli/src/utils/telemetry-datasource.provision.integration.test.ts @@ -0,0 +1,117 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #21733 — the ONE `telemetry` sibling provision every serving boot runs + * (ADR-0057 §3.6), and the primary a standalone stack keys it on. + * + * `provisionTelemetryDatasource` used to be inline in `serve.ts`'s config-load + * fallback, so the standalone stack — every plain `os dev` — never got the + * sibling `content/docs/deployment/cli.mdx` promises for a file-backed SQLite + * dev database. Both boots now call it; this file pins what it registers. + * + * Integration tier: it starts a real SQLite driver for the sibling (the + * `@objectstack/driver-sql` import below is what the tier partition measures). + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { provisionTelemetryDatasource, standaloneTelemetryPrimary } from './telemetry-datasource.js'; + +// [#10126] Pay the first transform of these dist-resolved workspace deps at +// MODULE LOAD: the helpers reach them through dynamic `import()`s inside +// clocked `it()` bodies (`scripts/check-test-source-alias.mjs`). +import '@objectstack/service-datasource'; +import '@objectstack/runtime'; + +const ENV_KEYS = ['OS_DATABASE_URL', 'DATABASE_URL', 'TURSO_DATABASE_URL', 'OS_DATABASE_DRIVER', 'OS_HOME'] as const; +const ORIGINAL_ENV: Record = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); +const dirs: string[] = []; +const opened: SqlDriver[] = []; + +afterEach(async () => { + for (const driver of opened.splice(0)) await driver.disconnect(); + for (const key of ENV_KEYS) { + const original = ORIGINAL_ENV[key]; + if (original === undefined) delete process.env[key]; + else process.env[key] = original; + } + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); +}); + +function scratch(): string { + const d = mkdtempSync(join(tmpdir(), 'os-21733-telemetry-')); + dirs.push(d); + return d; +} + +/** Provision against a recording `use`, and hand back what was registered. */ +async function provision(primaryPath: string | undefined, env: Record, dev: boolean) { + const used: any[] = []; + const path = await provisionTelemetryDatasource({ + primaryPath, + env, + dev, + use: (plugin) => { used.push(plugin); }, + warn: () => {}, + }); + for (const plugin of used) if (plugin?.driver instanceof SqlDriver) opened.push(plugin.driver); + return { path, used }; +} + +describe('provisionTelemetryDatasource — the one telemetry provision (#21733)', () => { + it('a dev boot on a file-backed primary registers `driver.telemetry` next to it, under the dev self-heal', async () => { + const dir = scratch(); + const { path, used } = await provision(join(dir, 'A.db'), {}, true); + expect(path).toBe(join(dir, 'A.telemetry.db')); + expect(used).toHaveLength(1); + expect(used[0].name).toBe('com.objectstack.driver.telemetry'); + expect(used[0].driver).toBeInstanceOf(SqlDriver); + expect(used[0].driver.name).toBe('telemetry'); + // The SAME decision the primary reads (`devAutoMigrateConfig`). + expect(used[0].driver.autoMigrate).toBe('safe'); + expect(existsSync(join(dir, 'A.telemetry.db'))).toBe(true); + }); + + it('OS_TELEMETRY_DB=0 opts out: nothing registered, no file', async () => { + const dir = scratch(); + const { path, used } = await provision(join(dir, 'A.db'), { OS_TELEMETRY_DB: '0' }, true); + expect(path).toBeUndefined(); + expect(used).toEqual([]); + expect(existsSync(join(dir, 'A.telemetry.db'))).toBe(false); + }); + + it('production provisions only on an explicit OS_TELEMETRY_DB path, and never self-heals', async () => { + const dir = scratch(); + expect((await provision(join(dir, 'A.db'), {}, false)).used).toEqual([]); + const explicit = join(dir, 'elsewhere.db'); + const { path, used } = await provision(join(dir, 'A.db'), { OS_TELEMETRY_DB: explicit }, false); + expect(path).toBe(explicit); + expect(used).toHaveLength(1); + expect(used[0].driver.autoMigrate).toBe('off'); + }); + + it('no file-backed primary, no sibling', async () => { + expect((await provision(undefined, {}, true)).used).toEqual([]); + expect((await provision(':memory:', {}, true)).used).toEqual([]); + }); +}); + +describe('standaloneTelemetryPrimary — the primary a standalone stack keys the sibling on (#21733)', () => { + it('is the native sqlite file the stack would open, and nothing for any other kind', async () => { + for (const key of ENV_KEYS) delete process.env[key]; + const dir = scratch(); + expect(await standaloneTelemetryPrimary({ projectRoot: dir, databaseUrl: `file:${join(dir, 'A.db')}` })) + .toBe(join(dir, 'A.db')); + // The unified default the stack falls back to, under the project root. + expect(await standaloneTelemetryPrimary({ projectRoot: dir })) + .toBe(join(dir, '.objectstack', 'data', 'objectstack.db')); + expect(await standaloneTelemetryPrimary({ projectRoot: dir, databaseUrl: ':memory:' })).toBeUndefined(); + expect(await standaloneTelemetryPrimary({ projectRoot: dir, databaseUrl: `wasm-sqlite://${join(dir, 'w.db')}` })) + .toBeUndefined(); + expect(await standaloneTelemetryPrimary({ projectRoot: dir, databaseUrl: 'postgres://u:p@localhost:5432/db' })) + .toBeUndefined(); + }); +}); diff --git a/packages/cli/src/utils/telemetry-datasource.ts b/packages/cli/src/utils/telemetry-datasource.ts index bb315918065..4324dd1224f 100644 --- a/packages/cli/src/utils/telemetry-datasource.ts +++ b/packages/cli/src/utils/telemetry-datasource.ts @@ -1,5 +1,7 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +import type { StandaloneStackConfig } from '@objectstack/runtime'; + /** * ADR-0057 §3.6 (#2834 ②) — where the dedicated telemetry datasource lives. * @@ -42,3 +44,89 @@ export function resolveTelemetryDbPath(opts: { } return `${primary}.telemetry.db`; } + +/** What {@link provisionTelemetryDatasource} needs from the boot that calls it. */ +export interface ProvisionTelemetryDatasourceOptions { + /** + * The primary datasource's on-disk SQLite path, or `undefined` when the + * primary is not a file-backed native SQLite database (another engine, + * `:memory:`) — then nothing is provisioned, whatever `OS_TELEMETRY_DB` says. + */ + primaryPath: string | undefined; + env: Record; + /** The host's dev declaration — the same value its primary datasource was built under. */ + dev: boolean; + /** Register a plugin with the booting kernel (`kernel.use`). */ + use: (plugin: unknown) => unknown; + /** Where the sqlite step-down's own warnings go. */ + warn: (message: string) => void; +} + +/** + * Provision the dedicated `telemetry` datasource next to a file-backed SQLite + * primary (ADR-0057 §3.6) — the ONE provision every serving boot runs. + * + * Returns the telemetry file it registered, or `undefined` when none was: + * {@link resolveTelemetryDbPath} declined (production without + * `OS_TELEMETRY_DB`, `OS_TELEMETRY_DB=0`, an in-memory primary), the sqlite + * step-down could only offer an in-memory engine, or the provision failed. + * Best-effort by design: a failed telemetry provision must never block boot — + * the lifecycle-classed objects simply stay on the primary datasource. + * + * Two boots call it (#21733): the config-load fallback (`resolveStorageDefinition`'s + * `sqliteFilePath`) and the standalone stack every plain `os dev` / `os serve` + * / `os start` composes (its resolved `default` database). It used to live + * inline in the first of them, so a plain `os dev` never got a telemetry + * sibling while `content/docs/deployment/cli.mdx` promised one for every dev + * boot on a file-backed SQLite database. + * + * The telemetry driver stays a pre-built `DriverPlugin` — the documented escape + * hatch for named auxiliary drivers (ADR-0062): the engine keys datasources by + * driver name, so naming the driver `telemetry` is the WHOLE wiring — + * `DriverPlugin.init` registers `driver.telemetry`, ObjectQL's discovery loop + * adopts it, and lifecycle-classed objects route to it. Its dev self-heal is + * the shared decision (`devAutoMigrateConfig`), the same one the primary reads. + */ +export async function provisionTelemetryDatasource( + opts: ProvisionTelemetryDatasourceOptions, +): Promise { + if (!opts.primaryPath) return undefined; + const telemetryPath = resolveTelemetryDbPath({ primaryPath: opts.primaryPath, env: opts.env, dev: opts.dev }); + if (!telemetryPath) return undefined; + try { + const { resolveSqliteDriver } = await import('@objectstack/service-datasource'); + const { DriverPlugin, devAutoMigrateConfig } = await import('@objectstack/runtime'); + const telemetry = await resolveSqliteDriver({ + filename: telemetryPath, + dev: opts.dev, + ...devAutoMigrateConfig('sqlite', opts.dev), + warn: opts.warn, + }); + // The ABI-broken step-down case: separating one in-memory store into + // another has no reclamation value, and would lose telemetry on restart. + if (telemetry.engine === 'memory') return undefined; + Object.defineProperty(telemetry.driver, 'name', { value: 'telemetry' }); + await opts.use(new DriverPlugin(telemetry.driver)); + return telemetryPath; + } catch { + return undefined; + } +} + +/** + * The file-backed native SQLite database a standalone stack built from `input` + * declares as its `default` datasource — the primary its `telemetry` sibling is + * keyed on — or `undefined` for every other kind and for `:memory:`. + * + * Read through the runtime's own pre-boot resolution (`resolveStandaloneDatabase`, + * the answer `createStandaloneStack` builds its definition from), so the sibling + * can never sit next to a different file than the one the boot opens. Only the + * native `sqlite` kind counts — the same kind the config-load fallback hands a + * `sqliteFilePath` (`resolveStorageDefinition`): a `sqlite-wasm` primary gets no + * sibling on either path. + */ +export async function standaloneTelemetryPrimary(input: StandaloneStackConfig): Promise { + const { resolveStandaloneDatabase } = await import('@objectstack/runtime'); + const db = resolveStandaloneDatabase(input); + return db.driver === 'sqlite' && db.sqliteFile ? db.sqliteFile : undefined; +} diff --git a/packages/cli/test/dev-standalone-self-heal.integration.test.ts b/packages/cli/test/dev-standalone-self-heal.integration.test.ts new file mode 100644 index 00000000000..a008bbf396e --- /dev/null +++ b/packages/cli/test/dev-standalone-self-heal.integration.test.ts @@ -0,0 +1,266 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #21733 — the card's acceptance, run against the BUILT CLI: a plain `os dev` + * restart on a no-plugins scaffold self-heals safe drift, and provisions the + * `telemetry` sibling, exactly as `content/docs/deployment/cli.mdx` says. + * + * ## The defect, measured at the public door before the fix + * + * A blank scaffold (one object, an org-scoped `unique` field, no plugins) boots + * through the STANDALONE stack (`shouldBootWithLibrary` → `createStandaloneStack`), + * whose `default` datasource never carried `autoMigrate: 'safe'` — only the + * config-load fallback a host config takes did. So on a database whose unique + * index an older release left non-NULL-safe: + * + * | boot | index after the boot | `A.telemetry.db` | + * |----------------------------------------|-------------------------------|------------------| + * | `os dev --no-watch -d file:A.db`, ×2 | bare `(organization_id, qa_code)` | absent | + * | same, `OS_MODE=off` (the host path) | NULL-safe `COALESCE(…)` | created | + * + * while the default boot's own drift line read "auto-applied at boot under dev + * autoMigrate: 'safe'". + * + * ## What each leg pins + * + * - boot 1 — a fresh `os dev` creates the NULL-safe index; with + * `OS_TELEMETRY_DB=0` it provisions no telemetry sibling (the opt-out). + * - boot 2 — after the card's step 3 restages the bare index, a plain `os dev` + * restart puts the NULL-safe one back, says `auto-reconciled`, and provisions + * `A.telemetry.db` (default-on in dev for a file-backed SQLite primary). + * - boot 3 — a production boot (`os serve`, `NODE_ENV` unset) on the restaged + * file reports the drift and leaves it: production never auto-migrates. + * + * The host-config path is pinned unchanged where it is decided + * (`src/utils/storage-driver.test.ts`), and both hosts are held to one answer + * per driver kind by `src/utils/dev-self-heal-host-parity.test.ts`; the + * one-shot fence (`os migrate plan`, write-mode boots under + * `NODE_ENV=development`) by `src/utils/schema-migrate.dev-self-heal-fence.integration.test.ts`. + * + * ## Spawn shape + * + * `bin/run.js` with `NODE_ENV` unset — the shipped entry, resolving commands + * from `dist/`, so the boots measure the built packages (`requireBuiltCli`). + * One process group per boot (`dev` supervises a `serve` grandchild). Every + * boot runs in `beforeAll` and every `it` reads what it recorded: clocked cases + * measure behaviour, never loading. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { spawn, type ChildProcess } from 'node:child_process'; +import { existsSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import Database from 'better-sqlite3'; +import { + childEnv, + E2E_SECRET_KEY, + portContentionError, + portDriftError, + randomPort, + requireBuiltCli, + RUN_JS_RESOLVES_FROM_DIST, +} from './helpers/serve-process.js'; +import { writeDefineStackConfig } from './helpers/define-stack-fixture.js'; + +const HERE = resolve(fileURLToPath(import.meta.url), '..'); +/** `bin/run.js` — the SHIPPED entrypoint. */ +const CLI = resolve(HERE, '../bin/run.js'); + +/** The banner's tail — every row above it has printed. */ +const READY = /Press Ctrl\+C to stop/; +const BOOT_TIMEOUT_MS = 180_000; +const ALL_BOOTS_TIMEOUT_MS = 3 * (BOOT_TIMEOUT_MS + 30_000); + +const INDEX = 'uniq_scaf_item_organization_id_qa_code'; + +/** The card's blank scaffold: one object, an org-scoped unique field, no plugins, no datasources. */ +const SCAFFOLD = { + manifest: { id: 'com.example.scaf', namespace: 'scaf', version: '1.0.0', type: 'app', name: 'scaf' }, + objects: [{ + name: 'scaf_item', + label: 'Item', + sharingModel: 'private', + fields: { qa_code: { type: 'text', label: 'QA code', unique: true } }, + }], +}; + +interface BootRecord { + /** The index DDL `sqlite_master` holds after the boot stopped. */ + indexSql: string | undefined; + telemetryExists: boolean; + output: string; +} + +const groups: ChildProcess[] = []; +let dir: string; +let db: string; +const record: Partial> = {}; + +function readIndexSql(): string | undefined { + const handle = new Database(db, { readonly: true }); + try { + const row = handle.prepare("SELECT sql FROM sqlite_master WHERE type = 'index' AND name = ?").get(INDEX) as + | { sql: string } + | undefined; + return row?.sql; + } finally { + handle.close(); + } +} + +/** The card's step 3: replace the NULL-safe unique index with the bare one an older release left. */ +function stageBareIndex(): void { + const handle = new Database(db); + try { + handle.exec(`DROP INDEX ${INDEX}; CREATE UNIQUE INDEX ${INDEX} ON scaf_item (organization_id, qa_code);`); + } finally { + handle.close(); + } +} + +/** Boot `os ` until its ready banner, then stop the whole group and return what it printed. */ +function bootUntilReady(argv: string[], port: string, env: Record): Promise { + return new Promise((resolveBoot, rejectBoot) => { + const child = spawn(process.execPath, [CLI, ...argv], { + cwd: dir, + // `childEnv`, never a bare `...process.env` — see its header. `NODE_ENV` + // unset, so the built entry resolves its commands from `dist/`. + env: childEnv({ + NO_COLOR: '1', + OS_CLOUD_URL: 'off', + OS_DISABLE_CONSOLE: '1', + OS_SECRET_KEY: E2E_SECRET_KEY, + OS_DATABASE_URL: undefined, + OS_ARTIFACT_URL: undefined, + OS_ARTIFACT_PATH: undefined, + OS_MODE: undefined, + NODE_ENV: undefined, + ...env, + }), + stdio: ['ignore', 'pipe', 'pipe'], + // Own process group: `dev` supervises a `serve` grandchild. + detached: true, + }); + groups.push(child); + let out = ''; + let settled = false; + const finish = (err: Error | null) => { + if (settled) return; + settled = true; + clearTimeout(timer); + if (err) { + stopGroup(child).then(() => rejectBoot(err), () => rejectBoot(err)); + return; + } + // Stop BEFORE the file is read: the boot's own writes are flushed on exit. + stopGroup(child).then(() => resolveBoot(out), rejectBoot); + }; + const timer = setTimeout( + () => finish(new Error(`os ${argv[0]} never printed ${READY}\n--- output ---\n${out.slice(-4000)}`)), + BOOT_TIMEOUT_MS, + ); + const onData = (d: unknown) => { + out += String(d); + if (READY.test(out)) finish(portDriftError(out, `os ${argv[0]}`, port)); + }; + child.stdout?.on('data', onData); + child.stderr?.on('data', onData); + child.on('exit', (code) => + finish(portContentionError(out, `os ${argv[0]}`, port) + ?? new Error(`os ${argv[0]} exited ${String(code)} before ${READY}\n--- output ---\n${out.slice(-4000)}`)), + ); + }); +} + +async function stopGroup(child: ChildProcess): Promise { + if (child.pid === undefined || child.exitCode !== null || child.signalCode !== null) return; + await new Promise((done) => { + const give = setTimeout(() => { + try { process.kill(-child.pid!, 'SIGKILL'); } catch { /* group already gone */ } + done(); + }, 15_000); + child.once('exit', () => { clearTimeout(give); done(); }); + try { process.kill(-child.pid!, 'SIGTERM'); } catch { clearTimeout(give); done(); } + }); +} + +/** + * One boot, recorded against its own leg so a failed boot cannot hide the + * others. Takes the boot already started, so every `bootUntilReady` call site + * hands its child environment over as a literal the env gate can read. + */ +async function recordBoot(leg: 'fresh' | 'restart' | 'production', boot: Promise): Promise { + try { + const output = await boot; + record[leg] = { indexSql: readIndexSql(), telemetryExists: existsSync(join(dir, 'A.telemetry.db')), output }; + } catch (error) { + record[leg] = error instanceof Error ? error : new Error(String(error)); + } +} + +function recorded(leg: 'fresh' | 'restart' | 'production'): BootRecord { + const r = record[leg]; + if (r === undefined) throw new Error(`the ${leg} boot was never run`); + if (r instanceof Error) throw r; + return r; +} + +const isNullSafe = (sql: string | undefined): boolean => /COALESCE\(`?organization_id`?/.test(sql ?? ''); + +beforeAll(async () => { + requireBuiltCli(RUN_JS_RESOLVES_FROM_DIST); + dir = mkdtempSync(join(tmpdir(), 'os-21733-dev-')); + writeDefineStackConfig(dir, SCAFFOLD); + db = join(dir, 'A.db'); + + // Boot 1: fresh, with the telemetry opt-out. + let port = randomPort(); + await recordBoot('fresh', bootUntilReady(['dev', '--no-watch', '-d', 'file:A.db', '-p', port], port, { OS_TELEMETRY_DB: '0' })); + + // Boot 2: the card's restart, on the restaged bare index, telemetry default-on. + if (existsSync(db)) stageBareIndex(); + port = randomPort(); + await recordBoot('restart', bootUntilReady(['dev', '--no-watch', '-d', 'file:A.db', '-p', port], port, {})); + + // Boot 3: production (`os serve`, NODE_ENV unset) on the restaged bare index. + if (existsSync(db)) stageBareIndex(); + port = randomPort(); + await recordBoot('production', bootUntilReady(['serve', 'objectstack.config.ts', '-p', port], port, { + OS_DATABASE_URL: 'file:A.db', + OS_AUTH_SECRET: 'e2e-21733-production-leg-secret-not-for-real-use', + OS_LOG_LEVEL: 'warn', + })); +}, ALL_BOOTS_TIMEOUT_MS); + +afterAll(async () => { + for (const child of groups) await stopGroup(child); + try { rmSync(dir, { recursive: true, force: true }); } catch { /* ignore */ } +}); + +describe('#21733 — a plain `os dev` (standalone stack) self-heals and provisions telemetry', () => { + it('a fresh `os dev` creates the NULL-safe index, and OS_TELEMETRY_DB=0 provisions no sibling', () => { + const fresh = recorded('fresh'); + expect(isNullSafe(fresh.indexSql), `fresh boot index: ${fresh.indexSql}`).toBe(true); + expect(fresh.telemetryExists, 'OS_TELEMETRY_DB=0 still created A.telemetry.db').toBe(false); + }); + + it('a plain `os dev` restart puts the staged bare unique index back NULL-safe', () => { + const restart = recorded('restart'); + expect(isNullSafe(restart.indexSql), `the restart left the bare index: ${restart.indexSql}`).toBe(true); + expect(restart.output).toContain('[schema-drift] auto-reconciled recreate_index on scaf_item'); + }); + + it('the same restart provisions the `telemetry` sibling next to the file-backed primary', () => { + const restart = recorded('restart'); + expect(restart.telemetryExists, 'no A.telemetry.db after a dev boot on file:A.db').toBe(true); + }); + + it('a production boot reports the staged drift and does not auto-migrate it', () => { + const production = recorded('production'); + expect(production.output).toContain(`index '${INDEX}' tightens`); + expect(production.output).not.toContain('[schema-drift] auto-reconciled'); + expect(isNullSafe(production.indexSql), `production auto-migrated: ${production.indexSql}`).toBe(false); + }); +}); diff --git a/packages/runtime/src/default-datasource-plugin.ts b/packages/runtime/src/default-datasource-plugin.ts index 21406fba69c..fe1f3b59cd2 100644 --- a/packages/runtime/src/default-datasource-plugin.ts +++ b/packages/runtime/src/default-datasource-plugin.ts @@ -62,7 +62,15 @@ export interface DefaultDatasourceDefinition { } export interface DefaultDatasourcePluginOptions { - /** Arms the shared factory's dev sqlite step-down (#2229) + loosen-only self-heal passthroughs. */ + /** + * Arms the shared factory's dev sqlite step-down (#2229): native + * better-sqlite3 → wasm → in-memory when the native binding cannot load. + * That is ALL it arms. The dev schema self-heal does not ride here: it rides + * in the definition's `config.autoMigrate`, which the host decides with + * `devAutoMigrateConfig` (`dev-auto-migrate.ts`, #21733) before it builds + * this plugin — so `dev: true` with a definition that carries no + * `autoMigrate` boots without the self-heal. + */ dev?: boolean; /** * Forwarded to the shared factory: what a `sqlite` default does when its diff --git a/packages/runtime/src/dev-auto-migrate.test.ts b/packages/runtime/src/dev-auto-migrate.test.ts new file mode 100644 index 00000000000..ed3bb9132a4 --- /dev/null +++ b/packages/runtime/src/dev-auto-migrate.test.ts @@ -0,0 +1,139 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// #21733 — the dev schema self-heal (`autoMigrate: 'safe'`, #2186) has ONE +// home, `dev-auto-migrate.ts`, and the standalone stack reads it. +// +// Three things are pinned here, from the inside out: +// +// 1. the decision's driver set IS the contract's: a kind gets `'safe'` iff +// its `@objectstack/spec` connection schema declares `autoMigrate` — so a +// driver cannot silently gain a key it ignores, nor lose one it honours; +// 2. the standalone stack's `default` definition carries it on a dev boot — +// the card's defect: it never did, so a plain `os dev` never self-healed; +// 3. only an EXPLICIT `dev: true` arms it. The `NODE_ENV` default the sqlite +// step-down takes does not: `bootSchemaStack` (every one-shot command) +// passes no `dev`, and must never auto-apply drift under +// `NODE_ENV=development`. The behavioural half of that fence — a real +// one-shot boot on a staged database — is +// `packages/cli/src/utils/schema-migrate.dev-self-heal-fence.integration.test.ts`. +// +// Nothing here connects to a database: `createStandaloneStack` builds a +// DEFINITION and hands it to `DefaultDatasourcePlugin` (ADR-0062 D1), and the +// definition is the whole of what this package decides. + +import { describe, it, expect, afterEach } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { BUILTIN_DRIVER_IDS, getDriverConfigJsonSchemaById } from '@objectstack/spec/data'; +import { devAutoMigrateConfig } from './dev-auto-migrate.js'; +import { createStandaloneStack } from './standalone-stack.js'; + +// [#10126] Pay the first transform of these dist-resolved workspace deps at +// MODULE LOAD: `createStandaloneStack` reaches them through dynamic +// `import()`s inside clocked `it()` bodies (`scripts/check-test-source-alias.mjs`). +import '@objectstack/service-datasource'; +import '@objectstack/objectql'; +import '@objectstack/metadata'; + +const ENV_KEYS = [ + 'OS_DATABASE_URL', + 'DATABASE_URL', + 'TURSO_DATABASE_URL', + 'OS_DATABASE_DRIVER', + 'OS_HOME', + 'NODE_ENV', +] as const; +const ORIGINAL_ENV: Record = Object.fromEntries( + ENV_KEYS.map((k) => [k, process.env[k]]), +); +const dirs: string[] = []; + +afterEach(() => { + for (const key of ENV_KEYS) { + const original = ORIGINAL_ENV[key]; + if (original === undefined) delete process.env[key]; + else process.env[key] = original; + } + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }); +}); + +function clearUrlEnv(): void { + for (const key of ENV_KEYS) if (key !== 'NODE_ENV') delete process.env[key]; +} + +function scratch(): string { + const d = mkdtempSync(join(tmpdir(), 'os-21733-')); + dirs.push(d); + return d; +} + +/** The `default` datasource DEFINITION a built stack carries. */ +async function defaultConfigOf(opts: Parameters[0]): Promise> { + const stack = await createStandaloneStack(opts); + const plugin = stack.plugins.find( + (p: any) => p?.name === 'com.objectstack.runtime.default-datasource', + ) as any; + expect(plugin, 'stack must carry the DefaultDatasourcePlugin').toBeDefined(); + return plugin.def.config ?? {}; +} + +const BOOT_TIMEOUT = 60_000; + +describe('devAutoMigrateConfig — the one decision (#21733)', () => { + it('answers `safe` on a dev boot for exactly the kinds whose spec contract declares `autoMigrate`', () => { + for (const id of BUILTIN_DRIVER_IDS) { + const declared = Object.keys( + (getDriverConfigJsonSchemaById(id) as { properties?: Record }).properties ?? {}, + ).includes('autoMigrate'); + expect(devAutoMigrateConfig(id, true), `driver "${id}" (contract declares autoMigrate: ${declared})`) + .toEqual(declared ? { autoMigrate: 'safe' } : {}); + } + }); + + it('the set is the three SQL kinds — sqlite-wasm, mongodb, turso and memory get nothing', () => { + const armed = BUILTIN_DRIVER_IDS.filter((id) => devAutoMigrateConfig(id, true).autoMigrate === 'safe'); + expect([...armed].sort()).toEqual(['mysql', 'postgres', 'sqlite']); + }); + + it('answers nothing outside a dev boot, for every kind', () => { + for (const id of BUILTIN_DRIVER_IDS) expect(devAutoMigrateConfig(id, false)).toEqual({}); + }); +}); + +describe('createStandaloneStack — the `default` definition reads the decision (#21733)', () => { + it('a dev boot on a file-backed sqlite database carries autoMigrate: safe', async () => { + clearUrlEnv(); + const dir = scratch(); + const config = await defaultConfigOf({ projectRoot: dir, databaseUrl: `file:${join(dir, 'A.db')}`, dev: true }); + expect(config).toEqual({ filename: join(dir, 'A.db'), autoMigrate: 'safe' }); + }, BOOT_TIMEOUT); + + it('a dev boot on postgres and mysql carries it too; on sqlite-wasm and mongodb it does not', async () => { + clearUrlEnv(); + const dir = scratch(); + expect(await defaultConfigOf({ projectRoot: dir, databaseUrl: 'postgres://u:p@localhost:5432/db', dev: true })) + .toEqual({ url: 'postgres://u:p@localhost:5432/db', autoMigrate: 'safe' }); + expect(await defaultConfigOf({ projectRoot: dir, databaseUrl: 'mysql://u:p@localhost:3306/db', dev: true })) + .toEqual({ url: 'mysql://u:p@localhost:3306/db', autoMigrate: 'safe' }); + expect(await defaultConfigOf({ projectRoot: dir, databaseUrl: `wasm-sqlite://${join(dir, 'w.db')}`, dev: true })) + .not.toHaveProperty('autoMigrate'); + expect(await defaultConfigOf({ projectRoot: dir, databaseUrl: 'mongodb://localhost:27017/db', dev: true })) + .not.toHaveProperty('autoMigrate'); + }, BOOT_TIMEOUT); + + it('a production boot (`dev: false`, what `os start` passes) does not carry it', async () => { + clearUrlEnv(); + const dir = scratch(); + const config = await defaultConfigOf({ projectRoot: dir, databaseUrl: `file:${join(dir, 'A.db')}`, dev: false }); + expect(config).not.toHaveProperty('autoMigrate'); + }, BOOT_TIMEOUT); + + it('the NODE_ENV=development default does NOT arm it — a one-shot boot passes no `dev`', async () => { + clearUrlEnv(); + process.env.NODE_ENV = 'development'; + const dir = scratch(); + const config = await defaultConfigOf({ projectRoot: dir, databaseUrl: `file:${join(dir, 'A.db')}` }); + expect(config).not.toHaveProperty('autoMigrate'); + }, BOOT_TIMEOUT); +}); diff --git a/packages/runtime/src/dev-auto-migrate.ts b/packages/runtime/src/dev-auto-migrate.ts new file mode 100644 index 00000000000..af58809f7f8 --- /dev/null +++ b/packages/runtime/src/dev-auto-migrate.ts @@ -0,0 +1,65 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The dev schema self-heal decision (#2186) — ONE home, read by every boot host. + * + * `os dev` runs the SQL drivers with `autoMigrate: 'safe'`: on restart, the safe + * drift `os migrate plan` lists (a relaxed NOT NULL, a widened column, a unique + * index that becomes NULL-safe once its duplicate probe is clean) is applied to + * the existing dev database, and the destructive kind is only ever warned about. + * The decision has two halves, and both live here: + * + * - **When** — a dev boot, as its HOST declares it (`dev`). Production never + * gets it. The SQL driver also refuses it under `NODE_ENV=production`; that + * is the driver's own second guard, not a copy of this rule. + * - **For which drivers** — the kinds whose connection contract declares + * `autoMigrate` (`@objectstack/spec`'s `SqliteConfigSchema`, + * `PostgresConfigSchema` and `MysqlConfigSchema`), which are exactly the + * kinds the shared datasource factory hands it to `SqlDriver` for. + * `sqlite-wasm`, `mongodb` and `turso` declare no such key: a config key the + * driver silently ignores is the declared ≠ enforced shape, so they get + * nothing. `dev-auto-migrate.test.ts` holds this set equal to the contract. + * + * The readers, which must never carry the condition themselves: + * + * - `createStandaloneStack` — the `default` datasource of every plain + * `os dev` / `os serve` / `os start` boot, over a config or an artifact; + * - the CLI's config-load fallback (`resolveStorageDefinition`, reached by a + * host config or `OS_MODE=off`) and the `telemetry` sibling datasource the + * CLI provisions next to a file-backed SQLite primary. + * + * Until #21733 only the CLI's fallback carried it, as an inline + * `isDev ? { autoMigrate: 'safe' } : {}`. The standalone stack built its + * definition without it, so a plain `os dev` on a non-host config never + * self-healed, while the plan and drift lines the driver prints said the + * change was "auto-applied at boot under dev autoMigrate: 'safe'". + */ + +import type { BuiltinDriverId } from '@objectstack/spec/data'; + +/** + * The datasource-config fragment the self-heal rides in — `config.autoMigrate`, + * the host-composition passthrough ADR-0062 records — or nothing. + */ +export type DevAutoMigrateConfig = { readonly autoMigrate?: 'safe' }; + +/** The driver kinds whose connection contract declares `autoMigrate`. */ +const DEV_AUTO_MIGRATE_DRIVERS: ReadonlySet = new Set([ + 'sqlite', + 'postgres', + 'mysql', +]); + +/** + * The self-heal fragment for one datasource definition: `{ autoMigrate: 'safe' }` + * on a dev boot for a driver whose contract declares the key, `{}` otherwise. + * Spread it into the definition's `config`. + * + * @param driver the CANONICAL driver id the definition is built for + * @param dev the host's declaration that this is a dev boot. A one-shot + * command boot is never one, whatever `NODE_ENV` says: it must + * apply only what its operator confirmed. + */ +export function devAutoMigrateConfig(driver: BuiltinDriverId, dev: boolean): DevAutoMigrateConfig { + return dev && DEV_AUTO_MIGRATE_DRIVERS.has(driver) ? { autoMigrate: 'safe' } : {}; +} diff --git a/packages/runtime/src/index.ts b/packages/runtime/src/index.ts index 84138defe7f..5d42ad0c478 100644 --- a/packages/runtime/src/index.ts +++ b/packages/runtime/src/index.ts @@ -11,6 +11,12 @@ export type { RuntimeConfig } from './runtime.js'; export { createStandaloneStack, resolveObjectStackHome, resolveStandaloneDatabase } from './standalone-stack.js'; export type { StandaloneStackConfig, StandaloneStackResult, ResolvedStandaloneDatabase } from './standalone-stack.js'; +// The dev schema self-heal decision (#21733) — public because `@objectstack/cli` +// is a reader, not a second home: its config-load fallback and its `telemetry` +// sibling read the SAME answer the standalone stack's `default` datasource does. +export { devAutoMigrateConfig } from './dev-auto-migrate.js'; +export type { DevAutoMigrateConfig } from './dev-auto-migrate.js'; + // The ONE libSQL/Turso loader (commit 68f5eccb1). Public because `@objectstack/cli` is a // consumer, not a second implementation: `utils/storage-driver.ts` delegates to // `loadTursoDriverFactory` and RE-EXPORTS `MissingDriverPackageError`, so diff --git a/packages/runtime/src/standalone-stack.ts b/packages/runtime/src/standalone-stack.ts index a330cab1293..69295ae4dcb 100644 --- a/packages/runtime/src/standalone-stack.ts +++ b/packages/runtime/src/standalone-stack.ts @@ -77,6 +77,7 @@ import { import type { IDatasourceDriverFactory } from '@objectstack/service-datasource'; import { loadArtifactBundle, isHttpUrl } from './load-artifact-bundle.js'; import { loadTursoDriverFactory } from './turso-driver-factory.js'; +import { devAutoMigrateConfig } from './dev-auto-migrate.js'; import { namesRetiredMemoryEngine, resolveProjectDatabaseUrl, @@ -219,6 +220,14 @@ export const StandaloneStackConfigSchema = z.object({ * in-memory step-down (#2229). When omitted, defaults to * `process.env.NODE_ENV === 'development'`. In production a native load * failure is NOT silently swapped for wasm/mingo (fail-closed). + * + * An EXPLICIT `true` also arms the dev schema self-heal on the `default` + * datasource (`autoMigrate: 'safe'` for the SQL kinds, #21733 — decided in + * `dev-auto-migrate.ts`). The `NODE_ENV` default does not: the serving + * boots (`os dev` / `os serve` / `os start`) always pass this key, and the + * one-shot command boot (`bootSchemaStack`) never does, so a one-shot + * command cannot apply drift its operator never confirmed, whatever + * `NODE_ENV` says. */ dev: z.boolean().optional(), /** @@ -728,9 +737,10 @@ export async function createStandaloneStack(config?: StandaloneStackConfig): Pro // a step-down to SQLite, which would open an empty local database while // the operator's libSQL data stays untouched (#3276). // - // No `autoMigrate` passthrough: `TursoDriverConfig` declares no such - // key, and handing the driver a config it silently ignores is the kind - // of "declared ≠ enforced" the CLI side deliberately avoided too. + // No `autoMigrate` for this kind: `TursoDriverConfig` declares no such + // key, so the shared self-heal decision below answers `{}` for it — + // handing the driver a config it silently ignores would be the kind of + // "declared ≠ enforced" both hosts avoid. driverId = 'turso'; const authToken = resolveDatabaseAuthToken(cfg); driverConfig = { url: dbUrl, ...(authToken ? { authToken } : {}) }; @@ -771,8 +781,21 @@ export async function createStandaloneStack(config?: StandaloneStackConfig): Pro `would hand the caller a database engine they never selected.` ); } + // #21733 — the dev schema self-heal (#2186), read from its ONE home, the + // same answer the CLI's config-load fallback reads (`resolveStorageDefinition`). + // Before this line the definition never carried it, so a plain `os dev` on + // a non-host config left safe drift in place on every restart while the + // drift line said it was "auto-applied at boot". + // + // Keyed on an EXPLICIT `dev: true`, never on `factoryDev`'s NODE_ENV + // default: `bootSchemaStack` passes no `dev`, and a non-deferred one-shot + // boot (`os meta resync --yes`, the data commands' `--apply`) runs schema + // sync — under `NODE_ENV=development` it would apply safe drift its + // operator never confirmed. The step-down keeps the default; the self-heal + // is a serving host's declaration. + const devSelfHeal = devAutoMigrateConfig(dbDriver, cfg.dev === true); const defaultDatasourcePlugin = new DefaultDatasourcePlugin( - { driver: driverId, config: driverConfig }, + { driver: driverId, config: { ...driverConfig, ...devSelfHeal } }, { dev: factoryDev, ...(cfg.sqliteAbsentFile ? { sqliteAbsentFile: cfg.sqliteAbsentFile } : {}), diff --git a/scripts/check-cli-test-child-env.mjs b/scripts/check-cli-test-child-env.mjs index 45c40db8077..9a3bf3c78bf 100644 --- a/scripts/check-cli-test-child-env.mjs +++ b/scripts/check-cli-test-child-env.mjs @@ -2635,8 +2635,15 @@ export function selfTest() { .filter((abs) => builtEntrypointSpawns(abs, readFileSync(abs, 'utf8')).spawns > 0) .map((abs) => relative(REPO_ROOT, abs).split(sep).join('/')) .sort(); - t('the built-entrypoint population is exactly the six files that spawn bin/run.js', + t('the built-entrypoint population is exactly the seven files that spawn bin/run.js', JSON.stringify(builtFiles) === JSON.stringify([ + // [#21733] Joined the population deliberately: the card's acceptance is + // a plain `os dev` restart on the BUILT packages, and its production leg + // (`os serve` with `NODE_ENV` unset must NOT auto-migrate) is only + // reachable in production posture — `bin/run-dev.js` sets + // `NODE_ENV=development` before argv is parsed. Every spawn hands its + // child `NODE_ENV: undefined`. + 'packages/cli/test/dev-standalone-self-heal.integration.test.ts', // [#12271] Joined the population deliberately, and it is the one member // whose subject is the ENTRY rather than `serve`: it spawns `bin/run.js` // with an ambient `development` / `test` child to assert that the