From 9c73f3702b16ea3fac006c8565c1b1807f280b0b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:19:43 +0000 Subject: [PATCH 1/8] test(rest): temporary probe measuring write doors that serve undeclared columns Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- packages/rest/src/zz-probe-21613.test.ts | 175 +++++++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100644 packages/rest/src/zz-probe-21613.test.ts diff --git a/packages/rest/src/zz-probe-21613.test.ts b/packages/rest/src/zz-probe-21613.test.ts new file mode 100644 index 00000000000..3962cdc92e4 --- /dev/null +++ b/packages/rest/src/zz-probe-21613.test.ts @@ -0,0 +1,175 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21613] TEMPORARY measurement probe — reverted before the fix lands. + * + * Records, on the composed REST harness (RestServer, then the protocol, then + * ObjectQL, then a real SqlDriver on better-sqlite3), which write doors serve a + * column no metadata declares, and which in-process carriers of a write result + * (data events, hook `previous` / `result`) carry one. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { RestServer } from './rest-server'; + +const OBJECT = 'rq_contact'; +const RETIRED = ['mailing_street', 'mailing_city']; + +const CONTACT_V1 = { + name: OBJECT, + label: 'Contact', + fields: { + name: { name: 'name', type: 'text' as const }, + email: { name: 'email', type: 'text' as const }, + mailing_street: { name: 'mailing_street', type: 'text' as const }, + mailing_city: { name: 'mailing_city', type: 'text' as const }, + }, +}; +const CONTACT_V2 = { + name: OBJECT, + label: 'Contact', + enable: { apiMethods: ['get', 'list', 'create', 'update', 'delete', 'bulk', 'upsert'] }, + fields: { + name: { name: 'name', type: 'text' as const }, + email: { name: 'email', type: 'text' as const }, + }, +}; + +function createMockServer() { + const noop = () => {}; + return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; +} +function makeRes() { + const res: any = { + write: () => true, end: () => {}, + header: () => res, + status: (code: number) => { res._status = code; return res; }, + json: (body: any) => { res._json = body; return res; }, + }; + return res; +} + +const dir = mkdtempSync(join(tmpdir(), 'os-21613-probe-')); +const filename = join(dir, 'retired-columns.sqlite'); +const engines: ObjectQL[] = []; +afterAll(async () => { + for (const e of engines) { try { await e.destroy(); } catch { /* noop */ } } + rmSync(dir, { recursive: true, force: true }); +}); +const sqlDriver = () => new SqlDriver({ client: 'better-sqlite3', connection: { filename }, useNullAsDefault: true }) as any; + +const notes: Record = {}; +const retiredIn = (v: any): string[] => { + if (!v || typeof v !== 'object') return []; + return RETIRED.filter((k) => k in v).map((k) => `${k}=${JSON.stringify(v[k])}`); +}; + +describe('[#21613] probe', () => { + let rest: RestServer; + let engine: ObjectQL; + const events: any[] = []; + const hooks: Array<{ event: string; previous: string[]; result: string[] }> = []; + + beforeAll(async () => { + const one = new ObjectQL(); + one.registerDriver(sqlDriver(), true); + await one.init(); + one.registry.registerObject(CONTACT_V1 as any); + await one.syncSchemas(); + await one.insert(OBJECT, [ + { id: 'c1', name: 'Ada', email: 'ada@example.com', mailing_street: '1 Retired Way', mailing_city: 'Oldtown' }, + { id: 'c2', name: 'Bo', email: 'bo@example.com', mailing_street: '2 Retired Way', mailing_city: 'Oldtown' }, + { id: 'c3', name: 'Cy', email: 'cy@example.com', mailing_street: '3 Retired Way', mailing_city: 'Oldtown' }, + { id: 'c4', name: 'Di', email: 'di@example.com', mailing_street: '4 Retired Way', mailing_city: 'Oldtown' }, + { id: 'c5', name: 'Ed', email: 'bulk@example.com', mailing_street: '5 Retired Way', mailing_city: 'Oldtown' }, + { id: 'c6', name: 'Fi', email: 'bulk@example.com', mailing_street: '6 Retired Way', mailing_city: 'Oldtown' }, + { id: 'c7', name: 'Gu', email: 'gone@example.com', mailing_street: '7 Retired Way', mailing_city: 'Oldtown' }, + ] as any); + await one.destroy(); + + engine = new ObjectQL(); + engines.push(engine); + engine.registerDriver(sqlDriver(), true); + await engine.init(); + engine.registry.registerObject(CONTACT_V2 as any); + await engine.syncSchemas(); + engine.setRealtimeService({ + publish: async (e: any) => { events.push(e); }, + subscribe: async () => 'x', + unsubscribe: async () => {}, + } as any); + for (const ev of ['beforeUpdate', 'afterUpdate', 'beforeDelete', 'afterDelete', 'afterInsert']) { + engine.registerHook(ev, async (ctx: any) => { + hooks.push({ event: `${ev}:${ctx.input?.id ?? ''}`, previous: retiredIn(ctx.previous), result: retiredIn(ctx.result) }); + }, { object: OBJECT }); + } + const protocol = new ObjectStackProtocolImplementation(engine as any); + rest = new RestServer(createMockServer() as any, protocol as any, { api: { requireAuth: false } } as any); + (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' }); + rest.registerRoutes(); + }); + + const route = (method: string, path: string) => { + const r = rest.getRoutes().find((x: any) => x.method === method && x.path === path); + if (!r) throw new Error(`no route ${method} ${path}`); + return r; + }; + const call = async (method: string, path: string, req: any) => { + const res = makeRes(); + await route(method, path).handler({ query: {}, headers: {}, ...req }, res); + return res; + }; + + it('measures every write door', async () => { + const p = '/api/v1/data/:object'; + let res = await call('PATCH', `${p}/:id`, { params: { object: OBJECT, id: 'c1' }, body: { name: 'Ada 2' } }); + notes['PATCH /data/:object/:id'] = { status: res._status ?? 200, retired: retiredIn(res._json?.record) }; + + res = await call('POST', p, { params: { object: OBJECT }, body: { id: 'n1', name: 'New' } }); + notes['POST /data/:object (create)'] = { status: res._status ?? 200, retired: retiredIn(res._json?.record) }; + + res = await call('POST', `${p}/:id/clone`, { params: { object: OBJECT, id: 'c2' }, body: { name: 'Bo copy' } }); + notes['POST /data/:object/:id/clone'] = { status: res._status ?? 200, retired: retiredIn(res._json?.record), err: res._json?.error }; + + res = await call('POST', `${p}/createMany`, { params: { object: OBJECT }, body: [{ id: 'm1', name: 'M1' }, { id: 'm2', name: 'M2' }] }); + notes['POST /data/:object/createMany'] = { status: res._status ?? 200, retired: (res._json?.records ?? []).map(retiredIn), err: res._json?.error }; + + res = await call('POST', `${p}/batch`, { params: { object: OBJECT }, body: { operation: 'update', records: [{ id: 'c3', data: { name: 'Cy 2' } }] } }); + notes['POST /data/:object/batch update'] = { status: res._status ?? 200, retired: (res._json?.results ?? []).map((r: any) => retiredIn(r.data)), err: res._json?.error ?? res._json?.results?.[0]?.errors }; + + res = await call('POST', `${p}/batch`, { params: { object: OBJECT }, body: { operation: 'upsert', records: [{ id: 'c4', data: { name: 'Di 2' } }, { id: 'u1', data: { name: 'Up new' } }] } }); + notes['POST /data/:object/batch upsert'] = { status: res._status ?? 200, retired: (res._json?.results ?? []).map((r: any) => retiredIn(r.data)), err: res._json?.error ?? res._json?.results?.map((r: any) => r.errors) }; + + res = await call('POST', `${p}/batch`, { params: { object: OBJECT }, body: { operation: 'create', records: [{ data: { id: 'b1', name: 'B1' } }] } }); + notes['POST /data/:object/batch create'] = { status: res._status ?? 200, retired: (res._json?.results ?? []).map((r: any) => retiredIn(r.data)), err: res._json?.error ?? res._json?.results?.map((r: any) => r.errors) }; + + res = await call('POST', `${p}/updateMany`, { params: { object: OBJECT }, body: { records: [{ id: 'c1', data: { name: 'Ada 3' } }] } }); + notes['POST /data/:object/updateMany'] = { status: res._status ?? 200, retired: (res._json?.results ?? []).map((r: any) => retiredIn(r.data)), err: res._json?.error ?? res._json?.results?.map((r: any) => r.errors) }; + + res = await call('DELETE', `${p}/:id`, { params: { object: OBJECT, id: 'c7' } }); + notes['DELETE /data/:object/:id'] = { status: res._status ?? 200, body: res._json }; + + // In-process verbs. + const updated = await engine.update(OBJECT, { id: 'c2', name: 'Bo 2' }); + notes['engine.update by id'] = retiredIn(updated); + const inserted = await engine.insert(OBJECT, { id: 'n2', name: 'New 2' }); + notes['engine.insert single'] = retiredIn(inserted); + const outcomes: any[] = await (engine as any).insertMany(OBJECT, [{ id: 'n3', name: 'New 3' }]); + notes['engine.insertMany'] = outcomes.map((o) => retiredIn(o.record)); + const count = await engine.update(OBJECT, { name: 'bulk' } as any, { where: { email: 'bulk@example.com' }, multi: true } as any); + notes['engine.update multi (count)'] = count; + const del = await engine.delete(OBJECT, { where: { email: 'bulk@example.com' }, multi: true } as any); + notes['engine.delete multi (count)'] = del; + + notes.events = events.map((e) => ({ type: e.type, recordId: e.payload?.recordId, after: retiredIn(e.payload?.after), changes: retiredIn(e.payload?.changes) })); + notes.hooks = hooks; + writeFileSync(process.env.OS_PROBE_OUT ?? join(tmpdir(), 'probe-21613.json'), JSON.stringify(notes, null, 2)); + expect(true).toBe(true); + }); +}); From b923112e062bd3e6945405a277659b9671c9e861 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:20:35 +0000 Subject: [PATCH 2/8] test(plugin-audit): temporary probe measuring what the audit ledger records of a write's prior read Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- .../plugin-audit/src/zz-probe-21613.test.ts | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 packages/plugins/plugin-audit/src/zz-probe-21613.test.ts diff --git a/packages/plugins/plugin-audit/src/zz-probe-21613.test.ts b/packages/plugins/plugin-audit/src/zz-probe-21613.test.ts new file mode 100644 index 00000000000..b1b903497fa --- /dev/null +++ b/packages/plugins/plugin-audit/src/zz-probe-21613.test.ts @@ -0,0 +1,128 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21613] TEMPORARY measurement probe — reverted before the fix lands. + * + * Does the audit ledger serve the engine's `previous` (the write path's prior + * read) or `result`, and what happens to it when only the RESULT is shaped? + */ + +import { describe, it, expect } from 'vitest'; +import { writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { ObjectQL } from '@objectstack/objectql'; +import { installAuditWriters } from './audit-writers.js'; + +const f = (name: string, type: string, extra: Record = {}) => + ({ name, label: name, type, ...extra }) as any; + +const sysAuditLog = { + name: 'sys_audit_log', label: 'Audit Log', + fields: { + id: f('id', 'text', { primaryKey: true }), action: f('action', 'text'), + user_id: f('user_id', 'text'), object_name: f('object_name', 'text'), + record_id: f('record_id', 'text'), old_value: f('old_value', 'textarea'), + new_value: f('new_value', 'textarea'), tenant_id: f('tenant_id', 'text'), + }, +}; +const sysActivity = { + name: 'sys_activity', label: 'Activity', + fields: { + id: f('id', 'text', { primaryKey: true }), type: f('type', 'text'), + timestamp: f('timestamp', 'datetime'), summary: f('summary', 'text'), + actor_id: f('actor_id', 'text'), object_name: f('object_name', 'text'), + record_id: f('record_id', 'text'), record_label: f('record_label', 'text'), + metadata: f('metadata', 'textarea'), + }, +}; +const contact = { + name: 'rq_contact', label: 'Contact', + fields: { name: f('name', 'text'), email: f('email', 'text') }, +}; +const RETIRED = ['mailing_street', 'mailing_city']; + +function makeDriver(shapeWriteResult: boolean) { + const stores = new Map>>(); + const storeFor = (o: string) => { let s = stores.get(o); if (!s) { s = new Map(); stores.set(o, s); } return s; }; + const copy = (r: T): T => (r == null ? r : JSON.parse(JSON.stringify(r))); + const strip = (r: Record) => { + if (!shapeWriteResult) return r; + const out = { ...r }; + for (const k of RETIRED) delete out[k]; + return out; + }; + const matches = (row: Record, where: any): boolean => { + if (!where || typeof where !== 'object') return true; + for (const [k, v] of Object.entries(where)) { + if (k.startsWith('$')) continue; + if (v && typeof v === 'object' && '$in' in v) { if (!(v.$in as unknown[]).includes(row[k])) return false; continue; } + if ((row[k] ?? null) !== (v ?? null)) return false; + } + return true; + }; + let n = 0; + const driver: any = { + name: 'memory', version: '0.0.0', supports: {} as any, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, + async find(o: string, ast: any) { return Array.from(storeFor(o).values()).filter((r) => matches(r, ast?.where)).map(copy); }, + async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return copy(r); return null; }, + async create(o: string, data: Record) { + n += 1; const id = (data.id as string) ?? `r_${n}`; + // A SQL table's `returning('*')`: every column, the retired ones null on a new row. + const row: Record = { ...data, id }; + if (o === 'rq_contact') for (const k of RETIRED) if (!(k in row)) row[k] = null; + storeFor(o).set(id, row); return strip(copy(row)); + }, + async update(o: string, id: string, data: Record) { + const s = storeFor(o); const cur = s.get(id); if (!cur) return null; + const next = { ...cur, ...data, id }; s.set(id, next); return strip(copy(next)); + }, + async delete(o: string, id: string) { return storeFor(o).delete(id); }, + async count(o: string, ast: any) { return (await this.find(o, ast)).length; }, + async bulkCreate(o: string, rows: Record[]) { return Promise.all(rows.map((r) => this.create(o, r))); }, + async updateMany(o: string, ast: any, data: Record) { + const rows = await this.find(o, ast); const s = storeFor(o); + for (const r of rows) s.set(r.id as string, { ...s.get(r.id as string), ...data, id: r.id }); + return rows.length; + }, + async deleteMany(o: string, ast: any) { const rows = await this.find(o, ast); for (const r of rows) storeFor(o).delete(r.id as string); return rows.length; }, + }; + return { driver, storeFor }; +} + +async function boot(shapeWriteResult: boolean) { + const engine = new ObjectQL(); + const d = makeDriver(shapeWriteResult); + engine.registerDriver(d.driver, true); + await engine.init(); + for (const o of [sysAuditLog, sysActivity, contact]) engine.registry.registerObject(o as any, 'probe'); + installAuditWriters(engine as any); + for (const id of ['c1', 'c2', 'c3', 'c4']) { + d.storeFor('rq_contact').set(id, { id, name: id, email: id === 'c3' || id === 'c4' ? 'bulk@x' : `${id}@x`, mailing_street: `${id} Retired Way`, mailing_city: 'Oldtown' }); + } + return { engine, ...d }; +} + +const ledger = (storeFor: any) => Array.from(storeFor('sys_audit_log').values()).map((r: any) => ({ + action: r.action, record: r.record_id, + old: r.old_value ? RETIRED.filter((k) => k in JSON.parse(r.old_value)).map((k) => `${k}=${JSON.stringify(JSON.parse(r.old_value)[k])}`) : null, + new: r.new_value ? RETIRED.filter((k) => k in JSON.parse(r.new_value)).map((k) => `${k}=${JSON.stringify(JSON.parse(r.new_value)[k])}`) : null, +})); + +describe('[#21613] audit probe', () => { + it('records what the ledger holds, today and with only the result shaped', async () => { + const out: Record = {}; + for (const shaped of [false, true]) { + const { engine, storeFor } = await boot(shaped); + await engine.update('rq_contact', { name: 'c1 renamed' }, { where: { id: 'c1' } } as any); + await engine.delete('rq_contact', { where: { id: 'c2' } } as any); + await engine.insert('rq_contact', { id: 'n1', name: 'new' }); + await engine.update('rq_contact', { name: 'bulk renamed' } as any, { where: { email: 'bulk@x' }, multi: true } as any); + await new Promise((r) => setTimeout(r, 20)); + out[shaped ? 'result shaped, previous raw' : 'today (driver rows whole)'] = ledger(storeFor); + } + writeFileSync(process.env.OS_PROBE_OUT ?? join(tmpdir(), 'probe-21613-audit.json'), JSON.stringify(out, null, 2)); + expect(true).toBe(true); + }); +}); From 10d7f333d977965d12aae8310355a046ed48b820 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:23:28 +0000 Subject: [PATCH 3/8] fix(objectql)!: a write's returned row and its prior read serve the declared fields, never an orphaned column Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- packages/objectql/src/engine.ts | 63 ++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/packages/objectql/src/engine.ts b/packages/objectql/src/engine.ts index aadf4018201..6008f2bc6b6 100644 --- a/packages/objectql/src/engine.ts +++ b/packages/objectql/src/engine.ts @@ -13628,6 +13628,25 @@ export class ObjectQL implements IObjectQLEngine { ); } + // [#21613] A write's returned row is a served row, so it carries the + // read verbs' default projection: the DECLARED field set + // (`declared-read-columns.ts`). A table column no metadata declares — + // a field retired in an upgrade, whose column additive sync leaves + // behind — rides back in `returning('*')` on driver-sql, and on a new + // row it reads null. Shaped HERE, on the rows as the driver returned + // them and after the one-row-per-input guard, so the formulas, the + // `afterInsert` hooks, the roll-up, the `data.record.created` events + // (and the webhooks that deliver them), the partial-mode outcomes and + // the door's own ingress strip all see the declared record — one + // decision for every door, never a per-door strip. Declared fields + // keep their treatment: `internal: true` fields stay whole on this + // engine-level result (the A-prime ruling, below), because an + // undeclared column is no field at all. + const declaredInsertColumns = declaredColumnSet(schemaForValidation); + result = isBatch + ? rowsWithDeclaredColumnsOnly(result as unknown[], declaredInsertColumns) + : withDeclaredColumnsOnly(result, declaredInsertColumns); + // Coerce `boolean` fields (SQLite/libsql return 0/1) to real booleans on // the after-hook view so flow trigger conditions (`record.is_escalated // != true`) and `{record.}` interpolation see JS booleans, not @@ -14433,7 +14452,16 @@ export class ObjectQL implements IObjectQLEngine { // the same for the same reason. const priorAst: QueryAST = { object, where: { id }, limit: 1 }; const preOpts = this.buildDriverOptions(object, opCtx.context, hookContext.input.options as any); - priorRecord = await driver.findOne(object, priorAst, preOpts); + // [#21613] The pre-image is shaped like every other row this engine + // reads off a driver: the declared field set, so `previous` (bound + // for the hooks below, and the audit ledger's side of every update + // diff) and the write's returned row are ONE view. Shaping only the + // returned row would make every update of a row carrying a retired + // column record that column as changed, its stored value included. + priorRecord = withDeclaredColumnsOnly( + await driver.findOne(object, priorAst, preOpts) as Record | null, + declaredColumnSet(updateSchema), + ); // ── [#7867] The not-found gate ────────────────────────────────── // // A by-id update whose id names no row was a SILENT NO-OP that @@ -14556,7 +14584,13 @@ export class ObjectQL implements IObjectQLEngine { readPriorRows = async () => { if (!priorRowsRead) { priorRowsRead = true; - priorRows = (await driver.find(object, ast, preOpts) as Record[]) ?? []; + // [#21613] The matched rows' pre-images, shaped like the + // by-id prior read: each per-row `afterUpdate` composes its + // `result` from one of them, so both sides stay declared. + priorRows = rowsWithDeclaredColumnsOnly( + (await driver.find(object, ast, preOpts) as Record[]) ?? [], + declaredColumnSet(updateSchema), + ); } return priorRows; }; @@ -15082,6 +15116,15 @@ export class ObjectQL implements IObjectQLEngine { } catch (driverError) { throw envelopeUniqueViolation(driverError, object); } + // [#21613] The by-id update returns the driver's post-write + // readback (`select *` on driver-sql), so a column no metadata + // declares rode back in the PATCH 200 `record` with its stored + // value. Same default projection as the read verbs and the + // insert result, applied on the row as the driver returned it: + // before the formulas, the `afterUpdate` hooks, the roll-up, the + // `data.record.updated` event and the door's ingress strip. A + // `null` readback (the row left the caller's scope) stays `null`. + result = withDeclaredColumnsOnly(result, declaredColumnSet(updateSchema)); } else { // [#6262] A bulk SET clause must not carry `id`. Reaching this // branch AT ALL means `resolveEngineUpdateDispatch` returned @@ -16735,10 +16778,18 @@ export class ObjectQL implements IObjectQLEngine { // tenant scope onto a raw driver read. Skipping it here would read // outside this write's transaction and across the tenant boundary — // `update()`'s prior read passes the same bag for the same reason. + // [#21613] Both pre-image reads below are shaped to the declared field + // set, as `update()`'s are: the pre-image is the delete's `previous`, and + // the audit ledger records it whole as the delete's `old_value`, so a + // column no metadata declares reached that served row with its value. + const declaredDeleteColumns = declaredColumnSet(deleteSchema); const readPreImage = async (targetId: unknown): Promise | null> => { const preAst: QueryAST = { object, where: { id: targetId }, limit: 1 }; const preOpts = this.buildDriverOptions(object, opCtx.context, hookContext.input.options as any); - return (await driver.findOne(object, preAst, preOpts)) as Record | null; + return withDeclaredColumnsOnly( + (await driver.findOne(object, preAst, preOpts)) as Record | null, + declaredDeleteColumns, + ); }; const bindPreImage = (row: Record | null): void => { // Never fabricate: a row that is not there leaves `previous` UNBOUND @@ -16878,7 +16929,11 @@ export class ObjectQL implements IObjectQLEngine { const perRowAfterHooks = this.hasHooksFor('afterDelete', object); if (perRowBeforeHooks || perRowAfterHooks) { const preOpts = this.buildDriverOptions(object, opCtx.context, hookContext.input.options as any); - const doomed = (await driver.find(object, ast, preOpts) as Record[]) ?? []; + // [#21613] Each doomed row is its per-row `previous`: declared set. + const doomed = rowsWithDeclaredColumnsOnly( + (await driver.find(object, ast, preOpts) as Record[]) ?? [], + declaredDeleteColumns, + ); // [D6] One ceiling, both phases, BEFORE the first per-row dispatch // and before the driver call. this.assertBulkPerRowHookBudget( From f082789802bf97951dd4a5dad2970eed11d03222 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:31:26 +0000 Subject: [PATCH 4/8] test(objectql): fixtures declare the fields their readonlyWhen and validation rules read Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- packages/objectql/src/engine.test.ts | 12 ++++++++++-- packages/objectql/src/plugin.integration.test.ts | 3 +++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/objectql/src/engine.test.ts b/packages/objectql/src/engine.test.ts index 46f1a3397fb..983c340b9df 100644 --- a/packages/objectql/src/engine.test.ts +++ b/packages/objectql/src/engine.test.ts @@ -1406,6 +1406,10 @@ describe('ObjectQL Engine', () => { name: 'invoice', fields: { amount: { type: 'number', readonlyWhen: 'record.locked == true' }, + // [#21613] Declared: the rules below read them, and a + // write's prior read now carries the declared fields only. + locked: { type: 'boolean' }, + limit: { type: 'number' }, }, validations: [{ type: 'cross_field', name: 'amount_cap', message: 'amount exceeds limit', condition: 'record.amount > record.limit', fields: ['amount'] }], } as any); @@ -1466,7 +1470,9 @@ describe('ObjectQL Engine', () => { it('reports a readonlyWhen-locked field on a single-id update (reason readonly_when)', async () => { vi.mocked(SchemaRegistry.getObject).mockReturnValue({ name: 'invoice', - fields: { amount: { type: 'number', readonlyWhen: 'record.locked == true' } }, + // [#21613] `locked` is declared: the prior read the rule + // reads carries the declared fields only. + fields: { amount: { type: 'number', readonlyWhen: 'record.locked == true' }, locked: { type: 'boolean' } }, } as any); vi.mocked(mockDriver.findOne).mockResolvedValue({ id: '1', locked: true, amount: 100 } as any); @@ -1483,7 +1489,9 @@ describe('ObjectQL Engine', () => { it('reports bulk-path strips too (multi update — locked in ≥1 matched row)', async () => { vi.mocked(SchemaRegistry.getObject).mockReturnValue({ name: 'invoice', - fields: { amount: { type: 'number', readonlyWhen: 'record.locked == true' } }, + // [#21613] `locked` is declared: the prior read the rule + // reads carries the declared fields only. + fields: { amount: { type: 'number', readonlyWhen: 'record.locked == true' }, locked: { type: 'boolean' } }, } as any); vi.mocked(mockDriver.find).mockResolvedValue([ { id: 'a', locked: false, amount: 10 }, diff --git a/packages/objectql/src/plugin.integration.test.ts b/packages/objectql/src/plugin.integration.test.ts index 694764425d9..3edc3524bad 100644 --- a/packages/objectql/src/plugin.integration.test.ts +++ b/packages/objectql/src/plugin.integration.test.ts @@ -1725,6 +1725,9 @@ describe('ObjectQLPlugin - Metadata Service Integration', () => { fields: { status: { name: 'status', label: 'Status', type: 'text' }, amount: { name: 'amount', label: 'Amount', type: 'number', readonlyWhen: "record.status == 'paid'" } as any, + // [#21613] Declared: the rule reads it off the prior read, which + // carries the declared fields only. + limit: { name: 'limit', label: 'Limit', type: 'number' }, }, validations: [{ type: 'cross_field', name: 'amount_cap', message: 'amount exceeds limit', condition: 'record.amount > record.limit', fields: ['amount'] }] as any, }, From dfbb80ec2e00fe06552532c3ec571862cdd5cc83 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 18:59:52 +0000 Subject: [PATCH 5/8] test: revert the two temporary measurement probes (9c73f3702b, b923112e06) The probes recorded which write doors, data events and hook contexts carried a column no metadata declares, before and after the engine shaping; their readings are in the PR body. The net diff carries none of them. Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- .../plugin-audit/src/zz-probe-21613.test.ts | 128 ------------- packages/rest/src/zz-probe-21613.test.ts | 175 ------------------ 2 files changed, 303 deletions(-) delete mode 100644 packages/plugins/plugin-audit/src/zz-probe-21613.test.ts delete mode 100644 packages/rest/src/zz-probe-21613.test.ts diff --git a/packages/plugins/plugin-audit/src/zz-probe-21613.test.ts b/packages/plugins/plugin-audit/src/zz-probe-21613.test.ts deleted file mode 100644 index b1b903497fa..00000000000 --- a/packages/plugins/plugin-audit/src/zz-probe-21613.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. - -/** - * [#21613] TEMPORARY measurement probe — reverted before the fix lands. - * - * Does the audit ledger serve the engine's `previous` (the write path's prior - * read) or `result`, and what happens to it when only the RESULT is shaped? - */ - -import { describe, it, expect } from 'vitest'; -import { writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { ObjectQL } from '@objectstack/objectql'; -import { installAuditWriters } from './audit-writers.js'; - -const f = (name: string, type: string, extra: Record = {}) => - ({ name, label: name, type, ...extra }) as any; - -const sysAuditLog = { - name: 'sys_audit_log', label: 'Audit Log', - fields: { - id: f('id', 'text', { primaryKey: true }), action: f('action', 'text'), - user_id: f('user_id', 'text'), object_name: f('object_name', 'text'), - record_id: f('record_id', 'text'), old_value: f('old_value', 'textarea'), - new_value: f('new_value', 'textarea'), tenant_id: f('tenant_id', 'text'), - }, -}; -const sysActivity = { - name: 'sys_activity', label: 'Activity', - fields: { - id: f('id', 'text', { primaryKey: true }), type: f('type', 'text'), - timestamp: f('timestamp', 'datetime'), summary: f('summary', 'text'), - actor_id: f('actor_id', 'text'), object_name: f('object_name', 'text'), - record_id: f('record_id', 'text'), record_label: f('record_label', 'text'), - metadata: f('metadata', 'textarea'), - }, -}; -const contact = { - name: 'rq_contact', label: 'Contact', - fields: { name: f('name', 'text'), email: f('email', 'text') }, -}; -const RETIRED = ['mailing_street', 'mailing_city']; - -function makeDriver(shapeWriteResult: boolean) { - const stores = new Map>>(); - const storeFor = (o: string) => { let s = stores.get(o); if (!s) { s = new Map(); stores.set(o, s); } return s; }; - const copy = (r: T): T => (r == null ? r : JSON.parse(JSON.stringify(r))); - const strip = (r: Record) => { - if (!shapeWriteResult) return r; - const out = { ...r }; - for (const k of RETIRED) delete out[k]; - return out; - }; - const matches = (row: Record, where: any): boolean => { - if (!where || typeof where !== 'object') return true; - for (const [k, v] of Object.entries(where)) { - if (k.startsWith('$')) continue; - if (v && typeof v === 'object' && '$in' in v) { if (!(v.$in as unknown[]).includes(row[k])) return false; continue; } - if ((row[k] ?? null) !== (v ?? null)) return false; - } - return true; - }; - let n = 0; - const driver: any = { - name: 'memory', version: '0.0.0', supports: {} as any, - async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, - async find(o: string, ast: any) { return Array.from(storeFor(o).values()).filter((r) => matches(r, ast?.where)).map(copy); }, - async findOne(o: string, ast: any) { for (const r of storeFor(o).values()) if (matches(r, ast?.where)) return copy(r); return null; }, - async create(o: string, data: Record) { - n += 1; const id = (data.id as string) ?? `r_${n}`; - // A SQL table's `returning('*')`: every column, the retired ones null on a new row. - const row: Record = { ...data, id }; - if (o === 'rq_contact') for (const k of RETIRED) if (!(k in row)) row[k] = null; - storeFor(o).set(id, row); return strip(copy(row)); - }, - async update(o: string, id: string, data: Record) { - const s = storeFor(o); const cur = s.get(id); if (!cur) return null; - const next = { ...cur, ...data, id }; s.set(id, next); return strip(copy(next)); - }, - async delete(o: string, id: string) { return storeFor(o).delete(id); }, - async count(o: string, ast: any) { return (await this.find(o, ast)).length; }, - async bulkCreate(o: string, rows: Record[]) { return Promise.all(rows.map((r) => this.create(o, r))); }, - async updateMany(o: string, ast: any, data: Record) { - const rows = await this.find(o, ast); const s = storeFor(o); - for (const r of rows) s.set(r.id as string, { ...s.get(r.id as string), ...data, id: r.id }); - return rows.length; - }, - async deleteMany(o: string, ast: any) { const rows = await this.find(o, ast); for (const r of rows) storeFor(o).delete(r.id as string); return rows.length; }, - }; - return { driver, storeFor }; -} - -async function boot(shapeWriteResult: boolean) { - const engine = new ObjectQL(); - const d = makeDriver(shapeWriteResult); - engine.registerDriver(d.driver, true); - await engine.init(); - for (const o of [sysAuditLog, sysActivity, contact]) engine.registry.registerObject(o as any, 'probe'); - installAuditWriters(engine as any); - for (const id of ['c1', 'c2', 'c3', 'c4']) { - d.storeFor('rq_contact').set(id, { id, name: id, email: id === 'c3' || id === 'c4' ? 'bulk@x' : `${id}@x`, mailing_street: `${id} Retired Way`, mailing_city: 'Oldtown' }); - } - return { engine, ...d }; -} - -const ledger = (storeFor: any) => Array.from(storeFor('sys_audit_log').values()).map((r: any) => ({ - action: r.action, record: r.record_id, - old: r.old_value ? RETIRED.filter((k) => k in JSON.parse(r.old_value)).map((k) => `${k}=${JSON.stringify(JSON.parse(r.old_value)[k])}`) : null, - new: r.new_value ? RETIRED.filter((k) => k in JSON.parse(r.new_value)).map((k) => `${k}=${JSON.stringify(JSON.parse(r.new_value)[k])}`) : null, -})); - -describe('[#21613] audit probe', () => { - it('records what the ledger holds, today and with only the result shaped', async () => { - const out: Record = {}; - for (const shaped of [false, true]) { - const { engine, storeFor } = await boot(shaped); - await engine.update('rq_contact', { name: 'c1 renamed' }, { where: { id: 'c1' } } as any); - await engine.delete('rq_contact', { where: { id: 'c2' } } as any); - await engine.insert('rq_contact', { id: 'n1', name: 'new' }); - await engine.update('rq_contact', { name: 'bulk renamed' } as any, { where: { email: 'bulk@x' }, multi: true } as any); - await new Promise((r) => setTimeout(r, 20)); - out[shaped ? 'result shaped, previous raw' : 'today (driver rows whole)'] = ledger(storeFor); - } - writeFileSync(process.env.OS_PROBE_OUT ?? join(tmpdir(), 'probe-21613-audit.json'), JSON.stringify(out, null, 2)); - expect(true).toBe(true); - }); -}); diff --git a/packages/rest/src/zz-probe-21613.test.ts b/packages/rest/src/zz-probe-21613.test.ts deleted file mode 100644 index 3962cdc92e4..00000000000 --- a/packages/rest/src/zz-probe-21613.test.ts +++ /dev/null @@ -1,175 +0,0 @@ -// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. - -/** - * [#21613] TEMPORARY measurement probe — reverted before the fix lands. - * - * Records, on the composed REST harness (RestServer, then the protocol, then - * ObjectQL, then a real SqlDriver on better-sqlite3), which write doors serve a - * column no metadata declares, and which in-process carriers of a write result - * (data events, hook `previous` / `result`) carry one. - */ - -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { ObjectQL } from '@objectstack/objectql'; -import { SqlDriver } from '@objectstack/driver-sql'; -import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; -import { RestServer } from './rest-server'; - -const OBJECT = 'rq_contact'; -const RETIRED = ['mailing_street', 'mailing_city']; - -const CONTACT_V1 = { - name: OBJECT, - label: 'Contact', - fields: { - name: { name: 'name', type: 'text' as const }, - email: { name: 'email', type: 'text' as const }, - mailing_street: { name: 'mailing_street', type: 'text' as const }, - mailing_city: { name: 'mailing_city', type: 'text' as const }, - }, -}; -const CONTACT_V2 = { - name: OBJECT, - label: 'Contact', - enable: { apiMethods: ['get', 'list', 'create', 'update', 'delete', 'bulk', 'upsert'] }, - fields: { - name: { name: 'name', type: 'text' as const }, - email: { name: 'email', type: 'text' as const }, - }, -}; - -function createMockServer() { - const noop = () => {}; - return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; -} -function makeRes() { - const res: any = { - write: () => true, end: () => {}, - header: () => res, - status: (code: number) => { res._status = code; return res; }, - json: (body: any) => { res._json = body; return res; }, - }; - return res; -} - -const dir = mkdtempSync(join(tmpdir(), 'os-21613-probe-')); -const filename = join(dir, 'retired-columns.sqlite'); -const engines: ObjectQL[] = []; -afterAll(async () => { - for (const e of engines) { try { await e.destroy(); } catch { /* noop */ } } - rmSync(dir, { recursive: true, force: true }); -}); -const sqlDriver = () => new SqlDriver({ client: 'better-sqlite3', connection: { filename }, useNullAsDefault: true }) as any; - -const notes: Record = {}; -const retiredIn = (v: any): string[] => { - if (!v || typeof v !== 'object') return []; - return RETIRED.filter((k) => k in v).map((k) => `${k}=${JSON.stringify(v[k])}`); -}; - -describe('[#21613] probe', () => { - let rest: RestServer; - let engine: ObjectQL; - const events: any[] = []; - const hooks: Array<{ event: string; previous: string[]; result: string[] }> = []; - - beforeAll(async () => { - const one = new ObjectQL(); - one.registerDriver(sqlDriver(), true); - await one.init(); - one.registry.registerObject(CONTACT_V1 as any); - await one.syncSchemas(); - await one.insert(OBJECT, [ - { id: 'c1', name: 'Ada', email: 'ada@example.com', mailing_street: '1 Retired Way', mailing_city: 'Oldtown' }, - { id: 'c2', name: 'Bo', email: 'bo@example.com', mailing_street: '2 Retired Way', mailing_city: 'Oldtown' }, - { id: 'c3', name: 'Cy', email: 'cy@example.com', mailing_street: '3 Retired Way', mailing_city: 'Oldtown' }, - { id: 'c4', name: 'Di', email: 'di@example.com', mailing_street: '4 Retired Way', mailing_city: 'Oldtown' }, - { id: 'c5', name: 'Ed', email: 'bulk@example.com', mailing_street: '5 Retired Way', mailing_city: 'Oldtown' }, - { id: 'c6', name: 'Fi', email: 'bulk@example.com', mailing_street: '6 Retired Way', mailing_city: 'Oldtown' }, - { id: 'c7', name: 'Gu', email: 'gone@example.com', mailing_street: '7 Retired Way', mailing_city: 'Oldtown' }, - ] as any); - await one.destroy(); - - engine = new ObjectQL(); - engines.push(engine); - engine.registerDriver(sqlDriver(), true); - await engine.init(); - engine.registry.registerObject(CONTACT_V2 as any); - await engine.syncSchemas(); - engine.setRealtimeService({ - publish: async (e: any) => { events.push(e); }, - subscribe: async () => 'x', - unsubscribe: async () => {}, - } as any); - for (const ev of ['beforeUpdate', 'afterUpdate', 'beforeDelete', 'afterDelete', 'afterInsert']) { - engine.registerHook(ev, async (ctx: any) => { - hooks.push({ event: `${ev}:${ctx.input?.id ?? ''}`, previous: retiredIn(ctx.previous), result: retiredIn(ctx.result) }); - }, { object: OBJECT }); - } - const protocol = new ObjectStackProtocolImplementation(engine as any); - rest = new RestServer(createMockServer() as any, protocol as any, { api: { requireAuth: false } } as any); - (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' }); - rest.registerRoutes(); - }); - - const route = (method: string, path: string) => { - const r = rest.getRoutes().find((x: any) => x.method === method && x.path === path); - if (!r) throw new Error(`no route ${method} ${path}`); - return r; - }; - const call = async (method: string, path: string, req: any) => { - const res = makeRes(); - await route(method, path).handler({ query: {}, headers: {}, ...req }, res); - return res; - }; - - it('measures every write door', async () => { - const p = '/api/v1/data/:object'; - let res = await call('PATCH', `${p}/:id`, { params: { object: OBJECT, id: 'c1' }, body: { name: 'Ada 2' } }); - notes['PATCH /data/:object/:id'] = { status: res._status ?? 200, retired: retiredIn(res._json?.record) }; - - res = await call('POST', p, { params: { object: OBJECT }, body: { id: 'n1', name: 'New' } }); - notes['POST /data/:object (create)'] = { status: res._status ?? 200, retired: retiredIn(res._json?.record) }; - - res = await call('POST', `${p}/:id/clone`, { params: { object: OBJECT, id: 'c2' }, body: { name: 'Bo copy' } }); - notes['POST /data/:object/:id/clone'] = { status: res._status ?? 200, retired: retiredIn(res._json?.record), err: res._json?.error }; - - res = await call('POST', `${p}/createMany`, { params: { object: OBJECT }, body: [{ id: 'm1', name: 'M1' }, { id: 'm2', name: 'M2' }] }); - notes['POST /data/:object/createMany'] = { status: res._status ?? 200, retired: (res._json?.records ?? []).map(retiredIn), err: res._json?.error }; - - res = await call('POST', `${p}/batch`, { params: { object: OBJECT }, body: { operation: 'update', records: [{ id: 'c3', data: { name: 'Cy 2' } }] } }); - notes['POST /data/:object/batch update'] = { status: res._status ?? 200, retired: (res._json?.results ?? []).map((r: any) => retiredIn(r.data)), err: res._json?.error ?? res._json?.results?.[0]?.errors }; - - res = await call('POST', `${p}/batch`, { params: { object: OBJECT }, body: { operation: 'upsert', records: [{ id: 'c4', data: { name: 'Di 2' } }, { id: 'u1', data: { name: 'Up new' } }] } }); - notes['POST /data/:object/batch upsert'] = { status: res._status ?? 200, retired: (res._json?.results ?? []).map((r: any) => retiredIn(r.data)), err: res._json?.error ?? res._json?.results?.map((r: any) => r.errors) }; - - res = await call('POST', `${p}/batch`, { params: { object: OBJECT }, body: { operation: 'create', records: [{ data: { id: 'b1', name: 'B1' } }] } }); - notes['POST /data/:object/batch create'] = { status: res._status ?? 200, retired: (res._json?.results ?? []).map((r: any) => retiredIn(r.data)), err: res._json?.error ?? res._json?.results?.map((r: any) => r.errors) }; - - res = await call('POST', `${p}/updateMany`, { params: { object: OBJECT }, body: { records: [{ id: 'c1', data: { name: 'Ada 3' } }] } }); - notes['POST /data/:object/updateMany'] = { status: res._status ?? 200, retired: (res._json?.results ?? []).map((r: any) => retiredIn(r.data)), err: res._json?.error ?? res._json?.results?.map((r: any) => r.errors) }; - - res = await call('DELETE', `${p}/:id`, { params: { object: OBJECT, id: 'c7' } }); - notes['DELETE /data/:object/:id'] = { status: res._status ?? 200, body: res._json }; - - // In-process verbs. - const updated = await engine.update(OBJECT, { id: 'c2', name: 'Bo 2' }); - notes['engine.update by id'] = retiredIn(updated); - const inserted = await engine.insert(OBJECT, { id: 'n2', name: 'New 2' }); - notes['engine.insert single'] = retiredIn(inserted); - const outcomes: any[] = await (engine as any).insertMany(OBJECT, [{ id: 'n3', name: 'New 3' }]); - notes['engine.insertMany'] = outcomes.map((o) => retiredIn(o.record)); - const count = await engine.update(OBJECT, { name: 'bulk' } as any, { where: { email: 'bulk@example.com' }, multi: true } as any); - notes['engine.update multi (count)'] = count; - const del = await engine.delete(OBJECT, { where: { email: 'bulk@example.com' }, multi: true } as any); - notes['engine.delete multi (count)'] = del; - - notes.events = events.map((e) => ({ type: e.type, recordId: e.payload?.recordId, after: retiredIn(e.payload?.after), changes: retiredIn(e.payload?.changes) })); - notes.hooks = hooks; - writeFileSync(process.env.OS_PROBE_OUT ?? join(tmpdir(), 'probe-21613.json'), JSON.stringify(notes, null, 2)); - expect(true).toBe(true); - }); -}); From b2ce222a29f33687c642219670965390dba1cb4b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 19:04:38 +0000 Subject: [PATCH 6/8] test: pin the declared write result on every door, event, webhook and the audit ledger; changeset Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- .../21613-write-result-declared-fields.md | 39 +++ ...result-declared-fields-conformance.test.ts | 261 +++++++++++++++++ .../src/audit-ledger-declared-fields.test.ts | 163 +++++++++++ .../webhook-payload-declared-fields.test.ts | 154 ++++++++++ .../data-write-result-declared-fields.test.ts | 262 ++++++++++++++++++ 5 files changed, 879 insertions(+) create mode 100644 .changeset/21613-write-result-declared-fields.md create mode 100644 packages/objectql/src/write-result-declared-fields-conformance.test.ts create mode 100644 packages/plugins/plugin-audit/src/audit-ledger-declared-fields.test.ts create mode 100644 packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts create mode 100644 packages/rest/src/data-write-result-declared-fields.test.ts diff --git a/.changeset/21613-write-result-declared-fields.md b/.changeset/21613-write-result-declared-fields.md new file mode 100644 index 00000000000..7900c93c53d --- /dev/null +++ b/.changeset/21613-write-result-declared-fields.md @@ -0,0 +1,39 @@ +--- +'@objectstack/objectql': minor +--- + +fix(objectql)!: the record a write returns, and the prior read it binds as `previous`, serve the object's declared fields and the platform's system columns, never a column no metadata declares (#21613) + +**BREAKING (narrowing)** — what a released write door returns shrinks. A column that +no metadata declares, typically a field retired in an upgrade whose column additive +schema sync leaves in the table until `os migrate apply --allow-destructive`, is no +longer returned by any write through the engine. This completes the read-side rule of +the previous release (#21571) for writes. + +| write | before | now | +| --- | --- | --- | +| `PATCH /api/v1/data/:object/:id`, batch `update`, `updateMany` | the post-write row read back with `select *`: retired columns with their stored values | the declared fields, the registry's system columns, `id`, `created_at`, `updated_at` | +| `POST /api/v1/data/:object`, `POST /api/v1/data/:object/:id/clone`, `createMany`, batch `create` / `upsert`, `insertMany` outcomes | the inserted row from `returning('*')`: retired columns as `null` | the same declared set | +| `data.record.created` / `data.record.updated` events, and the webhook deliveries that carry them as `after` | the same whole row | the declared set | +| `engine.insert` / `engine.update` in process (actions, flows, plugins), and a hook's `ctx.result` | the whole row | the declared set | +| a hook's `ctx.previous` on update and delete (by id and per row), and the audit ledger's delete `old_value` and create `new_value` | the whole stored row, retired columns and their values included | the declared set | + +**Unchanged:** declared fields keep their treatment. An `internal: true` field is still +returned whole on the engine-level write result to the privileged writer that just +wrote it, and still stripped from every data-door response; formulas are still +hydrated onto the result; the registry-injected tenant, owner and audit columns are +still returned. No driver changed: the engine shapes the rows any driver returns, so +the answer is the same on every driver and every door. A retired column's values stay +in the table. + +**If you still read a retired column's values off a write** (for example a hook, a +flow or a webhook receiver that copied an old column into its replacement): run that +conversion BEFORE upgrading to this release, while the old field is still declared, or, +once it lands, read the unmapped columns through the operator-only `os migrate` read +(objectstack#21573). There is no flag that re-opens undeclared columns on a write. A +reader that needs a column must declare it as a field; a validation rule, a +`readonlyWhen` or a hook condition that reads a column must name a declared field too. + +Clause-②: no (narrowing) + + diff --git a/packages/objectql/src/write-result-declared-fields-conformance.test.ts b/packages/objectql/src/write-result-declared-fields-conformance.test.ts new file mode 100644 index 00000000000..857acb1cffc --- /dev/null +++ b/packages/objectql/src/write-result-declared-fields-conformance.test.ts @@ -0,0 +1,261 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21613] The rows a write verb hands back — and the prior-read rows it binds + * as `previous` — carry only the object's DECLARED fields plus the + * platform-provisioned columns, whatever the driver returned. The write-side + * half of #21571's default projection, decided in the same place: the engine, + * on the rows as the driver returned them, with `declared-read-columns.ts`. + * + * ## The driver shape this pins against + * + * A SQL table's write verbs answer with every physical column: driver-sql + * builds create's row from `returning('*')` and update's from a `select *` + * readback. A column no metadata declares — a field retired in an upgrade, + * whose column additive sync leaves in the table — therefore rode back on + * every write: with its stored value on an update, as `null` on a new row. + * The store driver below behaves exactly that way: `rq_contact`'s physical + * columns include `mailing_street` / `mailing_city`, which the current + * declaration retired. + * + * ## What each case measures + * + * Every write verb that returns a row, every carrier of a write result the + * engine produces (the hooks' `result` and `previous`, the data event), the + * protocol doors built on them, and the declared treatment that must not + * move: an `internal: true` field stays WHOLE on the engine-level result (the + * A-prime ruling — the door strips it, not the engine) and a formula is still + * hydrated onto the result. + */ + +import { describe, it, expect, beforeEach } from 'vitest'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; + +import { ObjectQL } from './engine.js'; + +type Row = Record; + +const CONTACT = 'rq_contact'; +const RETIRED = ['mailing_street', 'mailing_city'] as const; + +/** A SQL-shaped store: every write answers with the whole stored row. */ +function makeSqlShapedStore() { + const tables = new Map>(); + const tableFor = (o: string): Map => { + let t = tables.get(o); + if (!t) { t = new Map(); tables.set(o, t); } + return t; + }; + /** The table's physical columns: the retired pair is still among them. */ + const physical = (o: string): string[] => (o === CONTACT ? [...RETIRED] : []); + const matches = (row: Row, where: Row | undefined): boolean => { + if (!where) return true; + for (const [k, v] of Object.entries(where)) { + if (k.startsWith('$')) continue; + if (v !== null && typeof v === 'object' && '$in' in (v as Row)) { + if (!((v as { $in: unknown[] }).$in).includes(row[k])) return false; + continue; + } + if ((row[k] ?? null) !== (v ?? null)) return false; + } + return true; + }; + const run = (o: string, ast?: { where?: Row }): Row[] => + Array.from(tableFor(o).values()).filter((r) => matches(r, ast?.where)).map((r) => ({ ...r })); + let seq = 0; + const driver = { + name: 'store-sql-shaped', version: '0.0.0', supports: {}, + async connect(): Promise {}, + async disconnect(): Promise {}, + async checkHealth(): Promise { return true; }, + async execute(): Promise { return null; }, + async find(o: string, ast?: { where?: Row }): Promise { return run(o, ast); }, + async findOne(o: string, ast?: { where?: Row }): Promise { return run(o, ast)[0] ?? null; }, + async create(o: string, data: Row): Promise { + seq += 1; + // `returning('*')`: every physical column, an unwritten one null. + const row: Row = { ...Object.fromEntries(physical(o).map((c) => [c, null])), ...data, id: data.id ?? `new_${seq}` }; + tableFor(o).set(String(row.id), row); + return { ...row }; + }, + async bulkCreate(o: string, rows: Row[]): Promise { + const out: Row[] = []; + for (const r of rows) out.push(await driver.create(o, r)); + return out; + }, + async update(o: string, id: string, data: Row): Promise { + const cur = tableFor(o).get(id); + if (!cur) return null; + const next: Row = { ...cur, ...data, id }; + tableFor(o).set(id, next); + // The post-write readback: `select *`. + return { ...next }; + }, + async updateMany(o: string, ast: { where?: Row }, data: Row): Promise { + const rows = run(o, ast); + for (const r of rows) tableFor(o).set(String(r.id), { ...tableFor(o).get(String(r.id)), ...data }); + return rows.length; + }, + async delete(o: string, id: string): Promise { return tableFor(o).delete(id); }, + async deleteMany(o: string, ast: { where?: Row }): Promise { + const rows = run(o, ast); + for (const r of rows) tableFor(o).delete(String(r.id)); + return rows.length; + }, + async count(o: string, ast?: { where?: Row }): Promise { return run(o, ast).length; }, + }; + return { + driver, + seed: (o: string, row: Row) => { tableFor(o).set(String(row.id), { ...row }); }, + stored: (o: string, id: string) => tableFor(o).get(id), + }; +} + +async function boot() { + const engine = new ObjectQL(); + const store = makeSqlShapedStore(); + engine.registerDriver(store.driver as never, true); + await engine.init(); + engine.registry.registerObject({ + name: CONTACT, + label: 'Contact', + fields: { + name: { type: 'text' }, + email: { type: 'text' }, + score: { type: 'number' }, + token: { type: 'text', internal: true }, + double_score: { type: 'formula', expression: { dialect: 'cel', source: 'record.score * 2' } }, + }, + } as never, 'test'); + for (const [id, email] of [['con_1', 'ada@x'], ['con_2', 'bulk@x'], ['con_3', 'bulk@x'], ['con_4', 'gone@x']] as const) { + store.seed(CONTACT, { + id, name: id, email, score: 21, token: `hash_${id}`, + mailing_street: `${id} Retired Way`, mailing_city: 'Oldtown', + }); + } + const published: Array<{ type: string; payload: Row }> = []; + engine.setRealtimeService({ + publish: async (e: { type: string; payload: Row }) => { published.push(e); }, + subscribe: async () => 'sub', + unsubscribe: async () => undefined, + } as never); + const hooks: Array<{ event: string; id: unknown; previous: unknown; result: unknown }> = []; + for (const event of ['beforeUpdate', 'afterUpdate', 'beforeDelete', 'afterDelete', 'afterInsert']) { + engine.registerHook(event, async (ctx: any) => { + hooks.push({ event, id: ctx.input?.id, previous: ctx.previous, result: ctx.result }); + }, { object: CONTACT }); + } + return { engine, store, published, hooks, protocol: new ObjectStackProtocolImplementation(engine as never) }; +} + +function expectDeclaredOnly(row: unknown): void { + expect(row).toBeTruthy(); + expect(typeof row).toBe('object'); + for (const retired of RETIRED) expect(Object.keys(row as Row)).not.toContain(retired); +} + +describe('[#21613] a write\'s returned row serves the declared fields only', () => { + let h: Awaited>; + beforeEach(async () => { h = await boot(); }); + + it('the fixture is real: the driver answers a write with the retired columns', async () => { + expect(await h.store.driver.update(CONTACT, 'con_1', { name: 'raw' })).toMatchObject({ mailing_street: 'con_1 Retired Way' }); + expect(await h.store.driver.create(CONTACT, { id: 'raw_new', name: 'raw' })).toMatchObject({ mailing_street: null }); + }); + + it('verb: insert, one row', async () => { + const row = await h.engine.insert(CONTACT, { id: 'n1', name: 'New', score: 1 }); + expect(row).toMatchObject({ id: 'n1', name: 'New' }); + expectDeclaredOnly(row); + }); + + it('verb: insert, a batch', async () => { + const rows = await h.engine.insert(CONTACT, [{ id: 'n1', name: 'A' }, { id: 'n2', name: 'B' }]); + expect(rows.map((r: Row) => r.id)).toEqual(['n1', 'n2']); + for (const row of rows) expectDeclaredOnly(row); + }); + + it('verb: insertMany — each ok outcome\'s record', async () => { + const outcomes = await h.engine.insertMany(CONTACT, [{ id: 'n1', name: 'A' }]); + expect(outcomes[0]).toMatchObject({ ok: true, record: { id: 'n1', name: 'A' } }); + expectDeclaredOnly((outcomes[0] as { record: Row }).record); + }); + + it('verb: update by id — the readback', async () => { + const row = await h.engine.update(CONTACT, { id: 'con_1', name: 'Ada 2' }); + expect(row).toMatchObject({ id: 'con_1', name: 'Ada 2', email: 'ada@x' }); + expectDeclaredOnly(row); + }); + + it('hooks: update by id binds a declared `previous` and `result`, both phases', async () => { + await h.engine.update(CONTACT, { id: 'con_1', name: 'Ada 2' }); + const before = h.hooks.find((x) => x.event === 'beforeUpdate'); + const after = h.hooks.find((x) => x.event === 'afterUpdate'); + expect(before?.previous).toMatchObject({ id: 'con_1', name: 'con_1' }); + expectDeclaredOnly(before?.previous); + expect(after?.previous).toMatchObject({ id: 'con_1', name: 'con_1' }); + expectDeclaredOnly(after?.previous); + expect(after?.result).toMatchObject({ id: 'con_1', name: 'Ada 2' }); + expectDeclaredOnly(after?.result); + }); + + it('hooks: a predicate update\'s per-row `previous` and composed `result`', async () => { + const count = await h.engine.update(CONTACT, { name: 'bulk' } as never, { where: { email: 'bulk@x' }, multi: true } as never); + expect(count).toBe(2); + const perRow = h.hooks.filter((x) => x.event === 'afterUpdate'); + expect(perRow.map((x) => x.id).sort()).toEqual(['con_2', 'con_3']); + for (const x of perRow) { + expect(x.result).toMatchObject({ name: 'bulk' }); + expectDeclaredOnly(x.previous); + expectDeclaredOnly(x.result); + } + }); + + it('hooks: delete by id and by predicate bind a declared `previous`', async () => { + await h.engine.delete(CONTACT, { where: { id: 'con_4' } } as never); + await h.engine.delete(CONTACT, { where: { email: 'bulk@x' }, multi: true } as never); + const afterDelete = h.hooks.filter((x) => x.event === 'afterDelete'); + expect(afterDelete.map((x) => x.id).sort()).toEqual(['con_2', 'con_3', 'con_4']); + for (const x of afterDelete) { + expect(x.previous).toMatchObject({ email: expect.any(String) }); + expectDeclaredOnly(x.previous); + } + }); + + it('events: data.record.created and data.record.updated carry the declared record as `after`', async () => { + await h.engine.insert(CONTACT, { id: 'n1', name: 'New' }); + await h.engine.update(CONTACT, { id: 'con_1', name: 'Ada 2' }); + const created = h.published.find((e) => e.type === 'data.record.created'); + const updated = h.published.find((e) => e.type === 'data.record.updated'); + expect(created?.payload).toMatchObject({ recordId: 'n1', after: { id: 'n1', name: 'New' } }); + expectDeclaredOnly(created?.payload.after); + expect(updated?.payload).toMatchObject({ recordId: 'con_1', after: { id: 'con_1', name: 'Ada 2' } }); + expectDeclaredOnly(updated?.payload.after); + }); + + it('doors: createData, updateData and cloneData answer the declared record', async () => { + const created: any = await h.protocol.createData({ object: CONTACT, data: { id: 'n1', name: 'New' } }); + expect(created.record).toMatchObject({ id: 'n1', name: 'New' }); + expectDeclaredOnly(created.record); + const updated: any = await h.protocol.updateData({ object: CONTACT, id: 'con_1', data: { name: 'Ada 2' } }); + expect(updated.record).toMatchObject({ id: 'con_1', name: 'Ada 2' }); + expectDeclaredOnly(updated.record); + const cloned: any = await h.protocol.cloneData({ object: CONTACT, id: 'con_1', overrides: { name: 'Ada copy' } }); + expect(cloned.record).toMatchObject({ name: 'Ada copy' }); + expectDeclaredOnly(cloned.record); + }); + + it('declared fields keep their treatment: `internal` whole on the engine result, stripped at the door; formula hydrated', async () => { + const engineRow = await h.engine.update(CONTACT, { id: 'con_1', name: 'Ada 2' }); + // A-prime: the privileged writer that just wrote the row gets it whole. + expect(engineRow).toMatchObject({ token: 'hash_con_1', double_score: 42 }); + const doorRow: any = await h.protocol.updateData({ object: CONTACT, id: 'con_1', data: { name: 'Ada 3' } }); + expect(doorRow.record).toMatchObject({ id: 'con_1', name: 'Ada 3', double_score: 42 }); + expect(doorRow.record).not.toHaveProperty('token'); + }); + + it('the driver\'s stored row is never mutated by the shaping', async () => { + await h.engine.update(CONTACT, { id: 'con_1', name: 'Ada 2' }); + expect(h.store.stored(CONTACT, 'con_1')).toMatchObject({ name: 'Ada 2', mailing_street: 'con_1 Retired Way' }); + }); +}); diff --git a/packages/plugins/plugin-audit/src/audit-ledger-declared-fields.test.ts b/packages/plugins/plugin-audit/src/audit-ledger-declared-fields.test.ts new file mode 100644 index 00000000000..2cc85077f82 --- /dev/null +++ b/packages/plugins/plugin-audit/src/audit-ledger-declared-fields.test.ts @@ -0,0 +1,163 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21613] The audit ledger records the DECLARED record of a write — never a + * column no metadata declares — on every side it takes: a create's + * `new_value`, both halves of an update diff, and a delete's `old_value`. + * + * ## Why this ledger is the measured door for the engine's `previous` + * + * `sys_audit_log` is a served object: its `old_value` / `new_value` are read + * back through the data door. The writer records the engine's own rows — the + * write result (`ctx.result`) and the prior read bound as `ctx.previous` — so + * whatever the engine hands it is what an auditor reads. On a table carrying a + * column of a field an upgrade retired, measured before the engine shaped + * those rows: + * + * - a delete's `old_value` recorded the retired column WITH its stored value; + * - a create's `new_value` recorded it as `null`; + * - an update recorded nothing of it, only because both sides carried it. + * + * The third line is why the prior read is shaped with the write result: with + * only the result shaped, every update of such a row recorded the retired + * column as CHANGED — `'1 Retired Way' → null` — putting its stored value + * into the ledger on every write. The update case below goes red on exactly + * that. + * + * The store driver is SQL-shaped: every write answers with the whole stored + * row, as driver-sql's `returning('*')` and its `select *` readback do. + */ + +import { describe, it, expect } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import { installAuditWriters } from './audit-writers.js'; + +type Row = Record; + +const f = (name: string, type: string, extra: Record = {}) => + ({ name, label: name, type, ...extra }) as any; + +const sysAuditLog = { + name: 'sys_audit_log', label: 'Audit Log', + fields: { + id: f('id', 'text', { primaryKey: true }), action: f('action', 'text'), + user_id: f('user_id', 'text'), object_name: f('object_name', 'text'), + record_id: f('record_id', 'text'), old_value: f('old_value', 'textarea'), + new_value: f('new_value', 'textarea'), tenant_id: f('tenant_id', 'text'), + }, +}; + +const sysActivity = { + name: 'sys_activity', label: 'Activity', + fields: { + id: f('id', 'text', { primaryKey: true }), type: f('type', 'text'), + timestamp: f('timestamp', 'datetime'), summary: f('summary', 'text'), + actor_id: f('actor_id', 'text'), object_name: f('object_name', 'text'), + record_id: f('record_id', 'text'), record_label: f('record_label', 'text'), + metadata: f('metadata', 'textarea'), + }, +}; + +const CONTACT = 'rq_contact'; +const RETIRED = ['mailing_street', 'mailing_city'] as const; + +const contact = { + name: CONTACT, label: 'Contact', + fields: { name: f('name', 'text'), email: f('email', 'text') }, +}; + +function makeSqlShapedStore() { + const tables = new Map>(); + const tableFor = (o: string): Map => { + let t = tables.get(o); + if (!t) { t = new Map(); tables.set(o, t); } + return t; + }; + const copy = (r: T): T => (r == null ? r : JSON.parse(JSON.stringify(r))); + const run = (o: string, where?: Row): Row[] => + Array.from(tableFor(o).values()) + .filter((r) => Object.entries(where ?? {}).every(([k, v]) => k.startsWith('$') || (r[k] ?? null) === (v ?? null))) + .map(copy); + let seq = 0; + const driver: any = { + name: 'store-sql-shaped', version: '0.0.0', supports: {}, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, + async execute() { return null; }, + async find(o: string, ast?: { where?: Row }) { return run(o, ast?.where); }, + async findOne(o: string, ast?: { where?: Row }) { return run(o, ast?.where)[0] ?? null; }, + async create(o: string, data: Row) { + seq += 1; + const retired = o === CONTACT ? Object.fromEntries(RETIRED.map((c) => [c, null])) : {}; + const row: Row = { ...retired, ...data, id: data.id ?? `r_${seq}` }; + tableFor(o).set(String(row.id), row); + return copy(row); + }, + async update(o: string, id: string, data: Row) { + const cur = tableFor(o).get(id); + if (!cur) return null; + const next: Row = { ...cur, ...data, id }; + tableFor(o).set(id, next); + return copy(next); + }, + async delete(o: string, id: string) { return tableFor(o).delete(id); }, + async count(o: string, ast?: { where?: Row }) { return run(o, ast?.where).length; }, + }; + return { driver, tableFor }; +} + +async function boot() { + const engine = new ObjectQL(); + const store = makeSqlShapedStore(); + engine.registerDriver(store.driver, true); + await engine.init(); + for (const o of [sysAuditLog, sysActivity, contact]) { + engine.registry.registerObject(o as any, 'com.objectstack.test.audit-ledger-declared-fields'); + } + installAuditWriters(engine as any); + for (const id of ['c1', 'c2']) { + store.tableFor(CONTACT).set(id, { + id, name: id, email: `${id}@x`, mailing_street: `${id} Retired Way`, mailing_city: 'Oldtown', + }); + } + const ledger = (action: string, recordId: string) => { + const row = Array.from(store.tableFor('sys_audit_log').values()) + .find((r) => r.object_name === CONTACT && r.action === action && r.record_id === recordId); + expect(row, `${action} ${recordId} audit row`).toBeDefined(); + return { + old: row!.old_value == null ? null : JSON.parse(String(row!.old_value)) as Row, + next: row!.new_value == null ? null : JSON.parse(String(row!.new_value)) as Row, + }; + }; + return { engine, ledger }; +} + +function expectDeclaredOnly(row: Row | null): void { + expect(row).toBeTruthy(); + for (const retired of RETIRED) expect(Object.keys(row!)).not.toContain(retired); +} + +describe('[#21613] the audit ledger records the declared record of a write', () => { + it('update: the diff records the real change and no retired column', async () => { + const { engine, ledger } = await boot(); + await engine.update(CONTACT, { name: 'c1 renamed' }, { where: { id: 'c1' } } as any); + const { old, next } = ledger('update', 'c1'); + expect(old).toEqual({ name: 'c1' }); + expect(next).toEqual({ name: 'c1 renamed' }); + }); + + it('delete: `old_value` is the declared pre-image', async () => { + const { engine, ledger } = await boot(); + await engine.delete(CONTACT, { where: { id: 'c2' } } as any); + const { old } = ledger('delete', 'c2'); + expect(old).toMatchObject({ id: 'c2', name: 'c2', email: 'c2@x' }); + expectDeclaredOnly(old); + }); + + it('create: `new_value` is the declared record', async () => { + const { engine, ledger } = await boot(); + await engine.insert(CONTACT, { id: 'n1', name: 'new' }); + const { next } = ledger('create', 'n1'); + expect(next).toMatchObject({ id: 'n1', name: 'new' }); + expectDeclaredOnly(next); + }); +}); diff --git a/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts b/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts new file mode 100644 index 00000000000..29b5cd1409d --- /dev/null +++ b/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts @@ -0,0 +1,154 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21613] A webhook delivers the declared record — never a column no + * metadata declares — because the engine shapes a write's returned row before + * the `data.record.*` event that carries it is published. + * + * The chain this drives is the real one on both sides of the hand-off: a real + * `ObjectQL` engine writes through a SQL-shaped store driver (every write + * answers with the whole stored row, as driver-sql's `returning('*')` and its + * `select *` readback do), publishes its event to the realtime service, and + * the real {@link AutoEnqueuer} copies that event into the outbox payload a + * receiver gets. `rq_contact`'s table still carries two columns of fields an + * upgrade retired, `mailing_street` and `mailing_city`. + * + * The enqueuer copies the event's `after` verbatim (`payload: { ...payload }`), + * so no edit here is involved: the engine's shaping is what this pins. + */ + +import { describe, expect, it } from 'vitest'; +import { ObjectQL } from '@objectstack/objectql'; +import type { + IRealtimeService, + RealtimeEventHandler, + RealtimeEventPayload, +} from '@objectstack/spec/contracts'; +import { AutoEnqueuer } from './auto-enqueuer.js'; + +type Row = Record; + +const CONTACT = 'rq_contact'; +const RETIRED = ['mailing_street', 'mailing_city'] as const; + +class FakeRealtime implements IRealtimeService { + private subs = new Map(); + private n = 0; + async publish(event: RealtimeEventPayload): Promise { + for (const sub of this.subs.values()) { + if (sub.opts?.object && event.object !== sub.opts.object) continue; + await sub.handler(event); + } + } + async subscribe(_channel: string, handler: any, opts?: any): Promise { + const id = `s-${++this.n}`; + this.subs.set(id, { handler, opts }); + return id; + } + async unsubscribe(id: string): Promise { + this.subs.delete(id); + } +} + +/** A SQL-shaped store: every write answers with the whole stored row. */ +function makeSqlShapedStore() { + const tables = new Map>(); + const tableFor = (o: string): Map => { + let t = tables.get(o); + if (!t) { t = new Map(); tables.set(o, t); } + return t; + }; + const run = (o: string, where?: Row): Row[] => + Array.from(tableFor(o).values()) + .filter((r) => Object.entries(where ?? {}).every(([k, v]) => k.startsWith('$') || (r[k] ?? null) === (v ?? null))) + .map((r) => ({ ...r })); + let seq = 0; + const driver = { + name: 'store-sql-shaped', version: '0.0.0', supports: {}, + async connect(): Promise {}, + async disconnect(): Promise {}, + async checkHealth(): Promise { return true; }, + async execute(): Promise { return null; }, + async find(o: string, ast?: { where?: Row }): Promise { return run(o, ast?.where); }, + async findOne(o: string, ast?: { where?: Row }): Promise { return run(o, ast?.where)[0] ?? null; }, + async create(o: string, data: Row): Promise { + seq += 1; + const retired = o === CONTACT ? Object.fromEntries(RETIRED.map((c) => [c, null])) : {}; + const row: Row = { ...retired, ...data, id: data.id ?? `new_${seq}` }; + tableFor(o).set(String(row.id), row); + return { ...row }; + }, + async update(o: string, id: string, data: Row): Promise { + const cur = tableFor(o).get(id); + if (!cur) return null; + const next: Row = { ...cur, ...data, id }; + tableFor(o).set(id, next); + return { ...next }; + }, + async delete(o: string, id: string): Promise { return tableFor(o).delete(id); }, + async count(o: string, ast?: { where?: Row }): Promise { return run(o, ast?.where).length; }, + }; + return { driver, seed: (o: string, row: Row) => { tableFor(o).set(String(row.id), { ...row }); } }; +} + +async function boot() { + const engine = new ObjectQL(); + const store = makeSqlShapedStore(); + engine.registerDriver(store.driver as never, true); + await engine.init(); + engine.registry.registerObject({ + name: CONTACT, + label: 'Contact', + fields: { name: { type: 'text' }, email: { type: 'text' } }, + } as never, 'test'); + // Just the columns the enqueuer's subscription cache reads. + engine.registry.registerObject({ + name: 'sys_webhook', + label: 'Webhook', + fields: { + name: { type: 'text' }, active: { type: 'boolean' }, object_name: { type: 'text' }, + triggers: { type: 'text' }, url: { type: 'text' }, method: { type: 'text' }, + }, + } as never, 'test'); + store.seed('sys_webhook', { + id: 'wh_1', name: 'crm_hook', active: true, object_name: CONTACT, + triggers: 'create,update', url: 'https://receiver.example/hook', method: 'POST', + }); + store.seed(CONTACT, { + id: 'c1', name: 'Ada', email: 'ada@example.com', + mailing_street: '1 Retired Way', mailing_city: 'Oldtown', + }); + + const realtime = new FakeRealtime(); + engine.setRealtimeService(realtime); + const delivered: Array<{ label?: string; payload: Row }> = []; + const enqueuer = new AutoEnqueuer(engine as never, realtime, async (input: any) => { + delivered.push({ label: input.label, payload: input.payload }); + }); + await enqueuer.start(); + return { engine, enqueuer, delivered }; +} + +function expectDeclaredOnly(row: unknown): void { + expect(row).toBeTruthy(); + for (const retired of RETIRED) expect(Object.keys(row as Row)).not.toContain(retired); +} + +describe('[#21613] a webhook delivers the declared record of a write', () => { + it('data.record.created and data.record.updated payloads carry no retired column', async () => { + const { engine, enqueuer, delivered } = await boot(); + try { + await engine.insert(CONTACT, { id: 'n1', name: 'New' }); + await engine.update(CONTACT, { id: 'c1', name: 'Ada 2' }); + } finally { + await enqueuer.stop(); + } + + const created = delivered.find((d) => d.label === 'data.record.created'); + const updated = delivered.find((d) => d.label === 'data.record.updated'); + expect(created?.payload).toMatchObject({ recordId: 'n1', action: 'created', after: { id: 'n1', name: 'New' } }); + expectDeclaredOnly(created?.payload.after); + expect(updated?.payload).toMatchObject({ recordId: 'c1', action: 'updated', after: { id: 'c1', name: 'Ada 2', email: 'ada@example.com' } }); + expectDeclaredOnly(updated?.payload.after); + }); +}); diff --git a/packages/rest/src/data-write-result-declared-fields.test.ts b/packages/rest/src/data-write-result-declared-fields.test.ts new file mode 100644 index 00000000000..cd17c750d15 --- /dev/null +++ b/packages/rest/src/data-write-result-declared-fields.test.ts @@ -0,0 +1,262 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21613] A write's response serves the object's DECLARED fields plus the + * platform's own system columns — never a column no metadata declares. The + * write-side half of #21571's read rule, decided in the same place: the + * engine shapes the rows its write verbs return, and the rows it reads before + * a write, with `declared-read-columns.ts`. + * + * The door the card measured, on the composed REST harness #21571's reach pin + * uses (`RestServer` → `ObjectStackProtocolImplementation` → `ObjectQL` → a + * real `SqlDriver` on better-sqlite3): + * + * 1. Boot ONE declares `rq_contact` with two mailing fields, syncs, and writes + * rows carrying values in them. + * 2. Boot TWO declares the same object with those two fields retired — the + * upgrade. Additive sync leaves both columns and their values in place. + * 3. Every write door that answers with a record: PATCH, POST, clone, and the + * bulk faces (createMany, batch, updateMany). + * + * Before the engine shaped its write results, step 3 answered PATCH with + * `record.mailing_street = '1 Retired Way'` (driver-sql's `select *` + * readback), and every create face with both retired columns as `null` + * (`returning('*')`). The `data.record.*` events, which the webhook outbox + * delivers verbatim, carried the same row as `after`; and the prior read bound + * as a hook's `previous` carried the stored values — the row the audit ledger + * records as a delete's `old_value`. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { ObjectQL } from '@objectstack/objectql'; +import { SqlDriver } from '@objectstack/driver-sql'; +import { ObjectStackProtocolImplementation } from '@objectstack/metadata-protocol'; +import { RestServer } from './rest-server'; + +const OBJECT = 'rq_contact'; +const RETIRED = ['mailing_street', 'mailing_city'] as const; + +const DECLARED = { + name: { name: 'name', type: 'text' as const }, + email: { name: 'email', type: 'text' as const }, + // Declared `internal`: the A-prime ruling keeps it on the engine-level + // result and strips it at the data door. Not this card's field class. + api_token: { name: 'api_token', type: 'text' as const, internal: true }, +}; + +const CONTACT_V1 = { + name: OBJECT, + label: 'Contact', + fields: { + ...DECLARED, + mailing_street: { name: 'mailing_street', type: 'text' as const }, + mailing_city: { name: 'mailing_city', type: 'text' as const }, + }, +}; + +/** The upgrade: the two mailing fields are retired from the declaration. */ +const CONTACT_V2 = { + name: OBJECT, + label: 'Contact', + enable: { apiMethods: ['get', 'list', 'create', 'update', 'delete', 'bulk'] }, + fields: { ...DECLARED }, +}; + +function createMockServer() { + const noop = () => {}; + return { get: noop, post: noop, put: noop, delete: noop, patch: noop, use: noop, listen: async () => {}, close: async () => {} }; +} + +function makeRes() { + const res: any = { + write: () => true, end: () => {}, + header: () => res, + status: (code: number) => { res._status = code; return res; }, + json: (body: any) => { res._json = body; return res; }, + }; + return res; +} + +const dir = mkdtempSync(join(tmpdir(), 'os-21613-')); +const filename = join(dir, 'retired-columns.sqlite'); +const engines: ObjectQL[] = []; +afterAll(async () => { + for (const e of engines) { + try { await e.destroy(); } catch { /* noop */ } + } + rmSync(dir, { recursive: true, force: true }); +}); + +const sqlDriver = () => + new SqlDriver({ client: 'better-sqlite3', connection: { filename }, useNullAsDefault: true }) as any; + +const SEEDED = ['c1', 'c2', 'c3', 'c4', 'c5'] as const; + +/** Boot one: the old declaration creates the table and writes the values. */ +async function bootOne(): Promise { + const engine = new ObjectQL(); + engine.registerDriver(sqlDriver(), true); + await engine.init(); + engine.registry.registerObject(CONTACT_V1 as any); + await engine.syncSchemas(); + await engine.insert(OBJECT, SEEDED.map((id) => ({ + id, name: id, email: `${id}@example.com`, api_token: `tok_${id}`, + mailing_street: `${id} Retired Way`, mailing_city: 'Oldtown', + })) as any); + await engine.destroy(); +} + +type Captured = { type: string; payload: Record }; +type HookSeen = { event: string; id: unknown; previous: unknown }; + +/** Boot two: the new declaration on the database boot one left behind. */ +async function bootTwo() { + const engine = new ObjectQL(); + engines.push(engine); + engine.registerDriver(sqlDriver(), true); + await engine.init(); + engine.registry.registerObject(CONTACT_V2 as any); + await engine.syncSchemas(); + + const events: Captured[] = []; + engine.setRealtimeService({ + publish: async (e: Captured) => { events.push(e); }, + subscribe: async () => 'sub', + unsubscribe: async () => undefined, + } as any); + const hooks: HookSeen[] = []; + for (const event of ['afterUpdate', 'afterDelete']) { + engine.registerHook(event, async (ctx: any) => { + hooks.push({ event, id: ctx.input?.id, previous: ctx.previous }); + }, { object: OBJECT }); + } + + const protocol = new ObjectStackProtocolImplementation(engine as any); + const rest = new RestServer(createMockServer() as any, protocol as any, { api: { requireAuth: false } } as any); + (rest as any).resolveExecCtx = async () => ({ userId: 'test-user' }); + rest.registerRoutes(); + const routes = rest.getRoutes(); + const call = async (method: string, path: string, req: Record) => { + const route = routes.find((r: any) => r.method === method && r.path === `/api/v1/data${path}`); + expect(route, `${method} /api/v1/data${path}`).toBeDefined(); + const res = makeRes(); + await route!.handler({ query: {}, headers: {}, ...req } as any, res); + return res; + }; + return { engine, events, hooks, call }; +} + +function expectDeclaredOnly(row: unknown): void { + expect(row).toBeTruthy(); + for (const retired of RETIRED) expect(Object.keys(row as object)).not.toContain(retired); +} + +describe('[#21613] a write\'s response serves the declared fields, never a retired column', () => { + let h: Awaited>; + beforeAll(async () => { + await bootOne(); + h = await bootTwo(); + }); + + it('the fixture is real: the table still stores the retired columns and their values', async () => { + // A raw driver read, below the engine: the upgrade left the data in place. + const driver = (h.engine as any).getDriver(OBJECT); + const stored = await driver.findOne(OBJECT, { object: OBJECT, where: { id: 'c5' }, limit: 1 }); + expect(stored).toMatchObject({ mailing_street: 'c5 Retired Way', mailing_city: 'Oldtown' }); + }); + + it('PATCH /data/:object/:id answers the record without the retired columns', async () => { + const res = await h.call('PATCH', '/:object/:id', { params: { object: OBJECT, id: 'c1' }, body: { name: 'Ada' } }); + expect(res._status ?? 200).toBe(200); + expect(res._json.record).toMatchObject({ id: 'c1', name: 'Ada', email: 'c1@example.com' }); + expectDeclaredOnly(res._json.record); + }); + + it('POST /data/:object answers 201 without them (a new row reads them as null)', async () => { + const res = await h.call('POST', '/:object', { params: { object: OBJECT }, body: { id: 'n1', name: 'New' } }); + expect(res._status).toBe(201); + expect(res._json.record).toMatchObject({ id: 'n1', name: 'New' }); + expectDeclaredOnly(res._json.record); + }); + + it('POST /data/:object/:id/clone answers 201 without them', async () => { + const res = await h.call('POST', '/:object/:id/clone', { params: { object: OBJECT, id: 'c2' }, body: { name: 'c2 copy' } }); + expect(res._status).toBe(201); + expect(res._json).toMatchObject({ sourceId: 'c2', record: { name: 'c2 copy', email: 'c2@example.com' } }); + expectDeclaredOnly(res._json.record); + }); + + it('the bulk faces: createMany, batch create / update / upsert, updateMany', async () => { + const createMany = await h.call('POST', '/:object/createMany', { params: { object: OBJECT }, body: [{ id: 'm1', name: 'M1' }, { id: 'm2', name: 'M2' }] }); + expect(createMany._status).toBe(201); + expect(createMany._json.records.map((r: any) => r.id)).toEqual(['m1', 'm2']); + for (const row of createMany._json.records) expectDeclaredOnly(row); + + const batch = await h.call('POST', '/:object/batch', { + params: { object: OBJECT }, + body: { operation: 'upsert', records: [{ id: 'c3', data: { name: 'c3 up' } }, { id: 'u1', data: { name: 'U1' } }] }, + }); + expect(batch._json.results.map((r: any) => [r.success, r.data?.name])).toEqual([[true, 'c3 up'], [true, 'U1']]); + for (const r of batch._json.results) expectDeclaredOnly(r.data); + + for (const operation of ['create', 'update'] as const) { + const records = operation === 'create' ? [{ data: { id: 'b1', name: 'B1' } }] : [{ id: 'c3', data: { name: 'c3 batch' } }]; + const res = await h.call('POST', '/:object/batch', { params: { object: OBJECT }, body: { operation, records } }); + expect(res._json.results[0]).toMatchObject({ success: true, data: { name: records[0].data.name } }); + expectDeclaredOnly(res._json.results[0].data); + } + + const updateMany = await h.call('POST', '/:object/updateMany', { params: { object: OBJECT }, body: { records: [{ id: 'c1', data: { name: 'Ada many' } }] } }); + expect(updateMany._json.results[0]).toMatchObject({ success: true, data: { id: 'c1', name: 'Ada many' } }); + expectDeclaredOnly(updateMany._json.results[0].data); + }); + + it('the data events those writes publish carry the declared record as `after`', async () => { + const created = h.events.filter((e) => e.type === 'data.record.created'); + const updated = h.events.filter((e) => e.type === 'data.record.updated'); + // n1, the clone, m1, m2, u1, b1 — and c3 (twice), c1 (twice): the writes above. + expect(created.length).toBeGreaterThanOrEqual(6); + expect(updated.length).toBeGreaterThanOrEqual(4); + for (const e of [...created, ...updated]) { + expect(e.payload.after).toMatchObject({ id: e.payload.recordId }); + expectDeclaredOnly(e.payload.after); + } + }); + + it('the prior read a write binds as `previous` is the declared record (update and delete)', async () => { + await h.call('PATCH', '/:object/:id', { params: { object: OBJECT, id: 'c4' }, body: { name: 'c4 edited' } }); + const del = await h.call('DELETE', '/:object/:id', { params: { object: OBJECT, id: 'c5' } }); + expect(del._status ?? 200).toBe(200); + const update = h.hooks.find((x) => x.event === 'afterUpdate' && x.id === 'c4'); + const remove = h.hooks.find((x) => x.event === 'afterDelete' && x.id === 'c5'); + expect(update?.previous).toMatchObject({ id: 'c4', name: 'c4' }); + expectDeclaredOnly(update?.previous); + expect(remove?.previous).toMatchObject({ id: 'c5', email: 'c5@example.com' }); + expectDeclaredOnly(remove?.previous); + }); + + it('declared fields and the system columns are served as before; `internal` stays the door\'s strip', async () => { + const res = await h.call('PATCH', '/:object/:id', { params: { object: OBJECT, id: 'c2' }, body: { name: 'c2 again' } }); + const record = res._json.record; + expect(record).toMatchObject({ id: 'c2', name: 'c2 again', email: 'c2@example.com' }); + // The registry-injected system columns, named one by one, and the + // platform-provisioned three. + for (const system of [ + 'id', 'created_at', 'updated_at', 'created_by', 'updated_by', + 'organization_id', 'owner_id', 'owning_business_unit_id', + ]) { + expect(record).toHaveProperty(system); + } + // Every key is a declared field or a platform-provisioned column. + const declared = new Set([...Object.keys(h.engine.registry.getObject(OBJECT)!.fields as object), 'id', 'created_at', 'updated_at']); + expect(Object.keys(record).filter((key) => !declared.has(key))).toEqual([]); + // `internal: true`: omitted by the data door, whole on the engine result. + expect(record).not.toHaveProperty('api_token'); + const engineRow = await h.engine.update(OBJECT, { id: 'c2', name: 'c2 engine' }); + expect(engineRow).toMatchObject({ id: 'c2', api_token: 'tok_c2' }); + expectDeclaredOnly(engineRow); + }); +}); From e7638365836d149207d3229ca49ed913a54ef4b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 19:43:42 +0000 Subject: [PATCH 7/8] test(plugin-webhooks): type the payload pin's enqueue double as HttpEnqueueFn Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- .../src/webhook-payload-declared-fields.test.ts | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts b/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts index 29b5cd1409d..219ab7aaf10 100644 --- a/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts +++ b/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts @@ -24,7 +24,7 @@ import type { RealtimeEventHandler, RealtimeEventPayload, } from '@objectstack/spec/contracts'; -import { AutoEnqueuer } from './auto-enqueuer.js'; +import { AutoEnqueuer, type HttpEnqueueFn } from './auto-enqueuer.js'; type Row = Record; @@ -122,9 +122,11 @@ async function boot() { const realtime = new FakeRealtime(); engine.setRealtimeService(realtime); const delivered: Array<{ label?: string; payload: Row }> = []; - const enqueuer = new AutoEnqueuer(engine as never, realtime, async (input: any) => { - delivered.push({ label: input.label, payload: input.payload }); - }); + const enqueue: HttpEnqueueFn = async (input) => { + delivered.push({ label: input.label, payload: input.payload as Row }); + return `delivery_${delivered.length}`; + }; + const enqueuer = new AutoEnqueuer(engine as never, realtime, enqueue); await enqueuer.start(); return { engine, enqueuer, delivered }; } From bfbeffed43bf59c7ce97b6049961e6068216aa3a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 3 Oct 2026 20:01:36 +0000 Subject: [PATCH 8/8] test: the pins' store doubles hold the caller's limit (check:objectql-double-limit) Claude-Session: https://claude.ai/code/session_017ErfyP2Rx7XWHJA27QjyUi Co-authored-by: Claude --- ...ite-result-declared-fields-conformance.test.ts | 12 ++++++++---- .../src/audit-ledger-declared-fields.test.ts | 15 +++++++++------ .../src/webhook-payload-declared-fields.test.ts | 15 +++++++++------ 3 files changed, 26 insertions(+), 16 deletions(-) diff --git a/packages/objectql/src/write-result-declared-fields-conformance.test.ts b/packages/objectql/src/write-result-declared-fields-conformance.test.ts index 857acb1cffc..a08381e51c1 100644 --- a/packages/objectql/src/write-result-declared-fields-conformance.test.ts +++ b/packages/objectql/src/write-result-declared-fields-conformance.test.ts @@ -60,8 +60,12 @@ function makeSqlShapedStore() { } return true; }; - const run = (o: string, ast?: { where?: Row }): Row[] => - Array.from(tableFor(o).values()).filter((r) => matches(r, ast?.where)).map((r) => ({ ...r })); + const run = (o: string, ast?: { where?: Row; limit?: number }): Row[] => { + const rows = Array.from(tableFor(o).values()).filter((r) => matches(r, ast?.where)); + // The caller's bound, after the filter, by presence. + const page = typeof ast?.limit === 'number' ? rows.slice(0, ast.limit) : rows; + return page.map((r) => ({ ...r })); + }; let seq = 0; const driver = { name: 'store-sql-shaped', version: '0.0.0', supports: {}, @@ -69,8 +73,8 @@ function makeSqlShapedStore() { async disconnect(): Promise {}, async checkHealth(): Promise { return true; }, async execute(): Promise { return null; }, - async find(o: string, ast?: { where?: Row }): Promise { return run(o, ast); }, - async findOne(o: string, ast?: { where?: Row }): Promise { return run(o, ast)[0] ?? null; }, + async find(o: string, ast?: { where?: Row; limit?: number }): Promise { return run(o, ast); }, + async findOne(o: string, ast?: { where?: Row; limit?: number }): Promise { return run(o, ast)[0] ?? null; }, async create(o: string, data: Row): Promise { seq += 1; // `returning('*')`: every physical column, an unwritten one null. diff --git a/packages/plugins/plugin-audit/src/audit-ledger-declared-fields.test.ts b/packages/plugins/plugin-audit/src/audit-ledger-declared-fields.test.ts index 2cc85077f82..30e5663b861 100644 --- a/packages/plugins/plugin-audit/src/audit-ledger-declared-fields.test.ts +++ b/packages/plugins/plugin-audit/src/audit-ledger-declared-fields.test.ts @@ -74,17 +74,20 @@ function makeSqlShapedStore() { return t; }; const copy = (r: T): T => (r == null ? r : JSON.parse(JSON.stringify(r))); - const run = (o: string, where?: Row): Row[] => - Array.from(tableFor(o).values()) - .filter((r) => Object.entries(where ?? {}).every(([k, v]) => k.startsWith('$') || (r[k] ?? null) === (v ?? null))) - .map(copy); + const run = (o: string, where?: Row, limit?: number): Row[] => { + const rows = Array.from(tableFor(o).values()) + .filter((r) => Object.entries(where ?? {}).every(([k, v]) => k.startsWith('$') || (r[k] ?? null) === (v ?? null))); + // The caller's bound, after the filter, by presence. + const page = typeof limit === 'number' ? rows.slice(0, limit) : rows; + return page.map(copy); + }; let seq = 0; const driver: any = { name: 'store-sql-shaped', version: '0.0.0', supports: {}, async connect() {}, async disconnect() {}, async checkHealth() { return true; }, async execute() { return null; }, - async find(o: string, ast?: { where?: Row }) { return run(o, ast?.where); }, - async findOne(o: string, ast?: { where?: Row }) { return run(o, ast?.where)[0] ?? null; }, + async find(o: string, ast?: { where?: Row; limit?: number }) { return run(o, ast?.where, ast?.limit); }, + async findOne(o: string, ast?: { where?: Row; limit?: number }) { return run(o, ast?.where, ast?.limit)[0] ?? null; }, async create(o: string, data: Row) { seq += 1; const retired = o === CONTACT ? Object.fromEntries(RETIRED.map((c) => [c, null])) : {}; diff --git a/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts b/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts index 219ab7aaf10..0deeb155754 100644 --- a/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts +++ b/packages/plugins/plugin-webhooks/src/webhook-payload-declared-fields.test.ts @@ -58,10 +58,13 @@ function makeSqlShapedStore() { if (!t) { t = new Map(); tables.set(o, t); } return t; }; - const run = (o: string, where?: Row): Row[] => - Array.from(tableFor(o).values()) - .filter((r) => Object.entries(where ?? {}).every(([k, v]) => k.startsWith('$') || (r[k] ?? null) === (v ?? null))) - .map((r) => ({ ...r })); + const run = (o: string, where?: Row, limit?: number): Row[] => { + const rows = Array.from(tableFor(o).values()) + .filter((r) => Object.entries(where ?? {}).every(([k, v]) => k.startsWith('$') || (r[k] ?? null) === (v ?? null))); + // The caller's bound, after the filter, by presence. + const page = typeof limit === 'number' ? rows.slice(0, limit) : rows; + return page.map((r) => ({ ...r })); + }; let seq = 0; const driver = { name: 'store-sql-shaped', version: '0.0.0', supports: {}, @@ -69,8 +72,8 @@ function makeSqlShapedStore() { async disconnect(): Promise {}, async checkHealth(): Promise { return true; }, async execute(): Promise { return null; }, - async find(o: string, ast?: { where?: Row }): Promise { return run(o, ast?.where); }, - async findOne(o: string, ast?: { where?: Row }): Promise { return run(o, ast?.where)[0] ?? null; }, + async find(o: string, ast?: { where?: Row; limit?: number }): Promise { return run(o, ast?.where, ast?.limit); }, + async findOne(o: string, ast?: { where?: Row; limit?: number }): Promise { return run(o, ast?.where, ast?.limit)[0] ?? null; }, async create(o: string, data: Row): Promise { seq += 1; const retired = o === CONTACT ? Object.fromEntries(RETIRED.map((c) => [c, null])) : {};