fix(objectql): droppedFields on a create names only keys the caller sent, never a middleware fill (#21682)

.changeset/21682-dropped-fields-platform-stamp.md

@@ -0,0 +1,22 @@
+---
+'@objectstack/objectql': patch
+---
+
+`droppedFields` on a create names only keys the caller sent, never a value a write middleware filled in
+
+Clause-②: no
+
+With `@objectstack/organizations` mounted (the walled tenancy postures), a non-system create that names no `organization_id` has that column filled with the caller's active organization by the organizations write middleware. `ObjectQL.insert` took its record of what the caller sent after that middleware had run. So the static `readonly` strip treated the fill as a caller write, took it, and reported it:
+
+- Before: `POST /api/v1/data/<object>` with a body that names no `organization_id` answered 201 with `droppedFields: [{ object, fields: ['organization_id'], reason: 'readonly' }]`. `onFieldsDropped` fired with the same event. The console showed it as a warning toast on every such create.
+- After: the same create answers 201 with no `droppedFields`, and `onFieldsDropped` does not fire. The row is stored in the active organization, as before.
+
+`insert` now records which keys each row carries before any write middleware runs. Only those keys count as sent by the caller. No field is exempted by name, so this covers every write middleware fill, including the `owner_id` fill of `@objectstack/plugin-security`. The referential-integrity check reads the same record, so it no longer checks a middleware-filled reference as if the caller had sent it.
+
+Unchanged:
+
+- A key the caller does send is judged and reported as before. That includes `organization_id` itself, and a key whose value a middleware rewrote.
+- The array insert (`createMany`) and the `single` posture already reported nothing for an absent `organization_id`, and they still do.
+- The stored row is the same. Before, the filled column was stripped and then filled again from the same active organization further down. Now the fill is kept.
+
+No export, type, error code or status changes.

packages/objectql/src/engine-insert-static-readonly-strip.test.ts

@@ -228,6 +228,53 @@ describe('#14147 — the exemptions, each one load-bearing', () => {
     expect(o.created?.completed_at, 'the hook wrote it — provenance, not value equality').toBe('2026-01-01T00:00:00Z');
   });
 
+  // [#21682] A write MIDDLEWARE runs before the caller snapshot is taken, so
+  // its fill used to read as a key the caller sent: stripped, and reported in
+  // `droppedFields` (`@objectstack/organizations`' `organization_id` fill, on
+  // every walled create naming no organization). The column here is an
+  // arbitrary author-declared one: what makes a key the platform's is WHEN it
+  // appeared, never its name.
+  const fillAbsent = (key: string, value: unknown) => (engine: ObjectQL) => {
+    engine.registerMiddleware(async (opCtx: any, next: () => Promise<void>) => {
+      if (opCtx.operation === 'insert') {
+        for (const row of Array.isArray(opCtx.data) ? opCtx.data : [opCtx.data]) {
+          if (row && !(key in row)) row[key] = value;
+        }
+      }
+      await next();
+    });
+  };
+
+  it('[#21682] a write middleware’s fill is not caller-supplied either: it lands, and nothing is reported', async () => {
+    const o = await observeInsert({ title: 'T' }, { context: { userId: 'u1' } }, fillAbsent('completed_at', '2026-01-01T00:00:00Z'));
+    expect(o.created?.completed_at, 'the platform’s value reaches the driver').toBe('2026-01-01T00:00:00Z');
+    expect(o.dropped, 'droppedFields names only keys the caller sent').toEqual([]);
+  });
+
+  it('[#21682] …per row on the batch path, beside a key the caller DID send, which is still taken and reported', async () => {
+    const { engine, creates } = await makeEngine();
+    fillAbsent('locked_note', 'stamp')(engine);
+    const dropped: DroppedFieldsEvent[] = [];
+    await engine.insert('duly_task', [
+      { title: 'A' },
+      { title: 'B', completed_at: 'forged' },
+    ] as any, { context: { userId: 'u1' }, onFieldsDropped: (e: DroppedFieldsEvent) => { dropped.push(e); } } as any);
+    expect(creates.map((c) => c.locked_note), 'the fill lands on every row').toEqual(['stamp', 'stamp']);
+    expect(creates[1]).not.toHaveProperty('completed_at');
+    expect(dropped).toEqual([{ object: 'duly_task', fields: ['completed_at'], reason: 'readonly' }]);
+  });
+
+  it('[#21682] a key the caller sent is judged as before even when a middleware rewrote its value', async () => {
+    const o = await observeInsert({ title: 'T', completed_at: 'forged' }, { context: { userId: 'u1' } }, (engine) => {
+      engine.registerMiddleware(async (opCtx: any, next: () => Promise<void>) => {
+        if (opCtx.operation === 'insert') opCtx.data.completed_at = 'rewritten';
+        await next();
+      });
+    });
+    expect(o.created, 'the caller named the key, so the strip still owns it').not.toHaveProperty('completed_at');
+    expect(o.dropped).toEqual([{ object: 'duly_task', fields: ['completed_at'], reason: 'readonly' }]);
+  });
+
   it('a PLATFORM-INTERNAL object is left to its own 403 guard (ADR-0086 / #3004, carried over)', async () => {
     // The reserved `sys_` namespace, and a `managedBy` bucket whose columns
     // carry their own fail-closed refusal, have dedicated write governance —

packages/objectql/src/engine.ts

@@ -2063,6 +2063,63 @@ function droppedFieldEvents(
   return events;
 }
 
+/**
+ * [#21682] The keys the CALLER sent on each row of an insert, recorded at
+ * `insert`'s entry, BEFORE the middleware chain runs, and keyed by the row
+ * OBJECT.
+ *
+ * `insert` takes its caller snapshot (`suppliedPerRow`) inside the middleware
+ * chain's innermost step, so by then a write middleware may already have
+ * filled the payload: `@objectstack/organizations` fills an absent
+ * `organization_id` with the active organization, and
+ * `@objectstack/plugin-security` fills an absent `owner_id` with the acting
+ * user. Both write IN PLACE, onto the very row objects recorded here. Without
+ * this record the snapshot reads those fills as keys the caller sent, so the
+ * static-`readonly` strip took the platform's own `organization_id` and
+ * `droppedFields` reported it, on every walled create that named no
+ * organization. The console announces every non-empty `droppedFields` as a
+ * warning toast. This is the insert-side twin of the update path's #8093
+ * (ADDRESSING IS NOT PAYLOAD): a value the platform put on the payload is
+ * not one the caller lost.
+ *
+ * Keyed by IDENTITY, not by index, so the answer survives a middleware that
+ * reorders a batch. A row a middleware REPLACED wholesale has no entry, and
+ * {@link callerSuppliedRow} then keeps every key it carries: the verdict from
+ * before this record existed. That is the over-reporting direction, never the
+ * under-stripping one.
+ *
+ * ⛔ No name list. Which keys are the platform's is answered by WHEN they
+ * appeared, so a new stamping middleware is covered without being named here.
+ */
+function callerKeySets(data: unknown): WeakMap<object, ReadonlySet<string>> {
+  const sets = new WeakMap<object, ReadonlySet<string>>();
+  for (const row of Array.isArray(data) ? data : [data]) {
+    if (row !== null && typeof row === 'object' && !sets.has(row)) {
+      sets.set(row, new Set(Object.keys(row)));
+    }
+  }
+  return sets;
+}
+
+/**
+ * One row of `insert`'s caller snapshot: a shallow COPY of the row as the
+ * middleware chain handed it on, keeping only the keys the caller sent
+ * (`sent`, from {@link callerKeySets}).
+ *
+ * Only WHICH keys is narrowed. A kept key keeps the value the chain handed
+ * on, so every key the caller did send is judged exactly as it was before
+ * #21682. `sent` undefined means "this row has no record" and keeps every key.
+ */
+function callerSuppliedRow(row: unknown, sent: ReadonlySet<string> | undefined): Record<string, unknown> {
+  const copy: Record<string, unknown> = { ...((row ?? {}) as Record<string, unknown>) };
+  if (sent) {
+    for (const key of Object.keys(copy)) {
+      if (!sent.has(key)) delete copy[key];
+    }
+  }
+  return copy;
+}
+
 /**
  * Evaluate formula virtual fields against the raw rows a driver handed back —
  * the read path (`find` / `findOne`) and, since #5504, the write path's
@@ -12831,6 +12888,10 @@ export class ObjectQL implements IObjectQLEngine {
     data = normalizeBlankTypedValues(this._registry.getObject(object), data);
     data = normalizeNumericStringValues(this._registry.getObject(object), data);
 
+    // [#21682] What the CALLER sent, recorded before any write middleware
+    // fills the payload. See `callerKeySets`.
+    const callerKeys = callerKeySets(data);
+
     const opCtx: OperationContext = {
       object,
       operation: 'insert',
@@ -12858,6 +12919,14 @@ export class ObjectQL implements IObjectQLEngine {
       // untouched, hooks run after and may override.
       const nowSnap = new Date();
       const isBatch = Array.isArray(opCtx.data);
+      // [#21682] Each row's caller key set, looked up NOW, on the row objects
+      // the middleware chain handed on. The computed-field door below may
+      // replace a row with a copy. Index-aligned from here on: every pass
+      // between here and the snapshot keeps the rows' order and count.
+      const callerKeysPerRow: Array<ReadonlySet<string> | undefined> =
+        (isBatch ? (opCtx.data as unknown[]) : [opCtx.data]).map(
+          (row) => (row !== null && typeof row === 'object' ? callerKeys.get(row) : undefined),
+        );
       // [#8682] The declared-field door — see `undeclaredWriteFieldErrors` for
       // what used to run below it for a request that was already refused.
       // FIRST, so nothing downstream (defaults, summary seeding, the hooks, the
@@ -12893,10 +12962,17 @@ export class ObjectQL implements IObjectQLEngine {
       }
       // [#4441] The RAW caller payload per row — before `applyFieldDefaults`
       // resolves any `defaultValue` / `current_user` token and before the
-      // beforeInsert hooks stamp `owner_id` / `organization_id` /
-      // `created_by`. The reference check consults it to decide WHAT THE
-      // CALLER ACTUALLY SENT, so neither a platform stamp nor a backfilled
-      // default is ever reported as the caller's bad reference.
+      // beforeInsert hooks stamp `created_by`. The reference check consults it
+      // to decide WHAT THE CALLER ACTUALLY SENT, so neither a platform stamp
+      // nor a backfilled default is ever reported as the caller's bad
+      // reference.
+      //
+      // [#21682] ...and without the keys a write MIDDLEWARE filled, which ran
+      // before this step: `organization_id` (`@objectstack/organizations`) and
+      // `owner_id` (`@objectstack/plugin-security`) are filled there, not by a
+      // hook. Each row keeps only the keys the caller sent (`callerKeySets`,
+      // recorded at entry), so the strips below report and take only those.
+      // The values stay as the chain handed them on.
       //
       // [#6339] It carries the caller's VALUES, and it is taken HERE — ahead of
       // the hooks — as an explicit shallow COPY. Both halves are load-bearing:
@@ -12918,7 +12994,7 @@ export class ObjectQL implements IObjectQLEngine {
       //    (#5591).
       const suppliedPerRow: Array<Record<string, unknown>> =
         (isBatch ? (opCtx.data as any[]) : [opCtx.data]).map(
-          (row) => ({ ...((row ?? {}) as Record<string, unknown>) }),
+          (row, i) => callerSuppliedRow(row, callerKeysPerRow[i]),
         );
       // [#20082] The write's ONE permission resolution, shared by every consumer
       // below that needs the map: the CEL defaults here, the re-default after

packages/plugins/organizations/src/create-explicit-organization-wall.test.ts

@@ -42,13 +42,23 @@
  *   every app object. The engine strips that column from a non-system payload
  *   as `readonly` AFTER the wall has judged it, so here an admitted value is
  *   re-derived, and a refused one never reaches the strip.
+ *
+ * ## [#21682] The stamp is not a caller write
+ *
+ * The last block pins the report half of the same seam: a create that names
+ * no organization is filled by Middleware A, and `droppedFields` (the REST
+ * create doors collect it through the `onFieldsDropped` listener driven here)
+ * must not name that fill as a key the caller sent. A key the caller DID send
+ * is still reported, and the array insert and the `single` posture answer
+ * as they did before.
  */
 
 import { describe, it, expect, afterEach, vi } from 'vitest';
 import { ObjectQL } from '@objectstack/objectql';
 import { SqliteWasmDriver } from '@objectstack/driver-sqlite-wasm';
 import { SecurityPlugin } from '@objectstack/plugin-security';
 import type { PermissionSet } from '@objectstack/spec/security';
+import type { DroppedFieldsEvent } from '@objectstack/spec/data';
 import { OrganizationsPlugin } from './organizations-plugin.js';
 
 /** The caller's active organization. */
@@ -108,7 +118,7 @@ const ADMIN_CTX = {
   posture: 'PLATFORM_ADMIN',
 };
 
-type Posture = 'isolated' | 'group';
+type Posture = 'isolated' | 'group' | 'single';
 
 const engines: ObjectQL[] = [];
 afterEach(async () => {
@@ -125,7 +135,11 @@ interface Booted {
   table: (name: string) => Promise<Array<Record<string, unknown>>>;
 }
 
-async function boot(posture: Posture = 'isolated'): Promise<Booted> {
+/**
+ * `organizations: false` boots without this package, the way `serve` boots the
+ * `single` posture — the control for the [#21682] block.
+ */
+async function boot(posture: Posture = 'isolated', { organizations: mountOrganizations = true } = {}): Promise<Booted> {
   const engine = new ObjectQL();
   engine.registerDriver(new SqliteWasmDriver({ filename: ':memory:' } as never) as never, true);
   await engine.init();
@@ -170,14 +184,18 @@ async function boot(posture: Posture = 'isolated'): Promise<Booted> {
     // not to the insert path under test.
     hook: vi.fn(),
   };
-  const organizations = new OrganizationsPlugin({ ensureDefaultOrganization: false });
+  const organizations = mountOrganizations ? new OrganizationsPlugin({ ensureDefaultOrganization: false }) : null;
   const security = new SecurityPlugin({ fallbackPermissionSet: 'member_default' });
   // Kernel order: every init, then every start — organizations first, as `serve` mounts it.
-  await organizations.init(ctx as never);
+  await organizations?.init(ctx as never);
   await security.init(ctx as never);
-  await organizations.start(ctx as never);
+  await organizations?.start(ctx as never);
   await security.start(ctx as never);
-  expect(services['org-scoping'], 'the real runtime is the org-scoping service').toBe(organizations);
+  if (organizations) {
+    expect(services['org-scoping'], 'the real runtime is the org-scoping service').toBe(organizations);
+  } else {
+    expect('org-scoping' in services, 'the control boots without the runtime').toBe(false);
+  }
   // The expected refusals log at WARN through the engine's own logger.
   vi.spyOn((engine as unknown as { logger: { warn: () => void } }).logger, 'warn').mockImplementation(() => undefined);
 
@@ -326,3 +344,85 @@ describe('[#21666] a create naming an organization meets the Layer 0 write wall,
     });
   });
 });
+
+/**
+ * One create with the listener the REST create doors wire
+ * (`ObjectStackProtocolImplementation.createData` / `createManyData` hand
+ * `onFieldsDropped` to `engine.insert` and answer what it collected as
+ * `droppedFields`), so `dropped` is the response's `droppedFields`.
+ */
+const createReporting = async (b: Booted, object: string, data: unknown, context: object) => {
+  const dropped: DroppedFieldsEvent[] = [];
+  const outcome = await attempt(() => b.engine.insert(object, data as never, {
+    context,
+    onFieldsDropped: (e: DroppedFieldsEvent) => { dropped.push(e); },
+  } as never));
+  return { outcome, dropped };
+};
+
+describe('[#21682] droppedFields names only keys the caller sent, so the organization stamp is not one', () => {
+  for (const [who, context] of CALLERS) {
+    it(`${who}: a walled create naming no organization reports no dropped field, and is stored in the active organization`, async () => {
+      const b = await boot();
+
+      const { outcome, dropped } = await createReporting(b, INJECTED, { id: 'r2', name: 'new' }, context);
+
+      expect(outcome.ok, outcome.message).toBe(true);
+      expect(dropped, 'the fill is the platform\'s write, not a key the caller lost').toEqual([]);
+      expect(b.seenByHooks, 'the beforeInsert chain still sees the stamp').toEqual([OWN_ORG]);
+      expect(await b.table(INJECTED)).toEqual([...SEEDED, { id: 'r2', organization_id: OWN_ORG }]);
+    });
+  }
+
+  it('a caller that sends the readonly key itself still sees it reported, the same key the stamp fills', async () => {
+    const b = await boot();
+
+    const { outcome, dropped } = await createReporting(b, INJECTED, { id: 'r2', name: 'new', organization_id: OWN_ORG }, MEMBER_CTX);
+
+    expect(outcome.ok, outcome.message).toBe(true);
+    expect(dropped).toEqual([{ object: INJECTED, fields: ['organization_id'], reason: 'readonly' }]);
+    expect(await b.table(INJECTED)).toEqual([...SEEDED, { id: 'r2', organization_id: OWN_ORG }]);
+  });
+
+  it('a readonly key the caller sends beside the fill is reported alone', async () => {
+    // `created_by` is an injected readonly column. No audit hook stamps it in
+    // this composition, so the caller's value is the one the strip takes.
+    const b = await boot();
+
+    const { outcome, dropped } = await createReporting(b, INJECTED, { id: 'r2', name: 'new', created_by: 'usr_forged' }, MEMBER_CTX);
+
+    expect(outcome.ok, outcome.message).toBe(true);
+    expect(dropped).toEqual([{ object: INJECTED, fields: ['created_by'], reason: 'readonly' }]);
+  });
+
+  it('the array insert is unchanged: a row naming no organization reports nothing', async () => {
+    const b = await boot();
+
+    const { outcome, dropped } = await createReporting(b, INJECTED, [{ id: 'r2', name: 'new' }], MEMBER_CTX);
+
+    expect(outcome.ok, outcome.message).toBe(true);
+    expect(dropped).toEqual([]);
+    expect(await b.table(INJECTED)).toEqual([...SEEDED, { id: 'r2', organization_id: OWN_ORG }]);
+  });
+
+  describe('the `single` posture, booted without this package, is unchanged', () => {
+    it('a create naming no organization reports nothing', async () => {
+      const b = await boot('single', { organizations: false });
+
+      const { outcome, dropped } = await createReporting(b, INJECTED, { id: 'r2', name: 'new' }, MEMBER_CTX);
+
+      expect(outcome.ok, outcome.message).toBe(true);
+      expect(dropped).toEqual([]);
+      expect(b.seenByHooks, 'no runtime, so no stamp').toEqual([undefined]);
+    });
+
+    it('a create naming an organization still reports it as readonly', async () => {
+      const b = await boot('single', { organizations: false });
+
+      const { outcome, dropped } = await createReporting(b, INJECTED, { id: 'r2', name: 'new', organization_id: OWN_ORG }, MEMBER_CTX);
+
+      expect(outcome.ok, outcome.message).toBe(true);
+      expect(dropped).toEqual([{ object: INJECTED, fields: ['organization_id'], reason: 'readonly' }]);
+    });
+  });
+});