From 2beb1441f012fb734b163d9da76824e06c012c59 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 06:36:49 +0000 Subject: [PATCH 1/7] fix(service-datasource): Import as Object saves through the metadata door's save persistObject called metadata.register('object', ...), which held the definition in memory only: no sys_metadata row, no engine schema sync, no external-object registration. An object imported under a name that differs from its remote table answered 500 'no such table', and every import was gone after a restart. It now calls saveMetaItem on the 'protocol' service, the save PUT /meta/object/:name makes, resolved when the import runs. Claude-Session: https://claude.ai/code/session_011K3zqE8Pv1Evw5hc8tZCnN Co-authored-by: Claude --- ...import-saves-through-metadata-door.test.ts | 152 ++++++++++++++++++ .../services/service-datasource/src/plugin.ts | 57 ++++++- 2 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 packages/services/service-datasource/src/__tests__/external-import-saves-through-metadata-door.test.ts diff --git a/packages/services/service-datasource/src/__tests__/external-import-saves-through-metadata-door.test.ts b/packages/services/service-datasource/src/__tests__/external-import-saves-through-metadata-door.test.ts new file mode 100644 index 0000000000..6cb9d1e40e --- /dev/null +++ b/packages/services/service-datasource/src/__tests__/external-import-saves-through-metadata-door.test.ts @@ -0,0 +1,152 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21788] "Import as Object" saves through the metadata door's own save. + * + * ## The defect this pins closed + * + * `ExternalDatasourceServicePlugin` wired `persistObject` to + * `metadata.register('object', name, definition)`. That put the definition in + * the metadata service's memory and nothing else: no `sys_metadata` row, no + * engine schema sync, no external-object registration with the driver. So an + * object imported under a name that differs from its remote table answered + * `201` and then `500 no such table: ` (the SQL driver resolved the + * table by the object's name), and every import was gone after a restart. + * `PUT /api/v1/meta/object/:name` with the same body was durable and served + * the rows, because it saves through `saveMetaItem` on the `'protocol'` + * service — the one save that persists the row, writes it through to the + * engine registry and syncs the object's storage (`syncObjectSchema`, which + * for a federated object maps the remote table). + * + * ## What each case pins + * + * - the import reaches `saveMetaItem` with the request the metadata door + * sends for an `object` (env-wide: `object` is not org-overridable), and + * registers through no second path; + * - the save door is resolved when the import runs, not at `init()` — a + * protocol registered after this plugin still receives the save; + * - a save the door refuses refuses the import with the door's own error; + * - with no save door at all, the import is refused before any remote + * introspection and nothing is saved. + * + * The plugin is imported through a RELATIVE specifier, so these cases measure + * `src/` and need no build. The booted-stack half (import, read rows, restart, + * read again) is the dogfood pin's. + */ + +import { describe, it, expect, vi } from 'vitest'; +import type { PluginContext } from '@objectstack/core'; +import type { IntrospectedSchema, IExternalDatasourceService } from '@objectstack/spec/contracts'; +import { ExternalDatasourceServicePlugin } from '../plugin.js'; + +const remoteSchema = (): IntrospectedSchema => + ({ + dialect: 'sqlite', + tables: { + customers: { + name: 'customers', + columns: [ + { name: 'id', type: 'text', nullable: false, primaryKey: true }, + { name: 'name', type: 'text', nullable: true, primaryKey: false }, + ], + }, + }, + }) as unknown as IntrospectedSchema; + +interface Harness { + ctx: PluginContext; + services: Map; + introspect: ReturnType; + register: ReturnType; +} + +/** A kernel context whose `getService` throws on an unregistered name, as the kernel's does. */ +function harness(): Harness { + const services = new Map(); + const introspect = vi.fn(async () => remoteSchema()); + const register = vi.fn(async () => undefined); + services.set('data', { introspectDatasource: introspect }); + services.set('metadata', { + get: async (type: string, name: string) => + type === 'datasource' ? { name, schemaMode: 'external' } : undefined, + register, + }); + const ctx = { + getService: (name: string) => { + if (!services.has(name)) throw new Error(`Service '${name}' not found`); + return services.get(name); + }, + registerService: (name: string, service: unknown) => { + services.set(name, service); + }, + trigger: async () => undefined, + logger: { info() {}, warn() {}, error() {}, debug() {} }, + } as unknown as PluginContext; + return { ctx, services, introspect, register }; +} + +async function federation(h: Harness): Promise { + await new ExternalDatasourceServicePlugin().init(h.ctx); + return h.services.get('external-datasource') as IExternalDatasourceService; +} + +describe('importObject saves through the metadata door (#21788)', () => { + it('reaches saveMetaItem with the metadata door\'s object request, and registers through no second path', async () => { + const h = harness(); + const saveMetaItem = vi.fn(async () => ({ success: true })); + h.services.set('protocol', { saveMetaItem }); + + const result = await (await federation(h)).importObject('warehouse', 'customers', { name: 'ext_cust' }); + + expect(saveMetaItem).toHaveBeenCalledTimes(1); + expect(saveMetaItem).toHaveBeenCalledWith({ type: 'object', name: 'ext_cust', item: result.definition }); + expect(result.definition).toMatchObject({ + name: 'ext_cust', + datasource: 'warehouse', + external: { remoteName: 'customers' }, + }); + expect(h.register).not.toHaveBeenCalled(); + }); + + it('resolves the save door when the import runs, so a protocol registered after init still receives it', async () => { + const h = harness(); + const service = await federation(h); + const saveMetaItem = vi.fn(async () => ({ success: true })); + h.services.set('protocol', { saveMetaItem }); + + await service.importObject('warehouse', 'customers', { name: 'ext_cust' }); + + expect(saveMetaItem).toHaveBeenCalledTimes(1); + expect(h.register).not.toHaveBeenCalled(); + }); + + it('refuses the import with the door\'s own refusal when the save is refused', async () => { + const h = harness(); + const refusal = Object.assign(new Error('object/ext_cust failed validation'), { + code: 'INVALID_METADATA', + status: 422, + }); + h.services.set('protocol', { saveMetaItem: vi.fn(async () => { throw refusal; }) }); + + const outcome = await (await federation(h)) + .importObject('warehouse', 'customers', { name: 'ext_cust' }) + .catch((e: unknown) => e); + + expect(outcome).toBe(refusal); + expect(outcome).toMatchObject({ code: 'INVALID_METADATA', status: 422 }); + expect(h.register).not.toHaveBeenCalled(); + }); + + it('with no save door, refuses before any remote introspection and saves nothing', async () => { + const h = harness(); + + const outcome = await (await federation(h)) + .importObject('warehouse', 'customers', { name: 'ext_cust' }) + .catch((e: unknown) => e); + + expect(outcome).toBeInstanceOf(Error); + expect((outcome as Error).message).toMatch(/requires a writable metadata store/); + expect(h.introspect).not.toHaveBeenCalled(); + expect(h.register).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/services/service-datasource/src/plugin.ts b/packages/services/service-datasource/src/plugin.ts index 80d5d74f98..d1d6a11488 100644 --- a/packages/services/service-datasource/src/plugin.ts +++ b/packages/services/service-datasource/src/plugin.ts @@ -1,6 +1,7 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. import type { Plugin, PluginContext } from '@objectstack/core'; +import type { MetadataProtocol } from '@objectstack/spec/api'; import type { IDataEngine, IntrospectedSchema } from '@objectstack/spec/contracts'; import { ExternalDatasourceService, @@ -30,6 +31,9 @@ interface MetadataServiceLike { register?: (type: string, name: string, data: unknown) => Promise | void; } +/** The metadata door's save, as the `'protocol'` service declares it. */ +type MetadataSaveDoor = Pick; + export interface ExternalDatasourceServicePluginOptions { /** Override the introspection function (mainly for tests). */ introspect?: (datasource: string) => Promise; @@ -77,6 +81,20 @@ export class ExternalDatasourceServicePlugin implements Plugin { ); }); + /** + * [#21788] The metadata door's own save: `saveMetaItem` on the `'protocol'` + * service, the call `PUT /api/v1/meta/object/:name` makes. + * + * Resolved where it is used, never at `init()`: the protocol registers in + * another plugin's `init()`, which may run after this one, and a verdict + * drawn here would be kept for the life of the process (AGENTS.md, "Startup + * registry reads"). + */ + const metadataSaveDoor = (): MetadataSaveDoor | undefined => { + const protocol = safeGetService>(ctx, 'protocol'); + return typeof protocol?.saveMetaItem === 'function' ? (protocol as MetadataSaveDoor) : undefined; + }; + const config: ExternalDatasourceServiceConfig = { introspect, getDatasource: async (n) => (await metadata?.get('datasource', n)) as DatasourceLike | undefined, @@ -94,13 +112,42 @@ export class ExternalDatasourceServicePlugin implements Plugin { persistCatalog: async (catalog) => { await metadata.register!('external_catalog', catalog.name, catalog); }, - // Runtime "Import as Object": persist a federated object so it's - // immediately queryable, no git commit required (ADR-0015 Addendum). - persistObject: async (name, definition) => { - await metadata.register!('object', name, definition); - }, } : {}), + /** + * Runtime "Import as Object" (ADR-0015 Addendum): save the federated + * object through the metadata door's own save, so it is exactly what a + * `PUT /meta/object/:name` of the same body makes it — a `sys_metadata` + * row the next boot binds, written through to the engine registry, its + * storage synced. For a federated object that sync is what maps the + * object onto its `external.remoteName` table in the driver. + * + * [#21788] This used to be `metadata.register('object', …)`, which only + * held the definition in the metadata service's memory: an object named + * differently from its remote table answered `500 no such table`, and + * every import was gone after a restart. ⛔ No second registration path + * beside the save — the save already writes the registry through. + * + * The request is the one that door sends for an `object`, field for + * field: no `organizationId`, because `object` is not org-overridable and + * that door's `organizationIdForMetaWrite` resolves none for it; no + * `packageId`, `mode` or `force`, because the import route takes no + * `?package`, `?mode` or `?force`. + * + * A GETTER, so the save door is asked for when an import runs: the + * service reads this slot before the draft and refuses with its own + * "requires a writable metadata store" when it is absent — before any + * remote introspection, and only when no save door is registered by + * then. ⛔ Do not move it into a spread: spreading reads the getter + * once, here, at `init()`. + */ + get persistObject() { + const door = metadataSaveDoor(); + if (!door) return undefined; + return async (name: string, definition: Record) => { + await door.saveMetaItem({ type: 'object', name, item: definition }); + }; + }, /** * Where a generated object's `${namespace}_` prefix comes from (ADR-0028). * From 0bc13c54dd726c9e119daa09dc35d2afee786c39 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 06:50:08 +0000 Subject: [PATCH 2/7] chore(changeset): service-datasource patch for the import save path Claude-Session: https://claude.ai/code/session_011K3zqE8Pv1Evw5hc8tZCnN Co-authored-by: Claude --- .changeset/21788-external-import-saves-like-meta.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .changeset/21788-external-import-saves-like-meta.md diff --git a/.changeset/21788-external-import-saves-like-meta.md b/.changeset/21788-external-import-saves-like-meta.md new file mode 100644 index 0000000000..f7d2db0b2e --- /dev/null +++ b/.changeset/21788-external-import-saves-like-meta.md @@ -0,0 +1,13 @@ +--- +'@objectstack/service-datasource': patch +--- + +"Import as Object" saves the imported federated object through the metadata door's own save, so it is durable and reads from its remote table (#21788). + +Clause-②: no + +- `POST /api/v1/datasources/:name/external/tables/:remote/import` used to hold the generated object in the metadata service's memory only. No `sys_metadata` row was written, the object's storage was not synced, and the driver was never told the object's remote table. An object imported under a name that differs from its remote table answered `201` and then `500 DATABASE_ERROR` (`no such table: `) on its first read. Every import was gone after a restart (`404 OBJECT_NOT_FOUND`). +- The import now calls `saveMetaItem` on the `protocol` service, the same save `PUT /api/v1/meta/object/:name` makes, with the request that door sends for an `object`. The object is a `sys_metadata` row the next boot binds, it is written through to the engine registry, and it is mapped onto its `external.remoteName` table. Both an import under a different name and one under the remote table's own name serve the remote rows, before and after a restart. +- The save door is looked up when an import runs, not when the plugin starts. A deployment with no metadata save door still refuses the import with "requires a writable metadata store", before any remote introspection. +- A save the metadata door refuses now refuses the import. The route answers it as `400 EXTERNAL_IMPORT_ERROR` with the door's message, as it answers every refused import. +- The route's request and response shapes are unchanged. From 99b5420d94589a324ff92154c5f410afab91e019 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 07:47:01 +0000 Subject: [PATCH 3/7] test(dogfood): Import as Object persists like the metadata door, across a cold boot Declares @objectstack/service-datasource (lockfile importer regenerated by pnpm install) and aliases it to source, so the pin mounts ExternalDatasourceServicePlugin from this checkout. Claude-Session: https://claude.ai/code/session_011K3zqE8Pv1Evw5hc8tZCnN Co-authored-by: Claude --- packages/qa/dogfood/package.json | 1 + ...nal-import-saves-like-meta.dogfood.test.ts | 127 ++++++++++++++++++ packages/qa/dogfood/vitest.config.ts | 9 ++ pnpm-lock.yaml | 15 +++ 4 files changed, 152 insertions(+) create mode 100644 packages/qa/dogfood/test/external-import-saves-like-meta.dogfood.test.ts diff --git a/packages/qa/dogfood/package.json b/packages/qa/dogfood/package.json index 38e84a896e..7e32e308e6 100644 --- a/packages/qa/dogfood/package.json +++ b/packages/qa/dogfood/package.json @@ -30,6 +30,7 @@ "@objectstack/plugin-sharing": "workspace:*", "@objectstack/plugin-webhooks": "workspace:*", "@objectstack/service-analytics": "workspace:*", + "@objectstack/service-datasource": "workspace:*", "@objectstack/service-messaging": "workspace:*", "@objectstack/service-storage": "workspace:*", "@objectstack/spec": "workspace:*", diff --git a/packages/qa/dogfood/test/external-import-saves-like-meta.dogfood.test.ts b/packages/qa/dogfood/test/external-import-saves-like-meta.dogfood.test.ts new file mode 100644 index 0000000000..00ad71e8fd --- /dev/null +++ b/packages/qa/dogfood/test/external-import-saves-like-meta.dogfood.test.ts @@ -0,0 +1,127 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// [#21788, ADR-0015 Addendum] "Import as Object" keeps what it answers `201` +// for — over the real showcase composition, across a cold boot on one +// database file. +// +// ## What was broken +// +// `POST /datasources/:name/external/tables/:remote/import` persisted the +// generated object with `metadata.register('object', …)`, which held it in the +// metadata service's memory only: no `sys_metadata` row, no engine schema +// sync, no external-object registration with the driver. Measured on the +// showcase before the fix: an import under a name that differs from its remote +// table answered `201` and then `500 DATABASE_ERROR` on its first read (the +// SQL driver resolved the table by the object's name — `no such table`), an +// import whose name equals the remote table answered `200` only because the +// two names coincide, and after a restart on the same database both answered +// `404 OBJECT_NOT_FOUND`. `PUT /meta/object/:name` with the same binding was +// durable and served the rows. The import now saves through that door's save. +// +// ## Why a booted stack +// +// The seam pins sit beside the plugin +// (`packages/services/service-datasource/src/__tests__/external-import-saves-through-metadata-door.test.ts`). +// What they cannot answer is whether, on the stack an operator runs, the save +// maps the object onto its remote table and lands a row the next boot binds. +// Only a read through the data door and a restart on the same file show that. +// +// The verify harness does not mount the federation service, so this file +// mounts `ExternalDatasourceServicePlugin` itself, as `objectstack dev` and +// `serve` do. The working directory is a temporary one because the showcase's +// external datasource and its fixture both name a cwd-relative SQLite file: +// this file's remote database is its own, not one a parallel file writes. + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import showcaseStack, { onEnable } from '@objectstack/example-showcase'; +import { ExternalDatasourceServicePlugin } from '@objectstack/service-datasource'; +import { bootStack, type VerifyStack } from '@objectstack/verify'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +/** The showcase's external datasource and its two remote tables (`external-fixture.ts`). */ +const DATASOURCE = 'showcase_external'; +/** Imported under a name that differs from its remote table — the defect's shape. */ +const RENAMED = 'dogfood_ext_cust_21788'; +/** Imported under the remote table's own name — the shape that hid the defect. */ +const SAME_NAME = 'orders'; + +const importPath = (remote: string) => `/datasources/${DATASOURCE}/external/tables/${remote}/import`; + +function recordsOf(json: unknown): Array> { + const body = json as { records?: unknown[] } | undefined; + return (body?.records ?? []) as Array>; +} + +async function boot(databaseFile: string): Promise { + return bootStack(showcaseStack, { databaseFile, extraPlugins: [new ExternalDatasourceServicePlugin()] }); +} + +describe('Import as Object persists like the metadata door (showcase, cold boot)', () => { + let stack: VerifyStack; + let token: string; + let prevCwd: string; + let dir: string; + let dbFile: string; + + const call = async (method: string, path: string, body?: unknown) => { + const res = await stack.apiAs(token, method, path, body); + const json: unknown = await res.json().catch(() => ({})); + return { status: res.status, json }; + }; + + beforeAll(async () => { + prevCwd = process.cwd(); + dir = mkdtempSync(join(tmpdir(), 'dogfood-21788-')); + process.chdir(dir); + dbFile = join(dir, 'showcase.db'); + // Provision the remote tables, exactly as `os dev` does at boot (the + // harness imports only the stack's default export, so `onEnable` never + // runs on its own). + await onEnable({ logger: { info() {}, warn() {} } } as never); + stack = await boot(dbFile); + token = await stack.signIn(); + }, 180_000); + + afterAll(async () => { + await stack?.stop(); + if (prevCwd) process.chdir(prevCwd); + if (dir) rmSync(dir, { recursive: true, force: true }); + }); + + it('an object imported under a name that differs from its remote table serves the remote rows', async () => { + const imported = await call('POST', importPath('customers'), { name: RENAMED }); + expect(imported.status, JSON.stringify(imported.json)).toBe(201); + + const read = await call('GET', `/data/${RENAMED}`); + expect(read.status, JSON.stringify(read.json)).toBe(200); + expect(recordsOf(read.json).map((r) => r.name)).toEqual( + expect.arrayContaining(['Aurora Labs', 'Borealis GmbH', 'Cyan Pacific']), + ); + }); + + it('control: an object imported under its remote table\'s own name serves the remote rows', async () => { + const imported = await call('POST', importPath('orders'), { name: SAME_NAME }); + expect(imported.status, JSON.stringify(imported.json)).toBe(201); + + const read = await call('GET', `/data/${SAME_NAME}`); + expect(read.status, JSON.stringify(read.json)).toBe(200); + expect(recordsOf(read.json)).toHaveLength(4); + }); + + it('after a cold boot on the same database file, both imported objects still serve their rows', async () => { + await stack.stop(); + stack = await boot(dbFile); + token = await stack.signIn(); + + // Independent facts about one restart, so each is reported on its own. + const renamed = await call('GET', `/data/${RENAMED}`); + expect.soft(renamed.status, JSON.stringify(renamed.json)).toBe(200); + expect.soft(recordsOf(renamed.json)).toHaveLength(3); + + const sameName = await call('GET', `/data/${SAME_NAME}`); + expect.soft(sameName.status, JSON.stringify(sameName.json)).toBe(200); + expect.soft(recordsOf(sameName.json)).toHaveLength(4); + }, 180_000); +}); diff --git a/packages/qa/dogfood/vitest.config.ts b/packages/qa/dogfood/vitest.config.ts index e3c09b5d9e..6fad20fa15 100644 --- a/packages/qa/dogfood/vitest.config.ts +++ b/packages/qa/dogfood/vitest.config.ts @@ -264,6 +264,15 @@ export default defineConfig({ find: /^@objectstack\/cloud-connection$/, replacement: path.resolve(__dirname, '../../cloud-connection/src/index.ts'), }, + // [#21788] `external-import-saves-like-meta.dogfood.test.ts` mounts + // `ExternalDatasourceServicePlugin` on a real boot (the harness + // does not) and drives the import door. The plugin's save path is + // the pin's subject, so the verdict is aliased to THIS checkout's + // source, not to the last `pnpm build`. + { + find: /^@objectstack\/service-datasource$/, + replacement: path.resolve(__dirname, '../../services/service-datasource/src/index.ts'), + }, ], }, test: { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d5561ea713..a72e6e19fe 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -2078,6 +2078,9 @@ importers: '@objectstack/service-analytics': specifier: workspace:* version: link:../../services/service-analytics + '@objectstack/service-datasource': + specifier: workspace:* + version: link:../../services/service-datasource '@objectstack/service-messaging': specifier: workspace:* version: link:../../services/service-messaging @@ -5530,67 +5533,79 @@ packages: resolution: {integrity: sha512-pzJ++UMCZEV4s6SP3Ryvj+snWP8s7aTXFkSXRyeBF4RSALftPzfqYssHdGOmOH2QnRAtyOhhg64tdjeSGJvYRg==} cpu: [arm64] os: [android] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-darwin-arm64@0.8.7': resolution: {integrity: sha512-vymR7Us3i/HJvUxA1N5sKCrrn2mXw/Xvzbt66zlRyGPDmkUzFCrn34OqodR/zctmD1JhaFlv+Ur6xNNPRIid/w==} cpu: [arm64] os: [darwin] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-darwin-x64@0.8.7': resolution: {integrity: sha512-aX1xy6wJeI2QN/ipu9B6/dzz6RmHQ5+Ty6uyf9csqYZDnY4/U2GsDskYQIRysc2Cuh+GNnvuO5xCXxiqkmVEcw==} cpu: [x64] os: [darwin] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-freebsd-x64@0.8.7': resolution: {integrity: sha512-5LT2KkjK0zBI2s3VoSyGPSs9Ey7rWPxWkCyFgZXoszQOkAk2z0biP+DLzb9zw6flmwwCCiyeZRqM65srfo8l1Q==} cpu: [x64] os: [freebsd] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-linux-arm-gnu@0.8.7': resolution: {integrity: sha512-4haNlVk624QoNSKIneoH9JKu5SvfD+Hkxg490HUS5pfFuWwoXT3zOmAdfwPMsSH0bNIkFO7GqtwDZ9EVpyzepw==} cpu: [arm] os: [linux] libc: [glibc] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-linux-arm-musl@0.8.7': resolution: {integrity: sha512-7HwJHVFtrufB5qHHL1PSDPr/j6uoNLwbwxa04QzsbpcbbzfDUbT37loHPu5u0NuetRUlV+TqXDlX6OpXcM8hKQ==} cpu: [arm] os: [linux] libc: [musl] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-linux-arm64-gnu@0.8.7': resolution: {integrity: sha512-yUEgxEPuDVBO+nkDw8qbssYA8oHu82Q0da+C7rGyVplmjlKa5DhBnMMagTEjFZx4jNDVWnGHJreUCSeGL0x/gQ==} cpu: [arm64] os: [linux] libc: [glibc] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-linux-arm64-musl@0.8.7': resolution: {integrity: sha512-2X7EwxPbgdNRqgMwtxOnNOGEmdm1RS8PD2Q5cOxj8cEZD4fy7yHHeSDoEdBOyrJtHzbG6jQB6CeReO1okb/S7Q==} cpu: [arm64] os: [linux] libc: [musl] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-linux-x64-gnu@0.8.7': resolution: {integrity: sha512-k/iQFK1gAvaHLzXXZ3/+g48wT5YB6MfikPb+juGCd9HzyPMUSBCy44rz6nT+xoWnnxmBoeBUywA4CvWCWZFTtg==} cpu: [x64] os: [linux] libc: [glibc] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-linux-x64-musl@0.8.7': resolution: {integrity: sha512-gTfYHx3cg8FERTYsg1dQrQFTutcWJ7wTp8YToyAJnZMbUCkcyuwUiWNYaEHyF0xIb+PsG7HfD+BLWhRRom5qKg==} cpu: [x64] os: [linux] libc: [musl] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-win32-arm64@0.8.7': resolution: {integrity: sha512-XDCWZZztOvdTtPNaU5EzrrV0dmMugdZ+Qdq5INeiGhGW5hD0TuCBIIXK7wTmRM6NcKarGlyBP5SYkiAuw6+slg==} cpu: [arm64] os: [win32] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-analyzer/binding-win32-x64@0.8.7': resolution: {integrity: sha512-VtSH1pWuk3bo+BiUAtzYa4Ku1r/10CO14QvkGpRhDcFck8sIf7ox+wiucyKNKcf+srWCYxR1hO+wn5N3BdtFdw==} cpu: [x64] os: [win32] + deprecated: yuku-analyzer runs on yuku-core since 0.14 '@yuku-toolchain/types@0.8.7': resolution: {integrity: sha512-2Z53dNxAJL6UvFoIrDZvYf3zlO8s4VJK4O2hhaB4mXVwwpX/7ajtss3cmfqKvamlNLWyt9FSWs4eoYdlbxpnHA==} From f2e3d7ff362ff6d1da04f2de4d402aaa69231393 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 07:47:01 +0000 Subject: [PATCH 4/7] fix(service-datasource): federated validation skips the platform's unprovisioned injected anchors A stored federated object is read with the anchors the platform injects (organization_id, created_by, updated_by, owner_id, owning_business_unit_id). Gate 2 compared them with the remote and reported each as missing_column at error severity, so a datasource with the default onMismatch 'fail' refused to boot. validateObjectUsing now skips the columns unprovisionedInjectedColumns names. Claude-Session: https://claude.ai/code/session_011K3zqE8Pv1Evw5hc8tZCnN Co-authored-by: Claude --- .../src/external-datasource-service.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/services/service-datasource/src/external-datasource-service.ts b/packages/services/service-datasource/src/external-datasource-service.ts index 2f1730e3b7..4e7b236ead 100644 --- a/packages/services/service-datasource/src/external-datasource-service.ts +++ b/packages/services/service-datasource/src/external-datasource-service.ts @@ -29,6 +29,7 @@ import { suggestFieldTypeForSqlType, isCompatible, ExternalCatalogSchema, + unprovisionedInjectedColumns, type ExternalCatalog, type SqlDialect, type FieldType, @@ -661,8 +662,19 @@ export class ExternalDatasourceService implements IExternalDatasourceService { fieldToRemote.set(fieldName, remoteCol); } + // [#21788] The anchors the platform injects with NO storage behind them + // (`organization_id`, the audit pair, `owner_id`, + // `owning_business_unit_id` on a federated object) are not remote columns + // and never were: the remote owns the schema. An object read as the + // registry holds it — every stored object the metadata door or the import + // saved, rehydrated at boot — carries them in `fields`, and comparing them + // reported each as a `missing_column` error, which aborted the boot under + // the default `onMismatch: 'fail'`. The spec's own provenance verdict + // (#7865) names exactly those anchors; an author-declared field of the same + // name is the author's and is still compared. + const unprovisioned = new Set(unprovisionedInjectedColumns(obj)); for (const [fieldName, field] of Object.entries(obj.fields ?? {})) { - if (BUILTIN_COLUMNS.has(fieldName)) continue; + if (BUILTIN_COLUMNS.has(fieldName) || unprovisioned.has(fieldName)) continue; const remoteCol = fieldToRemote.get(fieldName) ?? fieldName; if (ignore.has(remoteCol)) continue; From 86c01b91588912fb3022ecbb2db2ce99cc54f0cd Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 07:48:29 +0000 Subject: [PATCH 5/7] chore(changeset): the import narrows (minor, BREAKING banner) and validation skips injected anchors Claude-Session: https://claude.ai/code/session_011K3zqE8Pv1Evw5hc8tZCnN Co-authored-by: Claude --- .../21788-external-import-saves-like-meta.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.changeset/21788-external-import-saves-like-meta.md b/.changeset/21788-external-import-saves-like-meta.md index f7d2db0b2e..522acac924 100644 --- a/.changeset/21788-external-import-saves-like-meta.md +++ b/.changeset/21788-external-import-saves-like-meta.md @@ -1,13 +1,15 @@ --- -'@objectstack/service-datasource': patch +'@objectstack/service-datasource': minor --- -"Import as Object" saves the imported federated object through the metadata door's own save, so it is durable and reads from its remote table (#21788). +fix(service-datasource)!: "Import as Object" saves the imported federated object through the metadata door's own save, so it is durable and reads from its remote table; federated validation stops reporting the platform's injected anchors as missing remote columns (#21788) -Clause-②: no +**BREAKING** — the import route now answers `400` to some re-imports that used to answer `201`. -- `POST /api/v1/datasources/:name/external/tables/:remote/import` used to hold the generated object in the metadata service's memory only. No `sys_metadata` row was written, the object's storage was not synced, and the driver was never told the object's remote table. An object imported under a name that differs from its remote table answered `201` and then `500 DATABASE_ERROR` (`no such table: `) on its first read. Every import was gone after a restart (`404 OBJECT_NOT_FOUND`). -- The import now calls `saveMetaItem` on the `protocol` service, the same save `PUT /api/v1/meta/object/:name` makes, with the request that door sends for an `object`. The object is a `sys_metadata` row the next boot binds, it is written through to the engine registry, and it is mapped onto its `external.remoteName` table. Both an import under a different name and one under the remote table's own name serve the remote rows, before and after a restart. -- The save door is looked up when an import runs, not when the plugin starts. A deployment with no metadata save door still refuses the import with "requires a writable metadata store", before any remote introspection. -- A save the metadata door refuses now refuses the import. The route answers it as `400 EXTERNAL_IMPORT_ERROR` with the door's message, as it answers every refused import. -- The route's request and response shapes are unchanged. +Clause-②: no (narrowing) + +- **The import was neither durable nor mapped.** `POST /api/v1/datasources/:name/external/tables/:remote/import` held the generated object in the metadata service's memory only. No `sys_metadata` row was written, the object's storage was not synced, and the driver was never told the object's remote table. An object imported under a name that differs from its remote table answered `201` and then `500 DATABASE_ERROR` (`no such table: `) on its first read. Every import was gone after a restart (`404 OBJECT_NOT_FOUND`). +- **It now saves like `PUT /api/v1/meta/object/:name`.** The import calls `saveMetaItem` on the `protocol` service with the request that door sends for an `object`. The object becomes a `sys_metadata` row the next boot binds, it is written through to the engine registry, and it is mapped onto its `external.remoteName` table. An import under a different name and one under the remote table's own name both serve the remote rows, before and after a restart. The save door is looked up when an import runs. A deployment with no metadata save door still refuses the import with "requires a writable metadata store", before any remote introspection. +- **What narrows.** The metadata door's refusals now apply to the import, and the route relays each one as `400 EXTERNAL_IMPORT_ERROR` with the door's message. A re-import that would drop a field the stored object already has, or change its type, is refused as a destructive change. It used to answer `201` with an in-memory overwrite that a restart discarded. An import whose `name` collides with an object the metadata door will not overwrite is refused the same way. The route's request and response shapes are unchanged. +- **If a re-import or a name is refused:** import the table again under a new `name`. To change an object you already imported, save its new definition through `PUT /api/v1/meta/object/:name?force=true`, which accepts the destructive change on purpose. Re-submitting the import with `?force=true` does not help, because the import route reads no `force`. +- **Federated validation compares only what the remote owns.** A stored federated object is read back with the anchors the platform injects without storage (`organization_id`, `created_by`, `updated_by`, `owner_id`, `owning_business_unit_id`). The boot validation gate and `POST …/external/validate` reported each of them as a `missing_column` at error severity. So once a datasource with the default `external.validation.onMismatch: 'fail'` held such an object, it refused to boot. Validation now skips the columns `unprovisionedInjectedColumns` names. This closes the boot abort for objects saved through `PUT /api/v1/meta/object/:name`, not only for imports. A declared column the remote lacks is still a `missing_column` at error severity, and `fail` still aborts on it. From b9690f1852d4187fac480423931c29ba2fc669d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 07:48:48 +0000 Subject: [PATCH 6/7] chore(changeset): ADR-0087 disposition for the import narrowing Claude-Session: https://claude.ai/code/session_011K3zqE8Pv1Evw5hc8tZCnN Co-authored-by: Claude --- .changeset/21788-external-import-saves-like-meta.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.changeset/21788-external-import-saves-like-meta.md b/.changeset/21788-external-import-saves-like-meta.md index 522acac924..9937a0395f 100644 --- a/.changeset/21788-external-import-saves-like-meta.md +++ b/.changeset/21788-external-import-saves-like-meta.md @@ -13,3 +13,5 @@ Clause-②: no (narrowing) - **What narrows.** The metadata door's refusals now apply to the import, and the route relays each one as `400 EXTERNAL_IMPORT_ERROR` with the door's message. A re-import that would drop a field the stored object already has, or change its type, is refused as a destructive change. It used to answer `201` with an in-memory overwrite that a restart discarded. An import whose `name` collides with an object the metadata door will not overwrite is refused the same way. The route's request and response shapes are unchanged. - **If a re-import or a name is refused:** import the table again under a new `name`. To change an object you already imported, save its new definition through `PUT /api/v1/meta/object/:name?force=true`, which accepts the destructive change on purpose. Re-submitting the import with `?force=true` does not help, because the import route reads no `force`. - **Federated validation compares only what the remote owns.** A stored federated object is read back with the anchors the platform injects without storage (`organization_id`, `created_by`, `updated_by`, `owner_id`, `owning_business_unit_id`). The boot validation gate and `POST …/external/validate` reported each of them as a `missing_column` at error severity. So once a datasource with the default `external.validation.onMismatch: 'fail'` held such an object, it refused to boot. Validation now skips the columns `unprovisionedInjectedColumns` names. This closes the boot abort for objects saved through `PUT /api/v1/meta/object/:name`, not only for imports. A declared column the remote lacks is still a `missing_column` at error severity, and `fail` still aborts on it. + + From 8d640fa2829a734c2e58ad1a4702c1ec73b7ce49 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 5 Oct 2026 07:49:04 +0000 Subject: [PATCH 7/7] test(service-datasource): validation skips injected anchors, and real drift stays an error Claude-Session: https://claude.ai/code/session_011K3zqE8Pv1Evw5hc8tZCnN Co-authored-by: Claude --- ...external-validate-injected-anchors.test.ts | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 packages/services/service-datasource/src/__tests__/external-validate-injected-anchors.test.ts diff --git a/packages/services/service-datasource/src/__tests__/external-validate-injected-anchors.test.ts b/packages/services/service-datasource/src/__tests__/external-validate-injected-anchors.test.ts new file mode 100644 index 0000000000..c654dea723 --- /dev/null +++ b/packages/services/service-datasource/src/__tests__/external-validate-injected-anchors.test.ts @@ -0,0 +1,126 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#21788] Federated validation compares what the remote owns, and nothing the + * platform injected. + * + * ## The defect this pins closed + * + * The registry injects anchors onto every object it registers — on a federated + * one `organization_id`, `created_by`, `updated_by`, `owner_id` and + * `owning_business_unit_id`, with no storage behind them (the remote owns the + * schema). A stored federated object (saved through `PUT /meta/object` or the + * external-table import, rehydrated at boot) is read back as the registry + * holds it, anchors included, and `validateObjectUsing` compared every field + * but `id` / `created_at` / `updated_at` with the remote table. So each anchor + * came back as a `missing_column` at error severity, and the boot validation + * gate aborted any datasource with the default `onMismatch: 'fail'`. Measured + * on the showcase: "Object '…' does not match its remote table" listing exactly + * those five columns, for an object whose remote table was intact. + * + * ## What each case pins + * + * - an object carrying the platform's own anchor definitions validates `ok`, + * on the single-object read and on the per-datasource sweep the gate runs; + * - the NEGATIVE CONTROL: a declared business column the remote lacks is + * still a `missing_column` at error severity — the only diff, so the gate's + * `fail` policy (which aborts on any measured diff) still aborts on real + * drift; + * - an author-declared field that only shares an anchor's NAME is the + * author's (`resolveInjectedColumnProvenance` answers `'author'`) and is + * still compared. + * + * The anchors are built with the spec's own `injectedSystemColumnDefs`, so the + * fixture carries the bytes the registry injects rather than a copy of them. + */ + +import { describe, it, expect } from 'vitest'; +import type { IntrospectedSchema } from '@objectstack/spec/contracts'; +import { injectedSystemColumnDefs } from '@objectstack/spec/data'; +import { + ExternalDatasourceService, + type DatasourceLike, + type ObjectLike, +} from '../external-datasource-service.js'; + +const ANCHORS = ['organization_id', 'created_by', 'updated_by', 'owner_id', 'owning_business_unit_id']; + +const remoteSchema = (): IntrospectedSchema => + ({ + dialect: 'sqlite', + tables: { + customers: { + name: 'customers', + indexes: [], + columns: [ + { name: 'id', type: 'text', nullable: false, primaryKey: true }, + { name: 'name', type: 'text', nullable: true, primaryKey: false }, + { name: 'email', type: 'text', nullable: true, primaryKey: false }, + ], + }, + }, + }) as unknown as IntrospectedSchema; + +/** A federated object as the import authors it. */ +function authored(extraFields: Record> = {}): Record { + return { + name: 'ext_cust', + label: 'Ext Cust', + datasource: 'ext', + external: { remoteName: 'customers' }, + fields: { id: { type: 'text' }, name: { type: 'text' }, email: { type: 'text' }, ...extraFields }, + sharingModel: 'private', + }; +} + +/** The same object as the registry holds it: the platform's anchors injected into `fields`. */ +function stored(def: Record): ObjectLike { + const fields = def.fields as Record; + return { ...def, fields: { ...injectedSystemColumnDefs(def), ...fields } } as unknown as ObjectLike; +} + +function service(objects: ObjectLike[]): ExternalDatasourceService { + const ds: DatasourceLike = { name: 'ext', schemaMode: 'external' }; + return new ExternalDatasourceService({ + introspect: async () => remoteSchema(), + getDatasource: async (n) => (n === ds.name ? ds : undefined), + getObject: async (n) => objects.find((o) => o.name === n), + listObjects: async () => objects, + }); +} + +describe('federated validation skips the platform\'s unprovisioned anchors (#21788)', () => { + it('the fixture carries the anchors the platform injects (so the cases below are not vacuous)', () => { + expect(Object.keys(stored(authored()).fields ?? {})).toEqual(expect.arrayContaining(ANCHORS)); + }); + + it('a stored federated object carrying the injected anchors validates ok', async () => { + const svc = service([stored(authored())]); + + expect(await svc.validateObject('ext_cust')).toEqual({ ok: true, datasource: 'ext', object: 'ext_cust', diffs: [] }); + expect(await svc.validateDatasource('ext')).toEqual({ + ok: true, + results: [{ ok: true, datasource: 'ext', object: 'ext_cust', diffs: [] }], + }); + }); + + it('negative control: a declared business column the remote lacks is still a missing_column at error severity, and the only diff', async () => { + const svc = service([stored(authored({ loyalty_tier: { type: 'text' } }))]); + + const result = await svc.validateObject('ext_cust'); + + expect(result.ok).toBe(false); + expect(result.diffs).toEqual([ + { kind: 'missing_column', remoteName: 'customers', column: 'loyalty_tier', severity: 'error' }, + ]); + expect((await svc.validateDatasource('ext')).ok).toBe(false); + }); + + it('an author-declared field that only shares an anchor\'s name is still compared', async () => { + const svc = service([stored(authored({ owner_id: { type: 'text', label: 'Account owner' } }))]); + + expect((await svc.validateObject('ext_cust')).diffs).toEqual([ + { kind: 'missing_column', remoteName: 'customers', column: 'owner_id', severity: 'error' }, + ]); + }); +});