From 1ab95ee229cb6d2a933f510cf4efebd6b061599b Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Sep 2026 15:13:27 +0000 Subject: [PATCH 01/10] feat(spec): declare the query transport dialect and make it the single source MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `FindDataRequestSchema.query` declared the canonical QueryAST while the shipped `findData` door also folded a transport dialect no schema named (`$filter` / `$top` / `$skip` / `$orderby` / `$select` / `$expand` and the plural `filters`). Two dialects, one slot, one of them declared. `@objectstack/spec/data` now declares the transport spelling once — `QueryTransportParamsSchema` plus `QUERY_TRANSPORT_ALIAS_SLOTS` / `QUERY_TRANSPORT_DOLLAR_ALIASES` / `QUERY_TRANSPORT_DOLLAR_PARAMS` — and `@objectstack/metadata-protocol` folds by that export instead of its own `WIRE_QUERY_ALIAS_SLOTS` / `WIRE_DOLLAR_ALIASES`. Claude-Session: https://claude.ai/code/session_01LvwGppdonww4zGLWZo5rho Co-authored-by: Claude --- packages/metadata-protocol/src/protocol.ts | 74 +++---- .../rest-server-canonical-query-ast.test.ts | 70 ++++-- packages/spec/authorable-surface/data.json | 10 + packages/spec/json-schema.manifest/data.json | 2 + packages/spec/src/api/protocol.zod.ts | 30 ++- packages/spec/src/data/data-engine.zod.ts | 199 ++++++++++++++++++ 6 files changed, 310 insertions(+), 75 deletions(-) diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 2a53643c381..6088ee06424 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -90,8 +90,9 @@ import { SEARCHABLE_TEXTUAL_TYPES, SEARCHABLE_ENUM_TYPES, SEARCH_AUTO_EXCLUDED_FIELDS, isVirtualSearchField, classifyDottedFilterHead, - RPC_QUERY_ALIAS_SLOTS, foldQueryAliasSlots, - type QueryAliasConflict, type QueryAliasSlot, + foldQueryAliasSlots, + QUERY_TRANSPORT_ALIAS_SLOTS, QUERY_TRANSPORT_DOLLAR_ALIASES, QUERY_TRANSPORT_DOLLAR_PARAMS, + type QueryAliasConflict, type DroppedFieldsEvent, type QueryAST, type EngineQueryOptionsParsed, } from '@objectstack/spec/data'; import { PLURAL_TO_SINGULAR, SINGULAR_TO_PLURAL, canonicalMetaUrlType, metaUrlSpellingRefusal, unrecognisedMetaTypeRefusal, METADATA_ITEM_NAME_PATTERN } from '@objectstack/spec/shared'; @@ -3028,45 +3029,16 @@ const ODATA_SPELLING: Readonly> = { filter: '$filter', select: '$select', expand: '$expand', }; -/** - * [#3795] The spec's alias table ({@link RPC_QUERY_ALIAS_SLOTS}) extended with - * the wire-only spellings no schema declares: `filters` (documented plural - * alias of the `filter` transport param) and the OData `$filter` / `$expand`. - * Every spelling of one QueryAST slot resolves through ONE fold — the four - * slots that used to resolve backwards (canonical consulted last), each in its - * own open-coded way, are the reason the table lives in the spec and not here. - */ -const WIRE_QUERY_ALIAS_SLOTS: readonly QueryAliasSlot[] = (() => { - const extra: Record = { - where: ['filters', '$filter'], - expand: ['$expand'], - }; - return RPC_QUERY_ALIAS_SLOTS.map((slot) => ({ - canonical: slot.canonical, - aliases: [...slot.aliases, ...(extra[slot.canonical] ?? [])], - })); -})(); - -/** - * The OData `$`-prefixed spelling of each bare wire parameter this normalizer - * consumes, hoisted out of the loop in `findData` that used to own it so the - * arity survey below and that loop read ONE table (#7321). Adding a `$` alias - * in one place and not the other is exactly how a parameter ends up folded but - * unchecked. - * - * `$filter` / `$expand` are deliberately absent: they are declared as slot - * aliases on {@link WIRE_QUERY_ALIAS_SLOTS} instead, because they fold straight - * to a canonical key rather than to a bare wire spelling. - */ -const WIRE_DOLLAR_ALIASES: readonly (readonly [string, string])[] = [ - ['$top', 'top'], - ['$skip', 'skip'], - ['$orderby', 'orderBy'], - ['$select', 'select'], - ['$count', 'count'], - ['$search', 'search'], - ['$searchFields', 'searchFields'], -]; +// [#16066] The transport alias tables this normalizer folds by are DECLARED, +// and they are declared in the spec — `QUERY_TRANSPORT_ALIAS_SLOTS` and +// `QUERY_TRANSPORT_DOLLAR_ALIASES` (`@objectstack/spec/data`), imported above. +// +// They used to live here as module-private `WIRE_QUERY_ALIAS_SLOTS` / +// `WIRE_DOLLAR_ALIASES`: the spec's own table extended, in this file, with +// spellings no schema named. That extension is what made the `findData` door +// accept a second vocabulary through a slot declaring only the first — two +// dialects, one slot, one of them declared. Folding by the spec's export is +// the single source that closes it; ⛔ do not re-extend a table here. /** * [#7321] The list-query slots whose DECLARED value type admits an array, by @@ -3121,18 +3093,18 @@ const ARRAY_VALUED_QUERY_SLOTS: readonly string[] = [ ]; /** - * [#7321] {@link ARRAY_VALUED_QUERY_SLOTS} expanded to every WIRE spelling that + * [#7321] {@link ARRAY_VALUED_QUERY_SLOTS} expanded to every TRANSPORT spelling that * reaches it, derived from the same two tables the fold uses so a new alias * cannot silently lose its array arm — the failure mode would be `?$select=a& * $select=b` starting to 400, i.e. the damage case this card exists to avoid. */ const ARRAY_VALUED_LIST_QUERY_PARAMS: ReadonlySet = (() => { const names = new Set(ARRAY_VALUED_QUERY_SLOTS); - for (const slot of WIRE_QUERY_ALIAS_SLOTS) { + for (const slot of QUERY_TRANSPORT_ALIAS_SLOTS) { if (!names.has(slot.canonical)) continue; for (const alias of slot.aliases) names.add(alias); } - for (const [dollar, bare] of WIRE_DOLLAR_ALIASES) { + for (const [dollar, bare] of QUERY_TRANSPORT_DOLLAR_ALIASES) { if (names.has(bare)) names.add(dollar); } return names; @@ -10609,10 +10581,10 @@ export class ObjectStackProtocolImplementation implements // "'orderBy' is invalid" names a parameter absent from their request. // // [#7321] The table itself now lives at module scope - // ({@link WIRE_DOLLAR_ALIASES}) so the arity survey above and this fold + // ({@link QUERY_TRANSPORT_DOLLAR_ALIASES}) so the arity survey above and this fold // cannot drift apart on which `$` spellings exist. const wireSpelling: Record = {}; - for (const [dollar, bare] of WIRE_DOLLAR_ALIASES) { + for (const [dollar, bare] of QUERY_TRANSPORT_DOLLAR_ALIASES) { if (options[dollar] != null && options[bare] == null) { options[bare] = options[dollar]; wireSpelling[bare] = dollar; @@ -10635,7 +10607,7 @@ export class ObjectStackProtocolImplementation implements // composed with `wireSpelling` it names the parameter the caller // actually wrote in every rejection below (#4226). const spellingFor = (name: string): string => wireSpelling[name] ?? name; - const arrivedAs = foldQueryAliasSlots(options, WIRE_QUERY_ALIAS_SLOTS, (conflict) => { + const arrivedAs = foldQueryAliasSlots(options, QUERY_TRANSPORT_ALIAS_SLOTS, (conflict) => { throw conflictingQueryParamsError(conflict, spellingFor); }); const slotParam = (canonical: string): string => spellingFor(arrivedAs[canonical] ?? canonical); @@ -10830,9 +10802,13 @@ export class ObjectStackProtocolImplementation implements // UNFILTERED page — a footgun for scripts resolving ids by name). const unsupportedDollarParams = Object.keys(options).filter((k) => k.startsWith('$')); if (unsupportedDollarParams.length > 0) { + // [#16066] The supported set is QUOTED from the declaration, never + // re-typed here: a spelling added to the spec table used to leave + // this sentence naming a set the door no longer had, so the caller + // was told to use a parameter list that was already wrong. const err: any = new Error( `Unsupported query parameter(s): ${unsupportedDollarParams.join(', ')}. ` + - 'Supported $-prefixed parameters: $top, $skip, $orderby, $select, $count, $search, $searchFields, $filter, $expand.', + `Supported $-prefixed parameters: ${QUERY_TRANSPORT_DOLLAR_PARAMS.join(', ')}.`, ); err.status = 400; err.code = 'UNSUPPORTED_QUERY_PARAM'; @@ -10979,7 +10955,7 @@ export class ObjectStackProtocolImplementation implements // it, because the strip is what kept it from ever being honoured: the // parameter has been declared (`ODataQuerySchema.$count`, // `packages/spec/src/api/odata.zod.ts`), aliased (`$count` → `count`, - // {@link WIRE_DOLLAR_ALIASES}), reserved from the implicit-field-filter + // {@link QUERY_TRANSPORT_DOLLAR_ALIASES}), reserved from the implicit-field-filter // bucket ({@link RESERVED_LIST_QUERY_PARAMS}), arity-checked and boolean- // coerced — and then deleted unread, so every list request paid for the // COUNT query below whether or not the caller wanted a `total`. diff --git a/packages/rest/src/rest-server-canonical-query-ast.test.ts b/packages/rest/src/rest-server-canonical-query-ast.test.ts index 6e9062f1b63..021495779c3 100644 --- a/packages/rest/src/rest-server-canonical-query-ast.test.ts +++ b/packages/rest/src/rest-server-canonical-query-ast.test.ts @@ -38,6 +38,18 @@ * the type-level half that says the canonical shapes are the ones the contract * actually declares. * + * ⭐ [#16066] §2 moved when the SPEC half of this pair landed. The transport + * spelling is declared now — `QueryTransportParamsSchema` plus the alias table + * in `@objectstack/spec/data`, with `FindDataRequest['query']` declaring the + * AST or that spelling as its input and the AST as its output — so §2 no longer + * asserts that `$filter` / `$top` / `filters` are refused. It asserts the + * BOUNDARY of the widening instead: the spellings the table names assign, and + * a `$` name it does not (`$sort`) still does not. ⚠️ §1 is untouched by that + * and must stay untouched: what `rest-server.ts` and `import-runner.ts` are + * allowed to BUILD is a separate question from what the contract admits, and + * the answer is still the canonical AST — a server-built literal has no wire to + * arrive from. + * * ⚠️ The FOURTH literal is the reason §1 is phrased over the whole file rather * than over three named sites. `loadImportJob` built `{ $filter, $top }` and * handed it to a `p: any` handle — type-checked by nothing, named by no card, @@ -362,7 +374,7 @@ describe('[#16952] §1b the exported `ImportProtocolLike` declares what it is ha type Query = NonNullable; describe('[#16337] §2 the declared `FindDataRequest[\'query\']` contract', () => { - it('admits the canonical AST and refuses every wire spelling (compile-time)', () => { + it('admits the canonical AST and the declared transport spelling, and refuses what the table does not name (compile-time)', () => { const canonical: Query = { object: 'sys_import_job', where: { status: 'queued' }, @@ -374,27 +386,36 @@ describe('[#16337] §2 the declared `FindDataRequest[\'query\']` contract', () = }; expect(canonical.limit).toBe(50); - // Each directive below is LIVE: `tsconfig.test.json` compiles this - // layer, and an unused `@ts-expect-error` is TS2578 there. So these are - // assertions, not decoration — if the wire dialect were ever declared - // on `QuerySchema`, this block reds rather than going quiet. - // @ts-expect-error `$top` is not a declared QueryAST key + // ⭐ [#16066] The transport spelling is DECLARED now — the second half + // of the split this file's card names. `QuerySchema` itself is + // untouched (nothing below is an AST member); what widened is the + // `FindDataRequest['query']` INPUT, which admits the canonical AST, its + // transport spelling, or a bag carrying both, and whose OUTPUT is still + // the AST. So these three are assignments, not `@ts-expect-error`s. const dollarTop: Query = { object: 'x', $top: 5 }; - // @ts-expect-error [#16638] `$filter` is not a declared QueryAST key const dollarFilter: Query = { object: 'x', $filter: { id: '1' } }; - // @ts-expect-error `filters` is not a declared QueryAST key - const wireFilters: Query = { object: 'x', filters: [] }; - // @ts-expect-error `select` is the alias; the declared key is `fields` + const wireFilters: Query = { object: 'x', filters: '{"id":"1"}' }; + // ⭐ …and a bag mixing the two spellings, which is what a boundary that + // merges a caller's parameters with its own actually assembles. + const mixed: Query = { object: 'x', where: { status: 'queued' }, $top: 5 }; + + // Each directive below is LIVE: `tsconfig.test.json` compiles this + // layer, and an unused `@ts-expect-error` is TS2578 there. So these are + // assertions, not decoration — and they are the boundary of the + // widening: what the transport alias table does NOT name stays refused. + // @ts-expect-error `select` is an RPC alias, not a transport spelling; the declared key is `fields` const wireSelect: Query = { object: 'x', select: ['id'] }; - // @ts-expect-error `sort` is the alias; the declared key is `orderBy` + // @ts-expect-error `sort` is an RPC alias, not a transport spelling; the declared key is `orderBy` const wireSort: Query = { object: 'x', sort: [{ field: 'a', order: 'asc' }] }; - // @ts-expect-error the `{field: direction}` record is not `SortNode[]` + // @ts-expect-error the `{field: direction}` record is not `SortNode[]` — `$orderby` is the slot that takes it const recordSort: Query = { object: 'x', orderBy: { created_at: 'desc' } }; - // @ts-expect-error a comma list is not `Record` + // @ts-expect-error a comma list is not `Record` — `$expand` is the slot that takes it const commaExpand: Query = { object: 'x', expand: 'owner_id' }; - // @ts-expect-error `object` is REQUIRED on the declared query + // @ts-expect-error `object` is REQUIRED on the declared query, in either spelling const noObject: Query = { limit: 1 }; - expect([dollarTop, dollarFilter, wireFilters, wireSelect, wireSort, recordSort, commaExpand, noObject]).toHaveLength(8); + // @ts-expect-error `$sort` is not a spelling the transport table names — declaring the dialect did not open `$*` + const unnamedDollar: Query = { object: 'x', $sort: 'name' }; + expect([dollarTop, dollarFilter, wireFilters, mixed, wireSelect, wireSort, recordSort, commaExpand, noObject, unnamedDollar]).toHaveLength(10); }); }); @@ -412,7 +433,7 @@ type CreateArgs = Parameters[0]; type UpdateArgs = Parameters[0]; describe('[#16952] §2 the declared `ImportProtocolLike` parameter contract', () => { - it('admits what the runner sends, and refuses the wire dialect (compile-time)', () => { + it('admits what the runner sends and the declared transport spelling, and refuses what the table does not name (compile-time)', () => { // Exactly the three literals `import-runner.ts` builds, envelope included. const find: FindArgs = { object: 'sys_user', @@ -424,20 +445,25 @@ describe('[#16952] §2 the declared `ImportProtocolLike` parameter contract', () const update: UpdateArgs = { object: 'task', id: 'id_1', data: { name: 'r0' }, context: {} }; expect([find.object, create.object, update.id]).toEqual(['sys_user', 'task', 'id_1']); - // ⭐ The dialect an implementor used to freeze on, now refused at the - // extension point rather than observed from it. Each directive is LIVE - // — an unused `@ts-expect-error` is TS2578 under `tsconfig.test.json`. - // @ts-expect-error `$filter` is the wire spelling; the declared key is `where` + // ⭐ [#16066] The dialect an implementor used to freeze on reaches the + // extension point DECLARED: it inherits `FindDataRequest['query']`, so + // the transport spelling type-checks here for the same reason it does + // on the request itself. The extension point is not restating the + // dialect — it is reading the one declaration. const wireFilter: FindArgs = { object: 'sys_user', query: { object: 'sys_user', $filter: { email: 'a@b.c' } } }; - // @ts-expect-error `$top` is the wire spelling; the declared key is `limit` const wireTop: FindArgs = { object: 'sys_user', query: { object: 'sys_user', $top: 2 } }; + + // Each directive below is LIVE — an unused `@ts-expect-error` is TS2578 + // under `tsconfig.test.json`. + // @ts-expect-error a spelling the transport table does not name stays refused at the extension point too + const unnamedDollar: FindArgs = { object: 'sys_user', query: { object: 'sys_user', $sort: 'email' } }; // @ts-expect-error `object` is REQUIRED on every declared request const noObject: FindArgs = { query: { object: 'sys_user', where: {} } }; // @ts-expect-error `data` is REQUIRED on a create const noData: CreateArgs = { object: 'task' }; // @ts-expect-error `id` is REQUIRED on an update const noId: UpdateArgs = { object: 'task', data: { name: 'r0' } }; - expect([wireFilter, wireTop, noObject, noData, noId]).toHaveLength(5); + expect([wireFilter, wireTop, unnamedDollar, noObject, noData, noId]).toHaveLength(6); }); it('an implementor written against the declaration needs no annotation of its own', () => { diff --git a/packages/spec/authorable-surface/data.json b/packages/spec/authorable-surface/data.json index ce9a8eafbcc..3a05812244e 100644 --- a/packages/spec/authorable-surface/data.json +++ b/packages/spec/authorable-surface/data.json @@ -784,6 +784,16 @@ "data/Query:where", "data/Query:windowFunctions [RETIRED]", "data/QueryFilter:where", + "data/QueryTransportParams:$count", + "data/QueryTransportParams:$expand", + "data/QueryTransportParams:$filter", + "data/QueryTransportParams:$orderby", + "data/QueryTransportParams:$search", + "data/QueryTransportParams:$searchFields", + "data/QueryTransportParams:$select", + "data/QueryTransportParams:$skip", + "data/QueryTransportParams:$top", + "data/QueryTransportParams:filters", "data/RangeOperator:$between", "data/ReferenceResolution:field", "data/ReferenceResolution:fieldType", diff --git a/packages/spec/json-schema.manifest/data.json b/packages/spec/json-schema.manifest/data.json index 8fa311f473b..53ebe1d6f17 100644 --- a/packages/spec/json-schema.manifest/data.json +++ b/packages/spec/json-schema.manifest/data.json @@ -138,6 +138,8 @@ "data/PostgresConfig", "data/Query", "data/QueryFilter", + "data/QueryTransportParams", + "data/QueryWithTransport", "data/RangeOperator", "data/ReferenceIdValue", "data/ReferenceResolution", diff --git a/packages/spec/src/api/protocol.zod.ts b/packages/spec/src/api/protocol.zod.ts index 98faecbf8c9..c21f9e144c6 100644 --- a/packages/spec/src/api/protocol.zod.ts +++ b/packages/spec/src/api/protocol.zod.ts @@ -10,10 +10,10 @@ import { DeleteManyRequestSchema, } from './batch.zod'; import { MetadataCacheRequestSchema, MetadataCacheResponseSchema } from './http-cache.zod'; -import { QuerySchema, QUERY_DISTINCT_REMOVED } from '../data/query.zod'; +import { QUERY_DISTINCT_REMOVED } from '../data/query.zod'; import { retiredKey } from '../shared/retired-key'; import { MetadataItemNameSchema } from '../shared/identifiers.zod'; -import { DroppedFieldsEventSchema } from '../data/data-engine.zod'; +import { DroppedFieldsEventSchema, QueryWithTransportSchema } from '../data/data-engine.zod'; import { AnalyticsQueryRequestSchema, AnalyticsResultResponseSchema, @@ -1851,7 +1851,26 @@ export const GetUiViewResponseSchema = lazySchema(() => ViewSchema); * Find Data Request * Defines a query to retrieve records from a specific object. * Supports filtering, sorting, pagination, and field selection. - * + * + * ## Two spellings, one semantics + * + * `query` declares its INPUT as the canonical QueryAST **or** its transport + * spelling ({@link QueryTransportParamsSchema} — `$filter` / `$top` / `$skip` / + * `$orderby` / `$select` / `$expand` and the plural `filters`), and its OUTPUT + * as the AST: the transport keys are folded onto their canonical slots by + * {@link QUERY_TRANSPORT_ALIAS_SLOTS} / {@link QUERY_TRANSPORT_DOLLAR_ALIASES} + * at parse. The transport form is the FLATTENED SPELLING of the same AST with + * a 1:1 alias table — ⛔ never a second query semantics, and nothing is + * admitted into `QuerySchema` itself. + * + * This records what the door already accepted. `@objectstack/metadata-protocol` + * has folded these spellings since #3795 while the slot declared the AST alone, + * so every caller speaking them — objectui's published `QueryParams`, the + * `GET /data/:object` querystring — was unverifiable at build time and + * unrejected at runtime. A `$`-prefixed name the table does NOT carry is + * refused at the boundary with `400 UNSUPPORTED_QUERY_PARAM` + * ({@link QUERY_TRANSPORT_DOLLAR_PARAMS} is the quoted set). + * * @example * { * "object": "customers", @@ -1861,10 +1880,13 @@ export const GetUiViewResponseSchema = lazySchema(() => ViewSchema); * "limit": 10 * } * } + * @example + * // the same query in the transport spelling + * { "object": "customers", "query": { "$filter": { "status": "active" }, "$top": 10 } } */ export const FindDataRequestSchema = lazySchema(() => z.object({ object: z.string().describe('The unique machine name of the object to query (e.g. "account").'), - query: QuerySchema.optional().describe('Structured query definition (filter, sort, select, pagination).'), + query: QueryWithTransportSchema.optional().describe('Structured query definition (filter, sort, select, pagination) — the canonical QueryAST or its transport spelling.'), })); /** diff --git a/packages/spec/src/data/data-engine.zod.ts b/packages/spec/src/data/data-engine.zod.ts index 22a38a55b49..32b63d0810a 100644 --- a/packages/spec/src/data/data-engine.zod.ts +++ b/packages/spec/src/data/data-engine.zod.ts @@ -3,6 +3,7 @@ import { z } from 'zod'; import { FilterConditionSchema } from './filter.zod'; import { SortNodeSchema, QuerySchema, FullTextSearchSchema, FieldNodeSchema, AggregationNodeSchema, GroupByNodeSchema, QUERY_CURSOR_REMOVED, QUERY_DISTINCT_REMOVED } from './query.zod'; +import type { QueryAST, QueryInput } from './query.zod'; import { retiredKey } from '../shared/retired-key'; import { ExecutionContextSchema } from '../kernel/execution-context.zod'; @@ -674,6 +675,204 @@ const RpcQueryOptionsSchema = EngineQueryOptionsSchema.extend({ populate: z.array(z.string()).optional(), }).transform((options, ctx) => foldRpcQueryOptions(options, ctx)); +// ========================================================================== +// 2c. The QUERY TRANSPORT dialect — the FLATTENED SPELLING of the QueryAST +// ========================================================================== +// +// One vocabulary, two spellings, ONE semantics. Everything below is a +// key-level alias of a canonical QueryAST slot; nothing here means anything +// the AST cannot already say, which is the whole reason it can be declared +// without creating the second de-facto contract Prime Directive #12 warns +// about. ⛔ A spelling that would need its own meaning does not belong here — +// it belongs in the AST or nowhere. + +/** + * The transport-only alias spellings, by canonical QueryAST key — the ones a + * transport boundary speaks that {@link RPC_QUERY_ALIAS_SLOTS} does not carry. + * + * `filters` is the documented plural of the `filter` transport parameter; + * `$filter` / `$expand` are the OData spellings that fold STRAIGHT onto a + * canonical key rather than onto a bare one, which is why they are slot + * aliases here and not rows in {@link QUERY_TRANSPORT_DOLLAR_ALIASES}. + */ +const QUERY_TRANSPORT_ONLY_SLOT_ALIASES: Readonly> = { + where: ['filters', '$filter'], + expand: ['$expand'], +}; + +/** + * {@link RPC_QUERY_ALIAS_SLOTS} extended with the transport-only spellings — + * the alias table every transport boundary folds by, declared ONCE. + * + * Before this export the extension was re-declared inside + * `@objectstack/metadata-protocol` as a module-private `WIRE_QUERY_ALIAS_SLOTS`, + * so the `findData` door accepted a vocabulary no schema named: two dialects + * on one slot, one of them declared. A caller speaking the transport spelling + * was unverifiable at build time and unrejected at runtime. The table is the + * single source now; the normalizer folds by THIS array. + */ +export const QUERY_TRANSPORT_ALIAS_SLOTS: readonly QueryAliasSlot[] = RPC_QUERY_ALIAS_SLOTS.map( + (slot) => ({ + canonical: slot.canonical, + aliases: [...slot.aliases, ...(QUERY_TRANSPORT_ONLY_SLOT_ALIASES[slot.canonical] ?? [])], + }), +); + +/** + * The OData `$`-prefixed spelling of each BARE transport parameter, as + * `[dollarSpelling, bareSpelling]` pairs. + * + * NOT derivable by prefixing `$`: `sort` is `$orderby`. These fold in TWO hops + * — `$top` lands on `top`, which {@link QUERY_TRANSPORT_ALIAS_SLOTS} then folds + * onto `limit` — so a reader comparing only the slot table would miss them. + * + * `count` is the one bare spelling here that is not a QueryAST slot at all: it + * is the response's total-count flag, read beside the query rather than inside + * it. It is declared here because the transport spells it `$count` and the fold + * has to consume it; it does not become an AST member by being named. + */ +export const QUERY_TRANSPORT_DOLLAR_ALIASES: readonly (readonly [string, string])[] = [ + ['$top', 'top'], + ['$skip', 'skip'], + ['$orderby', 'orderBy'], + ['$select', 'select'], + ['$count', 'count'], + ['$search', 'search'], + ['$searchFields', 'searchFields'], +]; + +/** + * Every `$`-prefixed spelling the transport declares — the two-hop ones in + * table order, then the slot aliases in slot order. + * + * This is the set a boundary quotes when it refuses an UNDECLARED `$` name. A + * `$`-prefixed key can never be a field name, so anything outside this set is + * refused loudly instead of falling into the implicit-field-filter bucket, + * where it matched zero rows under a 200 and no caller could see it. + */ +export const QUERY_TRANSPORT_DOLLAR_PARAMS: readonly string[] = [ + ...QUERY_TRANSPORT_DOLLAR_ALIASES.map(([dollar]) => dollar), + ...QUERY_TRANSPORT_ALIAS_SLOTS.flatMap((slot) => slot.aliases.filter((a) => a.startsWith('$'))), +]; + +/** + * The transport query parameters — the `$`-prefixed spellings and the plural + * `filters`, each carrying the value of the canonical slot it folds onto. + * + * ⛔ This is NOT the hint table `@objectstack/metadata-protocol` keeps for + * near-miss suggestions; nothing in that table is accepted as input and it + * stays that way. This one IS accepted — it is what the `findData` door has + * folded since #3795, declared at last. + * + * Every member is optional and every value type is the union of the shapes the + * boundary already serves on that slot: the structured form a JSON body sends + * and the string form a querystring sends (`?$top=50` arrives as `'50'`, + * `?$select=a,b` as a comma list). Declaring the narrower structured form alone + * would have turned live traffic into a `400`, which is the repair #15866's + * parent thread closed for exactly this reason. + */ +export const QueryTransportParamsSchema = lazySchema(() => z.object({ + /** OData spelling of `where`. */ + $filter: z.union([z.string(), DataEngineFilterSchema]).optional() + .describe('Transport spelling of `where` (OData `$filter`)'), + /** Documented plural of the `filter` transport parameter; folds onto `where`. */ + filters: z.union([z.string(), DataEngineFilterSchema]).optional() + .describe('Transport spelling of `where` (plural of `filter`)'), + /** OData spelling of `limit` (two hops: `$top` to `top` to `limit`). */ + $top: z.union([z.number(), z.string()]).optional() + .describe('Transport spelling of `limit` (OData `$top`)'), + /** OData spelling of `offset` (two hops: `$skip` to `skip` to `offset`). */ + $skip: z.union([z.number(), z.string()]).optional() + .describe('Transport spelling of `offset` (OData `$skip`)'), + /** OData spelling of `orderBy`. */ + $orderby: z.union([z.string(), z.array(z.string()), DataEngineSortSchema]).optional() + .describe('Transport spelling of `orderBy` (OData `$orderby`)'), + /** OData spelling of `fields` (two hops: `$select` to `select` to `fields`). */ + $select: z.union([z.string(), z.array(FieldNodeSchema)]).optional() + .describe('Transport spelling of `fields` (OData `$select`)'), + /** OData spelling of `expand`. */ + $expand: z.union([z.string(), z.array(z.string()), z.record(z.string(), QuerySchema)]).optional() + .describe('Transport spelling of `expand` (OData `$expand`)'), + /** OData spelling of `search`. */ + $search: z.union([z.string(), FullTextSearchSchema]).optional() + .describe('Transport spelling of `search` (OData `$search`)'), + /** OData spelling of `searchFields`. */ + $searchFields: z.union([z.string(), z.array(z.string())]).optional() + .describe('Transport spelling of `searchFields` (OData `$searchFields`)'), + /** OData spelling of the response total-count flag. */ + $count: z.union([z.boolean(), z.string()]).optional() + .describe('Transport spelling of the response total-count flag (OData `$count`)'), +}).describe('Transport spellings of the QueryAST slots — folded to canonical keys at the boundary')); + +/** + * Fold a transport bag onto canonical QueryAST keys, by the two tables above. + * + * The same two steps, in the same order, that the protocol normalizer runs on + * raw wire input: `$` spellings onto their bare parameter first (a `$` alias + * never overwrites a bare spelling that is already present), then every alias + * onto its canonical slot via {@link foldQueryAliasSlots}. Conflicting + * spellings of ONE slot are reported, never resolved. + * + * Deliberately a fold over the shared TABLES rather than a second copy of the + * normalizer: this is the parse-time application, exactly as + * `foldRpcQueryOptions` is the parse-time application of + * {@link RPC_QUERY_ALIAS_SLOTS}. Value SHAPES are moved verbatim — lowering a + * comma list or a `{field: direction}` record is the boundary's job, one layer + * down, and doing it twice is how two implementations of one rule start. + */ +function foldQueryTransportBag( + input: Record, + onConflict: (conflict: QueryAliasConflict) => void, +): Record { + const bag: Record = { ...input }; + for (const [dollar, bare] of QUERY_TRANSPORT_DOLLAR_ALIASES) { + if (bag[dollar] != null && bag[bare] == null) bag[bare] = bag[dollar]; + delete bag[dollar]; + } + foldQueryAliasSlots(bag, QUERY_TRANSPORT_ALIAS_SLOTS, onConflict); + return bag; +} + +/** The transport query parameters, as an input shape. */ +export type QueryTransportParams = z.input; + +/** + * What a query slot that serves a transport boundary DECLARES as its input: + * the canonical QueryAST, the transport spelling, or a bag carrying both. + * + * The third arm is not generosity — it is what the door does. A boundary + * assembles one bag out of a caller's parameters and its own, so `where` and + * `$filter` can arrive together; the fold collapses them when they agree and + * refuses them when they do not. Declaring only the two pure arms would have + * re-created this card's own defect one size down: a shape the door accepts + * that the declaration denies. + */ +export type QueryWithTransportInput = QueryInput & Partial; + +/** + * A query slot whose declared INPUT is the canonical AST or its transport + * spelling, and whose declared OUTPUT is the AST. + * + * The first arm folds by {@link QUERY_TRANSPORT_ALIAS_SLOTS} / + * {@link QUERY_TRANSPORT_DOLLAR_ALIASES} and then validates the result as a + * real {@link QuerySchema}, so a transport bag parses to canonical keys. The + * second arm is `QuerySchema` itself, unchanged — which is what makes this + * widening provably unable to narrow anything: every input that parsed before + * still parses, by the same schema, with the same issues. + */ +export const QueryWithTransportSchema: z.ZodType = lazySchema(() => + z.union([ + z.record(z.string(), z.unknown()) + .transform((bag, ctx) => foldQueryTransportBag( + bag as Record, + (conflict) => ctx.addIssue(aliasConflictIssue(conflict)), + )) + .pipe(QuerySchema as unknown as z.ZodType>), + QuerySchema, + ]) as unknown as z.ZodType, +); + + export const DataEngineFindRequestSchema = lazySchema(() => z.object({ method: z.literal('find'), object: z.string(), From d92ffb73fb2876439f077ba03ff4ab98d2018c1a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Sep 2026 15:38:46 +0000 Subject: [PATCH 02/10] test(metadata-protocol): pin the transport fold byte-equal across the hoist Claude-Session: https://claude.ai/code/session_01LvwGppdonww4zGLWZo5rho Co-authored-by: Claude --- content/docs/references/api/protocol.mdx | 26 +- content/docs/references/data/data-engine.mdx | 82 +++++- content/docs/references/index.mdx | 10 +- ...07-unknown-key-strictness-ledger.counts.md | 22 +- .../protocol.query-transport-dialect.test.ts | 252 ++++++++++++++++++ packages/spec/api-surface/data.json | 8 + packages/spec/declaration-map/data.json | 4 + packages/spec/export-origins/data.json | 8 + packages/spec/src/data/data-engine.zod.ts | 7 + scripts/check-filter-alias-parity.mjs | 139 +++++----- 10 files changed, 474 insertions(+), 84 deletions(-) create mode 100644 packages/metadata-protocol/src/protocol.query-transport-dialect.test.ts diff --git a/content/docs/references/api/protocol.mdx b/content/docs/references/api/protocol.mdx index 5047ed154e5..d4ba842016d 100644 --- a/content/docs/references/api/protocol.mdx +++ b/content/docs/references/api/protocol.mdx @@ -867,9 +867,31 @@ Enable package response | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | **object** | `string` | ✅ | The unique machine name of the object to query (e.g. "account"). | -| **query** | `{ object: string; fields?: string[]; where?: any; search?: string \| object; … }` | optional | Structured query definition (filter, sort, select, pagination). | +| **query** | `{ object: string; fields?: string[]; where?: any; search?: string \| object; … } \| … +1 more` | optional | Structured query definition (filter, sort, select, pagination) — the canonical QueryAST or its transport spelling. | -### Nested Shape: `FindDataRequest.query` +### Nested Shape: `FindDataRequest.query[option 1]` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **object** | `string` | ✅ | Object name (e.g. account) | +| **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | +| **where** | `any` | optional | Filtering criteria (WHERE) | +| **search** | `string \| { query: string; fields?: string[]; fuzzy: boolean; operator: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | +| **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | +| **orderBy** | `{ field: string; order: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | +| **limit** | `number` | optional | Max records to return (LIMIT) | +| **offset** | `number` | optional | Records to skip (OFFSET) | +| **top** | `number` | optional | Alias for limit (OData compatibility) | +| **cursor** | `never` | optional | [REMOVED] `query.cursor` was removed in @objectstack/spec 17 (ADR-0049) — no driver ever implemented keyset pagination, so the cursor was accepted and ignored and every page came back identical (a caller looping "until hasMore is false" never terminates). Delete the key; `QueryBuilder.cursor()` was removed with it. Express the keyset as an ordinary `where` predicate on your sort key — `where: { created_at: { $gt: last.created_at } }` with the matching `orderBy` — which every driver executes with canonicalised comparands. A first-class cursor, if ever built, will be a response-minted opaque token, not this caller-built record. | +| **joins** | `never` | optional | [REMOVED] `query.joins` was removed in @objectstack/spec 17 (ADR-0049) — no engine or driver ever read it: a query carrying `joins` behaved exactly as if the key were absent, while its name squatted on the reserved REST parameter set. Delete the key. Related records are read through `expand` — `expand: { owner_id: { object: 'user', fields: ['name'] } }` — which the engine resolves via batch $in queries, and whose nested query selects the related record's own columns. Keep the foreign key in your own projection (`fields: ['title', 'owner_id']`): the relation is carried by that column, so projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement — no driver ever resolved one and the ingress refuses it (`400 INVALID_FIELD`). | +| **aggregations** | `{ function: Enum<'count' \| 'sum' \| 'avg' \| 'min' \| 'max' \| 'count_distinct'>; field?: string; alias: string; filter?: any }[]` | optional | Aggregation functions | +| **groupBy** | `(string \| { field: string; dateGranularity?: Enum<'day' \| 'week' \| 'month' \| 'quarter' \| 'year'>; alias?: string })[]` | optional | GROUP BY targets (strings or `{field, dateGranularity?}` objects for date bucketing) | +| **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | +| **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | +| **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | +| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select (`fields`) and filter (`where`, AND-merged with the batch $in), plus further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3; per-parent `limit`/`offset`/`orderBy` are NOT applied on this path. | + +### Nested Shape: `FindDataRequest.query[option 2]` | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | diff --git a/content/docs/references/data/data-engine.mdx b/content/docs/references/data/data-engine.mdx index 266477b513f..b71c9c52146 100644 --- a/content/docs/references/data/data-engine.mdx +++ b/content/docs/references/data/data-engine.mdx @@ -21,8 +21,8 @@ The Data Engine acts as the "Driver" layer in the Hexagonal Architecture. ## TypeScript Usage ```typescript -import { BaseEngineOptionsSchema, DataEngineAggregateOptionsSchema, DataEngineAggregateRequestSchema, DataEngineCountOptionsSchema, DataEngineCountRequestSchema, DataEngineDeleteOptionsSchema, DataEngineDeleteRequestSchema, DataEngineExecuteRequestSchema, DataEngineFilterSchema, DataEngineFindOneRequestSchema, DataEngineFindRequestSchema, DataEngineInsertOptionsSchema, DataEngineInsertRequestSchema, DataEngineQueryOptionsSchema, DataEngineRequestSchema, DataEngineSortSchema, DataEngineUpdateOptionsSchema, DataEngineUpdateRequestSchema, DataEngineVectorFindRequestSchema, DroppedFieldsEventSchema, EngineAggregateOptionsSchema, EngineCountOptionsSchema, EngineDeleteOptionsSchema, EngineQueryOptionsSchema, EngineUpdateOptionsSchema } from '@objectstack/spec/data'; -import type { BaseEngineOptions, DataEngineAggregateOptions, DataEngineCountOptions, DataEngineDeleteOptions, DataEngineExecuteRequest, DataEngineFilter, DataEngineInsertOptions, DataEngineInsertRequest, DataEngineQueryOptions, DataEngineRequest, DataEngineSort, DataEngineUpdateOptions, DataEngineVectorFindRequest, DroppedFieldsEvent, EngineAggregateOptions, EngineCountOptions, EngineDeleteOptions, EngineQueryOptions, EngineUpdateOptions } from '@objectstack/spec/data'; +import { BaseEngineOptionsSchema, DataEngineAggregateOptionsSchema, DataEngineAggregateRequestSchema, DataEngineCountOptionsSchema, DataEngineCountRequestSchema, DataEngineDeleteOptionsSchema, DataEngineDeleteRequestSchema, DataEngineExecuteRequestSchema, DataEngineFilterSchema, DataEngineFindOneRequestSchema, DataEngineFindRequestSchema, DataEngineInsertOptionsSchema, DataEngineInsertRequestSchema, DataEngineQueryOptionsSchema, DataEngineRequestSchema, DataEngineSortSchema, DataEngineUpdateOptionsSchema, DataEngineUpdateRequestSchema, DataEngineVectorFindRequestSchema, DroppedFieldsEventSchema, EngineAggregateOptionsSchema, EngineCountOptionsSchema, EngineDeleteOptionsSchema, EngineQueryOptionsSchema, EngineUpdateOptionsSchema, QueryTransportParamsSchema, QueryWithTransportSchema } from '@objectstack/spec/data'; +import type { BaseEngineOptions, DataEngineAggregateOptions, DataEngineCountOptions, DataEngineDeleteOptions, DataEngineExecuteRequest, DataEngineFilter, DataEngineInsertOptions, DataEngineInsertRequest, DataEngineQueryOptions, DataEngineRequest, DataEngineSort, DataEngineUpdateOptions, DataEngineVectorFindRequest, DroppedFieldsEvent, EngineAggregateOptions, EngineCountOptions, EngineDeleteOptions, EngineQueryOptions, EngineUpdateOptions, QueryTransportParams, QueryWithTransport } from '@objectstack/spec/data'; // Validate data const result = BaseEngineOptionsSchema.parse(data); @@ -1209,3 +1209,81 @@ QueryAST-aligned options for DataEngine.update operations --- +## QueryTransportParams + +Transport spellings of the QueryAST slots — folded to canonical keys at the boundary + +### Properties + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **$filter** | `string \| Record \| any` | optional | Transport spelling of `where` (OData `$filter`) | +| **filters** | `string \| Record \| any` | optional | Transport spelling of `where` (plural of `filter`) | +| **$top** | `number \| string` | optional | Transport spelling of `limit` (OData `$top`) | +| **$skip** | `number \| string` | optional | Transport spelling of `offset` (OData `$skip`) | +| **$orderby** | `string \| string[] \| Record> \| Record \| { field: string; order: Enum<'asc' \| 'desc'> }[]` | optional | Transport spelling of `orderBy` (OData `$orderby`) | +| **$select** | `string \| string[]` | optional | Transport spelling of `fields` (OData `$select`) | +| **$expand** | `string \| string[] \| Record` | optional | Transport spelling of `expand` (OData `$expand`) | +| **$search** | `string \| { query: string; fields?: string[]; fuzzy: boolean; operator: Enum<'and' \| 'or'>; … }` | optional | Transport spelling of `search` (OData `$search`) | +| **$searchFields** | `string \| string[]` | optional | Transport spelling of `searchFields` (OData `$searchFields`) | +| **$count** | `boolean \| string` | optional | Transport spelling of the response total-count flag (OData `$count`) | + +### Nested Shape: `QueryTransportParams.$expand[string]` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **object** | `string` | ✅ | Object name (e.g. account) | +| **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | +| **where** | `any` | optional | Filtering criteria (WHERE) | +| **search** | `string \| { query: string; fields?: string[]; fuzzy: boolean; operator: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | +| **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | +| **orderBy** | `{ field: string; order: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | +| **limit** | `number` | optional | Max records to return (LIMIT) | +| **offset** | `number` | optional | Records to skip (OFFSET) | +| **top** | `number` | optional | Alias for limit (OData compatibility) | +| **cursor** | `never` | optional | [REMOVED] `query.cursor` was removed in @objectstack/spec 17 (ADR-0049) — no driver ever implemented keyset pagination, so the cursor was accepted and ignored and every page came back identical (a caller looping "until hasMore is false" never terminates). Delete the key; `QueryBuilder.cursor()` was removed with it. Express the keyset as an ordinary `where` predicate on your sort key — `where: { created_at: { $gt: last.created_at } }` with the matching `orderBy` — which every driver executes with canonicalised comparands. A first-class cursor, if ever built, will be a response-minted opaque token, not this caller-built record. | +| **joins** | `never` | optional | [REMOVED] `query.joins` was removed in @objectstack/spec 17 (ADR-0049) — no engine or driver ever read it: a query carrying `joins` behaved exactly as if the key were absent, while its name squatted on the reserved REST parameter set. Delete the key. Related records are read through `expand` — `expand: { owner_id: { object: 'user', fields: ['name'] } }` — which the engine resolves via batch $in queries, and whose nested query selects the related record's own columns. Keep the foreign key in your own projection (`fields: ['title', 'owner_id']`): the relation is carried by that column, so projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement — no driver ever resolved one and the ingress refuses it (`400 INVALID_FIELD`). | +| **aggregations** | `{ function: Enum<'count' \| 'sum' \| 'avg' \| 'min' \| 'max' \| 'count_distinct'>; field?: string; alias: string; filter?: any }[]` | optional | Aggregation functions | +| **groupBy** | `(string \| { field: string; dateGranularity?: Enum<'day' \| 'week' \| 'month' \| 'quarter' \| 'year'>; alias?: string })[]` | optional | GROUP BY targets (strings or `{field, dateGranularity?}` objects for date bucketing) | +| **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | +| **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | +| **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | +| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select (`fields`) and filter (`where`, AND-merged with the batch $in), plus further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3; per-parent `limit`/`offset`/`orderBy` are NOT applied on this path. | + +### Nested Shape: `QueryTransportParams.$search` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **query** | `string` | ✅ | Search query text | +| **fields** | `string[]` | optional | Fields to search in (if not specified, searches all text fields) | +| **fuzzy** | `boolean` | optional (default: `false`) | [EXPERIMENTAL — not enforced] Fuzzy matching (tolerate typos). The ADR-0061 expansion reads only `query` + `fields`; no executor receives this flag. | +| **operator** | `Enum<'and' \| 'or'>` | optional (default: `"or"`) | [EXPERIMENTAL — not enforced] Logical operator between terms. The ADR-0061 expansion applies its own term semantics; no executor receives this flag. | +| **boost** | `Record` | optional | [EXPERIMENTAL — not enforced] Field-specific relevance boosting (field name -> boost factor). No executor scores results. | +| **minScore** | `number` | optional | [EXPERIMENTAL — not enforced] Minimum relevance score threshold. No executor scores results. | +| **language** | `string` | optional | [EXPERIMENTAL — not enforced] Language for text analysis (e.g., "en", "zh", "es"). No executor selects an analyzer. | +| **highlight** | `boolean` | optional (default: `false`) | [EXPERIMENTAL — not enforced] Search result highlighting. No executor emits highlights. | + + +--- + +## QueryWithTransport + +### Union Options + +This schema accepts one of the following structures: + +#### Option 1 + +Reference: { object: string; fields?: string[]; where?: any; search?: string | object; … } + +--- + +#### Option 2 + +Reference: { object: string; fields?: string[]; where?: any; search?: string | object; … } + +--- + + +--- + diff --git a/content/docs/references/index.mdx b/content/docs/references/index.mdx index e913a27704a..68d32a4b054 100644 --- a/content/docs/references/index.mdx +++ b/content/docs/references/index.mdx @@ -1,6 +1,6 @@ --- title: Protocol Reference -description: Every schema published by @objectstack/spec — 1527 schemas across 14 protocol modules +description: Every schema published by @objectstack/spec — 1529 schemas across 14 protocol modules --- {/* ⚠️ AUTO-GENERATED — DO NOT EDIT. Run build-docs.ts to regenerate. Hand-written docs live in the module folders under content/docs/. */} @@ -22,7 +22,7 @@ counts are sums of the rows they head. Regenerate with | [AI Protocol](/docs/references/ai) | 12 | 68 | Agents, tools, skills, RAG and knowledge sources, model registry, conversations. | | [API Protocol](/docs/references/api) | 31 | 440 | REST contracts, endpoints, routing, realtime, batch, discovery. | | [Automation Protocol](/docs/references/automation) | 14 | 74 | Flows and their nodes, approvals, ETL pipelines, webhooks, state machines, execution records. | -| [Data Protocol](/docs/references/data) | 29 | 173 | Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. | +| [Data Protocol](/docs/references/data) | 29 | 175 | Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. | | [Identity Protocol](/docs/references/identity) | 5 | 27 | Users and accounts, organizations, positions, SCIM provisioning. | | [Integration Protocol](/docs/references/integration) | 1 | 24 | The single connector protocol (ADR-0097) — catalog descriptors and provider-bound instances. | | [Kernel Protocol](/docs/references/kernel) | 30 | 159 | Plugin lifecycle and manifests, capabilities and security, metadata loading, service registry. | @@ -33,7 +33,7 @@ counts are sums of the rows they head. Regenerate with | [Studio Protocol](/docs/references/studio) | 3 | 35 | Studio designer metadata — the authoring surfaces for the protocols above. | | [System Protocol](/docs/references/system) | 33 | 272 | The runtime environment — logging, jobs, cache, metrics, notifications, i18n and compliance. | | [UI Protocol](/docs/references/ui) | 16 | 156 | Apps, pages, views, dashboards, reports, actions and themes — the ObjectUI layer. | -| **Total** | **194** | **1527** | 14 protocol modules | +| **Total** | **194** | **1529** | 14 protocol modules | --- @@ -129,7 +129,7 @@ Flows and their nodes, approvals, ETL pipelines, webhooks, state machines, execu ## Data Protocol -**Source:** `packages/spec/src/data/` · **Import:** `@objectstack/spec/data` · **29 pages, 173 schemas** +**Source:** `packages/spec/src/data/` · **Import:** `@objectstack/spec/data` · **29 pages, 175 schemas** Objects, fields, queries, filters, datasources and drivers — the ObjectQL layer. @@ -137,7 +137,7 @@ Objects, fields, queries, filters, datasources and drivers — the ObjectQL laye | :--- | :--- | | [`analytics.zod.ts`](/docs/references/data/analytics) | `AggregationMetricType`, `AnalyticsDateRange`, `AnalyticsDateRangePreset`, `AnalyticsQuery`, `Cube`, `CubeJoin`, `Dimension`, `DimensionType`, `Metric`, `TimeUpdateInterval` | | [`context-tokens.zod.ts`](/docs/references/data/context-tokens) | `ContextToken`, `ContextTokenPlaceholder` | -| [`data-engine.zod.ts`](/docs/references/data/data-engine) | `BaseEngineOptions`, `DataEngineAggregateOptions`, `DataEngineAggregateRequest`, `DataEngineCountOptions`, `DataEngineCountRequest`, `DataEngineDeleteOptions`, `DataEngineDeleteRequest`, `DataEngineExecuteRequest`, `DataEngineFilter`, `DataEngineFindOneRequest`, `DataEngineFindRequest`, `DataEngineInsertOptions`, `DataEngineInsertRequest`, `DataEngineQueryOptions`, `DataEngineRequest`, `DataEngineSort`, `DataEngineUpdateOptions`, `DataEngineUpdateRequest`, `DataEngineVectorFindRequest`, `DroppedFieldsEvent`, `EngineAggregateOptions`, `EngineCountOptions`, `EngineDeleteOptions`, `EngineQueryOptions`, `EngineUpdateOptions` | +| [`data-engine.zod.ts`](/docs/references/data/data-engine) | `BaseEngineOptions`, `DataEngineAggregateOptions`, `DataEngineAggregateRequest`, `DataEngineCountOptions`, `DataEngineCountRequest`, `DataEngineDeleteOptions`, `DataEngineDeleteRequest`, `DataEngineExecuteRequest`, `DataEngineFilter`, `DataEngineFindOneRequest`, `DataEngineFindRequest`, `DataEngineInsertOptions`, `DataEngineInsertRequest`, `DataEngineQueryOptions`, `DataEngineRequest`, `DataEngineSort`, `DataEngineUpdateOptions`, `DataEngineUpdateRequest`, `DataEngineVectorFindRequest`, `DroppedFieldsEvent`, `EngineAggregateOptions`, `EngineCountOptions`, `EngineDeleteOptions`, `EngineQueryOptions`, `EngineUpdateOptions`, `QueryTransportParams`, `QueryWithTransport` | | [`datasource.zod.ts`](/docs/references/data/datasource) | `Datasource`, `DriverDefinition`, `DriverType`, `ExternalDatasourceSettings`, `SchemaMode` | | [`date-macros.zod.ts`](/docs/references/data/date-macros) | `DateMacroPlaceholder`, `DateMacroToken` | | [`document.zod.ts`](/docs/references/data/document) | `Document`, `DocumentTemplate`, `DocumentVersion`, `ESignatureConfig` | diff --git a/docs/audits/2026-07-unknown-key-strictness-ledger.counts.md b/docs/audits/2026-07-unknown-key-strictness-ledger.counts.md index f81ac8b9742..a34fd034eeb 100644 --- a/docs/audits/2026-07-unknown-key-strictness-ledger.counts.md +++ b/docs/audits/2026-07-unknown-key-strictness-ledger.counts.md @@ -21,8 +21,8 @@ regenerate. | Measure | Value | |---|---| | Triaged directories | 5 | -| Object sites in them | 447 | -| Still-open (strip) sites | 124 | +| Object sites in them | 448 | +| Still-open (strip) sites | 125 | | Files carrying at least one | 22 | Remaining strip sites by class: @@ -31,7 +31,7 @@ Remaining strip sites by class: |---|---| | authorable — the ruling's forced scope | 1 | | unresolved — needs a per-schema verdict | 0 | -| wire / open — out of forced scope | 119 | +| wire / open — out of forced scope | 120 | | no door — no carrier, ADR-0049 territory | 3 | | no gate — carrier live, no parse | 0 | | covered — no carrier, no parse, guarded at every consumer | 1 | @@ -45,11 +45,11 @@ The `strict` column is the one the campaign schedules against; it counts both th | Dir | Sites | strict | passthrough | catchall | strip | |---|---|---|---|---|---| | `ui/` | 175 | 165 | 3 | 0 | 7 | -| `data/` | 157 | 76 | 1 | 0 | 80 | +| `data/` | 158 | 76 | 1 | 0 | 81 | | `automation/` | 68 | 43 | 0 | 1 | 24 | | `security/` | 20 | 7 | 0 | 0 | 13 | | `studio/` | 27 | 27 | 0 | 0 | 0 | -| **total** | **447** | **318** | **4** | **1** | **124** | +| **total** | **448** | **318** | **4** | **1** | **125** | ## File-level triage — site counts @@ -83,7 +83,7 @@ classify and is not listed (it becomes reportable the day it grows its first sit | File | Sites | |---|---| | `analytics.zod.ts` | 7 | -| `data-engine.zod.ts` | 13 | +| `data-engine.zod.ts` | 14 | | `datasource.zod.ts` | 6 | | `document.zod.ts` | 8 | | `driver-nosql.zod.ts` | 10 | @@ -107,7 +107,7 @@ classify and is not listed (it becomes reportable the day it grows its first sit | `seed-loader.zod.ts` | 12 | | `seed.zod.ts` | 1 | | `validation.zod.ts` | 6 | -| **total** | **157** | +| **total** | **158** | ### `automation/` — sites @@ -176,11 +176,11 @@ over it is here. ### `data/` — open -**80 strip of 157**, in 11 file(s). +**81 strip of 158**, in 11 file(s). | File | Strip | Sites | |---|---|---| -| `data-engine.zod.ts` | 13 | 13 | +| `data-engine.zod.ts` | 14 | 14 | | `document.zod.ts` | 8 | 8 | | `driver-nosql.zod.ts` | 10 | 10 | | `driver-sql.zod.ts` | 2 | 2 | @@ -191,13 +191,13 @@ over it is here. | `hook.zod.ts` | 5 | 7 | | `query.zod.ts` | 4 | 5 | | `seed-loader.zod.ts` | 12 | 12 | -| **total** | **80** | **157** | +| **total** | **81** | **158** | | Bucket | Sites | |---|---| | authorable — the ruling's forced scope | 0 | | unresolved — needs a per-schema verdict | 0 | -| wire / open — out of forced scope | 78 | +| wire / open — out of forced scope | 79 | | no door — no carrier, ADR-0049 territory | 2 | | no gate — carrier live, no parse | 0 | | covered — no carrier, no parse, guarded at every consumer | 0 | diff --git a/packages/metadata-protocol/src/protocol.query-transport-dialect.test.ts b/packages/metadata-protocol/src/protocol.query-transport-dialect.test.ts new file mode 100644 index 00000000000..97f233ed428 --- /dev/null +++ b/packages/metadata-protocol/src/protocol.query-transport-dialect.test.ts @@ -0,0 +1,252 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#16066] The query transport dialect is DECLARED, and this package folds by + * the declaration rather than by a table of its own. + * + * ## What this pins + * + * `FindDataRequestSchema.query` declared the canonical QueryAST while the + * shipped `findData` door also accepted a transport dialect no schema named: + * `$filter` / `$top` / `$skip` / `$orderby` / `$select` / `$expand` and the + * plural `filters`. Two dialects, one slot, one of them declared — so every + * caller speaking the second was unverifiable at build time and unrejected at + * runtime. + * + * The ruling declared the transport form as the FLATTENED SPELLING of the same + * AST with a 1:1 alias table, exported once from `@objectstack/spec/data`, and + * replaced this package's module-private `WIRE_QUERY_ALIAS_SLOTS` / + * `WIRE_DOLLAR_ALIASES` with that export. + * + * ## §1 is the byte-equality receipt the ruling asks for + * + * A hoist that changes what the door folds is not a hoist. §1 holds the two + * exported tables against the values the module-private ones RESOLVED TO before + * the swap, transcribed verbatim from `origin/main` at `6dfa3ea772`. ⛔ These + * literals are not a restatement of the spec table to be "kept in sync" — they + * are a frozen BEFORE reading, and the only legitimate way to change one is a + * deliberate change to what the door accepts, with its own changeset. + * + * ⚠️ A table comparison alone would be a pin that cannot fail in the way that + * matters: equal tables prove nothing if the fold stopped reading them. §2 + * therefore drives every alias through the REAL normalizer and asserts the + * option bag `engine.find` receives is the one the canonical spelling produces + * — spelling-equivalence measured against the canonical, never against a + * hand-written expectation that would have to be re-derived on every change. + */ + +import { describe, it, expect, vi } from 'vitest'; +import { readFileSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { + QUERY_TRANSPORT_ALIAS_SLOTS, + QUERY_TRANSPORT_DOLLAR_ALIASES, + QUERY_TRANSPORT_DOLLAR_PARAMS, +} from '@objectstack/spec/data'; +import { ObjectStackProtocolImplementation } from './protocol.js'; + +const HERE = dirname(fileURLToPath(import.meta.url)); + +// --------------------------------------------------------------------------- +// §1 The BEFORE reading — what the module-private tables resolved to on +// `origin/main` @ 6dfa3ea772, immediately before the swap. +// --------------------------------------------------------------------------- + +/** `WIRE_QUERY_ALIAS_SLOTS` as it resolved before #16066. */ +const BEFORE_WIRE_QUERY_ALIAS_SLOTS = [ + { canonical: 'where', aliases: ['filter', 'filters', '$filter'] }, + { canonical: 'fields', aliases: ['select'] }, + { canonical: 'orderBy', aliases: ['sort'] }, + { canonical: 'offset', aliases: ['skip'] }, + { canonical: 'expand', aliases: ['populate', '$expand'] }, + { canonical: 'limit', aliases: ['top'] }, +]; + +/** `WIRE_DOLLAR_ALIASES` as it read before #16066. */ +const BEFORE_WIRE_DOLLAR_ALIASES = [ + ['$top', 'top'], + ['$skip', 'skip'], + ['$orderby', 'orderBy'], + ['$select', 'select'], + ['$count', 'count'], + ['$search', 'search'], + ['$searchFields', 'searchFields'], +]; + +/** + * The `$`-parameter list the refusal sentence carried before #16066 — then a + * string literal typed out beside the table, now derived from it. + */ +const BEFORE_SUPPORTED_DOLLAR_SENTENCE = + '$top, $skip, $orderby, $select, $count, $search, $searchFields, $filter, $expand'; + +describe('[#16066] §1 the fold is byte-equal before and after the single-source swap', () => { + it('the exported slot table resolves to exactly what the private one did', () => { + expect(QUERY_TRANSPORT_ALIAS_SLOTS.map((s) => ({ canonical: s.canonical, aliases: [...s.aliases] }))) + .toEqual(BEFORE_WIRE_QUERY_ALIAS_SLOTS); + }); + + it('the exported `$`-alias table reads exactly what the private one did', () => { + expect(QUERY_TRANSPORT_DOLLAR_ALIASES.map((p) => [...p])).toEqual(BEFORE_WIRE_DOLLAR_ALIASES); + }); + + it('the refusal sentence quotes the same set, in the same order', () => { + expect(QUERY_TRANSPORT_DOLLAR_PARAMS.join(', ')).toBe(BEFORE_SUPPORTED_DOLLAR_SENTENCE); + }); +}); + +// --------------------------------------------------------------------------- +// §2 The fold reads THOSE tables — every alias, through the real normalizer +// --------------------------------------------------------------------------- + +const SCHEMA = { + name: 'invoice', + nameField: 'name', + fields: { + name: { name: 'name', type: 'text' }, + status: { name: 'status', type: 'text' }, + owner_id: { name: 'owner_id', type: 'lookup', reference: 'sys_user' }, + }, +}; + +const USER_SCHEMA = { name: 'sys_user', nameField: 'name', fields: { name: { name: 'name', type: 'text' } } }; + +function makeProtocol() { + const find = vi.fn(async () => [] as unknown[]); + const count = vi.fn(async () => 0); + const engine = { + registry: { + getObject: (n: string) => (n === 'invoice' ? SCHEMA : n === 'sys_user' ? USER_SCHEMA : undefined), + }, + find, + count, + aggregate: vi.fn(async () => [] as unknown[]), + }; + return { p: new ObjectStackProtocolImplementation(engine as any), find, count }; +} + +/** + * Everything one query DID: the option bag `engine.find` received, whether the + * COUNT query was issued, and the response envelope. + * + * ⚠️ The bag alone is not enough. `count` is a protocol-layer flag stripped + * before the engine sees it, so `{$count: true}` and `{count: true}` both hand + * `engine.find` an empty bag — a comparison of bags would have declared that + * pair equal without folding anything. Widening the observation to the COUNT + * call and the envelope is what makes every pair below capable of failing. + */ +async function outcomeOf(query: Record): Promise { + const { p, find, count } = makeProtocol(); + let response: unknown; + try { + response = await (p as unknown as { findData: (r: unknown) => Promise }) + .findData({ object: 'invoice', query }); + } catch (error) { + return { threw: (error as Error).message }; + } + return { + bag: find.mock.calls[0]?.[1] ?? { neverCalled: true }, + counted: count.mock.calls.length, + response, + }; +} + +/** + * Every alias in the declared table, paired with the spelling it must agree + * with. Derived from the exported tables rather than typed out, so a spelling + * added to the declaration without a case here cannot slip through: the + * completeness assertion below counts the pairs the tables produce. + */ +const SLOT_VALUES: Record = { + where: { status: 'queued' }, + fields: ['name', 'status'], + orderBy: 'name', + offset: 5, + expand: ['owner_id'], + limit: 7, + count: true, + search: 'acme', + searchFields: ['name'], +}; + +describe('[#16066] §2 every alias the table declares folds onto its canonical slot', () => { + const slotPairs = QUERY_TRANSPORT_ALIAS_SLOTS.flatMap((slot) => + slot.aliases.map((alias) => [alias, slot.canonical] as const)); + const dollarPairs = QUERY_TRANSPORT_DOLLAR_ALIASES.map(([dollar, bare]) => [dollar, bare] as const); + + it('the declared table produces the pairs this suite drives (no silent shrink)', () => { + // 9 slot aliases (`where` carries three, `expand` two, the other four + // one each) and 7 `$` aliases. A spelling dropped from the declaration + // takes its case with it, so the count is pinned rather than inferred. + expect(slotPairs.length).toBe(9); + expect(dollarPairs.length).toBe(7); + expect(new Set([...slotPairs, ...dollarPairs].map(([a]) => a)).size).toBe(16); + }); + + for (const [alias, canonical] of [...slotPairs, ...dollarPairs]) { + it(`\`${alias}\` reaches the engine exactly as \`${canonical}\` does`, async () => { + const value = SLOT_VALUES[canonical] ?? SLOT_VALUES[resolveCanonical(canonical)]; + expect(value, `no probe value for the \`${canonical}\` slot`).toBeDefined(); + const viaAlias = await outcomeOf({ [alias]: value }); + const viaCanonical = await outcomeOf({ [canonical]: value }); + // ⛔ Not a pair that agrees by both failing: the canonical spelling + // has to have SERVED, or this case proves nothing about the fold. + expect(viaCanonical, JSON.stringify(viaCanonical)).not.toHaveProperty('threw'); + expect(viaAlias).toEqual(viaCanonical); + }); + } +}); + +/** A `$` alias' bare target may itself be a slot alias (`$top` -> `top` -> `limit`). */ +function resolveCanonical(name: string): string { + const slot = QUERY_TRANSPORT_ALIAS_SLOTS.find((s) => s.aliases.includes(name)); + return slot ? slot.canonical : name; +} + +// --------------------------------------------------------------------------- +// §3 This package declares no transport table of its own +// --------------------------------------------------------------------------- + +describe('[#16066] §3 the single source is the spec export', () => { + const source = readFileSync(resolve(HERE, 'protocol.ts'), 'utf8'); + + it('no local `WIRE_QUERY_ALIAS_SLOTS` / `WIRE_DOLLAR_ALIASES` declaration survives', () => { + expect(source).not.toMatch(/(?:const|let|var)\s+WIRE_QUERY_ALIAS_SLOTS\b/); + expect(source).not.toMatch(/(?:const|let|var)\s+WIRE_DOLLAR_ALIASES\b/); + }); + + it('the fold names the spec export at the call site', () => { + expect(source).toMatch(/foldQueryAliasSlots\(options,\s*QUERY_TRANSPORT_ALIAS_SLOTS\b/); + expect(source).toMatch(/for \(const \[dollar, bare\] of QUERY_TRANSPORT_DOLLAR_ALIASES\)/); + }); +}); + +// --------------------------------------------------------------------------- +// §4 A `$` spelling the table does NOT name is refused, loudly +// --------------------------------------------------------------------------- + +describe('[#16066] §4 an undeclared `$` spelling is refused, not folded', () => { + it('answers 400 UNSUPPORTED_QUERY_PARAM and names the parameter', async () => { + const { p, find } = makeProtocol(); + await expect( + (p as unknown as { findData: (r: unknown) => Promise }) + .findData({ object: 'invoice', query: { $sort: 'name' } }), + ).rejects.toMatchObject({ code: 'UNSUPPORTED_QUERY_PARAM', status: 400 }); + expect(find).not.toHaveBeenCalled(); + }); + + it('quotes the DECLARED set rather than a hand-copied sentence', async () => { + const { p } = makeProtocol(); + const error = await (p as unknown as { findData: (r: unknown) => Promise }) + .findData({ object: 'invoice', query: { $sort: 'name' } }) + .then(() => null, (e: Error) => e); + expect(error).toBeInstanceOf(Error); + expect(error!.message).toContain('$sort'); + // The whole declared set, in declaration order — the assertion that goes + // red the day a spelling is added to the table and the sentence is not. + expect(error!.message).toContain( + `Supported $-prefixed parameters: ${QUERY_TRANSPORT_DOLLAR_PARAMS.join(', ')}.`, + ); + }); +}); diff --git a/packages/spec/api-surface/data.json b/packages/spec/api-surface/data.json index 4a15d6d2f4d..326cb21ce27 100644 --- a/packages/spec/api-surface/data.json +++ b/packages/spec/api-surface/data.json @@ -498,6 +498,9 @@ "ProvisionPrimaryOptions (interface)", "QUERY_CURSOR_REMOVED (const)", "QUERY_DISTINCT_REMOVED (const)", + "QUERY_TRANSPORT_ALIAS_SLOTS (const)", + "QUERY_TRANSPORT_DOLLAR_ALIASES (const)", + "QUERY_TRANSPORT_DOLLAR_PARAMS (const)", "QueryAST (type)", "QueryAliasConflict (interface)", "QueryAliasSlot (interface)", @@ -505,6 +508,11 @@ "QueryFilterSchema (const)", "QueryInput (type)", "QuerySchema (const)", + "QueryTransportParams (type)", + "QueryTransportParamsSchema (const)", + "QueryWithTransport (type)", + "QueryWithTransportInput (type)", + "QueryWithTransportSchema (const)", "RAW_FILE_VALUES_CONTEXT_KEY (const)", "READ_ONLY_BELONGS_ON_DATASOURCE (const)", "RECORD_SURFACE_PAGE_THRESHOLD (const)", diff --git a/packages/spec/declaration-map/data.json b/packages/spec/declaration-map/data.json index 1db92a4e2db..e93c8746bd0 100644 --- a/packages/spec/declaration-map/data.json +++ b/packages/spec/declaration-map/data.json @@ -252,6 +252,10 @@ "QueryFilter": "data/QueryFilter", "QueryFilterSchema": "data/QueryFilter", "QuerySchema": "data/Query", + "QueryTransportParams": "data/QueryTransportParams", + "QueryTransportParamsSchema": "data/QueryTransportParams", + "QueryWithTransport": "data/QueryWithTransport", + "QueryWithTransportSchema": "data/QueryWithTransport", "RangeOperatorSchema": "data/RangeOperator", "ReferenceIdValue": "data/ReferenceIdValue", "ReferenceIdValueSchema": "data/ReferenceIdValue", diff --git a/packages/spec/export-origins/data.json b/packages/spec/export-origins/data.json index a07c5b8b169..f0c659a229f 100644 --- a/packages/spec/export-origins/data.json +++ b/packages/spec/export-origins/data.json @@ -487,6 +487,9 @@ "ProvisionPrimaryOptions": "src/data/display-name.ts#ProvisionPrimaryOptions (interface)", "QUERY_CURSOR_REMOVED": "src/data/query.zod.ts#QUERY_CURSOR_REMOVED (const)", "QUERY_DISTINCT_REMOVED": "src/data/query.zod.ts#QUERY_DISTINCT_REMOVED (const)", + "QUERY_TRANSPORT_ALIAS_SLOTS": "src/data/data-engine.zod.ts#QUERY_TRANSPORT_ALIAS_SLOTS (const)", + "QUERY_TRANSPORT_DOLLAR_ALIASES": "src/data/data-engine.zod.ts#QUERY_TRANSPORT_DOLLAR_ALIASES (const)", + "QUERY_TRANSPORT_DOLLAR_PARAMS": "src/data/data-engine.zod.ts#QUERY_TRANSPORT_DOLLAR_PARAMS (const)", "QueryAST": "src/data/query.zod.ts#QueryAST (type)", "QueryAliasConflict": "src/data/data-engine.zod.ts#QueryAliasConflict (interface)", "QueryAliasSlot": "src/data/data-engine.zod.ts#QueryAliasSlot (interface)", @@ -494,6 +497,11 @@ "QueryFilterSchema": "src/data/filter.zod.ts#QueryFilterSchema (const)", "QueryInput": "src/data/query.zod.ts#QueryInput (type)", "QuerySchema": "src/data/query.zod.ts#QuerySchema (const)", + "QueryTransportParams": "src/data/data-engine.zod.ts#QueryTransportParams (type)", + "QueryTransportParamsSchema": "src/data/data-engine.zod.ts#QueryTransportParamsSchema (const)", + "QueryWithTransport": "src/data/data-engine.zod.ts#QueryWithTransport (type)", + "QueryWithTransportInput": "src/data/data-engine.zod.ts#QueryWithTransportInput (type)", + "QueryWithTransportSchema": "src/data/data-engine.zod.ts#QueryWithTransportSchema (const)", "RAW_FILE_VALUES_CONTEXT_KEY": "src/data/field-value.zod.ts#RAW_FILE_VALUES_CONTEXT_KEY (const)", "READ_ONLY_BELONGS_ON_DATASOURCE": "src/data/driver/common.zod.ts#READ_ONLY_BELONGS_ON_DATASOURCE (const)", "RECORD_SURFACE_PAGE_THRESHOLD": "src/data/record-surface.ts#RECORD_SURFACE_PAGE_THRESHOLD (const)", diff --git a/packages/spec/src/data/data-engine.zod.ts b/packages/spec/src/data/data-engine.zod.ts index 32b63d0810a..8e66dbd9b16 100644 --- a/packages/spec/src/data/data-engine.zod.ts +++ b/packages/spec/src/data/data-engine.zod.ts @@ -872,6 +872,13 @@ export const QueryWithTransportSchema: z.ZodType, ); +/** + * What a transport-aware query slot PARSES TO — the canonical QueryAST, in + * every spelling. The declared output is deliberately the AST alone: a + * consumer reading a parsed query never sees a transport key. + */ +export type QueryWithTransport = z.infer; + export const DataEngineFindRequestSchema = lazySchema(() => z.object({ method: z.literal('find'), diff --git a/scripts/check-filter-alias-parity.mjs b/scripts/check-filter-alias-parity.mjs index f54a07e3f98..3e6af64aeba 100644 --- a/scripts/check-filter-alias-parity.mjs +++ b/scripts/check-filter-alias-parity.mjs @@ -2,29 +2,35 @@ // Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. /** - * Filter-slot wire-alias parity guard (#8002). + * Filter-slot transport-alias parity guard (#8002). * * ## The seam * - * The ONE filter slot has four wire spellings, and they are declared in two - * packages with different ownership: + * The ONE filter slot has four transport spellings. Since #16066 all four are + * declared, and declared in ONE file — `packages/spec/src/data/data-engine.zod.ts`: * - * - `where` / `filter` come from the spec's own table - * (`RPC_QUERY_ALIAS_SLOTS`, `packages/spec/src/data/data-engine.zod.ts`) — - * "the ONE place the alias to canonical mapping is declared". - * - `filters` / `$filter` are **wire-only**: no schema declares them. - * `packages/metadata-protocol` extends the spec table with them locally - * (`WIRE_QUERY_ALIAS_SLOTS`), which is what makes them fold into `where`. + * - `where` / `filter` come from `RPC_QUERY_ALIAS_SLOTS`, "the ONE place the + * alias to canonical mapping is declared". + * - `filters` / `$filter` are the TRANSPORT-only spellings, declared beside it + * as `QUERY_TRANSPORT_ONLY_SLOT_ALIASES` and folded into the exported + * `QUERY_TRANSPORT_ALIAS_SLOTS`. They used to be module-private in + * `packages/metadata-protocol` (`WIRE_QUERY_ALIAS_SLOTS`), which is what made + * the `findData` door accept a vocabulary no schema named; #16066 hoisted + * them onto the spec's export surface — the option this gate's original + * write-up left open for the spec seat. * - * #7390 added a third reader: `packages/rest` gates the filter slot's ARITY at + * #7390 added a second reader: `packages/rest` gates the filter slot's ARITY at * the querystring ingress (`FILTER_SLOT_QUERY_PARAMS`, * `packages/rest/src/query-multiplicity.ts`) and needs the same four spellings - * to do it. It derives `where` / `filter` from the spec table; it cannot derive - * `filters` / `$filter`, so it names them literally. + * to do it. It derives `where` / `filter` from the spec table; it does not derive + * `filters` / `$filter`, so it names them literally — which is why this gate is + * still load-bearing after the hoist, and why it was re-pointed rather than + * deleted. `packages/metadata-protocol` now folds by the spec export and adds + * nothing of its own; it is named below as the CONSUMER, never as a declarer. * * ## What goes wrong without this gate, and in which direction * - * A **fifth** wire-only spelling added to `metadata-protocol`'s table folds + * A **fifth** transport-only spelling added to the spec table folds * correctly in the normalizer and is silently UNGATED at the ingress: * repetition on that spelling falls back to the misdiagnosis #7390 exists to * remove — a 400 telling the caller their filter is malformed when every filter @@ -71,13 +77,12 @@ * * ## What it does NOT do * - * It does not decide whether the wire vocabulary belongs in `@objectstack/spec` - * at all — hoisting these spellings onto the spec's export surface is the other - * option on #8002 and is deliberately left open for the spec seat. This gate is - * the reversible move: it adds no public API and no dependency-graph edge, and - * it pays the drift risk down either way. If the hoist ever lands, both sides - * become derived, the sets stay equal by construction, and this script can be - * deleted rather than migrated. + * It does not decide anything about the REST side's own derivation. The hoist + * landed (#16066), so the ingress COULD now derive all four spellings from + * `QUERY_TRANSPORT_ALIAS_SLOTS` and make this script deletable — ⛔ but until it + * does, the ingress still names `filters` / `$filter` literally and the drift + * this gate exists for is still reachable. Deriving that side is its own change, + * in `packages/rest`, with this script's deletion as its second half. * * Run `--self-test` to prove the readers and the comparison against planted * fixtures before trusting a green run. @@ -109,7 +114,7 @@ import { isEntrypoint } from './invoked-as.mjs'; // remedy is to find what stopped registering. const SELF_TEST_BATTERIES = Object.freeze({ '1. The tree as it stands: both sides name the same four spellings.': 2, - '2. A fifth spelling on the NORMALIZER side only — the direction #8002 is about, and the one nothing else in the repo refuses.': 3, + '2. A fifth spelling on the DECLARED side only — the direction #8002 is about, and the one nothing else in the repo refuses.': 3, '3. The same fifth spelling on BOTH sides is green again — the gate judges parity, not the size of the set.': 1, '4. A fifth spelling on the INGRESS side only.': 2, '5. A `$` alias folding INTO a filter spelling is a filter spelling. It reaches `where` through two hops, which is exactly the shape a reader comparing only the slot tables would miss.': 2, @@ -135,8 +140,10 @@ const REST_FILE = 'packages/rest/src/query-multiplicity.ts'; /** The declarations read out of them. */ const SPEC_TABLE = 'RPC_QUERY_ALIAS_SLOTS'; -const WIRE_TABLE = 'WIRE_QUERY_ALIAS_SLOTS'; -const DOLLAR_TABLE = 'WIRE_DOLLAR_ALIASES'; +const WIRE_TABLE = 'QUERY_TRANSPORT_ALIAS_SLOTS'; +const DOLLAR_TABLE = 'QUERY_TRANSPORT_DOLLAR_ALIASES'; +/** The transport-only extras the exported slot table folds in. */ +const EXTRAS_TABLE = 'QUERY_TRANSPORT_ONLY_SLOT_ALIASES'; const REST_SET = 'FILTER_SLOT_QUERY_PARAMS'; /** The canonical key of the filter slot. Everything here is about this one slot. */ @@ -230,15 +237,16 @@ export function readSpecSlots(text, path = SPEC_FILE) { } /** - * The wire-only spellings `metadata-protocol` adds on top of the spec table, + * The transport-only spellings the spec declares on top of its own RPC table, * as `canonical -> extra spellings`, plus the `$`-alias table. * - * Shape read: an IIFE declaring `const extra: Record = { where: [...] }` - * and mapping over `RPC_QUERY_ALIAS_SLOTS`. The mapping is ASSERTED, not - * assumed: if `WIRE_QUERY_ALIAS_SLOTS` stops reading the spec table, the two - * sides no longer share a derived half and this gate's model of them is wrong. + * Shape read: a `QUERY_TRANSPORT_ONLY_SLOT_ALIASES` object literal + * (`{ where: [...] }`) and a `QUERY_TRANSPORT_ALIAS_SLOTS` that maps over + * `RPC_QUERY_ALIAS_SLOTS`. The mapping is ASSERTED, not assumed: if the + * transport table stops reading the RPC table, the two sides no longer share a + * derived half and this gate's model of them is wrong. */ -export function readProtocolExtras(text, path = PROTOCOL_FILE) { +export function readProtocolExtras(text, path = SPEC_FILE) { const source = parse(path, text); const decl = findDeclaration(source, WIRE_TABLE, path); if (!decl.initializer) throw new UnreadableShape(`${path}: \`${WIRE_TABLE}\` has no initializer.`); @@ -253,26 +261,22 @@ export function readProtocolExtras(text, path = PROTOCOL_FILE) { ); } - let extraObject = null; - for (const node of walk(decl.initializer)) { - if (ts.isVariableDeclaration(node) && ts.isIdentifier(node.name) && node.name.text === 'extra') { - extraObject = node.initializer; - } - } + const extraObject = findDeclaration(source, EXTRAS_TABLE, path).initializer; if (!extraObject || !ts.isObjectLiteralExpression(extraObject)) { throw new UnreadableShape( - `${path}: \`${WIRE_TABLE}\` no longer declares its wire-only spellings as an \`extra\` object literal.`, + `${path}: \`${EXTRAS_TABLE}\` is no longer an object literal, so the transport-only ` + + `spellings \`${WIRE_TABLE}\` folds in cannot be read.`, ); } const extras = new Map(); for (const prop of extraObject.properties) { if (!ts.isPropertyAssignment(prop)) { - throw new UnreadableShape(`${path}: the \`extra\` table has a property this gate cannot read statically.`); + throw new UnreadableShape(`${path}: \`${EXTRAS_TABLE}\` has a property this gate cannot read statically.`); } const key = propertyName(prop); - if (key === null) throw new UnreadableShape(`${path}: the \`extra\` table has a computed key.`); - extras.set(key, stringLiterals(prop.initializer, `the \`extra\` table's \`${key}\` entry`, path)); + if (key === null) throw new UnreadableShape(`${path}: \`${EXTRAS_TABLE}\` has a computed key.`); + extras.set(key, stringLiterals(prop.initializer, `\`${EXTRAS_TABLE}\`'s \`${key}\` entry`, path)); } return { extras, dollarAliases: readDollarAliases(source, path) }; } @@ -394,7 +398,7 @@ export function readRestSpellings(text, specSlots, path = REST_FILE) { * Compare the two sides. Returns the problems (empty = parity holds) alongside * the sets, so the self-test can assert on both the verdict and what was read. */ -export function judge({ specText, protocolText, restText }) { +export function judge({ specText, restText }) { const problems = []; let protocolSet = null; let restSet = null; @@ -410,7 +414,7 @@ export function judge({ specText, protocolText, restText }) { + `silencing it.`, ); } else { - const { extras, dollarAliases } = readProtocolExtras(protocolText); + const { extras, dollarAliases } = readProtocolExtras(specText); protocolSet = new Set([...filterSlot, ...(extras.get(FILTER_SLOT) ?? [])]); for (const [dollar, bare] of dollarAliases) { if (protocolSet.has(bare)) protocolSet.add(dollar); @@ -445,8 +449,9 @@ export function judge({ specText, protocolText, restText }) { function formatDrift({ onlyProtocol, onlyRest, protocolSet, restSet }) { const lines = []; lines.push('The filter slot is spelled differently on its two sides:\n'); - lines.push(` normalizer ${PROTOCOL_FILE}`); + lines.push(` declared ${SPEC_FILE}`); lines.push(` ${WIRE_TABLE} -> ${[...protocolSet].sort().map((s) => `\`${s}\``).join(', ')}`); + lines.push(` (folded by the normalizer, ${PROTOCOL_FILE})`); lines.push(` ingress ${REST_FILE}`); lines.push(` ${REST_SET} -> ${[...restSet].sort().map((s) => `\`${s}\``).join(', ')}\n`); if (onlyProtocol.length) { @@ -468,8 +473,8 @@ function formatDrift({ onlyProtocol, onlyRest, protocolSet, restSet }) { lines.push(` The ingress refuses ${it} as a repeated filter parameter, but the normalizer does not`); lines.push(` fold ${it} into \`${FILTER_SLOT}\` at all — so a SINGLE occurrence is not a filter, and`); lines.push(' the arity refusal describes a parameter that does nothing.'); - lines.push(` Fix: add ${quoted} to the \`extra\` table in \`${WIRE_TABLE}\``); - lines.push(` (${PROTOCOL_FILE}), or drop ${it} from \`${REST_SET}\`.`); + lines.push(` Fix: add ${quoted} to \`${EXTRAS_TABLE}\``); + lines.push(` (${SPEC_FILE}), or drop ${it} from \`${REST_SET}\`.`); } lines.push(''); lines.push('Both sides must name the same set: one slot, one spelling set, two readers (#8002).'); @@ -489,21 +494,26 @@ export const RPC_QUERY_ALIAS_SLOTS: readonly QueryAliasSlot[] = [ ]; `; -/** A normalizer fixture. `extraWhere` / `dollars` are what the cases vary. */ +/** + * The transport half of the spec fixture. `extraWhere` / `dollars` are what the + * cases vary. Concatenated onto {@link FIXTURE_SPEC} by `run()`, because since + * #16066 both halves are declarations in the SAME file. + */ function fixtureProtocol(extraWhere = ['filters', '$filter'], dollars = [['$top', 'top'], ['$select', 'select']]) { return ` -const WIRE_QUERY_ALIAS_SLOTS: readonly QueryAliasSlot[] = (() => { - const extra: Record = { - where: [${extraWhere.map((s) => `'${s}'`).join(', ')}], - expand: ['$expand'], - }; - return RPC_QUERY_ALIAS_SLOTS.map((slot) => ({ - canonical: slot.canonical, - aliases: [...slot.aliases, ...(extra[slot.canonical] ?? [])], - })); -})(); - -const WIRE_DOLLAR_ALIASES: readonly (readonly [string, string])[] = [ +const QUERY_TRANSPORT_ONLY_SLOT_ALIASES: Readonly> = { + where: [${extraWhere.map((s) => `'${s}'`).join(', ')}], + expand: ['$expand'], +}; + +export const QUERY_TRANSPORT_ALIAS_SLOTS: readonly QueryAliasSlot[] = RPC_QUERY_ALIAS_SLOTS.map( + (slot) => ({ + canonical: slot.canonical, + aliases: [...slot.aliases, ...(QUERY_TRANSPORT_ONLY_SLOT_ALIASES[slot.canonical] ?? [])], + }), +); + +export const QUERY_TRANSPORT_DOLLAR_ALIASES: readonly (readonly [string, string])[] = [ ${dollars.map(([d, b]) => ` ['${d}', '${b}'],`).join('\n')} ]; `; @@ -544,8 +554,10 @@ function selfTest() { registerCase(); if (!condition) failures.push(`${name}${detail ? ` — ${detail}` : ''}`); }; + // One file carries both halves now, so the fixtures concatenate: the RPC + // table first, the transport declarations after it. const run = (protocolText, restText, specText = FIXTURE_SPEC) => - judge({ specText, protocolText, restText }); + judge({ specText: specText + protocolText, restText }); // 1. The tree as it stands: both sides name the same four spellings. battery('1. The tree as it stands: both sides name the same four spellings.'); @@ -560,9 +572,9 @@ function selfTest() { ); } - // 2. A fifth spelling on the NORMALIZER side only — the direction #8002 is + // 2. A fifth spelling on the DECLARED side only — the direction #8002 is // about, and the one nothing else in the repo refuses. - battery('2. A fifth spelling on the NORMALIZER side only — the direction #8002 is about, and the one nothing else in the repo refuses.'); + battery('2. A fifth spelling on the DECLARED side only — the direction #8002 is about, and the one nothing else in the repo refuses.'); { const { problems } = run(fixtureProtocol(['filters', '$filter', 'where_clause']), fixtureRest()); check('a normalizer-only fifth spelling is red', problems.length === 1, `saw ${problems.length}`); @@ -573,7 +585,7 @@ function selfTest() { ); check( 'the failure names both files', - problems[0]?.includes(PROTOCOL_FILE) && problems[0]?.includes(REST_FILE), + problems[0]?.includes(SPEC_FILE) && problems[0]?.includes(REST_FILE), problems[0], ); } @@ -626,7 +638,7 @@ function selfTest() { const detached = fixtureProtocol().replace('RPC_QUERY_ALIAS_SLOTS.map', 'SOME_OTHER_TABLE.map'); const { problems } = run(detached, fixtureRest()); check( - 'a normalizer that stops deriving from the spec table is red', + 'a transport table that stops deriving from the RPC table is red', problems.length === 1 && problems[0].includes(SPEC_TABLE), problems[0], ); @@ -748,13 +760,12 @@ function main() { const read = (rel) => readFileSync(join(ROOT, rel), 'utf8'); const { problems, protocolSet, restSet } = judge({ specText: read(SPEC_FILE), - protocolText: read(PROTOCOL_FILE), restText: read(REST_FILE), }); if (problems.length === 0) { console.log( - `check:filter-alias-parity: OK (${restSet.size} wire spelling(s) of the \`${FILTER_SLOT}\` slot, ` + `check:filter-alias-parity: OK (${restSet.size} transport spelling(s) of the \`${FILTER_SLOT}\` slot, ` + `identical on both sides: ${[...restSet].sort().join(', ')})`, ); return; From 3e4631dde1e4f5f5dde569f562e1b753dd2f5044 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Sep 2026 15:40:51 +0000 Subject: [PATCH 03/10] test(spec): declare-side pins for the query transport dialect, plus the changeset Claude-Session: https://claude.ai/code/session_01LvwGppdonww4zGLWZo5rho Co-authored-by: Claude --- .../16066-query-transport-dialect-declared.md | 26 +++ .../spec/src/data/query-transport.test.ts | 175 ++++++++++++++++++ 2 files changed, 201 insertions(+) create mode 100644 .changeset/16066-query-transport-dialect-declared.md create mode 100644 packages/spec/src/data/query-transport.test.ts diff --git a/.changeset/16066-query-transport-dialect-declared.md b/.changeset/16066-query-transport-dialect-declared.md new file mode 100644 index 00000000000..05201e20e88 --- /dev/null +++ b/.changeset/16066-query-transport-dialect-declared.md @@ -0,0 +1,26 @@ +--- +"@objectstack/spec": minor +"@objectstack/metadata-protocol": patch +--- + +The query TRANSPORT dialect is declared, and the `findData` fold now derives from that one declaration. + +`FindDataRequestSchema.query` declared `QuerySchema` — the canonical QueryAST — while the shipped `findData` door also accepted a second spelling of the same query through the same slot: `$filter` / `$top` / `$skip` / `$orderby` / `$select` / `$expand` and the plural `filters`. `@objectstack/metadata-protocol` folded them from a module-private table whose own comment called them "the wire-only spellings no schema declares". Two dialects, one slot, one of them declared — so every caller speaking the second was unverifiable at build time and unrejected at runtime. + +**New in `@objectstack/spec/data`** (8 exports, 0 removed): + +- `QueryTransportParamsSchema` / `QueryTransportParams` — the transport parameters, each carrying the value of the canonical slot it folds onto. +- `QUERY_TRANSPORT_ALIAS_SLOTS` — `RPC_QUERY_ALIAS_SLOTS` extended with the transport-only spellings (`filters` / `$filter` onto `where`, `$expand` onto `expand`). +- `QUERY_TRANSPORT_DOLLAR_ALIASES` — the `$`-to-bare pairs that fold in two hops (`$top` onto `top` onto `limit`). +- `QUERY_TRANSPORT_DOLLAR_PARAMS` — the `$` spellings a boundary quotes when it refuses an undeclared one. +- `QueryWithTransportSchema` / `QueryWithTransport` / `QueryWithTransportInput` — the query slot whose declared input is the AST or its transport spelling and whose declared output is the AST. + +**`FindDataRequestSchema.query` is that slot now.** Its `z.input` admits the canonical AST, the transport spelling, or a bag carrying both; its `z.output` is still the AST, reached by a transform driven by the two tables. The transport form is declared as the FLATTENED SPELLING of the canonical AST with a 1:1 alias table — never as a second semantics — so `QuerySchema` itself is untouched and still drops a `$` key as unknown. + +**Nothing is narrowed, and nothing about the served behaviour changed.** The canonical arm of the union is `QuerySchema` unchanged, so every body that parsed before still parses, by the same schema, with the same issues. `@objectstack/metadata-protocol` folds by the spec export instead of its own table, and both resolved tables — plus the `$`-parameter list its `UNSUPPORTED_QUERY_PARAM` refusal quotes — are pinned byte-equal to their pre-change values. + +**An undeclared `$` spelling is still refused loudly**, with the same `400 UNSUPPORTED_QUERY_PARAM`; the sentence now quotes `QUERY_TRANSPORT_DOLLAR_PARAMS` rather than a hand-copied list, so a spelling added to the table cannot leave the refusal naming a set the door no longer has. + +Measured and unchanged: `getData` takes `select` / `expand` directly and carries no `query` slot, and `updateManyData` / `deleteManyData` take `records[]` / `ids[]` — none of the three has a transport-dialect split to declare. + +Clause-②: yes (widening) diff --git a/packages/spec/src/data/query-transport.test.ts b/packages/spec/src/data/query-transport.test.ts new file mode 100644 index 00000000000..d23b84e3216 --- /dev/null +++ b/packages/spec/src/data/query-transport.test.ts @@ -0,0 +1,175 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#16066] The query transport dialect, as the spec declares it. + * + * `FindDataRequestSchema.query` declared the canonical QueryAST while the + * shipped `findData` door also folded a transport dialect no schema named. The + * ruling declared that dialect here — as the FLATTENED SPELLING of the same + * AST with a 1:1 alias table, ⛔ never as a second semantics — so the declared + * INPUT is the AST or its transport spelling and the declared OUTPUT is the AST. + * + * ## Three things this file holds apart + * + * 1. **What the transport declares** — the `$`-spellings and `filters`, and + * nothing else. §1 pins the schema's key set against the alias tables, so + * a spelling can neither be declared without a fold nor folded without a + * declaration. + * 2. **What it parses to** — §2 drives every alias through + * `QueryWithTransportSchema` and asserts the canonical slot it lands on. + * 3. **What `QuerySchema` still is** — §4. The AST did NOT grow these keys; + * `QuerySchema` is byte-for-byte the schema it was, which is the half of + * Prime Directive #12 the ruling kept. + */ + +import { describe, it, expect } from 'vitest'; +import { + QUERY_TRANSPORT_ALIAS_SLOTS, + QUERY_TRANSPORT_DOLLAR_ALIASES, + QUERY_TRANSPORT_DOLLAR_PARAMS, + QueryTransportParamsSchema, + QueryWithTransportSchema, + RPC_QUERY_ALIAS_SLOTS, +} from './data-engine.zod'; +import { QuerySchema } from './query.zod'; + +describe('[#16066] §1 what the transport declares', () => { + it('declares exactly the `$`-spellings and the plural `filters`', () => { + const declared = Object.keys((QueryTransportParamsSchema as unknown as { shape: object }).shape).sort(); + expect(declared).toEqual([ + '$count', '$expand', '$filter', '$orderby', '$search', '$searchFields', '$select', '$skip', '$top', + 'filters', + ]); + }); + + it('every `$` spelling it declares is one the alias tables actually fold', () => { + const folded = new Set([ + ...QUERY_TRANSPORT_DOLLAR_ALIASES.map(([dollar]) => dollar), + ...QUERY_TRANSPORT_ALIAS_SLOTS.flatMap((slot) => slot.aliases), + ]); + const declared = Object.keys((QueryTransportParamsSchema as unknown as { shape: object }).shape); + // Both directions: a declared spelling nothing folds would be a promise the + // door does not keep, and a folded `$` spelling nothing declares is the + // defect this card was filed about. + for (const key of declared) expect(folded.has(key), `declared but never folded: ${key}`).toBe(true); + for (const key of folded) { + if (!key.startsWith('$') && key !== 'filters') continue; + expect(declared.includes(key), `folded but never declared: ${key}`).toBe(true); + } + }); + + it('extends the RPC table rather than replacing it — same slots, same order', () => { + expect(QUERY_TRANSPORT_ALIAS_SLOTS.map((s) => s.canonical)) + .toEqual(RPC_QUERY_ALIAS_SLOTS.map((s) => s.canonical)); + for (const [i, slot] of QUERY_TRANSPORT_ALIAS_SLOTS.entries()) { + expect(slot.aliases.slice(0, RPC_QUERY_ALIAS_SLOTS[i].aliases.length)) + .toEqual([...RPC_QUERY_ALIAS_SLOTS[i].aliases]); + } + }); + + it('the refusal set is the `$` spellings, in declaration order', () => { + expect([...QUERY_TRANSPORT_DOLLAR_PARAMS]).toEqual([ + '$top', '$skip', '$orderby', '$select', '$count', '$search', '$searchFields', '$filter', '$expand', + ]); + }); +}); + +describe('[#16066] §2 every alias parses to its canonical slot', () => { + const parse = (query: Record) => QueryWithTransportSchema.parse(query); + + const CASES: Array<[string, Record, Record]> = [ + ['$filter -> where', { $filter: { status: 'open' } }, { where: { status: 'open' } }], + ['filters -> where', { filters: { status: 'open' } }, { where: { status: 'open' } }], + ['filter -> where', { filter: { status: 'open' } }, { where: { status: 'open' } }], + ['$top -> limit', { $top: 10 }, { limit: 10 }], + ['top -> limit', { top: 10 }, { limit: 10 }], + ['$skip -> offset', { $skip: 20 }, { offset: 20 }], + ['skip -> offset', { skip: 20 }, { offset: 20 }], + ['$orderby -> orderBy', { $orderby: [{ field: 'name', order: 'asc' }] }, { orderBy: [{ field: 'name', order: 'asc' }] }], + ['sort -> orderBy', { sort: [{ field: 'name', order: 'asc' }] }, { orderBy: [{ field: 'name', order: 'asc' }] }], + ['$select -> fields', { $select: ['name'] }, { fields: ['name'] }], + ['select -> fields', { select: ['name'] }, { fields: ['name'] }], + ['$expand -> expand', { $expand: { owner: { object: 'owner' } } }, { expand: { owner: { object: 'owner' } } }], + ['$search -> search', { $search: 'acme' }, { search: 'acme' }], + ['$searchFields -> searchFields', { $searchFields: ['name'] }, { searchFields: ['name'] }], + ]; + + for (const [label, input, expected] of CASES) { + it(label, () => { + expect(parse({ object: 'account', ...input })).toEqual({ object: 'account', ...expected }); + }); + } + + it('the canonical AST parses to itself, untouched', () => { + const ast = { + object: 'account', + where: { status: 'open' }, + orderBy: [{ field: 'name', order: 'desc' }], + limit: 10, + offset: 0, + fields: ['name'], + expand: { owner: { object: 'owner' } }, + }; + expect(parse(ast)).toEqual(ast); + }); + + it('a bag carrying BOTH spellings of one slot folds when they agree', () => { + expect(parse({ object: 'account', where: { a: 1 }, $filter: { a: 1 } })) + .toEqual({ object: 'account', where: { a: 1 } }); + }); + + it('a bag mixing spellings of DIFFERENT slots folds all of them', () => { + expect(parse({ object: 'account', where: { a: 1 }, $top: 5, $select: ['name'] })) + .toEqual({ object: 'account', where: { a: 1 }, limit: 5, fields: ['name'] }); + }); + + it('`$count` is consumed, not smuggled into the AST — the AST has no count slot', () => { + expect(parse({ object: 'account', $count: true })).toEqual({ object: 'account' }); + }); +}); + +describe('[#16066] §3 the widening cannot narrow what already parsed', () => { + /** + * The canonical arm of the union is `QuerySchema` itself, unchanged, so + * anything that parsed before still parses — including shapes the transport + * arm declines. ⛔ This is the property that makes the declaration safe to + * land as a `minor`: it records what the door accepts and refuses nothing new. + */ + const stillParses = (query: Record) => + expect(QueryWithTransportSchema.safeParse(query).success).toBe(true); + + it('a querystring-shaped `$top` (a string) still parses', () => stillParses({ object: 'a', $top: '50' })); + it('a JSON-encoded `$filter` string still parses', () => stillParses({ object: 'a', $filter: '{"x":1}' })); + it('a `{field: direction}` `$orderby` record still parses', () => stillParses({ object: 'a', $orderby: { created_at: 'desc' } })); + it('a comma-list `$select` still parses', () => stillParses({ object: 'a', $select: 'a,b' })); + it('a comma-list `$expand` still parses', () => stillParses({ object: 'a', $expand: 'owner' })); + it('a leftover key destined for the implicit field filter still parses', () => stillParses({ object: 'a', status: 'open' })); + it('conflicting spellings of one slot still parse — the DOOR refuses them, where it can name the parameter the caller wrote', () => + stillParses({ object: 'a', where: { a: 1 }, $filter: { b: 2 } })); + + it('a non-object query is still refused', () => { + expect(QueryWithTransportSchema.safeParse('nope').success).toBe(false); + }); +}); + +describe('[#16066] §4 `QuerySchema` did not grow the dialect', () => { + /** + * The ruling's Prime Directive #12 half: the transport form is declared as a + * SPELLING of the AST, never admitted into the AST. A `$` key handed to + * `QuerySchema` is still an unknown key it drops on the floor — which is + * exactly why the slot, not the AST, is where the union lives. + */ + it('`QuerySchema` still strips `$filter` instead of reading it', () => { + expect(QuerySchema.parse({ object: 'account', $filter: { a: 1 } } as never)) + .toEqual({ object: 'account' }); + }); + + it('`QuerySchema` still strips `$top` instead of reading it', () => { + expect(QuerySchema.parse({ object: 'account', $top: 5 } as never)).toEqual({ object: 'account' }); + }); + + it('…while the transport-aware slot folds both', () => { + expect(QueryWithTransportSchema.parse({ object: 'account', $filter: { a: 1 }, $top: 5 })) + .toEqual({ object: 'account', where: { a: 1 }, limit: 5 }); + }); +}); From 2cfacff1ef0235fd310078dc4b00a4b1b247bc0e Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Sep 2026 16:04:55 +0000 Subject: [PATCH 04/10] fix(spec): follow ADR-0122's author/parsed alias convention on the new types `QueryWithTransport` is the author state and `QueryWithTransportParsed` the parsed one; `QueryTransportParamsParsed` names the parsed transport bag. `check:spec-parsed-alias` named both, and the test-side engine mock is typed with the parameters it is really called with. Claude-Session: https://claude.ai/code/session_01LvwGppdonww4zGLWZo5rho Co-authored-by: Claude --- .../16066-query-transport-dialect-declared.md | 4 ++-- .../protocol.query-transport-dialect.test.ts | 5 ++++- packages/spec/api-surface/data.json | 3 ++- packages/spec/export-origins/data.json | 3 ++- packages/spec/src/data/data-engine.zod.ts | 21 ++++++++++++------- 5 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.changeset/16066-query-transport-dialect-declared.md b/.changeset/16066-query-transport-dialect-declared.md index 05201e20e88..910e28f6dae 100644 --- a/.changeset/16066-query-transport-dialect-declared.md +++ b/.changeset/16066-query-transport-dialect-declared.md @@ -9,11 +9,11 @@ The query TRANSPORT dialect is declared, and the `findData` fold now derives fro **New in `@objectstack/spec/data`** (8 exports, 0 removed): -- `QueryTransportParamsSchema` / `QueryTransportParams` — the transport parameters, each carrying the value of the canonical slot it folds onto. +- `QueryTransportParamsSchema` / `QueryTransportParams` / `QueryTransportParamsParsed` — the transport parameters, each carrying the value of the canonical slot it folds onto. - `QUERY_TRANSPORT_ALIAS_SLOTS` — `RPC_QUERY_ALIAS_SLOTS` extended with the transport-only spellings (`filters` / `$filter` onto `where`, `$expand` onto `expand`). - `QUERY_TRANSPORT_DOLLAR_ALIASES` — the `$`-to-bare pairs that fold in two hops (`$top` onto `top` onto `limit`). - `QUERY_TRANSPORT_DOLLAR_PARAMS` — the `$` spellings a boundary quotes when it refuses an undeclared one. -- `QueryWithTransportSchema` / `QueryWithTransport` / `QueryWithTransportInput` — the query slot whose declared input is the AST or its transport spelling and whose declared output is the AST. +- `QueryWithTransportSchema` / `QueryWithTransport` / `QueryWithTransportParsed` — the query slot whose declared input is the AST or its transport spelling and whose declared output is the AST. **`FindDataRequestSchema.query` is that slot now.** Its `z.input` admits the canonical AST, the transport spelling, or a bag carrying both; its `z.output` is still the AST, reached by a transform driven by the two tables. The transport form is declared as the FLATTENED SPELLING of the canonical AST with a 1:1 alias table — never as a second semantics — so `QuerySchema` itself is untouched and still drops a `$` key as unknown. diff --git a/packages/metadata-protocol/src/protocol.query-transport-dialect.test.ts b/packages/metadata-protocol/src/protocol.query-transport-dialect.test.ts index 97f233ed428..9c2e7f72b37 100644 --- a/packages/metadata-protocol/src/protocol.query-transport-dialect.test.ts +++ b/packages/metadata-protocol/src/protocol.query-transport-dialect.test.ts @@ -113,7 +113,10 @@ const SCHEMA = { const USER_SCHEMA = { name: 'sys_user', nameField: 'name', fields: { name: { name: 'name', type: 'text' } } }; function makeProtocol() { - const find = vi.fn(async () => [] as unknown[]); + // Typed with the parameters the engine is really called with, so the + // option-bag read below is `calls[0][1]` rather than an out-of-range index + // on a zero-arity tuple. + const find = vi.fn(async (_object?: string, _options?: unknown) => [] as unknown[]); const count = vi.fn(async () => 0); const engine = { registry: { diff --git a/packages/spec/api-surface/data.json b/packages/spec/api-surface/data.json index 326cb21ce27..70a13d7299f 100644 --- a/packages/spec/api-surface/data.json +++ b/packages/spec/api-surface/data.json @@ -509,9 +509,10 @@ "QueryInput (type)", "QuerySchema (const)", "QueryTransportParams (type)", + "QueryTransportParamsParsed (type)", "QueryTransportParamsSchema (const)", "QueryWithTransport (type)", - "QueryWithTransportInput (type)", + "QueryWithTransportParsed (type)", "QueryWithTransportSchema (const)", "RAW_FILE_VALUES_CONTEXT_KEY (const)", "READ_ONLY_BELONGS_ON_DATASOURCE (const)", diff --git a/packages/spec/export-origins/data.json b/packages/spec/export-origins/data.json index f0c659a229f..6cc002ebaa0 100644 --- a/packages/spec/export-origins/data.json +++ b/packages/spec/export-origins/data.json @@ -498,9 +498,10 @@ "QueryInput": "src/data/query.zod.ts#QueryInput (type)", "QuerySchema": "src/data/query.zod.ts#QuerySchema (const)", "QueryTransportParams": "src/data/data-engine.zod.ts#QueryTransportParams (type)", + "QueryTransportParamsParsed": "src/data/data-engine.zod.ts#QueryTransportParamsParsed (type)", "QueryTransportParamsSchema": "src/data/data-engine.zod.ts#QueryTransportParamsSchema (const)", "QueryWithTransport": "src/data/data-engine.zod.ts#QueryWithTransport (type)", - "QueryWithTransportInput": "src/data/data-engine.zod.ts#QueryWithTransportInput (type)", + "QueryWithTransportParsed": "src/data/data-engine.zod.ts#QueryWithTransportParsed (type)", "QueryWithTransportSchema": "src/data/data-engine.zod.ts#QueryWithTransportSchema (const)", "RAW_FILE_VALUES_CONTEXT_KEY": "src/data/field-value.zod.ts#RAW_FILE_VALUES_CONTEXT_KEY (const)", "READ_ONLY_BELONGS_ON_DATASOURCE": "src/data/driver/common.zod.ts#READ_ONLY_BELONGS_ON_DATASOURCE (const)", diff --git a/packages/spec/src/data/data-engine.zod.ts b/packages/spec/src/data/data-engine.zod.ts index 8e66dbd9b16..c8e7dc20eca 100644 --- a/packages/spec/src/data/data-engine.zod.ts +++ b/packages/spec/src/data/data-engine.zod.ts @@ -833,9 +833,16 @@ function foldQueryTransportBag( return bag; } -/** The transport query parameters, as an input shape. */ +/** The transport query parameters, as an author writes them (ADR-0122). */ export type QueryTransportParams = z.input; +/** + * The transport query parameters after a parse (ADR-0122). Not identical to + * {@link QueryTransportParams}: `$expand`'s record arm carries nested + * `QuerySchema` values, whose author and parsed states differ. + */ +export type QueryTransportParamsParsed = z.infer; + /** * What a query slot that serves a transport boundary DECLARES as its input: * the canonical QueryAST, the transport spelling, or a bag carrying both. @@ -847,7 +854,7 @@ export type QueryTransportParams = z.input; * re-created this card's own defect one size down: a shape the door accepts * that the declaration denies. */ -export type QueryWithTransportInput = QueryInput & Partial; +export type QueryWithTransport = QueryInput & Partial; /** * A query slot whose declared INPUT is the canonical AST or its transport @@ -860,7 +867,7 @@ export type QueryWithTransportInput = QueryInput & Partial * widening provably unable to narrow anything: every input that parsed before * still parses, by the same schema, with the same issues. */ -export const QueryWithTransportSchema: z.ZodType = lazySchema(() => +export const QueryWithTransportSchema: z.ZodType = lazySchema(() => z.union([ z.record(z.string(), z.unknown()) .transform((bag, ctx) => foldQueryTransportBag( @@ -869,15 +876,15 @@ export const QueryWithTransportSchema: z.ZodType>), QuerySchema, - ]) as unknown as z.ZodType, + ]) as unknown as z.ZodType, ); /** * What a transport-aware query slot PARSES TO — the canonical QueryAST, in - * every spelling. The declared output is deliberately the AST alone: a - * consumer reading a parsed query never sees a transport key. + * every spelling (ADR-0122). The declared output is deliberately the AST + * alone: a consumer reading a parsed query never sees a transport key. */ -export type QueryWithTransport = z.infer; +export type QueryWithTransportParsed = z.infer; export const DataEngineFindRequestSchema = lazySchema(() => z.object({ From 5c072bbe630519257dbec5ad5440b84d0c3c05c3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Sep 2026 17:00:37 +0000 Subject: [PATCH 05/10] fix(spec): declare the query slot as ONE extended object, not a union MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `z.union` on the slot put zod's own `invalid_union` issue at the head of the issue list for every malformed query body, and the REST ingress reports `fields[0]` off that list — `query.aggregations.0.function` became a bare `query`. Extending `QuerySchema` with the transport members instead leaves every canonical member's issue path exactly where it was. The transport members now carry every value shape the door serves on that slot, including the body-form filter AST array and an explicit null withdrawal, and the fold lowers the legacy shapes onto the canonical slot. Claude-Session: https://claude.ai/code/session_01LvwGppdonww4zGLWZo5rho Co-authored-by: Claude --- content/docs/references/api/protocol.mdx | 47 ++--- content/docs/references/data/data-engine.mdx | 150 +++++++++++++-- .../rest-server-canonical-query-ast.test.ts | 14 +- packages/spec/authorable-surface/data.json | 37 ++++ packages/spec/src/data/data-engine.zod.ts | 173 +++++++++++++----- .../spec/src/data/query-transport.test.ts | 38 +++- 6 files changed, 352 insertions(+), 107 deletions(-) diff --git a/content/docs/references/api/protocol.mdx b/content/docs/references/api/protocol.mdx index d4ba842016d..4d2a35861c3 100644 --- a/content/docs/references/api/protocol.mdx +++ b/content/docs/references/api/protocol.mdx @@ -867,18 +867,18 @@ Enable package response | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | **object** | `string` | ✅ | The unique machine name of the object to query (e.g. "account"). | -| **query** | `{ object: string; fields?: string[]; where?: any; search?: string \| object; … } \| … +1 more` | optional | Structured query definition (filter, sort, select, pagination) — the canonical QueryAST or its transport spelling. | +| **query** | `{ object: string; fields?: string[]; where?: any; search?: string \| object; … }` | optional | Structured query definition (filter, sort, select, pagination) — the canonical QueryAST or its transport spelling. | -### Nested Shape: `FindDataRequest.query[option 1]` +### Nested Shape: `FindDataRequest.query` | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | **object** | `string` | ✅ | Object name (e.g. account) | | **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | | **where** | `any` | optional | Filtering criteria (WHERE) | -| **search** | `string \| { query: string; fields?: string[]; fuzzy: boolean; operator: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | +| **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | | **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | -| **orderBy** | `{ field: string; order: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | +| **orderBy** | `{ field: string; order?: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | | **limit** | `number` | optional | Max records to return (LIMIT) | | **offset** | `number` | optional | Records to skip (OFFSET) | | **top** | `number` | optional | Alias for limit (OData compatibility) | @@ -889,29 +889,22 @@ Enable package response | **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | | **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | | **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | -| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select (`fields`) and filter (`where`, AND-merged with the batch $in), plus further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3; per-parent `limit`/`offset`/`orderBy` are NOT applied on this path. | - -### Nested Shape: `FindDataRequest.query[option 2]` - -| Property | Type | Required | Description | -| :--- | :--- | :--- | :--- | -| **object** | `string` | ✅ | Object name (e.g. account) | -| **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | -| **where** | `any` | optional | Filtering criteria (WHERE) | -| **search** | `string \| { query: string; fields?: string[]; fuzzy: boolean; operator: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | -| **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | -| **orderBy** | `{ field: string; order: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | -| **limit** | `number` | optional | Max records to return (LIMIT) | -| **offset** | `number` | optional | Records to skip (OFFSET) | -| **top** | `number` | optional | Alias for limit (OData compatibility) | -| **cursor** | `never` | optional | [REMOVED] `query.cursor` was removed in @objectstack/spec 17 (ADR-0049) — no driver ever implemented keyset pagination, so the cursor was accepted and ignored and every page came back identical (a caller looping "until hasMore is false" never terminates). Delete the key; `QueryBuilder.cursor()` was removed with it. Express the keyset as an ordinary `where` predicate on your sort key — `where: { created_at: { $gt: last.created_at } }` with the matching `orderBy` — which every driver executes with canonicalised comparands. A first-class cursor, if ever built, will be a response-minted opaque token, not this caller-built record. | -| **joins** | `never` | optional | [REMOVED] `query.joins` was removed in @objectstack/spec 17 (ADR-0049) — no engine or driver ever read it: a query carrying `joins` behaved exactly as if the key were absent, while its name squatted on the reserved REST parameter set. Delete the key. Related records are read through `expand` — `expand: { owner_id: { object: 'user', fields: ['name'] } }` — which the engine resolves via batch $in queries, and whose nested query selects the related record's own columns. Keep the foreign key in your own projection (`fields: ['title', 'owner_id']`): the relation is carried by that column, so projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement — no driver ever resolved one and the ingress refuses it (`400 INVALID_FIELD`). | -| **aggregations** | `{ function: Enum<'count' \| 'sum' \| 'avg' \| 'min' \| 'max' \| 'count_distinct'>; field?: string; alias: string; filter?: any }[]` | optional | Aggregation functions | -| **groupBy** | `(string \| { field: string; dateGranularity?: Enum<'day' \| 'week' \| 'month' \| 'quarter' \| 'year'>; alias?: string })[]` | optional | GROUP BY targets (strings or `{field, dateGranularity?}` objects for date bucketing) | -| **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | -| **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | -| **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | -| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select (`fields`) and filter (`where`, AND-merged with the batch $in), plus further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3; per-parent `limit`/`offset`/`orderBy` are NOT applied on this path. | +| **expand** | `Record` | optional | | +| **$filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (OData `$filter`) | +| **filters** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (plural of `filter`) | +| **$top** | `number \| string \| null` | optional | Transport spelling of `limit` (OData `$top`) | +| **$skip** | `number \| string \| null` | optional | Transport spelling of `offset` (OData `$skip`) | +| **$orderby** | `string \| string[] \| Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` (OData `$orderby`) | +| **$select** | `string \| string[] \| null` | optional | Transport spelling of `fields` (OData `$select`) | +| **$expand** | `string \| string[] \| Record \| null` | optional | Transport spelling of `expand` (OData `$expand`) | +| **$search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … } \| null` | optional | Transport spelling of `search` (OData `$search`) | +| **$searchFields** | `string \| string[] \| null` | optional | Transport spelling of `searchFields` (OData `$searchFields`) | +| **$count** | `boolean \| string \| null` | optional | Transport spelling of the response total-count flag (OData `$count`) | +| **filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` | +| **select** | `string \| string[] \| null` | optional | Transport spelling of `fields` | +| **sort** | `string \| string[] \| Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` | +| **skip** | `number \| string \| null` | optional | Transport spelling of `offset` | +| **populate** | `string \| string[] \| null` | optional | Transport spelling of `expand` | --- diff --git a/content/docs/references/data/data-engine.mdx b/content/docs/references/data/data-engine.mdx index b71c9c52146..dd73e687155 100644 --- a/content/docs/references/data/data-engine.mdx +++ b/content/docs/references/data/data-engine.mdx @@ -1217,16 +1217,21 @@ Transport spellings of the QueryAST slots — folded to canonical keys at the bo | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **$filter** | `string \| Record \| any` | optional | Transport spelling of `where` (OData `$filter`) | -| **filters** | `string \| Record \| any` | optional | Transport spelling of `where` (plural of `filter`) | -| **$top** | `number \| string` | optional | Transport spelling of `limit` (OData `$top`) | -| **$skip** | `number \| string` | optional | Transport spelling of `offset` (OData `$skip`) | -| **$orderby** | `string \| string[] \| Record> \| Record \| { field: string; order: Enum<'asc' \| 'desc'> }[]` | optional | Transport spelling of `orderBy` (OData `$orderby`) | -| **$select** | `string \| string[]` | optional | Transport spelling of `fields` (OData `$select`) | -| **$expand** | `string \| string[] \| Record` | optional | Transport spelling of `expand` (OData `$expand`) | -| **$search** | `string \| { query: string; fields?: string[]; fuzzy: boolean; operator: Enum<'and' \| 'or'>; … }` | optional | Transport spelling of `search` (OData `$search`) | -| **$searchFields** | `string \| string[]` | optional | Transport spelling of `searchFields` (OData `$searchFields`) | -| **$count** | `boolean \| string` | optional | Transport spelling of the response total-count flag (OData `$count`) | +| **$filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (OData `$filter`) | +| **filters** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (plural of `filter`) | +| **$top** | `number \| string \| null` | optional | Transport spelling of `limit` (OData `$top`) | +| **$skip** | `number \| string \| null` | optional | Transport spelling of `offset` (OData `$skip`) | +| **$orderby** | `string \| string[] \| Record> \| Record \| { field: string; order: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` (OData `$orderby`) | +| **$select** | `string \| string[] \| null` | optional | Transport spelling of `fields` (OData `$select`) | +| **$expand** | `string \| string[] \| Record \| null` | optional | Transport spelling of `expand` (OData `$expand`) | +| **$search** | `string \| { query: string; fields?: string[]; fuzzy: boolean; operator: Enum<'and' \| 'or'>; … } \| null` | optional | Transport spelling of `search` (OData `$search`) | +| **$searchFields** | `string \| string[] \| null` | optional | Transport spelling of `searchFields` (OData `$searchFields`) | +| **$count** | `boolean \| string \| null` | optional | Transport spelling of the response total-count flag (OData `$count`) | +| **filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` | +| **select** | `string \| string[] \| null` | optional | Transport spelling of `fields` | +| **sort** | `string \| string[] \| Record> \| Record \| { field: string; order: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` | +| **skip** | `number \| string \| null` | optional | Transport spelling of `offset` | +| **populate** | `string \| string[] \| null` | optional | Transport spelling of `expand` | ### Nested Shape: `QueryTransportParams.$expand[string]` @@ -1268,21 +1273,130 @@ Transport spellings of the QueryAST slots — folded to canonical keys at the bo ## QueryWithTransport -### Union Options +### Properties -This schema accepts one of the following structures: +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **object** | `string` | ✅ | Object name (e.g. account) | +| **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | +| **where** | `any` | optional | Filtering criteria (WHERE) | +| **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | +| **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | +| **orderBy** | `{ field: string; order?: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | +| **limit** | `number` | optional | Max records to return (LIMIT) | +| **offset** | `number` | optional | Records to skip (OFFSET) | +| **top** | `number` | optional | Alias for limit (OData compatibility) | +| **cursor** | `never` | optional | [REMOVED] `query.cursor` was removed in @objectstack/spec 17 (ADR-0049) — no driver ever implemented keyset pagination, so the cursor was accepted and ignored and every page came back identical (a caller looping "until hasMore is false" never terminates). Delete the key; `QueryBuilder.cursor()` was removed with it. Express the keyset as an ordinary `where` predicate on your sort key — `where: { created_at: { $gt: last.created_at } }` with the matching `orderBy` — which every driver executes with canonicalised comparands. A first-class cursor, if ever built, will be a response-minted opaque token, not this caller-built record. | +| **joins** | `never` | optional | [REMOVED] `query.joins` was removed in @objectstack/spec 17 (ADR-0049) — no engine or driver ever read it: a query carrying `joins` behaved exactly as if the key were absent, while its name squatted on the reserved REST parameter set. Delete the key. Related records are read through `expand` — `expand: { owner_id: { object: 'user', fields: ['name'] } }` — which the engine resolves via batch $in queries, and whose nested query selects the related record's own columns. Keep the foreign key in your own projection (`fields: ['title', 'owner_id']`): the relation is carried by that column, so projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement — no driver ever resolved one and the ingress refuses it (`400 INVALID_FIELD`). | +| **aggregations** | `{ function: Enum<'count' \| 'sum' \| 'avg' \| 'min' \| 'max' \| 'count_distinct'>; field?: string; alias: string; filter?: any }[]` | optional | Aggregation functions | +| **groupBy** | `(string \| { field: string; dateGranularity?: Enum<'day' \| 'week' \| 'month' \| 'quarter' \| 'year'>; alias?: string })[]` | optional | GROUP BY targets (strings or `{field, dateGranularity?}` objects for date bucketing) | +| **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | +| **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | +| **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | +| **expand** | `Record` | optional | | +| **$filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (OData `$filter`) | +| **filters** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (plural of `filter`) | +| **$top** | `number \| string \| null` | optional | Transport spelling of `limit` (OData `$top`) | +| **$skip** | `number \| string \| null` | optional | Transport spelling of `offset` (OData `$skip`) | +| **$orderby** | `string \| string[] \| Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` (OData `$orderby`) | +| **$select** | `string \| string[] \| null` | optional | Transport spelling of `fields` (OData `$select`) | +| **$expand** | `string \| string[] \| Record \| null` | optional | Transport spelling of `expand` (OData `$expand`) | +| **$search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … } \| null` | optional | Transport spelling of `search` (OData `$search`) | +| **$searchFields** | `string \| string[] \| null` | optional | Transport spelling of `searchFields` (OData `$searchFields`) | +| **$count** | `boolean \| string \| null` | optional | Transport spelling of the response total-count flag (OData `$count`) | +| **filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` | +| **select** | `string \| string[] \| null` | optional | Transport spelling of `fields` | +| **sort** | `string \| string[] \| Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` | +| **skip** | `number \| string \| null` | optional | Transport spelling of `offset` | +| **populate** | `string \| string[] \| null` | optional | Transport spelling of `expand` | + +### Nested Shape: `QueryWithTransport.search` -#### Option 1 +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **query** | `string` | ✅ | Search query text | +| **fields** | `string[]` | optional | Fields to search in (if not specified, searches all text fields) | +| **fuzzy** | `boolean` | optional (default: `false`) | [EXPERIMENTAL — not enforced] Fuzzy matching (tolerate typos). The ADR-0061 expansion reads only `query` + `fields`; no executor receives this flag. | +| **operator** | `Enum<'and' \| 'or'>` | optional (default: `"or"`) | [EXPERIMENTAL — not enforced] Logical operator between terms. The ADR-0061 expansion applies its own term semantics; no executor receives this flag. | +| **boost** | `Record` | optional | [EXPERIMENTAL — not enforced] Field-specific relevance boosting (field name -> boost factor). No executor scores results. | +| **minScore** | `number` | optional | [EXPERIMENTAL — not enforced] Minimum relevance score threshold. No executor scores results. | +| **language** | `string` | optional | [EXPERIMENTAL — not enforced] Language for text analysis (e.g., "en", "zh", "es"). No executor selects an analyzer. | +| **highlight** | `boolean` | optional (default: `false`) | [EXPERIMENTAL — not enforced] Search result highlighting. No executor emits highlights. | -Reference: { object: string; fields?: string[]; where?: any; search?: string | object; … } +### Nested Shape: `QueryWithTransport.aggregations[number]` ---- +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **function** | `Enum<'count' \| 'sum' \| 'avg' \| 'min' \| 'max' \| 'count_distinct'>` | ✅ | Aggregation function | +| **field** | `string` | optional | Field to aggregate (optional for COUNT(*)) | +| **alias** | `string` | ✅ | Result column alias | +| **distinct** | `never` | optional | [REMOVED] `query.aggregations[].distinct` was removed in @objectstack/spec 17 (ADR-0049) — exactly ONE of the six faces that read an aggregation honoured it. The objectql in-memory fallback deduplicated the values before applying the function, while `driver-sql`, `driver-turso`, `driver-mongodb`, `driver-memory` and the service-analytics SQL builder all ignored it — so `{ function: 'sum', field: 'amount', distinct: true }` answered a DEDUPLICATED sum when the engine fell back in memory and an ordinary sum on every SQL datasource: one query, two numbers, chosen by which backend happened to serve it. Both answers are plausible, so nothing surfaced the divergence. Delete the key. For a deduplicated COUNT the live spelling is the `count_distinct` aggregation function, which every SQL face compiles to `COUNT(DISTINCT field)` and the in-memory fallback computes identically. `SUM(DISTINCT …)` / `AVG(DISTINCT …)` get no replacement: no backend ever computed them here, and a per-row measure that needs deduplicating is a modelling problem to fix in the data, not a flag on the read. | +| **filter** | `any` | optional | Per-aggregation filter (SQL FILTER (WHERE …) semantics): narrows the source rows THIS aggregation reads, leaving sibling aggregations unfiltered. Enforced by engine.aggregate: lowered in memory for drivers without native conditional aggregation; a driver reached directly refuses rather than silently dropping it. | -#### Option 2 +### Nested Shape: `QueryWithTransport.groupBy[number]` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **field** | `string` | ✅ | Field to group by | +| **dateGranularity** | `Enum<'day' \| 'week' \| 'month' \| 'quarter' \| 'year'>` | optional | Bucket date values into uniform periods (day/week/month/quarter/year) | +| **alias** | `string` | optional | Alias for the projected group value | -Reference: { object: string; fields?: string[]; where?: any; search?: string | object; … } +### Nested Shape: `QueryWithTransport.expand[string]` ---- +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **object** | `string` | ✅ | Object name (e.g. account) | +| **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | +| **where** | `any` | optional | Filtering criteria (WHERE) | +| **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | +| **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | +| **orderBy** | `{ field: string; order?: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | +| **limit** | `number` | optional | Max records to return (LIMIT) | +| **offset** | `number` | optional | Records to skip (OFFSET) | +| **top** | `number` | optional | Alias for limit (OData compatibility) | +| **cursor** | `never` | optional | [REMOVED] `query.cursor` was removed in @objectstack/spec 17 (ADR-0049) — no driver ever implemented keyset pagination, so the cursor was accepted and ignored and every page came back identical (a caller looping "until hasMore is false" never terminates). Delete the key; `QueryBuilder.cursor()` was removed with it. Express the keyset as an ordinary `where` predicate on your sort key — `where: { created_at: { $gt: last.created_at } }` with the matching `orderBy` — which every driver executes with canonicalised comparands. A first-class cursor, if ever built, will be a response-minted opaque token, not this caller-built record. | +| **joins** | `never` | optional | [REMOVED] `query.joins` was removed in @objectstack/spec 17 (ADR-0049) — no engine or driver ever read it: a query carrying `joins` behaved exactly as if the key were absent, while its name squatted on the reserved REST parameter set. Delete the key. Related records are read through `expand` — `expand: { owner_id: { object: 'user', fields: ['name'] } }` — which the engine resolves via batch $in queries, and whose nested query selects the related record's own columns. Keep the foreign key in your own projection (`fields: ['title', 'owner_id']`): the relation is carried by that column, so projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement — no driver ever resolved one and the ingress refuses it (`400 INVALID_FIELD`). | +| **aggregations** | `{ function: Enum<'count' \| 'sum' \| 'avg' \| 'min' \| 'max' \| 'count_distinct'>; field?: string; alias: string; filter?: any }[]` | optional | Aggregation functions | +| **groupBy** | `(string \| { field: string; dateGranularity?: Enum<'day' \| 'week' \| 'month' \| 'quarter' \| 'year'>; alias?: string })[]` | optional | GROUP BY targets (strings or `{field, dateGranularity?}` objects for date bucketing) | +| **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | +| **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | +| **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | +| **expand** | `Record` | optional | | + +### Nested Shape: `QueryWithTransport.$expand[string]` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **object** | `string` | ✅ | Object name (e.g. account) | +| **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | +| **where** | `any` | optional | Filtering criteria (WHERE) | +| **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | +| **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | +| **orderBy** | `{ field: string; order?: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | +| **limit** | `number` | optional | Max records to return (LIMIT) | +| **offset** | `number` | optional | Records to skip (OFFSET) | +| **top** | `number` | optional | Alias for limit (OData compatibility) | +| **cursor** | `never` | optional | [REMOVED] `query.cursor` was removed in @objectstack/spec 17 (ADR-0049) — no driver ever implemented keyset pagination, so the cursor was accepted and ignored and every page came back identical (a caller looping "until hasMore is false" never terminates). Delete the key; `QueryBuilder.cursor()` was removed with it. Express the keyset as an ordinary `where` predicate on your sort key — `where: { created_at: { $gt: last.created_at } }` with the matching `orderBy` — which every driver executes with canonicalised comparands. A first-class cursor, if ever built, will be a response-minted opaque token, not this caller-built record. | +| **joins** | `never` | optional | [REMOVED] `query.joins` was removed in @objectstack/spec 17 (ADR-0049) — no engine or driver ever read it: a query carrying `joins` behaved exactly as if the key were absent, while its name squatted on the reserved REST parameter set. Delete the key. Related records are read through `expand` — `expand: { owner_id: { object: 'user', fields: ['name'] } }` — which the engine resolves via batch $in queries, and whose nested query selects the related record's own columns. Keep the foreign key in your own projection (`fields: ['title', 'owner_id']`): the relation is carried by that column, so projecting it away leaves expansion nothing to resolve. A dotted `fields` path is NOT a replacement — no driver ever resolved one and the ingress refuses it (`400 INVALID_FIELD`). | +| **aggregations** | `{ function: Enum<'count' \| 'sum' \| 'avg' \| 'min' \| 'max' \| 'count_distinct'>; field?: string; alias: string; filter?: any }[]` | optional | Aggregation functions | +| **groupBy** | `(string \| { field: string; dateGranularity?: Enum<'day' \| 'week' \| 'month' \| 'quarter' \| 'year'>; alias?: string })[]` | optional | GROUP BY targets (strings or `{field, dateGranularity?}` objects for date bucketing) | +| **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | +| **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | +| **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | +| **expand** | `Record` | optional | | + +### Nested Shape: `QueryWithTransport.$search` + +| Property | Type | Required | Description | +| :--- | :--- | :--- | :--- | +| **query** | `string` | ✅ | Search query text | +| **fields** | `string[]` | optional | Fields to search in (if not specified, searches all text fields) | +| **fuzzy** | `boolean` | optional (default: `false`) | [EXPERIMENTAL — not enforced] Fuzzy matching (tolerate typos). The ADR-0061 expansion reads only `query` + `fields`; no executor receives this flag. | +| **operator** | `Enum<'and' \| 'or'>` | optional (default: `"or"`) | [EXPERIMENTAL — not enforced] Logical operator between terms. The ADR-0061 expansion applies its own term semantics; no executor receives this flag. | +| **boost** | `Record` | optional | [EXPERIMENTAL — not enforced] Field-specific relevance boosting (field name -> boost factor). No executor scores results. | +| **minScore** | `number` | optional | [EXPERIMENTAL — not enforced] Minimum relevance score threshold. No executor scores results. | +| **language** | `string` | optional | [EXPERIMENTAL — not enforced] Language for text analysis (e.g., "en", "zh", "es"). No executor selects an analyzer. | +| **highlight** | `boolean` | optional (default: `false`) | [EXPERIMENTAL — not enforced] Search result highlighting. No executor emits highlights. | --- diff --git a/packages/rest/src/rest-server-canonical-query-ast.test.ts b/packages/rest/src/rest-server-canonical-query-ast.test.ts index 021495779c3..f3273f65024 100644 --- a/packages/rest/src/rest-server-canonical-query-ast.test.ts +++ b/packages/rest/src/rest-server-canonical-query-ast.test.ts @@ -399,15 +399,17 @@ describe('[#16337] §2 the declared `FindDataRequest[\'query\']` contract', () = // merges a caller's parameters with its own actually assembles. const mixed: Query = { object: 'x', where: { status: 'queued' }, $top: 5 }; + // The bare aliases the same table names assign for the same reason. + const wireSelect: Query = { object: 'x', select: ['id'] }; + const wireSort: Query = { object: 'x', sort: [{ field: 'a', order: 'asc' }] }; + // Each directive below is LIVE: `tsconfig.test.json` compiles this // layer, and an unused `@ts-expect-error` is TS2578 there. So these are // assertions, not decoration — and they are the boundary of the - // widening: what the transport alias table does NOT name stays refused. - // @ts-expect-error `select` is an RPC alias, not a transport spelling; the declared key is `fields` - const wireSelect: Query = { object: 'x', select: ['id'] }; - // @ts-expect-error `sort` is an RPC alias, not a transport spelling; the declared key is `orderBy` - const wireSort: Query = { object: 'x', sort: [{ field: 'a', order: 'asc' }] }; - // @ts-expect-error the `{field: direction}` record is not `SortNode[]` — `$orderby` is the slot that takes it + // widening: what the transport alias table does NOT name stays refused, + // and a VALUE shape the canonical slot does not declare stays refused on + // that slot (the transport spelling is where the legacy shape lives). + // @ts-expect-error the `{field: direction}` record is not `SortNode[]` — `$orderby` / `sort` are the slots that take it const recordSort: Query = { object: 'x', orderBy: { created_at: 'desc' } }; // @ts-expect-error a comma list is not `Record` — `$expand` is the slot that takes it const commaExpand: Query = { object: 'x', expand: 'owner_id' }; diff --git a/packages/spec/authorable-surface/data.json b/packages/spec/authorable-surface/data.json index 3a05812244e..779e172edfa 100644 --- a/packages/spec/authorable-surface/data.json +++ b/packages/spec/authorable-surface/data.json @@ -793,7 +793,44 @@ "data/QueryTransportParams:$select", "data/QueryTransportParams:$skip", "data/QueryTransportParams:$top", + "data/QueryTransportParams:filter", "data/QueryTransportParams:filters", + "data/QueryTransportParams:populate", + "data/QueryTransportParams:select", + "data/QueryTransportParams:skip", + "data/QueryTransportParams:sort", + "data/QueryWithTransport:$count", + "data/QueryWithTransport:$expand", + "data/QueryWithTransport:$filter", + "data/QueryWithTransport:$orderby", + "data/QueryWithTransport:$search", + "data/QueryWithTransport:$searchFields", + "data/QueryWithTransport:$select", + "data/QueryWithTransport:$skip", + "data/QueryWithTransport:$top", + "data/QueryWithTransport:aggregations", + "data/QueryWithTransport:cursor [RETIRED]", + "data/QueryWithTransport:distinct [RETIRED]", + "data/QueryWithTransport:expand", + "data/QueryWithTransport:fields", + "data/QueryWithTransport:filter", + "data/QueryWithTransport:filters", + "data/QueryWithTransport:groupBy", + "data/QueryWithTransport:having", + "data/QueryWithTransport:joins [RETIRED]", + "data/QueryWithTransport:limit", + "data/QueryWithTransport:object", + "data/QueryWithTransport:offset", + "data/QueryWithTransport:orderBy", + "data/QueryWithTransport:populate", + "data/QueryWithTransport:search", + "data/QueryWithTransport:searchFields", + "data/QueryWithTransport:select", + "data/QueryWithTransport:skip", + "data/QueryWithTransport:sort", + "data/QueryWithTransport:top", + "data/QueryWithTransport:where", + "data/QueryWithTransport:windowFunctions [RETIRED]", "data/RangeOperator:$between", "data/ReferenceResolution:field", "data/ReferenceResolution:fieldType", diff --git a/packages/spec/src/data/data-engine.zod.ts b/packages/spec/src/data/data-engine.zod.ts index c8e7dc20eca..e4614195437 100644 --- a/packages/spec/src/data/data-engine.zod.ts +++ b/packages/spec/src/data/data-engine.zod.ts @@ -756,52 +756,79 @@ export const QUERY_TRANSPORT_DOLLAR_PARAMS: readonly string[] = [ ]; /** - * The transport query parameters — the `$`-prefixed spellings and the plural - * `filters`, each carrying the value of the canonical slot it folds onto. + * The transport query parameters — every spelling the two tables above name + * that is not itself a QueryAST key: the `$`-prefixed forms, the plural + * `filters`, and the bare aliases (`filter` / `select` / `sort` / `skip` / + * `populate`). Each carries the value of the canonical slot it folds onto. + * + * `top` is the one alias the tables name that is NOT here: `BaseQuerySchema` + * already declares it beside `limit`, and re-declaring it would widen a + * canonical member rather than a transport one. * * ⛔ This is NOT the hint table `@objectstack/metadata-protocol` keeps for * near-miss suggestions; nothing in that table is accepted as input and it * stays that way. This one IS accepted — it is what the `findData` door has * folded since #3795, declared at last. * - * Every member is optional and every value type is the union of the shapes the - * boundary already serves on that slot: the structured form a JSON body sends - * and the string form a querystring sends (`?$top=50` arrives as `'50'`, - * `?$select=a,b` as a comma list). Declaring the narrower structured form alone - * would have turned live traffic into a `400`, which is the repair #15866's - * parent thread closed for exactly this reason. + * Every member is nullable-optional and every value type is the union of the + * shapes the boundary already serves on that slot: the structured form a JSON + * body sends, the string form a querystring sends (`?$top=50` arrives as + * `'50'`, `?$select=a,b` as a comma list), the ARRAY form the filter slot's AST + * takes (`['status', '=', 'open']`, which `isFilterAST` reads), and an explicit + * `null`, which the fold treats as a withdrawal rather than a value. + * + * ⛔ Declaring the narrower structured form alone would have turned live traffic + * into a `400` — measured: the body-form AST array on + * `POST /data/:object/query`, pinned by `#7390 §3` in + * `rest-server-repeated-filter-param.test.ts`, which is exactly the repair + * #15866's parent thread closed for this reason. */ export const QueryTransportParamsSchema = lazySchema(() => z.object({ /** OData spelling of `where`. */ - $filter: z.union([z.string(), DataEngineFilterSchema]).optional() + $filter: z.union([z.string(), z.array(z.unknown()), DataEngineFilterSchema]).nullable().optional() .describe('Transport spelling of `where` (OData `$filter`)'), /** Documented plural of the `filter` transport parameter; folds onto `where`. */ - filters: z.union([z.string(), DataEngineFilterSchema]).optional() + filters: z.union([z.string(), z.array(z.unknown()), DataEngineFilterSchema]).nullable().optional() .describe('Transport spelling of `where` (plural of `filter`)'), /** OData spelling of `limit` (two hops: `$top` to `top` to `limit`). */ - $top: z.union([z.number(), z.string()]).optional() + $top: z.union([z.number(), z.string()]).nullable().optional() .describe('Transport spelling of `limit` (OData `$top`)'), /** OData spelling of `offset` (two hops: `$skip` to `skip` to `offset`). */ - $skip: z.union([z.number(), z.string()]).optional() + $skip: z.union([z.number(), z.string()]).nullable().optional() .describe('Transport spelling of `offset` (OData `$skip`)'), /** OData spelling of `orderBy`. */ - $orderby: z.union([z.string(), z.array(z.string()), DataEngineSortSchema]).optional() + $orderby: z.union([z.string(), z.array(z.string()), DataEngineSortSchema]).nullable().optional() .describe('Transport spelling of `orderBy` (OData `$orderby`)'), /** OData spelling of `fields` (two hops: `$select` to `select` to `fields`). */ - $select: z.union([z.string(), z.array(FieldNodeSchema)]).optional() + $select: z.union([z.string(), z.array(FieldNodeSchema)]).nullable().optional() .describe('Transport spelling of `fields` (OData `$select`)'), /** OData spelling of `expand`. */ - $expand: z.union([z.string(), z.array(z.string()), z.record(z.string(), QuerySchema)]).optional() + $expand: z.union([z.string(), z.array(z.string()), z.record(z.string(), QuerySchema)]).nullable().optional() .describe('Transport spelling of `expand` (OData `$expand`)'), /** OData spelling of `search`. */ - $search: z.union([z.string(), FullTextSearchSchema]).optional() + $search: z.union([z.string(), FullTextSearchSchema]).nullable().optional() .describe('Transport spelling of `search` (OData `$search`)'), /** OData spelling of `searchFields`. */ - $searchFields: z.union([z.string(), z.array(z.string())]).optional() + $searchFields: z.union([z.string(), z.array(z.string())]).nullable().optional() .describe('Transport spelling of `searchFields` (OData `$searchFields`)'), /** OData spelling of the response total-count flag. */ - $count: z.union([z.boolean(), z.string()]).optional() + $count: z.union([z.boolean(), z.string()]).nullable().optional() .describe('Transport spelling of the response total-count flag (OData `$count`)'), + /** Bare transport spelling of `where` — the RPC table's own alias. */ + filter: z.union([z.string(), z.array(z.unknown()), DataEngineFilterSchema]).nullable().optional() + .describe('Transport spelling of `where`'), + /** Bare transport spelling of `fields`. */ + select: z.union([z.string(), z.array(z.string())]).nullable().optional() + .describe('Transport spelling of `fields`'), + /** Bare transport spelling of `orderBy`. */ + sort: z.union([z.string(), z.array(z.string()), DataEngineSortSchema]).nullable().optional() + .describe('Transport spelling of `orderBy`'), + /** Bare transport spelling of `offset`. */ + skip: z.union([z.number(), z.string()]).nullable().optional() + .describe('Transport spelling of `offset`'), + /** Bare transport spelling of `expand`. */ + populate: z.union([z.string(), z.array(z.string())]).nullable().optional() + .describe('Transport spelling of `expand`'), }).describe('Transport spellings of the QueryAST slots — folded to canonical keys at the boundary')); /** @@ -809,27 +836,77 @@ export const QueryTransportParamsSchema = lazySchema(() => z.object({ * * The same two steps, in the same order, that the protocol normalizer runs on * raw wire input: `$` spellings onto their bare parameter first (a `$` alias - * never overwrites a bare spelling that is already present), then every alias - * onto its canonical slot via {@link foldQueryAliasSlots}. Conflicting - * spellings of ONE slot are reported, never resolved. + * never overwrites a bare spelling that is already present, and an explicit + * `null` is a WITHDRAWAL rather than a value), then every alias onto its + * canonical slot via {@link foldQueryAliasSlots}. * * Deliberately a fold over the shared TABLES rather than a second copy of the * normalizer: this is the parse-time application, exactly as * `foldRpcQueryOptions` is the parse-time application of - * {@link RPC_QUERY_ALIAS_SLOTS}. Value SHAPES are moved verbatim — lowering a - * comma list or a `{field: direction}` record is the boundary's job, one layer - * down, and doing it twice is how two implementations of one rule start. + * {@link RPC_QUERY_ALIAS_SLOTS}. + * + * ⚠️ A CONFLICT is left unfolded here rather than reported. The boundary + * refuses it one layer down, and it is the only layer that can name the + * spelling the caller actually wrote — telling someone who sent `?$orderby=…` + * that "'orderBy' is invalid" names a parameter absent from their request. So + * the parse does not manufacture a second, worse diagnostic for a request the + * door already answers. */ -function foldQueryTransportBag( - input: Record, - onConflict: (conflict: QueryAliasConflict) => void, -): Record { +function foldQueryTransportBag(input: Record): Record { const bag: Record = { ...input }; for (const [dollar, bare] of QUERY_TRANSPORT_DOLLAR_ALIASES) { if (bag[dollar] != null && bag[bare] == null) bag[bare] = bag[dollar]; delete bag[dollar]; } - foldQueryAliasSlots(bag, QUERY_TRANSPORT_ALIAS_SLOTS, onConflict); + foldQueryAliasSlots(bag, QUERY_TRANSPORT_ALIAS_SLOTS, () => { /* the boundary refuses it */ }); + return lowerFoldedTransportValues(bag); +} + +/** + * Lower the legacy value SHAPES a transport spelling may carry onto the one + * shape the canonical slot declares — exactly the two `foldRpcQueryOptions` + * lowers, plus the querystring's stringly-typed numbers and comma lists. + * + * ⛔ Not a second normalizer. Everything here is a shape ALREADY declared on + * the member it came in on, lowered to the shape already declared on the slot + * it folded onto; nothing is parsed, nothing is looked up, nothing is + * validated. A JSON-encoded `$filter` string is deliberately left alone — that + * one IS parsing, it belongs at the boundary that owns the refusal, and doing + * it twice is how two implementations of one rule start. + */ +function lowerFoldedTransportValues(bag: Record): Record { + for (const key of ['limit', 'offset'] as const) { + if (typeof bag[key] === 'string' && bag[key] !== '') { + const n = Number(bag[key]); + if (Number.isFinite(n)) bag[key] = n; + } + } + for (const key of ['fields', 'searchFields'] as const) { + if (typeof bag[key] === 'string') { + bag[key] = (bag[key] as string).split(',').map((s) => s.trim()).filter(Boolean); + } + } + // A folded `sort` / `$orderby` may carry the record spellings + // `DataEngineSortSchema` allows; canonical `orderBy` declares `SortNode[]`. + if (bag.orderBy !== undefined && bag.orderBy !== null + && !Array.isArray(bag.orderBy) && typeof bag.orderBy === 'object') { + bag.orderBy = Object.entries(bag.orderBy as Record).map( + ([field, order]) => ({ field, order: order === 'asc' || order === 1 ? 'asc' : 'desc' }), + ); + } + // `count` is the one folded key whose canonical target is not an AST slot: + // it is the response's total-count flag, read beside the query. It is left on + // the bag rather than invented into the AST or silently dropped. + // + // A folded `populate` / `$expand` may be a relation-name list or a comma + // list; canonical `expand` is a `{name: QueryAST}` record. + const expandNames = typeof bag.expand === 'string' + ? bag.expand.split(',').map((s) => s.trim()).filter(Boolean) + : Array.isArray(bag.expand) ? (bag.expand as unknown[]).filter((r): r is string => typeof r === 'string') + : null; + if (expandNames) { + bag.expand = Object.fromEntries(expandNames.map((rel) => [rel, { object: rel }])); + } return bag; } @@ -860,24 +937,28 @@ export type QueryWithTransport = QueryInput & Partial; * A query slot whose declared INPUT is the canonical AST or its transport * spelling, and whose declared OUTPUT is the AST. * - * The first arm folds by {@link QUERY_TRANSPORT_ALIAS_SLOTS} / - * {@link QUERY_TRANSPORT_DOLLAR_ALIASES} and then validates the result as a - * real {@link QuerySchema}, so a transport bag parses to canonical keys. The - * second arm is `QuerySchema` itself, unchanged — which is what makes this - * widening provably unable to narrow anything: every input that parsed before - * still parses, by the same schema, with the same issues. + * ⛔ ONE object, not a `z.union` of two, and the difference is a measured + * contract rather than taste. Wrapping the slot in a union puts zod's own + * `invalid_union` issue at the head of the list for EVERY malformed query, and + * the REST ingress reports `fields[0]` off that list — so + * `query.aggregations.0.function` became a bare `query` and the sentence that + * says what to fix stopped being the first thing an AI caller reads + * (`rest-server.ts` `zodIssuesToFields`, pinned by `#5014` and by + * `list-view-grouping-query-door.test.ts` §9). Extending `QuerySchema` with the + * transport members instead leaves every canonical member's issue path exactly + * where it was. + * + * The transform folds by {@link QUERY_TRANSPORT_ALIAS_SLOTS} / + * {@link QUERY_TRANSPORT_DOLLAR_ALIASES} and lowers the legacy value shapes, so + * a transport bag parses to canonical keys. The canonical members keep + * `QuerySchema`'s own declarations verbatim: a body carrying no transport key + * is judged by the identical schema, with the identical issues. */ -export const QueryWithTransportSchema: z.ZodType = lazySchema(() => - z.union([ - z.record(z.string(), z.unknown()) - .transform((bag, ctx) => foldQueryTransportBag( - bag as Record, - (conflict) => ctx.addIssue(aliasConflictIssue(conflict)), - )) - .pipe(QuerySchema as unknown as z.ZodType>), - QuerySchema, - ]) as unknown as z.ZodType, -); +export const QueryWithTransportSchema: z.ZodType = lazySchema( + () => (QuerySchema as unknown as z.ZodObject) + .extend((QueryTransportParamsSchema as unknown as z.ZodObject).shape) + .transform((bag) => foldQueryTransportBag(bag as Record) as unknown as QueryAST), +) as unknown as z.ZodType; /** * What a transport-aware query slot PARSES TO — the canonical QueryAST, in diff --git a/packages/spec/src/data/query-transport.test.ts b/packages/spec/src/data/query-transport.test.ts index d23b84e3216..f7ba9f78e2e 100644 --- a/packages/spec/src/data/query-transport.test.ts +++ b/packages/spec/src/data/query-transport.test.ts @@ -34,26 +34,36 @@ import { import { QuerySchema } from './query.zod'; describe('[#16066] §1 what the transport declares', () => { - it('declares exactly the `$`-spellings and the plural `filters`', () => { + it('declares exactly the spellings the two tables name, minus the one the AST already carries', () => { const declared = Object.keys((QueryTransportParamsSchema as unknown as { shape: object }).shape).sort(); + // DERIVED, not transcribed: the set is every non-canonical spelling the + // tables fold, less `top` — which `BaseQuerySchema` declares beside `limit`, + // so re-declaring it here would widen a canonical member. + const fromTables = new Set([ + ...QUERY_TRANSPORT_ALIAS_SLOTS.flatMap((slot) => slot.aliases), + ...QUERY_TRANSPORT_DOLLAR_ALIASES.map(([dollar]) => dollar), + ]); + fromTables.delete('top'); + expect(declared).toEqual([...fromTables].sort()); + // …and a literal, so a table that shrank cannot make both sides agree at zero. expect(declared).toEqual([ '$count', '$expand', '$filter', '$orderby', '$search', '$searchFields', '$select', '$skip', '$top', - 'filters', + 'filter', 'filters', 'populate', 'select', 'skip', 'sort', ]); }); - it('every `$` spelling it declares is one the alias tables actually fold', () => { + it('every spelling it declares is one the alias tables actually fold', () => { const folded = new Set([ ...QUERY_TRANSPORT_DOLLAR_ALIASES.map(([dollar]) => dollar), ...QUERY_TRANSPORT_ALIAS_SLOTS.flatMap((slot) => slot.aliases), ]); const declared = Object.keys((QueryTransportParamsSchema as unknown as { shape: object }).shape); // Both directions: a declared spelling nothing folds would be a promise the - // door does not keep, and a folded `$` spelling nothing declares is the - // defect this card was filed about. + // door does not keep, and a folded spelling nothing declares is the defect + // this card was filed about. for (const key of declared) expect(folded.has(key), `declared but never folded: ${key}`).toBe(true); for (const key of folded) { - if (!key.startsWith('$') && key !== 'filters') continue; + if (key === 'top') continue; // carried by `BaseQuerySchema` itself expect(declared.includes(key), `folded but never declared: ${key}`).toBe(true); } }); @@ -82,9 +92,7 @@ describe('[#16066] §2 every alias parses to its canonical slot', () => { ['filters -> where', { filters: { status: 'open' } }, { where: { status: 'open' } }], ['filter -> where', { filter: { status: 'open' } }, { where: { status: 'open' } }], ['$top -> limit', { $top: 10 }, { limit: 10 }], - ['top -> limit', { top: 10 }, { limit: 10 }], ['$skip -> offset', { $skip: 20 }, { offset: 20 }], - ['skip -> offset', { skip: 20 }, { offset: 20 }], ['$orderby -> orderBy', { $orderby: [{ field: 'name', order: 'asc' }] }, { orderBy: [{ field: 'name', order: 'asc' }] }], ['sort -> orderBy', { sort: [{ field: 'name', order: 'asc' }] }, { orderBy: [{ field: 'name', order: 'asc' }] }], ['$select -> fields', { $select: ['name'] }, { fields: ['name'] }], @@ -92,6 +100,9 @@ describe('[#16066] §2 every alias parses to its canonical slot', () => { ['$expand -> expand', { $expand: { owner: { object: 'owner' } } }, { expand: { owner: { object: 'owner' } } }], ['$search -> search', { $search: 'acme' }, { search: 'acme' }], ['$searchFields -> searchFields', { $searchFields: ['name'] }, { searchFields: ['name'] }], + ['skip -> offset', { skip: 20 }, { offset: 20 }], + ['populate -> expand', { populate: ['owner'] }, { expand: { owner: { object: 'owner' } } }], + ['top -> limit', { top: 10 }, { limit: 10 }], ]; for (const [label, input, expected] of CASES) { @@ -123,8 +134,10 @@ describe('[#16066] §2 every alias parses to its canonical slot', () => { .toEqual({ object: 'account', where: { a: 1 }, limit: 5, fields: ['name'] }); }); - it('`$count` is consumed, not smuggled into the AST — the AST has no count slot', () => { - expect(parse({ object: 'account', $count: true })).toEqual({ object: 'account' }); + it('`$count` folds onto the bare `count` flag — the one target that is not an AST slot', () => { + // It is the response's total-count flag, read beside the query. Left on the + // bag rather than invented into the AST or silently dropped. + expect(parse({ object: 'account', $count: true })).toEqual({ object: 'account', count: true }); }); }); @@ -139,6 +152,11 @@ describe('[#16066] §3 the widening cannot narrow what already parsed', () => { expect(QueryWithTransportSchema.safeParse(query).success).toBe(true); it('a querystring-shaped `$top` (a string) still parses', () => stillParses({ object: 'a', $top: '50' })); + it('…and lowers to the number the AST slot declares', () => { + expect(QueryWithTransportSchema.parse({ object: 'a', $top: '50' })).toEqual({ object: 'a', limit: 50 }); + }); + it('an explicit null on a transport key is a withdrawal, not a value', () => + expect(QueryWithTransportSchema.parse({ object: 'a', $top: null })).toEqual({ object: 'a' })); it('a JSON-encoded `$filter` string still parses', () => stillParses({ object: 'a', $filter: '{"x":1}' })); it('a `{field: direction}` `$orderby` record still parses', () => stillParses({ object: 'a', $orderby: { created_at: 'desc' } })); it('a comma-list `$select` still parses', () => stillParses({ object: 'a', $select: 'a,b' })); From 83f94806067c4a4c4d0c59b9c623161556b5cc0c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Sep 2026 18:33:19 +0000 Subject: [PATCH 06/10] =?UTF-8?q?fix(spec):=20make=20the=20query=20transpo?= =?UTF-8?q?rt=20fold=20total=20=E2=80=94=20lower=20or=20refuse,=20never=20?= =?UTF-8?q?emit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01LvwGppdonww4zGLWZo5rho --- packages/spec/authorable-surface/data.json | 2 + packages/spec/src/data/data-engine.zod.ts | 371 ++++++++++++++---- .../spec/src/data/query-transport.test.ts | 271 +++++++++++-- 3 files changed, 532 insertions(+), 112 deletions(-) diff --git a/packages/spec/authorable-surface/data.json b/packages/spec/authorable-surface/data.json index 779e172edfa..2dea79eef10 100644 --- a/packages/spec/authorable-surface/data.json +++ b/packages/spec/authorable-surface/data.json @@ -793,6 +793,7 @@ "data/QueryTransportParams:$select", "data/QueryTransportParams:$skip", "data/QueryTransportParams:$top", + "data/QueryTransportParams:count", "data/QueryTransportParams:filter", "data/QueryTransportParams:filters", "data/QueryTransportParams:populate", @@ -809,6 +810,7 @@ "data/QueryWithTransport:$skip", "data/QueryWithTransport:$top", "data/QueryWithTransport:aggregations", + "data/QueryWithTransport:count", "data/QueryWithTransport:cursor [RETIRED]", "data/QueryWithTransport:distinct [RETIRED]", "data/QueryWithTransport:expand", diff --git a/packages/spec/src/data/data-engine.zod.ts b/packages/spec/src/data/data-engine.zod.ts index e4614195437..0b6364b9025 100644 --- a/packages/spec/src/data/data-engine.zod.ts +++ b/packages/spec/src/data/data-engine.zod.ts @@ -1,7 +1,8 @@ // Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license. import { z } from 'zod'; -import { FilterConditionSchema } from './filter.zod'; +import { FilterConditionSchema, FilterArraySchema, parseFilterAST } from './filter.zod'; +import type { FilterArray, FilterCondition } from './filter.zod'; import { SortNodeSchema, QuerySchema, FullTextSearchSchema, FieldNodeSchema, AggregationNodeSchema, GroupByNodeSchema, QUERY_CURSOR_REMOVED, QUERY_DISTINCT_REMOVED } from './query.zod'; import type { QueryAST, QueryInput } from './query.zod'; import { retiredKey } from '../shared/retired-key'; @@ -679,12 +680,18 @@ const RpcQueryOptionsSchema = EngineQueryOptionsSchema.extend({ // 2c. The QUERY TRANSPORT dialect — the FLATTENED SPELLING of the QueryAST // ========================================================================== // -// One vocabulary, two spellings, ONE semantics. Everything below is a -// key-level alias of a canonical QueryAST slot; nothing here means anything -// the AST cannot already say, which is the whole reason it can be declared -// without creating the second de-facto contract Prime Directive #12 warns -// about. ⛔ A spelling that would need its own meaning does not belong here — -// it belongs in the AST or nowhere. +// One vocabulary, two spellings, ONE semantics — and "one semantics" is a +// claim about VALUES, not only about keys. Everything below is a key-level +// alias of a canonical QueryAST slot, and every spelling of a slot accepts +// the SAME set of value shapes: each accepted shape is lowered to the +// canonical member's declared shape before the parse returns, and a shape +// that cannot be lowered without parsing the spec must not do is REFUSED at +// the parse rather than emitted under the AST type. +// +// ⛔ A spelling that would need its own meaning does not belong here — it +// belongs in the AST or nowhere. ⛔ A value shape that only ONE spelling of a +// slot admits does not belong here either: that is the same second contract +// one layer down, selected by spelling instead of by key. /** * The transport-only alias spellings, by canonical QueryAST key — the ones a @@ -728,8 +735,16 @@ export const QUERY_TRANSPORT_ALIAS_SLOTS: readonly QueryAliasSlot[] = RPC_QUERY_ * * `count` is the one bare spelling here that is not a QueryAST slot at all: it * is the response's total-count flag, read beside the query rather than inside - * it. It is declared here because the transport spells it `$count` and the fold - * has to consume it; it does not become an AST member by being named. + * it — but it is TRANSPORTED inside it (`findData` reads `options.count` off + * the same bag), so it is declared on the parsed output too. See + * {@link QueryWithTransportParsed}. + * + * ⚠️ A `$` spelling never overwrites a bare spelling already present, and the + * `$` key is dropped either way: `{top: 6, $top: 5}` folds to `limit: 6` and + * the `5` is discarded without a diagnostic. That is byte-equal to what the + * `findData` door has always done and it is left alone deliberately — raising + * it here and not there would make the POST body and the GET querystring + * answer the same request differently. */ export const QUERY_TRANSPORT_DOLLAR_ALIASES: readonly (readonly [string, string])[] = [ ['$top', 'top'], @@ -755,11 +770,41 @@ export const QUERY_TRANSPORT_DOLLAR_PARAMS: readonly string[] = [ ...QUERY_TRANSPORT_ALIAS_SLOTS.flatMap((slot) => slot.aliases.filter((a) => a.startsWith('$'))), ]; +/** + * The value set EVERY spelling of the `where` slot accepts — a filter + * condition, or the input-only `FilterArray` sugar. + * + * The array arm is here because the DOOR serves it on the canonical key too: + * `findData({query: {where: ['status', '=', 'open']}})` reaches the engine as + * `{status: 'open'}`. Declaring it on `$filter` alone would have left `where` + * refusing at the schema what the door accepts at runtime — two acceptance + * grammars for one slot, selected by spelling. + * + * ⛔ It is NOT declared on `QuerySchema.where`. #5158's ruling C is explicit: + * `FilterArray` is input-only sugar, the wire/storage contract stays exactly + * as it is, and every arrival is lowered through the ONE sink + * ({@link parseFilterAST}) — option A, widening `where` so every driver and + * transport maintains two filter compilers, was rejected. That negative is + * pinned in `filter-array-declaration.test.ts`. So the sugar is accepted HERE, + * at the transport boundary, and lowered before the AST is produced. + */ +const TransportFilterValueSchema = lazySchema(() => z.union([FilterArraySchema, DataEngineFilterSchema])); + +/** + * The value set the `count` spellings accept: the boolean, or the two strings + * a querystring can carry it as. Lowered to the boolean by the fold, because + * {@link QueryWithTransportParsed} declares a boolean — and because + * `findData`'s opt-out test is `options.count === false`, under which an + * unlowered `'false'` reads as "count anyway". + */ +const TransportCountValueSchema = lazySchema(() => z.union([z.boolean(), z.enum(['true', 'false'])])); + /** * The transport query parameters — every spelling the two tables above name * that is not itself a QueryAST key: the `$`-prefixed forms, the plural - * `filters`, and the bare aliases (`filter` / `select` / `sort` / `skip` / - * `populate`). Each carries the value of the canonical slot it folds onto. + * `filters`, the bare aliases (`filter` / `select` / `sort` / `skip` / + * `populate`) and the response-flag pair `count` / `$count`. Each carries the + * value of the canonical slot it folds onto. * * `top` is the one alias the tables name that is NOT here: `BaseQuerySchema` * already declares it beside `limit`, and re-declaring it would widen a @@ -770,34 +815,42 @@ export const QUERY_TRANSPORT_DOLLAR_PARAMS: readonly string[] = [ * stays that way. This one IS accepted — it is what the `findData` door has * folded since #3795, declared at last. * - * Every member is nullable-optional and every value type is the union of the - * shapes the boundary already serves on that slot: the structured form a JSON - * body sends, the string form a querystring sends (`?$top=50` arrives as - * `'50'`, `?$select=a,b` as a comma list), the ARRAY form the filter slot's AST - * takes (`['status', '=', 'open']`, which `isFilterAST` reads), and an explicit - * `null`, which the fold treats as a withdrawal rather than a value. + * Every member is nullable-optional (an explicit `null` is a WITHDRAWAL, which + * the fold deletes rather than folds) and every value type is the set the + * boundary already serves on that slot, LOWERED to the canonical member's + * declared shape by {@link foldQueryTransportBag}: the structured form a JSON + * body sends, the comma list a querystring sends (`?$select=a,b`), the + * stringly-typed number (`?$top=50` arrives as `'50'`), the relation-name list + * on `$expand`, and the `FilterArray` sugar on the filter spellings. * - * ⛔ Declaring the narrower structured form alone would have turned live traffic - * into a `400` — measured: the body-form AST array on + * ⛔ Three shapes are deliberately NOT declared, because lowering them means + * PARSING — and a second parser beside the door's is how one rule gets two + * implementations that disagree: a JSON-encoded `$filter` string, the OData + * `$orderby` / `sort` expression string (`'name desc'`, `'-created_at'`) and + * its `string[]` form. They fail the parse at the member that carries them + * rather than reaching the AST as a string the engine would have to re-read. + * + * ⛔ Declaring the narrower structured form ALONE would have turned live + * traffic into a `400` — measured: the body-form AST array on * `POST /data/:object/query`, pinned by `#7390 §3` in * `rest-server-repeated-filter-param.test.ts`, which is exactly the repair * #15866's parent thread closed for this reason. */ export const QueryTransportParamsSchema = lazySchema(() => z.object({ /** OData spelling of `where`. */ - $filter: z.union([z.string(), z.array(z.unknown()), DataEngineFilterSchema]).nullable().optional() + $filter: TransportFilterValueSchema.nullable().optional() .describe('Transport spelling of `where` (OData `$filter`)'), /** Documented plural of the `filter` transport parameter; folds onto `where`. */ - filters: z.union([z.string(), z.array(z.unknown()), DataEngineFilterSchema]).nullable().optional() + filters: TransportFilterValueSchema.nullable().optional() .describe('Transport spelling of `where` (plural of `filter`)'), /** OData spelling of `limit` (two hops: `$top` to `top` to `limit`). */ $top: z.union([z.number(), z.string()]).nullable().optional() - .describe('Transport spelling of `limit` (OData `$top`)'), + .describe('Transport spelling of `limit` (OData `$top`) — a number, or the digits a querystring carries it as'), /** OData spelling of `offset` (two hops: `$skip` to `skip` to `offset`). */ $skip: z.union([z.number(), z.string()]).nullable().optional() - .describe('Transport spelling of `offset` (OData `$skip`)'), + .describe('Transport spelling of `offset` (OData `$skip`) — a number, or the digits a querystring carries it as'), /** OData spelling of `orderBy`. */ - $orderby: z.union([z.string(), z.array(z.string()), DataEngineSortSchema]).nullable().optional() + $orderby: DataEngineSortSchema.nullable().optional() .describe('Transport spelling of `orderBy` (OData `$orderby`)'), /** OData spelling of `fields` (two hops: `$select` to `select` to `fields`). */ $select: z.union([z.string(), z.array(FieldNodeSchema)]).nullable().optional() @@ -812,16 +865,19 @@ export const QueryTransportParamsSchema = lazySchema(() => z.object({ $searchFields: z.union([z.string(), z.array(z.string())]).nullable().optional() .describe('Transport spelling of `searchFields` (OData `$searchFields`)'), /** OData spelling of the response total-count flag. */ - $count: z.union([z.boolean(), z.string()]).nullable().optional() + $count: TransportCountValueSchema.nullable().optional() .describe('Transport spelling of the response total-count flag (OData `$count`)'), + /** Bare spelling of the response total-count flag; only an explicit `false` opts out. */ + count: TransportCountValueSchema.nullable().optional() + .describe('Response total-count flag — only an explicit `false` skips the COUNT query'), /** Bare transport spelling of `where` — the RPC table's own alias. */ - filter: z.union([z.string(), z.array(z.unknown()), DataEngineFilterSchema]).nullable().optional() + filter: TransportFilterValueSchema.nullable().optional() .describe('Transport spelling of `where`'), /** Bare transport spelling of `fields`. */ select: z.union([z.string(), z.array(z.string())]).nullable().optional() .describe('Transport spelling of `fields`'), /** Bare transport spelling of `orderBy`. */ - sort: z.union([z.string(), z.array(z.string()), DataEngineSortSchema]).nullable().optional() + sort: DataEngineSortSchema.nullable().optional() .describe('Transport spelling of `orderBy`'), /** Bare transport spelling of `offset`. */ skip: z.union([z.number(), z.string()]).nullable().optional() @@ -832,7 +888,8 @@ export const QueryTransportParamsSchema = lazySchema(() => z.object({ }).describe('Transport spellings of the QueryAST slots — folded to canonical keys at the boundary')); /** - * Fold a transport bag onto canonical QueryAST keys, by the two tables above. + * Fold a transport bag onto canonical QueryAST keys, by the two tables above, + * and lower every folded value onto the canonical member's declared shape. * * The same two steps, in the same order, that the protocol normalizer runs on * raw wire input: `$` spellings onto their bare parameter first (a `$` alias @@ -845,41 +902,119 @@ export const QueryTransportParamsSchema = lazySchema(() => z.object({ * `foldRpcQueryOptions` is the parse-time application of * {@link RPC_QUERY_ALIAS_SLOTS}. * - * ⚠️ A CONFLICT is left unfolded here rather than reported. The boundary - * refuses it one layer down, and it is the only layer that can name the - * spelling the caller actually wrote — telling someone who sent `?$orderby=…` - * that "'orderBy' is invalid" names a parameter absent from their request. So - * the parse does not manufacture a second, worse diagnostic for a request the - * door already answers. + * A CONFLICT — two spellings of one slot carrying different values — fails the + * parse with {@link aliasConflictIssue} at the canonical path, quoting the + * spelling the caller actually WROTE. `wireSpelling` / `slotParam` here are the + * same composition the door's `findData` uses for the same reason (#4226): + * telling someone who sent `?$orderby=…` that "'orderBy' is invalid" names a + * parameter absent from their request. Leaving the conflict unfolded — the + * shape this fold shipped with — left BOTH spellings on the parsed bag, so the + * declared AST output carried a transport key. */ -function foldQueryTransportBag(input: Record): Record { +function foldQueryTransportBag( + input: Record, + ctx: z.core.$RefinementCtx, +): QueryAstWithCount { const bag: Record = { ...input }; + const wireSpelling: Record = {}; for (const [dollar, bare] of QUERY_TRANSPORT_DOLLAR_ALIASES) { - if (bag[dollar] != null && bag[bare] == null) bag[bare] = bag[dollar]; + if (bag[dollar] != null && bag[bare] == null) { + bag[bare] = bag[dollar]; + wireSpelling[bare] = dollar; + } delete bag[dollar]; } - foldQueryAliasSlots(bag, QUERY_TRANSPORT_ALIAS_SLOTS, () => { /* the boundary refuses it */ }); - return lowerFoldedTransportValues(bag); + const spellingFor = (name: string): string => wireSpelling[name] ?? name; + let refused = false; + const arrivedAs = foldQueryAliasSlots(bag, QUERY_TRANSPORT_ALIAS_SLOTS, (conflict) => { + refused = true; + ctx.addIssue(aliasConflictIssue({ + canonical: conflict.canonical, + spellings: conflict.spellings.map(spellingFor), + })); + }); + const slotParam = (canonical: string): string => spellingFor(arrivedAs[canonical] ?? canonical); + if (lowerFoldedTransportValues(bag, ctx, slotParam)) refused = true; + // One diagnostic per defect. A refusal above already named the parameter the + // caller wrote and why; running the AST parse over the value it refused would + // add a second issue about the same key, and the REST ingress reports + // `fields[0]` off that list (`zodIssuesToFields`, #5014). + if (refused) return z.NEVER; + // THE OUTPUT IS CONSTRUCTED HERE, and nowhere else. The folded bag is parsed + // by {@link QueryAstWithCountSchema} and that parse's RESULT is what leaves + // this transform, so a value the AST does not declare cannot be returned + // under the AST type — the shape this schema shipped with asserted its output + // with a cast and validated nothing, and `limit: 'abc'` / `orderBy: 'name'` / + // `where: 'not json'` / a `$filter` left behind by an unfolded conflict all + // left the parse wearing it. + // + // ⛔ NOT `.transform(fold).pipe(QueryAstWithCountSchema)`, which is the same + // construction one level out: `build-schemas.ts` publishes a schema's OUTPUT + // shape whenever that shape has a JSON form and only falls back to the INPUT + // when it does not, so adding the pipe makes `data/QueryWithTransport.json` + // publish the canonical AST and unpublishes all 32 transport keys from + // `authorable-surface/data.json` — measured, and refused by that file's own + // deletion gate. The authorable surface of this slot IS the transport + // vocabulary, so the parse is done here instead. + const parsed = QueryAstWithCountSchema.safeParse(bag); + if (!parsed.success) { + for (const issue of parsed.error.issues) ctx.addIssue({ ...issue, path: [...issue.path] }); + return z.NEVER; + } + return parsed.data; } /** - * Lower the legacy value SHAPES a transport spelling may carry onto the one - * shape the canonical slot declares — exactly the two `foldRpcQueryOptions` - * lowers, plus the querystring's stringly-typed numbers and comma lists. + * Lower every folded value SHAPE onto the one shape the canonical slot + * declares — or refuse it, at the canonical path, naming the spelling the + * caller wrote. * - * ⛔ Not a second normalizer. Everything here is a shape ALREADY declared on - * the member it came in on, lowered to the shape already declared on the slot - * it folded onto; nothing is parsed, nothing is looked up, nothing is - * validated. A JSON-encoded `$filter` string is deliberately left alone — that - * one IS parsing, it belongs at the boundary that owns the refusal, and doing - * it twice is how two implementations of one rule start. + * This function is what makes the fold TOTAL: after it, every key left on the + * bag is a QueryAST member (plus `count`) carrying a value that member + * declares, which is why {@link QueryWithTransportSchema} can PARSE the result + * with the AST schema instead of asserting it. Nothing here parses: a comma + * list is split, a numeric string is converted, a sort record is turned into + * sort nodes, a relation list is turned into an expand map, and the + * `FilterArray` sugar is handed to {@link parseFilterAST}, the ONE declared + * lowering sink (#5158 ruling C). Anything that would need a real parser was + * refused one level up, on the member that declared it. + * + * ⚠️ A refusal is reported and the value is LEFT AS IT IS rather than deleted: + * `ctx.addIssue` already fails the parse, so the bag is never observed — and + * deleting the key would turn a refused `$top` into a query with no limit, + * which is the unbounded read this whole declaration exists to close. + * + * Returns whether anything was refused, so the caller can skip the AST parse + * that would otherwise report the same key twice. */ -function lowerFoldedTransportValues(bag: Record): Record { +function lowerFoldedTransportValues( + bag: Record, + ctx: z.core.$RefinementCtx, + slotParam: (canonical: string) => string, +): boolean { + let refused = false; + const refuse = (canonical: string, detail: string): void => { + refused = true; + ctx.addIssue({ + code: 'custom', + path: [canonical], + message: `Query parameter '${slotParam(canonical)}' ${detail}`, + }); + }; + // `?$top=50` / `?$skip=10` arrive as digits; canonical `limit` / `offset` + // declare numbers. A string that is not a number is refused rather than + // passed on: `$top: 'abc'` used to reach the engine as `limit: null` — an + // UNBOUNDED read under a 200 — and `$top: ''` as `limit: 0`. for (const key of ['limit', 'offset'] as const) { - if (typeof bag[key] === 'string' && bag[key] !== '') { - const n = Number(bag[key]); - if (Number.isFinite(n)) bag[key] = n; + const value = bag[key]; + if (typeof value !== 'string') continue; + const trimmed = value.trim(); + const n = trimmed === '' ? Number.NaN : Number(trimmed); + if (!Number.isFinite(n)) { + refuse(key, `carries ${JSON.stringify(value)}, which is not a number.`); + continue; } + bag[key] = n; } for (const key of ['fields', 'searchFields'] as const) { if (typeof bag[key] === 'string') { @@ -894,10 +1029,24 @@ function lowerFoldedTransportValues(bag: Record): Record ({ field, order: order === 'asc' || order === 1 ? 'asc' : 'desc' }), ); } - // `count` is the one folded key whose canonical target is not an AST slot: - // it is the response's total-count flag, read beside the query. It is left on - // the bag rather than invented into the AST or silently dropped. - // + // The `FilterArray` sugar, on whichever spelling of the slot it arrived + // under, through the one declared sink. `parseFilterAST` THROWS on a + // comparand shape it refuses (`assertListComparandShapes`); a throw out of a + // transform escapes `safeParse` entirely, so it is converted to an issue. + if (Array.isArray(bag.where)) { + let lowered: unknown; + try { + lowered = parseFilterAST(bag.where); + } catch (error) { + refuse('where', `carries a filter array that cannot be lowered: ${(error as Error)?.message ?? String(error)}`); + lowered = bag.where; + } + if (lowered === undefined) { + refuse('where', 'carries a filter array that lowers to no condition at all.'); + } else { + bag.where = lowered; + } + } // A folded `populate` / `$expand` may be a relation-name list or a comma // list; canonical `expand` is a `{name: QueryAST}` record. const expandNames = typeof bag.expand === 'string' @@ -907,7 +1056,12 @@ function lowerFoldedTransportValues(bag: Record): Record [rel, { object: rel }])); } - return bag; + // `count` is the one folded key whose canonical target is not an AST slot: + // it is the response's total-count flag, read beside the query and declared + // on the output beside it ({@link QueryWithTransportParsed}). + if (bag.count === 'true') bag.count = true; + else if (bag.count === 'false') bag.count = false; + return refused; } /** The transport query parameters, as an author writes them (ADR-0122). */ @@ -930,12 +1084,70 @@ export type QueryTransportParamsParsed = z.infer + & { where?: FilterCondition | FilterArray } + & Partial; + +/** + * What a transport-aware query slot parses to: the canonical QueryAST, in every + * spelling, plus the `count` flag. Module-private because ADR-0122 names the + * exported pair {@link QueryWithTransport} / {@link QueryWithTransportParsed}, + * and the latter is inferred from the schema this type constructs. + * + * `count` is on the output because it is on the WIRE inside this slot and read + * out of it — `findData` takes its opt-out from `options.count` on the same bag + * it hands the engine. Folding `$count` and then dropping it would declare a + * parameter the parse silently discards, which is the phantom-key shape this + * card was filed about. It is not an AST member and never reaches the engine: + * the door deletes it before `engine.find`. */ -export type QueryWithTransport = QueryInput & Partial; +type QueryAstWithCount = QueryAST & { count?: boolean }; + +/** + * The AST plus the `count` flag — the schema that CONSTRUCTS + * {@link QueryWithTransportSchema}'s output. Every parsed query is the result + * of this schema's own parse, so the declared output is a measurement rather + * than an assertion. + * + * Written as an intersection rather than an `extend` so both type arguments + * come from `QuerySchema`'s own declaration instead of being restated: + * `QuerySchema` is annotated `z.ZodType` for its + * recursion, and `.extend` on it is reachable only through a cast that throws + * that annotation away. + */ +const QueryAstWithCountSchema: z.ZodType = + lazySchema(() => QuerySchema.and(z.object({ + count: z.boolean().optional() + .describe('Response total-count flag — only an explicit `false` skips the COUNT query'), + }))); + +/** + * The canonical query object's `ZodObject` face, which `QuerySchema`'s own + * recursion annotation (`z.ZodType`) hides. + * + * ⛔ This cast asserts nothing about what the schema accepts or emits — it + * restores the builder surface (`.extend`, `.shape`) on a value that IS a + * `ZodObject` at run time. The claim about the parsed OUTPUT is made by + * {@link QueryAstWithCountSchema}'s parse inside the fold, and by nothing else. + */ +const queryObjectFace = (): z.ZodObject => + QuerySchema as unknown as z.ZodObject; /** * A query slot whose declared INPUT is the canonical AST or its transport - * spelling, and whose declared OUTPUT is the AST. + * spelling, and whose OUTPUT is CONSTRUCTED: the folded bag is parsed by + * {@link QueryAstWithCountSchema} and that parse's result is what the transform + * returns, so a value the AST does not declare cannot leave this schema. The + * previous shape asserted the output type with a cast and validated nothing, + * and every non-AST value the fold could emit — `limit: 'abc'`, + * `orderBy: 'name'`, `where: 'not json'`, an unfolded `$filter` left behind by + * a conflict — left the parse wearing the AST type. * * ⛔ ONE object, not a `z.union` of two, and the difference is a measured * contract rather than taste. Wrapping the slot in a union puts zod's own @@ -949,25 +1161,42 @@ export type QueryWithTransport = QueryInput & Partial; * where it was. * * The transform folds by {@link QUERY_TRANSPORT_ALIAS_SLOTS} / - * {@link QUERY_TRANSPORT_DOLLAR_ALIASES} and lowers the legacy value shapes, so - * a transport bag parses to canonical keys. The canonical members keep - * `QuerySchema`'s own declarations verbatim: a body carrying no transport key - * is judged by the identical schema, with the identical issues. + * {@link QUERY_TRANSPORT_DOLLAR_ALIASES} and lowers every legacy value shape, + * so a transport bag parses to canonical keys. The canonical members keep + * `QuerySchema`'s own declarations verbatim — except `where`, which accepts the + * `FilterArray` sugar on every spelling of its slot because the door serves it + * on every spelling of its slot. */ -export const QueryWithTransportSchema: z.ZodType = lazySchema( - () => (QuerySchema as unknown as z.ZodObject) - .extend((QueryTransportParamsSchema as unknown as z.ZodObject).shape) - .transform((bag) => foldQueryTransportBag(bag as Record) as unknown as QueryAST), -) as unknown as z.ZodType; +export const QueryWithTransportSchema = lazySchema( + () => queryObjectFace() + .extend({ + ...(QueryTransportParamsSchema as unknown as z.ZodObject).shape, + where: TransportFilterValueSchema.optional().describe( + 'Filtering criteria (WHERE) — a filter condition, or the input-only `FilterArray` ' + + "sugar (`['status', '=', 'open']`), which is lowered through `parseFilterAST` " + + 'before the query is produced (#5158 ruling C).' + ), + }) + .transform((bag, ctx) => foldQueryTransportBag(bag as Record, ctx)), + // ⚠️ ONE cast, and it restates the INPUT only. The OUTPUT type is inferred + // from the transform's return type, which is the return type of + // `QueryAstWithCountSchema.safeParse` — so nothing here asserts what the + // schema emits. The input has to be restated because `.extend` on the erased + // object face types it as an open record, which would make `z.input` of this + // slot admit `$sort` and a query with no `object`. +) as unknown as z.ZodType; /** - * What a transport-aware query slot PARSES TO — the canonical QueryAST, in - * every spelling (ADR-0122). The declared output is deliberately the AST - * alone: a consumer reading a parsed query never sees a transport key. + * What a transport-aware query slot PARSES TO (ADR-0122): the canonical + * QueryAST plus the `count` flag, in every spelling. + * + * A consumer reading a parsed query never sees a transport key, and that is + * now a property of the construction rather than a promise: the fold removes + * every transport spelling, and {@link QueryAstWithCountSchema} — whose parse + * produces this value — strips anything that is somehow left. */ export type QueryWithTransportParsed = z.infer; - export const DataEngineFindRequestSchema = lazySchema(() => z.object({ method: z.literal('find'), object: z.string(), diff --git a/packages/spec/src/data/query-transport.test.ts b/packages/spec/src/data/query-transport.test.ts index f7ba9f78e2e..40b36a1c8c8 100644 --- a/packages/spec/src/data/query-transport.test.ts +++ b/packages/spec/src/data/query-transport.test.ts @@ -23,6 +23,7 @@ */ import { describe, it, expect } from 'vitest'; +import type { z } from 'zod'; import { QUERY_TRANSPORT_ALIAS_SLOTS, QUERY_TRANSPORT_DOLLAR_ALIASES, @@ -32,38 +33,46 @@ import { RPC_QUERY_ALIAS_SLOTS, } from './data-engine.zod'; import { QuerySchema } from './query.zod'; +import type { QueryAST } from './query.zod'; describe('[#16066] §1 what the transport declares', () => { - it('declares exactly the spellings the two tables name, minus the one the AST already carries', () => { + /** Every spelling the two tables touch — both halves of each `$` pair. */ + const foldedSpellings = (): Set => new Set([ + ...QUERY_TRANSPORT_ALIAS_SLOTS.flatMap((slot) => slot.aliases), + ...QUERY_TRANSPORT_DOLLAR_ALIASES.flatMap(([dollar, bare]) => [dollar, bare]), + ]); + + /** The canonical keys `QuerySchema` itself declares — read, never transcribed. */ + const canonicalKeys = (): Set => + new Set(Object.keys((QuerySchema as unknown as { shape: object }).shape)); + + it('declares exactly the folded spellings the AST does not already carry', () => { const declared = Object.keys((QueryTransportParamsSchema as unknown as { shape: object }).shape).sort(); - // DERIVED, not transcribed: the set is every non-canonical spelling the - // tables fold, less `top` — which `BaseQuerySchema` declares beside `limit`, - // so re-declaring it here would widen a canonical member. - const fromTables = new Set([ - ...QUERY_TRANSPORT_ALIAS_SLOTS.flatMap((slot) => slot.aliases), - ...QUERY_TRANSPORT_DOLLAR_ALIASES.map(([dollar]) => dollar), - ]); - fromTables.delete('top'); + // DERIVED, not transcribed: every spelling the tables fold, less the ones + // `BaseQuerySchema` already declares (`top`, `orderBy`, `search`, + // `searchFields`) — re-declaring one of those here would widen a CANONICAL + // member rather than a transport one. `count` survives the subtraction + // because it is the one bare half of a `$` pair the AST does not carry. + const canonical = canonicalKeys(); + const fromTables = [...foldedSpellings()].filter((k) => !canonical.has(k)); expect(declared).toEqual([...fromTables].sort()); // …and a literal, so a table that shrank cannot make both sides agree at zero. expect(declared).toEqual([ '$count', '$expand', '$filter', '$orderby', '$search', '$searchFields', '$select', '$skip', '$top', - 'filter', 'filters', 'populate', 'select', 'skip', 'sort', + 'count', 'filter', 'filters', 'populate', 'select', 'skip', 'sort', ]); }); it('every spelling it declares is one the alias tables actually fold', () => { - const folded = new Set([ - ...QUERY_TRANSPORT_DOLLAR_ALIASES.map(([dollar]) => dollar), - ...QUERY_TRANSPORT_ALIAS_SLOTS.flatMap((slot) => slot.aliases), - ]); + const folded = foldedSpellings(); + const canonical = canonicalKeys(); const declared = Object.keys((QueryTransportParamsSchema as unknown as { shape: object }).shape); // Both directions: a declared spelling nothing folds would be a promise the // door does not keep, and a folded spelling nothing declares is the defect // this card was filed about. for (const key of declared) expect(folded.has(key), `declared but never folded: ${key}`).toBe(true); for (const key of folded) { - if (key === 'top') continue; // carried by `BaseQuerySchema` itself + if (canonical.has(key)) continue; // carried by `BaseQuerySchema` itself expect(declared.includes(key), `folded but never declared: ${key}`).toBe(true); } }); @@ -135,38 +144,187 @@ describe('[#16066] §2 every alias parses to its canonical slot', () => { }); it('`$count` folds onto the bare `count` flag — the one target that is not an AST slot', () => { - // It is the response's total-count flag, read beside the query. Left on the - // bag rather than invented into the AST or silently dropped. + // It is the response's total-count flag, read beside the query and declared + // on the parsed output beside it. `findData` takes its opt-out from + // `options.count` on this very bag, so folding it and then dropping it would + // declare a parameter the parse silently discards. expect(parse({ object: 'account', $count: true })).toEqual({ object: 'account', count: true }); }); + + it('the bare `count` spelling is declared too, and keeps its value', () => { + expect(parse({ object: 'account', count: false })).toEqual({ object: 'account', count: false }); + }); }); -describe('[#16066] §3 the widening cannot narrow what already parsed', () => { +describe('[#16066] §3 the fold is TOTAL — every admitted value shape lowers, or the parse fails', () => { /** - * The canonical arm of the union is `QuerySchema` itself, unchanged, so - * anything that parsed before still parses — including shapes the transport - * arm declines. ⛔ This is the property that makes the declaration safe to - * land as a `minor`: it records what the door accepts and refuses nothing new. + * THE LOAD-BEARING SECTION, and every assertion in it reads the OUTPUT. + * + * "One semantics" is a claim about VALUES, not only about keys. This file's + * first shape asserted `.success` alone, and under that assertion the + * transport arm admitted a whole grammar the canonical arm of the SAME schema + * refuses — `$top: 'abc'` parsed to `limit: 'abc'`, `$orderby: 'name'` to + * `orderBy: 'name'`, `$filter: 'not json'` to `where: 'not json'`, and a + * conflicting bag parsed with `$filter` STILL ON THE OUTPUT. Each of those is + * a `.success === true`. ⛔ A test that asserts success alone cannot catch a + * wrong output; assert the output. + * + * What made them reachable is what the door does with them: `$top: 'abc'` + * passed POST validation and reached the engine as `limit: null` — an + * UNBOUNDED read under a 200 — and `$top: ''` as `limit: 0`. */ - const stillParses = (query: Record) => - expect(QueryWithTransportSchema.safeParse(query).success).toBe(true); - - it('a querystring-shaped `$top` (a string) still parses', () => stillParses({ object: 'a', $top: '50' })); - it('…and lowers to the number the AST slot declares', () => { - expect(QueryWithTransportSchema.parse({ object: 'a', $top: '50' })).toEqual({ object: 'a', limit: 50 }); - }); - it('an explicit null on a transport key is a withdrawal, not a value', () => - expect(QueryWithTransportSchema.parse({ object: 'a', $top: null })).toEqual({ object: 'a' })); - it('a JSON-encoded `$filter` string still parses', () => stillParses({ object: 'a', $filter: '{"x":1}' })); - it('a `{field: direction}` `$orderby` record still parses', () => stillParses({ object: 'a', $orderby: { created_at: 'desc' } })); - it('a comma-list `$select` still parses', () => stillParses({ object: 'a', $select: 'a,b' })); - it('a comma-list `$expand` still parses', () => stillParses({ object: 'a', $expand: 'owner' })); - it('a leftover key destined for the implicit field filter still parses', () => stillParses({ object: 'a', status: 'open' })); - it('conflicting spellings of one slot still parse — the DOOR refuses them, where it can name the parameter the caller wrote', () => - stillParses({ object: 'a', where: { a: 1 }, $filter: { b: 2 } })); - - it('a non-object query is still refused', () => { - expect(QueryWithTransportSchema.safeParse('nope').success).toBe(false); + const parsesTo = (query: Record, expected: Record) => + expect(QueryWithTransportSchema.parse(query)).toEqual(expected); + const refuses = (query: Record) => { + const result = QueryWithTransportSchema.safeParse(query); + expect(result.success, `expected a refusal, got ${JSON.stringify(result)}`).toBe(false); + return result.success ? [] : result.error.issues; + }; + + describe('the shapes that LOWER — the output is the canonical value, never the arrival shape', () => { + it('a querystring-shaped `$top` becomes the number the AST slot declares', () => + parsesTo({ object: 'a', $top: '50' }, { object: 'a', limit: 50 })); + it('a querystring-shaped `$skip` becomes a number too', () => + parsesTo({ object: 'a', $skip: '10' }, { object: 'a', offset: 10 })); + it('a `{field: direction}` `$orderby` record becomes sort nodes', () => + parsesTo({ object: 'a', $orderby: { created_at: 'desc' } }, + { object: 'a', orderBy: [{ field: 'created_at', order: 'desc' }] })); + it('a `{field: 1 | -1}` `$orderby` record becomes sort nodes', () => + parsesTo({ object: 'a', $orderby: { created_at: -1 } }, + { object: 'a', orderBy: [{ field: 'created_at', order: 'desc' }] })); + it('a comma-list `$select` becomes a field array', () => + parsesTo({ object: 'a', $select: 'a,b' }, { object: 'a', fields: ['a', 'b'] })); + it('a comma-list `$searchFields` becomes an array', () => + parsesTo({ object: 'a', $searchFields: 'a,b' }, { object: 'a', searchFields: ['a', 'b'] })); + it('a comma-list `$expand` becomes an expand map', () => + parsesTo({ object: 'a', $expand: 'owner,project' }, + { object: 'a', expand: { owner: { object: 'owner' }, project: { object: 'project' } } })); + it('a `$count` string becomes the boolean the flag is read as', () => + parsesTo({ object: 'a', $count: 'false' }, { object: 'a', count: false })); + it('an explicit null on a transport key is a withdrawal, not a value', () => + parsesTo({ object: 'a', $top: null }, { object: 'a' })); + it('a leftover key destined for the implicit field filter is dropped, not carried', () => + parsesTo({ object: 'a', status: 'open' }, { object: 'a' })); + }); + + describe('the `FilterArray` sugar — accepted on EVERY spelling of the slot, lowered through the one sink', () => { + /** + * The door serves the ObjectQL array on the canonical key too: `findData` + * hands `{where: ['status', '=', 'open']}` to the engine as + * `{status: 'open'}`. Declaring it on `$filter` alone left `where` refusing + * at the schema what the door accepts at run time — one slot, two grammars, + * selected by spelling. + * + * ⛔ It is lowered, never admitted: #5158 ruling C keeps `FilterArray` + * input-only sugar and `parseFilterAST` the one lowering sink. §4 below + * pins the other half — `QuerySchema.where` still refuses it. + */ + const LOWERED = { status: 'open' }; + it('`where`', () => parsesTo({ object: 'a', where: ['status', '=', 'open'] }, { object: 'a', where: LOWERED })); + it('`$filter`', () => parsesTo({ object: 'a', $filter: ['status', '=', 'open'] }, { object: 'a', where: LOWERED })); + it('`filter`', () => parsesTo({ object: 'a', filter: ['status', '=', 'open'] }, { object: 'a', where: LOWERED })); + it('`filters`', () => parsesTo({ object: 'a', filters: ['status', '=', 'open'] }, { object: 'a', where: LOWERED })); + it('a compound group lowers to the logical node', () => + parsesTo({ object: 'a', $filter: ['and', ['stage', '=', 'won'], ['amount', '>', 1000]] }, + { object: 'a', where: { $and: [{ stage: 'won' }, { amount: { $gt: 1000 } }] } })); + }); + + describe('the shapes that are REFUSED — because lowering them would mean parsing', () => { + /** + * A JSON-encoded filter, an OData sort expression and a non-numeric `$top` + * cannot be lowered without a parser, and a second parser beside the door's + * is how one rule ends up with two implementations that disagree. They fail + * the parse rather than leaving it under the AST type. + */ + it('a JSON-encoded `$filter` string', () => refuses({ object: 'a', $filter: '{"x":1}' })); + it('an OData `$orderby` expression string', () => refuses({ object: 'a', $orderby: 'name desc' })); + it('a `sort` expression string', () => refuses({ object: 'a', sort: '-created_at' })); + it('a `string[]` `$orderby`', () => refuses({ object: 'a', $orderby: ['name'] })); + it('a non-numeric `$top` — the unbounded-read case', () => { + const issues = refuses({ object: 'a', $top: 'abc' }); + // ONE diagnostic, at the canonical path, naming the parameter the caller + // actually wrote (#4226) — not `limit`, which is absent from the request. + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['limit']); + expect(issues[0].message).toContain("'$top'"); + }); + it('an empty-string `$top` — the `limit: 0` case', () => refuses({ object: 'a', $top: '' })); + it('a non-numeric `$skip`', () => refuses({ object: 'a', $skip: 'abc' })); + it('a `$count` that is neither the boolean nor its two spellings', () => + refuses({ object: 'a', $count: 'yes' })); + it('a non-object query', () => expect(QueryWithTransportSchema.safeParse('nope').success).toBe(false)); + }); + + describe('a CONFLICT fails the parse and names the spelling the caller wrote', () => { + /** + * Left unfolded, a conflict put BOTH spellings on the parsed bag, so the + * declared AST output carried a transport key — the in-source promise that + * "a consumer reading a parsed query never sees a transport key" was + * measured false on exactly this input. + */ + it('`where` against `$filter`', () => { + const issues = refuses({ object: 'a', where: { a: 1 }, $filter: { b: 2 } }); + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['where']); + expect(issues[0].message).toContain("'$filter'"); + }); + it('`sort` against `$orderby`, quoting the `$` spelling', () => { + const issues = refuses({ object: 'a', sort: { a: 'asc' }, $orderby: { b: 'desc' } }); + expect(issues).toHaveLength(1); + expect(issues[0].path).toEqual(['orderBy']); + expect(issues[0].message).toContain("'$orderby'"); + }); + it('the same value under two spellings is redundant, not a conflict', () => + parsesTo({ object: 'a', where: { a: 1 }, $filter: { a: 1 } }, { object: 'a', where: { a: 1 } })); + }); + + describe('the OUTPUT is the AST — by construction', () => { + /** + * The transform returns the result of the AST schema's own parse, so the + * only keys that can leave are the ones that schema declares. These two + * assertions are what a `.success`-only test could never make. + */ + it('no transport spelling survives a parse, on any slot', () => { + const parsed = QueryWithTransportSchema.parse({ + object: 'a', $filter: { s: 1 }, $top: 5, $skip: 1, $orderby: { n: 'asc' }, + $select: 'a', $expand: 'owner', $search: 'x', $searchFields: 'a', $count: true, + }) as Record; + for (const key of QUERY_TRANSPORT_DOLLAR_PARAMS) { + expect(key in parsed, `${key} survived the fold`).toBe(false); + } + for (const key of ['filter', 'filters', 'select', 'sort', 'skip', 'populate', 'top']) { + expect(key in parsed, `${key} survived the fold`).toBe(false); + } + expect(parsed).toEqual({ + object: 'a', where: { s: 1 }, limit: 5, offset: 1, + orderBy: [{ field: 'n', order: 'asc' }], fields: ['a'], + expand: { owner: { object: 'owner' } }, search: 'x', searchFields: ['a'], count: true, + }); + }); + + it('every parsed output is itself a valid QueryAST (plus the `count` flag)', () => { + const INPUTS: Array> = [ + { object: 'a', $top: '50' }, + { object: 'a', $skip: '10' }, + { object: 'a', $orderby: { n: -1 } }, + { object: 'a', $select: 'a,b' }, + { object: 'a', $expand: 'owner' }, + { object: 'a', $filter: ['status', '=', 'open'] }, + { object: 'a', where: ['status', '=', 'open'] }, + { object: 'a', $searchFields: 'a,b' }, + { object: 'a', $count: 'true' }, + ]; + for (const input of INPUTS) { + const parsed = QueryWithTransportSchema.parse(input) as Record; + const { count, ...ast } = parsed; + // The canonical schema — the wire/storage contract — must accept the + // whole of it. Before the fold was total this failed on six of the nine. + const round = QuerySchema.safeParse(ast); + expect(round.success, `${JSON.stringify(input)} -> ${JSON.stringify(parsed)}`).toBe(true); + expect(round.success && round.data).toEqual(ast); + expect(count === undefined || typeof count === 'boolean').toBe(true); + } + }); }); }); @@ -191,3 +349,34 @@ describe('[#16066] §4 `QuerySchema` did not grow the dialect', () => { .toEqual({ object: 'account', where: { a: 1 }, limit: 5 }); }); }); + +describe('[#16066] §5 the declared OUTPUT type, pinned where a runtime test cannot reach', () => { + /** + * `z.infer` of this slot is the canonical AST plus the `count` flag — and + * nothing else. The slot used to claim the bare `QueryAST` through a cast + * while the transform emitted whatever the fold produced; the claim is now + * made by the AST schema's own parse inside that transform, and this is the + * half of it a value-level assertion cannot see. + * + * ⚠️ `count` is part of the output on purpose: it rides inside this slot on + * the wire and `findData` reads it off the same bag. It is not an AST member + * and never reaches the engine. + */ + type Exact = (() => T extends A ? 1 : 2) extends (() => T extends B ? 1 : 2) ? true : false; + + it('is `QueryAST & { count?: boolean }`', () => { + const outputIsAstPlusCount: Exact< + z.infer, + QueryAST & { count?: boolean } + > = true; + expect(outputIsAstPlusCount).toBe(true); + }); + + it('…and the INPUT still admits the transport spelling and refuses a bag with no object', () => { + const transportBag: z.input = { object: 'a', $top: 5, $filter: { a: 1 } }; + const canonicalAst: z.input = { object: 'a', where: { a: 1 }, limit: 5 }; + // @ts-expect-error — `object` is required on every spelling of the slot. + const noObject: z.input = { $top: 5 }; + expect([transportBag, canonicalAst, noObject].length).toBe(3); + }); +}); From e695f895c8698cf78300c9df570ab18048e607d3 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Sep 2026 19:05:10 +0000 Subject: [PATCH 07/10] fix(spec): re-describe the recursive expand member, and state the value rule Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01LvwGppdonww4zGLWZo5rho --- .../16066-query-transport-dialect-declared.md | 23 +++++++--- content/docs/references/api/protocol.mdx | 23 +++++----- content/docs/references/data/data-engine.mdx | 42 ++++++++++--------- ...07-unknown-key-strictness-ledger.counts.md | 22 +++++----- .../rest-server-repeated-filter-param.test.ts | 28 ++++++++++++- packages/spec/src/api/protocol.zod.ts | 21 +++++++--- packages/spec/src/data/data-engine.zod.ts | 24 ++++++++++- 7 files changed, 126 insertions(+), 57 deletions(-) diff --git a/.changeset/16066-query-transport-dialect-declared.md b/.changeset/16066-query-transport-dialect-declared.md index 910e28f6dae..e07948906d9 100644 --- a/.changeset/16066-query-transport-dialect-declared.md +++ b/.changeset/16066-query-transport-dialect-declared.md @@ -3,23 +3,34 @@ "@objectstack/metadata-protocol": patch --- -The query TRANSPORT dialect is declared, and the `findData` fold now derives from that one declaration. +The query TRANSPORT dialect is declared — keys AND values — and the `findData` fold now derives from that one declaration. `FindDataRequestSchema.query` declared `QuerySchema` — the canonical QueryAST — while the shipped `findData` door also accepted a second spelling of the same query through the same slot: `$filter` / `$top` / `$skip` / `$orderby` / `$select` / `$expand` and the plural `filters`. `@objectstack/metadata-protocol` folded them from a module-private table whose own comment called them "the wire-only spellings no schema declares". Two dialects, one slot, one of them declared — so every caller speaking the second was unverifiable at build time and unrejected at runtime. -**New in `@objectstack/spec/data`** (8 exports, 0 removed): +**New in `@objectstack/spec/data`** (9 exports, 0 removed): - `QueryTransportParamsSchema` / `QueryTransportParams` / `QueryTransportParamsParsed` — the transport parameters, each carrying the value of the canonical slot it folds onto. - `QUERY_TRANSPORT_ALIAS_SLOTS` — `RPC_QUERY_ALIAS_SLOTS` extended with the transport-only spellings (`filters` / `$filter` onto `where`, `$expand` onto `expand`). - `QUERY_TRANSPORT_DOLLAR_ALIASES` — the `$`-to-bare pairs that fold in two hops (`$top` onto `top` onto `limit`). - `QUERY_TRANSPORT_DOLLAR_PARAMS` — the `$` spellings a boundary quotes when it refuses an undeclared one. -- `QueryWithTransportSchema` / `QueryWithTransport` / `QueryWithTransportParsed` — the query slot whose declared input is the AST or its transport spelling and whose declared output is the AST. +- `QueryWithTransportSchema` / `QueryWithTransport` / `QueryWithTransportParsed` — the query slot whose declared input is the AST or its transport spelling and whose parsed output is the AST plus the `count` flag. -**`FindDataRequestSchema.query` is that slot now.** Its `z.input` admits the canonical AST, the transport spelling, or a bag carrying both; its `z.output` is still the AST, reached by a transform driven by the two tables. The transport form is declared as the FLATTENED SPELLING of the canonical AST with a 1:1 alias table — never as a second semantics — so `QuerySchema` itself is untouched and still drops a `$` key as unknown. +**`FindDataRequestSchema.query` is that slot now.** Its `z.input` admits the canonical AST, the transport spelling, or a bag carrying both. Its `z.output` is `QueryAST & { count?: boolean }` — the canonical AST, plus the response total-count flag, which rides inside this slot on the wire and is read off it by `findData` rather than passed to the engine. The output is CONSTRUCTED: the fold's result is parsed by the AST schema and that parse's result is what leaves the transform, so a transport key or a non-AST value cannot reach a consumer. The transport form is the FLATTENED SPELLING of the canonical AST with a 1:1 alias table — never a second semantics — so `QuerySchema` itself is untouched and still drops a `$` key as unknown. -**Nothing is narrowed, and nothing about the served behaviour changed.** The canonical arm of the union is `QuerySchema` unchanged, so every body that parsed before still parses, by the same schema, with the same issues. `@objectstack/metadata-protocol` folds by the spec export instead of its own table, and both resolved tables — plus the `$`-parameter list its `UNSUPPORTED_QUERY_PARAM` refusal quotes — are pinned byte-equal to their pre-change values. +**One semantics means one set of VALUES, not only one set of keys, and that is what this declaration now enforces.** Every spelling of a slot accepts the same value shapes; each is lowered to the canonical member's declared shape, or refused. What lowers: a stringly-typed `$top` / `$skip` (`'50'` becomes `50`), a comma list on `$select` / `$searchFields` / `$expand`, a `{field: direction}` sort record, a relation-name list on `populate`, `'true'` / `'false'` on `$count`, and the input-only `FilterArray` sugar (`['status', '=', 'open']`) on every spelling of the filter slot — `where` included — lowered through `parseFilterAST`, the one declared sink (#5158 ruling C; `QuerySchema.where` still refuses the array). -**An undeclared `$` spelling is still refused loudly**, with the same `400 UNSUPPORTED_QUERY_PARAM`; the sentence now quotes `QUERY_TRANSPORT_DOLLAR_PARAMS` rather than a hand-copied list, so a spelling added to the table cannot leave the refusal naming a set the door no longer has. +**What is REFUSED at the parse**, because lowering it would mean parsing the spec must not do, and because emitting it would put a value under the AST type that the AST does not declare: + +- a non-numeric `$top` / `$skip` (`$top: 'abc'`, `$top: ''`) — `400` instead of an engine call with `limit: null`, i.e. an UNBOUNDED read under a `200`, or `limit: 0`; +- a JSON-encoded `$filter` string (`'{"status":"open"}'`); +- an OData sort EXPRESSION on `$orderby` / `sort` (`'name desc'`, `'-created_at'`, `['name']`) — the record and `SortNode[]` forms are unaffected; +- a filter array no lowering can express, such as the INFIX join `[condA, 'and', condB]` — the prefix form `['and', condA, condB]` is the one the platform reads, and the engine already answered `400` for the infix one; +- a `$count` that is neither the boolean nor `'true'` / `'false'`; +- two spellings of one slot carrying different values — reported at the canonical path, quoting the spelling the caller actually wrote (`$orderby`, not `orderBy`). + +These refusals narrow no DECLARED surface: none of these value shapes was ever declared, and each one the door could serve is still served. They narrow how far an unservable body travels before it is refused — from the engine, or from a wrong answer under a `200`, to the ingress that can name the parameter to fix. The GET querystring path is unchanged: it does not parse through this schema. + +**`@objectstack/metadata-protocol` folds by the spec export** instead of its own table, and both resolved tables — plus the `$`-parameter list its `UNSUPPORTED_QUERY_PARAM` refusal quotes — are pinned byte-equal to their pre-change values. An undeclared `$` spelling is still refused loudly with the same `400 UNSUPPORTED_QUERY_PARAM`; the sentence now quotes `QUERY_TRANSPORT_DOLLAR_PARAMS` rather than a hand-copied list, so a spelling added to the table cannot leave the refusal naming a set the door no longer has. Measured and unchanged: `getData` takes `select` / `expand` directly and carries no `query` slot, and `updateManyData` / `deleteManyData` take `records[]` / `ids[]` — none of the three has a transport-dialect split to declare. diff --git a/content/docs/references/api/protocol.mdx b/content/docs/references/api/protocol.mdx index 4d2a35861c3..fa6f1a7c08f 100644 --- a/content/docs/references/api/protocol.mdx +++ b/content/docs/references/api/protocol.mdx @@ -867,7 +867,7 @@ Enable package response | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | | **object** | `string` | ✅ | The unique machine name of the object to query (e.g. "account"). | -| **query** | `{ object: string; fields?: string[]; where?: any; search?: string \| object; … }` | optional | Structured query definition (filter, sort, select, pagination) — the canonical QueryAST or its transport spelling. | +| **query** | `{ object: string; fields?: string[]; where?: [string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any; search?: string \| object; … }` | optional | Structured query definition (filter, sort, select, pagination) — the canonical QueryAST or its transport spelling. | ### Nested Shape: `FindDataRequest.query` @@ -875,7 +875,7 @@ Enable package response | :--- | :--- | :--- | :--- | | **object** | `string` | ✅ | Object name (e.g. account) | | **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | -| **where** | `any` | optional | Filtering criteria (WHERE) | +| **where** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any` | optional | Filtering criteria (WHERE) — a filter condition, or the input-only `FilterArray` sugar (`['status', '=', 'open']`), which is lowered through `parseFilterAST` before the query is produced (#5158 ruling C). | | **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | | **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | | **orderBy** | `{ field: string; order?: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | @@ -889,20 +889,21 @@ Enable package response | **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | | **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | | **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | -| **expand** | `Record` | optional | | -| **$filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (OData `$filter`) | -| **filters** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (plural of `filter`) | -| **$top** | `number \| string \| null` | optional | Transport spelling of `limit` (OData `$top`) | -| **$skip** | `number \| string \| null` | optional | Transport spelling of `offset` (OData `$skip`) | -| **$orderby** | `string \| string[] \| Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` (OData `$orderby`) | +| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select (`fields`) and filter (`where`, AND-merged with the batch $in), plus further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3; per-parent `limit`/`offset`/`orderBy` are NOT applied on this path. | +| **$filter** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any \| null` | optional | Transport spelling of `where` (OData `$filter`) | +| **filters** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any \| null` | optional | Transport spelling of `where` (plural of `filter`) | +| **$top** | `number \| string \| null` | optional | Transport spelling of `limit` (OData `$top`) — a number, or the digits a querystring carries it as | +| **$skip** | `number \| string \| null` | optional | Transport spelling of `offset` (OData `$skip`) — a number, or the digits a querystring carries it as | +| **$orderby** | `Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` (OData `$orderby`) | | **$select** | `string \| string[] \| null` | optional | Transport spelling of `fields` (OData `$select`) | | **$expand** | `string \| string[] \| Record \| null` | optional | Transport spelling of `expand` (OData `$expand`) | | **$search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … } \| null` | optional | Transport spelling of `search` (OData `$search`) | | **$searchFields** | `string \| string[] \| null` | optional | Transport spelling of `searchFields` (OData `$searchFields`) | -| **$count** | `boolean \| string \| null` | optional | Transport spelling of the response total-count flag (OData `$count`) | -| **filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` | +| **$count** | `boolean \| Enum<'true' \| 'false'> \| null` | optional | Transport spelling of the response total-count flag (OData `$count`) | +| **count** | `boolean \| Enum<'true' \| 'false'> \| null` | optional | Response total-count flag — only an explicit `false` skips the COUNT query | +| **filter** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any \| null` | optional | Transport spelling of `where` | | **select** | `string \| string[] \| null` | optional | Transport spelling of `fields` | -| **sort** | `string \| string[] \| Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` | +| **sort** | `Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` | | **skip** | `number \| string \| null` | optional | Transport spelling of `offset` | | **populate** | `string \| string[] \| null` | optional | Transport spelling of `expand` | diff --git a/content/docs/references/data/data-engine.mdx b/content/docs/references/data/data-engine.mdx index dd73e687155..0a1bc12eb98 100644 --- a/content/docs/references/data/data-engine.mdx +++ b/content/docs/references/data/data-engine.mdx @@ -1217,19 +1217,20 @@ Transport spellings of the QueryAST slots — folded to canonical keys at the bo | Property | Type | Required | Description | | :--- | :--- | :--- | :--- | -| **$filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (OData `$filter`) | -| **filters** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (plural of `filter`) | -| **$top** | `number \| string \| null` | optional | Transport spelling of `limit` (OData `$top`) | -| **$skip** | `number \| string \| null` | optional | Transport spelling of `offset` (OData `$skip`) | -| **$orderby** | `string \| string[] \| Record> \| Record \| { field: string; order: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` (OData `$orderby`) | +| **$filter** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any \| null` | optional | Transport spelling of `where` (OData `$filter`) | +| **filters** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any \| null` | optional | Transport spelling of `where` (plural of `filter`) | +| **$top** | `number \| string \| null` | optional | Transport spelling of `limit` (OData `$top`) — a number, or the digits a querystring carries it as | +| **$skip** | `number \| string \| null` | optional | Transport spelling of `offset` (OData `$skip`) — a number, or the digits a querystring carries it as | +| **$orderby** | `Record> \| Record \| { field: string; order: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` (OData `$orderby`) | | **$select** | `string \| string[] \| null` | optional | Transport spelling of `fields` (OData `$select`) | | **$expand** | `string \| string[] \| Record \| null` | optional | Transport spelling of `expand` (OData `$expand`) | | **$search** | `string \| { query: string; fields?: string[]; fuzzy: boolean; operator: Enum<'and' \| 'or'>; … } \| null` | optional | Transport spelling of `search` (OData `$search`) | | **$searchFields** | `string \| string[] \| null` | optional | Transport spelling of `searchFields` (OData `$searchFields`) | -| **$count** | `boolean \| string \| null` | optional | Transport spelling of the response total-count flag (OData `$count`) | -| **filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` | +| **$count** | `boolean \| Enum<'true' \| 'false'> \| null` | optional | Transport spelling of the response total-count flag (OData `$count`) | +| **count** | `boolean \| Enum<'true' \| 'false'> \| null` | optional | Response total-count flag — only an explicit `false` skips the COUNT query | +| **filter** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any \| null` | optional | Transport spelling of `where` | | **select** | `string \| string[] \| null` | optional | Transport spelling of `fields` | -| **sort** | `string \| string[] \| Record> \| Record \| { field: string; order: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` | +| **sort** | `Record> \| Record \| { field: string; order: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` | | **skip** | `number \| string \| null` | optional | Transport spelling of `offset` | | **populate** | `string \| string[] \| null` | optional | Transport spelling of `expand` | @@ -1279,7 +1280,7 @@ Transport spellings of the QueryAST slots — folded to canonical keys at the bo | :--- | :--- | :--- | :--- | | **object** | `string` | ✅ | Object name (e.g. account) | | **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | -| **where** | `any` | optional | Filtering criteria (WHERE) | +| **where** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any` | optional | Filtering criteria (WHERE) — a filter condition, or the input-only `FilterArray` sugar (`['status', '=', 'open']`), which is lowered through `parseFilterAST` before the query is produced (#5158 ruling C). | | **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | | **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | | **orderBy** | `{ field: string; order?: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | @@ -1293,20 +1294,21 @@ Transport spellings of the QueryAST slots — folded to canonical keys at the bo | **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | | **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | | **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | -| **expand** | `Record` | optional | | -| **$filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (OData `$filter`) | -| **filters** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` (plural of `filter`) | -| **$top** | `number \| string \| null` | optional | Transport spelling of `limit` (OData `$top`) | -| **$skip** | `number \| string \| null` | optional | Transport spelling of `offset` (OData `$skip`) | -| **$orderby** | `string \| string[] \| Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` (OData `$orderby`) | +| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select (`fields`) and filter (`where`, AND-merged with the batch $in), plus further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3; per-parent `limit`/`offset`/`orderBy` are NOT applied on this path. | +| **$filter** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any \| null` | optional | Transport spelling of `where` (OData `$filter`) | +| **filters** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any \| null` | optional | Transport spelling of `where` (plural of `filter`) | +| **$top** | `number \| string \| null` | optional | Transport spelling of `limit` (OData `$top`) — a number, or the digits a querystring carries it as | +| **$skip** | `number \| string \| null` | optional | Transport spelling of `offset` (OData `$skip`) — a number, or the digits a querystring carries it as | +| **$orderby** | `Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` (OData `$orderby`) | | **$select** | `string \| string[] \| null` | optional | Transport spelling of `fields` (OData `$select`) | | **$expand** | `string \| string[] \| Record \| null` | optional | Transport spelling of `expand` (OData `$expand`) | | **$search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … } \| null` | optional | Transport spelling of `search` (OData `$search`) | | **$searchFields** | `string \| string[] \| null` | optional | Transport spelling of `searchFields` (OData `$searchFields`) | -| **$count** | `boolean \| string \| null` | optional | Transport spelling of the response total-count flag (OData `$count`) | -| **filter** | `string \| any[] \| Record \| any \| null` | optional | Transport spelling of `where` | +| **$count** | `boolean \| Enum<'true' \| 'false'> \| null` | optional | Transport spelling of the response total-count flag (OData `$count`) | +| **count** | `boolean \| Enum<'true' \| 'false'> \| null` | optional | Response total-count flag — only an explicit `false` skips the COUNT query | +| **filter** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any \| null` | optional | Transport spelling of `where` | | **select** | `string \| string[] \| null` | optional | Transport spelling of `fields` | -| **sort** | `string \| string[] \| Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` | +| **sort** | `Record> \| Record \| { field: string; order?: Enum<'asc' \| 'desc'> }[] \| null` | optional | Transport spelling of `orderBy` | | **skip** | `number \| string \| null` | optional | Transport spelling of `offset` | | **populate** | `string \| string[] \| null` | optional | Transport spelling of `expand` | @@ -1361,7 +1363,7 @@ Transport spellings of the QueryAST slots — folded to canonical keys at the bo | **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | | **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | | **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | -| **expand** | `Record` | optional | | +| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select (`fields`) and filter (`where`, AND-merged with the batch $in), plus further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3; per-parent `limit`/`offset`/`orderBy` are NOT applied on this path. | ### Nested Shape: `QueryWithTransport.$expand[string]` @@ -1383,7 +1385,7 @@ Transport spellings of the QueryAST slots — folded to canonical keys at the bo | **having** | `any` | optional | HAVING — filter over the AGGREGATED rows (aggregation aliases + groupBy projections); applied engine-side after aggregation | | **windowFunctions** | `never` | optional | [REMOVED] `query.windowFunctions` was removed in @objectstack/spec 17 (ADR-0049) — `find()` never applied it: no engine or driver read the key on the query path, so every OVER clause it declared was silently dropped. Delete the key. Window functions are a SQL-driver capability behind `SqlDriver.findWithWindowFunctions(object, query)` (embedder-level; not on the `IDataDriver` contract or the REST surface); request-level analytics are `aggregations` + `groupBy`. | | **distinct** | `never` | optional | [REMOVED] `query.distinct` was removed in @objectstack/spec 17 (ADR-0049 / ADR-0078) — no driver ever rendered SELECT DISTINCT; the flag's only observable effect was MIS-WIRED: the REST list path treated a distinct query as not countable and silently degraded `total`/`hasMore` to a page-local estimate while still returning duplicate rows. Delete the key; `QueryBuilder.distinct()` was removed with it, and the count suppression is gone (`total` is truthful again). For unique values of one column use the SQL/memory drivers' `distinct(object, field)` door; for unique combinations, `groupBy`; for a deduplicated count, the `count_distinct` aggregation. | -| **expand** | `Record` | optional | | +| **expand** | `Record` | optional | Recursive relation loading map. Keys are lookup/master_detail field names; values are nested QueryAST objects that control select (`fields`) and filter (`where`, AND-merged with the batch $in), plus further expansion on the related object. The engine resolves expand via batch $in queries (driver-agnostic) with a default max depth of 3; per-parent `limit`/`offset`/`orderBy` are NOT applied on this path. | ### Nested Shape: `QueryWithTransport.$search` diff --git a/docs/audits/2026-07-unknown-key-strictness-ledger.counts.md b/docs/audits/2026-07-unknown-key-strictness-ledger.counts.md index a34fd034eeb..2db86e40f4f 100644 --- a/docs/audits/2026-07-unknown-key-strictness-ledger.counts.md +++ b/docs/audits/2026-07-unknown-key-strictness-ledger.counts.md @@ -21,8 +21,8 @@ regenerate. | Measure | Value | |---|---| | Triaged directories | 5 | -| Object sites in them | 448 | -| Still-open (strip) sites | 125 | +| Object sites in them | 449 | +| Still-open (strip) sites | 126 | | Files carrying at least one | 22 | Remaining strip sites by class: @@ -31,7 +31,7 @@ Remaining strip sites by class: |---|---| | authorable — the ruling's forced scope | 1 | | unresolved — needs a per-schema verdict | 0 | -| wire / open — out of forced scope | 120 | +| wire / open — out of forced scope | 121 | | no door — no carrier, ADR-0049 territory | 3 | | no gate — carrier live, no parse | 0 | | covered — no carrier, no parse, guarded at every consumer | 1 | @@ -45,11 +45,11 @@ The `strict` column is the one the campaign schedules against; it counts both th | Dir | Sites | strict | passthrough | catchall | strip | |---|---|---|---|---|---| | `ui/` | 175 | 165 | 3 | 0 | 7 | -| `data/` | 158 | 76 | 1 | 0 | 81 | +| `data/` | 159 | 76 | 1 | 0 | 82 | | `automation/` | 68 | 43 | 0 | 1 | 24 | | `security/` | 20 | 7 | 0 | 0 | 13 | | `studio/` | 27 | 27 | 0 | 0 | 0 | -| **total** | **448** | **318** | **4** | **1** | **125** | +| **total** | **449** | **318** | **4** | **1** | **126** | ## File-level triage — site counts @@ -83,7 +83,7 @@ classify and is not listed (it becomes reportable the day it grows its first sit | File | Sites | |---|---| | `analytics.zod.ts` | 7 | -| `data-engine.zod.ts` | 14 | +| `data-engine.zod.ts` | 15 | | `datasource.zod.ts` | 6 | | `document.zod.ts` | 8 | | `driver-nosql.zod.ts` | 10 | @@ -107,7 +107,7 @@ classify and is not listed (it becomes reportable the day it grows its first sit | `seed-loader.zod.ts` | 12 | | `seed.zod.ts` | 1 | | `validation.zod.ts` | 6 | -| **total** | **158** | +| **total** | **159** | ### `automation/` — sites @@ -176,11 +176,11 @@ over it is here. ### `data/` — open -**81 strip of 158**, in 11 file(s). +**82 strip of 159**, in 11 file(s). | File | Strip | Sites | |---|---|---| -| `data-engine.zod.ts` | 14 | 14 | +| `data-engine.zod.ts` | 15 | 15 | | `document.zod.ts` | 8 | 8 | | `driver-nosql.zod.ts` | 10 | 10 | | `driver-sql.zod.ts` | 2 | 2 | @@ -191,13 +191,13 @@ over it is here. | `hook.zod.ts` | 5 | 7 | | `query.zod.ts` | 4 | 5 | | `seed-loader.zod.ts` | 12 | 12 | -| **total** | **81** | **158** | +| **total** | **82** | **159** | | Bucket | Sites | |---|---| | authorable — the ruling's forced scope | 0 | | unresolved — needs a per-schema verdict | 0 | -| wire / open — out of forced scope | 79 | +| wire / open — out of forced scope | 80 | | no door — no carrier, ADR-0049 territory | 2 | | no gate — carrier live, no parse | 0 | | covered — no carrier, no parse, guarded at every consumer | 0 | diff --git a/packages/rest/src/rest-server-repeated-filter-param.test.ts b/packages/rest/src/rest-server-repeated-filter-param.test.ts index 88c5a92cb35..da14e26e919 100644 --- a/packages/rest/src/rest-server-repeated-filter-param.test.ts +++ b/packages/rest/src/rest-server-repeated-filter-param.test.ts @@ -344,17 +344,41 @@ describe('#7390 §3 — `POST /data/:object/query` is untouched', () => { }); it('a body-form nested AST is forwarded too', async () => { + // [#16066] The PREFIX form — `['and', condA, condB]` — which is what the + // platform's one lowering sink actually reads. This case used to send + // the INFIX form and pass, because `findData` is mocked here and nothing + // downstream ran; the case below pins what happens to that one now. const { drive, protocol } = boot(); const answer = await drive('POST', `${DATA}/:object/query`, { params: { object: 'task' }, - body: { filter: [['status', '=', 'open'], 'and', ['done', '=', false]] }, + body: { filter: ['and', ['status', '=', 'open'], ['done', '=', false]] }, }); expect(answer.status).toBe(200); expect(protocol.findData.mock.calls[0][0].query.filter).toEqual( - [['status', '=', 'open'], 'and', ['done', '=', false]], + ['and', ['status', '=', 'open'], ['done', '=', false]], ); }); + it('[#16066] an INFIX join is refused at the declaration, not forwarded into a refusal', async () => { + // `isFilterAST` refuses `[condA, 'and', condB]`, `parseFilterAST` lowers + // it to nothing, and the engine answers `400` for it with the wording + // "Infix joins ([condA, "or", condB]) are NOT one of the shapes — write + // the prefix form" (`objectql/src/engine.ts`). So this body has always + // been a 400; declaring the filter slot's value set moves the refusal to + // the ingress, where the caller is told which parameter to fix. + // + // ⛔ Not a narrowing of what the platform serves — a narrowing of how far + // an unservable body travels before it is refused. + const { drive, protocol } = boot(); + const answer = await drive('POST', `${DATA}/:object/query`, { + params: { object: 'task' }, + body: { filter: [['status', '=', 'open'], 'and', ['done', '=', false]] }, + }); + expect(answer.status).toBe(400); + expect(answer.body?.code).toBe('VALIDATION_FAILED'); + expect(protocol.findData).not.toHaveBeenCalled(); + }); + it('the helper itself never reads a body — it is handed `req.query` only', () => { // Structural, not incidental: the gate takes the query bag as its whole // input, so there is no path by which a body could reach it. diff --git a/packages/spec/src/api/protocol.zod.ts b/packages/spec/src/api/protocol.zod.ts index c21f9e144c6..d65fb56a116 100644 --- a/packages/spec/src/api/protocol.zod.ts +++ b/packages/spec/src/api/protocol.zod.ts @@ -1857,11 +1857,22 @@ export const GetUiViewResponseSchema = lazySchema(() => ViewSchema); * `query` declares its INPUT as the canonical QueryAST **or** its transport * spelling ({@link QueryTransportParamsSchema} — `$filter` / `$top` / `$skip` / * `$orderby` / `$select` / `$expand` and the plural `filters`), and its OUTPUT - * as the AST: the transport keys are folded onto their canonical slots by - * {@link QUERY_TRANSPORT_ALIAS_SLOTS} / {@link QUERY_TRANSPORT_DOLLAR_ALIASES} - * at parse. The transport form is the FLATTENED SPELLING of the same AST with - * a 1:1 alias table — ⛔ never a second query semantics, and nothing is - * admitted into `QuerySchema` itself. + * as the AST plus the `count` flag: the transport keys are folded onto their + * canonical slots by {@link QUERY_TRANSPORT_ALIAS_SLOTS} / + * {@link QUERY_TRANSPORT_DOLLAR_ALIASES} at parse, and the folded bag is then + * PARSED by the AST schema, so the output is constructed rather than asserted. + * The transport form is the FLATTENED SPELLING of the same AST with a 1:1 alias + * table — ⛔ never a second query semantics, and nothing is admitted into + * `QuerySchema` itself. + * + * One semantics is a claim about VALUES too: every spelling of a slot accepts + * one set of value shapes, each of which is lowered to the canonical member's + * declared shape (`'50'` to `50`, a comma list to an array, the `FilterArray` + * sugar through `parseFilterAST`) or REFUSED at the parse. A shape that would + * need a parser the spec must not run — a JSON-encoded `$filter` string, an + * OData sort expression, a non-numeric `$top` — fails rather than reaching a + * consumer under the AST type; `$top: 'abc'` used to pass validation here and + * reach the engine as `limit: null`, an unbounded read under a 200. * * This records what the door already accepted. `@objectstack/metadata-protocol` * has folded these spellings since #3795 while the slot declared the AST alone, diff --git a/packages/spec/src/data/data-engine.zod.ts b/packages/spec/src/data/data-engine.zod.ts index 0b6364b9025..f388a53faae 100644 --- a/packages/spec/src/data/data-engine.zod.ts +++ b/packages/spec/src/data/data-engine.zod.ts @@ -1136,8 +1136,27 @@ const QueryAstWithCountSchema: z.ZodType => - QuerySchema as unknown as z.ZodObject; +const queryObjectFace = (): z.ZodObject> => + QuerySchema as unknown as z.ZodObject>; + +/** + * `expand` re-described on the extended shape, with the exact text + * `QuerySchema` gives it — READ from the member, ⛔ never re-typed, because two + * copies of a description drift. + * + * Not decoration. `expand` is RECURSIVE, so `z.toJSONSchema` hoists it into + * `$defs` and the property renders as a bare `$ref`; a `$ref` carries no + * sibling `description` unless the property node has one of its own. This slot + * publishes its INPUT shape — `build-schemas.ts` falls back to `io: 'input'` + * because the output of a transform has no JSON form — and in that direction + * the `FindDataRequest.query` / `expand` row rendered with an EMPTY description + * cell in `content/docs/references/api/protocol.mdx`. Re-describing the member + * puts the text back beside the `$ref`. + */ +const describedCanonicalExpand = (): z.ZodType => { + const member = queryObjectFace().shape.expand; + return member.describe(member.description ?? ''); +}; /** * A query slot whose declared INPUT is the canonical AST or its transport @@ -1171,6 +1190,7 @@ export const QueryWithTransportSchema = lazySchema( () => queryObjectFace() .extend({ ...(QueryTransportParamsSchema as unknown as z.ZodObject).shape, + expand: describedCanonicalExpand(), where: TransportFilterValueSchema.optional().describe( 'Filtering criteria (WHERE) — a filter condition, or the input-only `FilterArray` ' + "sugar (`['status', '=', 'open']`), which is lowered through `parseFilterAST` " From 137010a79cfcd2513c8cee61f2cecb81f2c77b06 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 17 Sep 2026 21:14:22 +0000 Subject: [PATCH 08/10] fix(rest,spec): type-check the transport declaration's consumers, keep the id out of customer prose Two CI reds, both this branch's own: - `packages/rest` `typecheck` was never run: its `test` script covers only the `local` vitest project, and the type check is a separate script. The test layer held four errors, every one of them the declared `FindDataRequest['query'].where` INPUT union gaining the `FilterArray` arm the door already serves. Narrowed at the three read sites -- by REFUSING the other arm, not by casting past it -- and pinned the JSON-encoded filter string as the refusal the transport table deliberately makes it. - `check:doc-authoring` refused an internal issue id inside `.describe()` prose, which is printed at the customer and resolves to nothing there. Moved to an adjacent `//` comment; regenerated the two reference pages the sentence projects into. Claude-Session: https://claude.ai/code/session_01LvwGppdonww4zGLWZo5rho Co-authored-by: Claude --- content/docs/references/api/protocol.mdx | 2 +- content/docs/references/data/data-engine.mdx | 2 +- packages/rest/src/import-runner-bulk.test.ts | 19 ++++++++++++++++++- .../src/import-runner-idempotency.test.ts | 11 +++++++++-- .../rest-server-canonical-query-ast.test.ts | 12 +++++++++--- packages/spec/src/data/data-engine.zod.ts | 5 ++++- 6 files changed, 42 insertions(+), 9 deletions(-) diff --git a/content/docs/references/api/protocol.mdx b/content/docs/references/api/protocol.mdx index fa6f1a7c08f..008b1df0061 100644 --- a/content/docs/references/api/protocol.mdx +++ b/content/docs/references/api/protocol.mdx @@ -875,7 +875,7 @@ Enable package response | :--- | :--- | :--- | :--- | | **object** | `string` | ✅ | Object name (e.g. account) | | **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | -| **where** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any` | optional | Filtering criteria (WHERE) — a filter condition, or the input-only `FilterArray` sugar (`['status', '=', 'open']`), which is lowered through `parseFilterAST` before the query is produced (#5158 ruling C). | +| **where** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any` | optional | Filtering criteria (WHERE) — a filter condition, or the input-only `FilterArray` sugar (`['status', '=', 'open']`), which is lowered through `parseFilterAST` before the query is produced. | | **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | | **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | | **orderBy** | `{ field: string; order?: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | diff --git a/content/docs/references/data/data-engine.mdx b/content/docs/references/data/data-engine.mdx index 0a1bc12eb98..32edc8f710b 100644 --- a/content/docs/references/data/data-engine.mdx +++ b/content/docs/references/data/data-engine.mdx @@ -1280,7 +1280,7 @@ Transport spellings of the QueryAST slots — folded to canonical keys at the bo | :--- | :--- | :--- | :--- | | **object** | `string` | ✅ | Object name (e.g. account) | | **fields** | `string[]` | optional | Fields to retrieve — names of the queried object's OWN columns. A dotted path (`owner.name`) is not a projection: no driver resolves one, and the ingress refuses it with `400 INVALID_FIELD`. Related data is read with `expand`, whose nested QueryAST both filters (`where`) and selects (`fields`) the related record's columns. The projection must RETAIN the foreign-key column: `fields: ['title']` with `expand: 'project_id'` resolves nothing, because the relation is carried by that key — add `'project_id'` and it works. Where the value is wanted on the queried object itself, denormalise it onto that object (a stored field, written when the source changes), the same remedy the sort axis prescribes. | -| **where** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any` | optional | Filtering criteria (WHERE) — a filter condition, or the input-only `FilterArray` sugar (`['status', '=', 'open']`), which is lowered through `parseFilterAST` before the query is produced (#5158 ruling C). | +| **where** | `[string, string, any] \| [string, string] \| [string, object, ...object[]] \| object[] \| Record \| any` | optional | Filtering criteria (WHERE) — a filter condition, or the input-only `FilterArray` sugar (`['status', '=', 'open']`), which is lowered through `parseFilterAST` before the query is produced. | | **search** | `string \| { query: string; fields?: string[]; fuzzy?: boolean; operator?: Enum<'and' \| 'or'>; … }` | optional | Full-text search — the query text (canonical, ADR-0061 D1), or a structured FullTextSearch configuration | | **searchFields** | `string[]` | optional | Narrow the search to these fields (server-intersected with the allowed searchable set — can only narrow, never widen; ADR-0061 D1) | | **orderBy** | `{ field: string; order?: Enum<'asc' \| 'desc'> }[]` | optional | Sorting instructions (ORDER BY) | diff --git a/packages/rest/src/import-runner-bulk.test.ts b/packages/rest/src/import-runner-bulk.test.ts index 333e8fce189..9e335bf936f 100644 --- a/packages/rest/src/import-runner-bulk.test.ts +++ b/packages/rest/src/import-runner-bulk.test.ts @@ -23,6 +23,23 @@ type FindArgs = Parameters[0]; type CreateArgs = Parameters[0]; import type { ExportFieldMeta } from './export-format.js'; +/** + * The CANONICAL object `where` out of the slot's declared input union. + * + * `FindDataRequest['query'].where` admits the input-only `FilterArray` sugar as + * well, because the transport door serves it on every spelling of that slot. + * `runImport` builds only the object form, so this narrows by REFUSING the + * other arm rather than by casting past it: a runner that started emitting the + * array sugar fails here loudly instead of type-checking into silence. + */ +function canonicalWhere(args: FindArgs): Record { + const where = args.query?.where; + if (where == null || Array.isArray(where)) { + throw new Error(`runImport must send a canonical object \`where\`; got ${JSON.stringify(where)}`); + } + return where; +} + const metaMap = new Map([ ['name', { name: 'name', type: 'text' }], ]); @@ -162,7 +179,7 @@ describe('runImport — bulk create batching (framework#2678)', () => { // Row 1 ('existing') matches an existing record → update; the rest are creates. // [#16638] Reads the CANONICAL `where` the runner sends, not `$filter`. const findData = vi.fn(async (args: FindArgs) => - (args.query!.where!.name === 'existing' ? [{ id: 'existing_id', name: 'existing' }] : [])); + (canonicalWhere(args).name === 'existing' ? [{ id: 'existing_id', name: 'existing' }] : [])); const p: ImportProtocolLike = { findData, createData: vi.fn(), updateData, createManyData }; const summary = await runImport({ diff --git a/packages/rest/src/import-runner-idempotency.test.ts b/packages/rest/src/import-runner-idempotency.test.ts index 1e4434f281e..a6dc4101d4d 100644 --- a/packages/rest/src/import-runner-idempotency.test.ts +++ b/packages/rest/src/import-runner-idempotency.test.ts @@ -187,8 +187,15 @@ describe('runImport — idempotent retry with natural keys (framework#3149)', () expectEveryProbeNarrowed(findData.mock.calls, appliedFilters); const probes = findData.mock.calls.map(([a]) => a.query!); expect(probes).toHaveLength(1); - expect(Object.keys(probes[0].where!)).toEqual(['id']); - expect([...(probes[0].where!.id as { $in: string[] }).$in].sort()).toEqual(store.map((r) => r.id).sort()); + // The slot's declared INPUT also admits the `FilterArray` sugar the + // transport door serves; the runner builds only the object form, so the + // other arm is REFUSED here rather than cast past. + const probeWhere = probes[0].where; + if (probeWhere == null || Array.isArray(probeWhere)) { + throw new Error(`the recheck must send a canonical object \`where\`; got ${JSON.stringify(probeWhere)}`); + } + expect(Object.keys(probeWhere)).toEqual(['id']); + expect([...(probeWhere.id as { $in: string[] }).$in].sort()).toEqual(store.map((r) => r.id).sort()); expect(probes[0].limit).toBe(store.length); }); diff --git a/packages/rest/src/rest-server-canonical-query-ast.test.ts b/packages/rest/src/rest-server-canonical-query-ast.test.ts index f3273f65024..f9dc1bff2f1 100644 --- a/packages/rest/src/rest-server-canonical-query-ast.test.ts +++ b/packages/rest/src/rest-server-canonical-query-ast.test.ts @@ -394,7 +394,7 @@ describe('[#16337] §2 the declared `FindDataRequest[\'query\']` contract', () = // the AST. So these three are assignments, not `@ts-expect-error`s. const dollarTop: Query = { object: 'x', $top: 5 }; const dollarFilter: Query = { object: 'x', $filter: { id: '1' } }; - const wireFilters: Query = { object: 'x', filters: '{"id":"1"}' }; + const wireFilters: Query = { object: 'x', filters: { id: '1' } }; // ⭐ …and a bag mixing the two spellings, which is what a boundary that // merges a caller's parameters with its own actually assembles. const mixed: Query = { object: 'x', where: { status: 'queued' }, $top: 5 }; @@ -417,7 +417,9 @@ describe('[#16337] §2 the declared `FindDataRequest[\'query\']` contract', () = const noObject: Query = { limit: 1 }; // @ts-expect-error `$sort` is not a spelling the transport table names — declaring the dialect did not open `$*` const unnamedDollar: Query = { object: 'x', $sort: 'name' }; - expect([dollarTop, dollarFilter, wireFilters, mixed, wireSelect, wireSort, recordSort, commaExpand, noObject, unnamedDollar]).toHaveLength(10); + // @ts-expect-error a JSON-ENCODED filter string is declared on NO filter spelling — lowering one means running a second parser beside the door's, which is the one thing the transport declaration refuses to do + const jsonEncodedFilter: Query = { object: 'x', filters: '{"id":"1"}' }; + expect([dollarTop, dollarFilter, wireFilters, mixed, wireSelect, wireSort, recordSort, commaExpand, noObject, unnamedDollar, jsonEncodedFilter]).toHaveLength(11); }); }); @@ -472,7 +474,11 @@ describe('[#16952] §2 the declared `ImportProtocolLike` parameter contract', () // ⭐ The whole point: leave the parameter unannotated and the contract // types it. This is the shape `plugin-auth`'s hand-written implementor // could not have while the declaration said `any`. - const probes: Array | undefined> = []; + // Typed FROM the declaration, like every alias in this block — the slot + // admits the `FilterArray` sugar as well as the object form, and a + // hand-written `Record` here would be a fourth + // restatement of the dialect, which is what this section exists to stop. + const probes: Array = []; const p: ImportProtocolLike = { findData: async (args) => { probes.push(args.query?.where); return { records: [] }; }, createData: async (args) => ({ id: String(args.data.name) }), diff --git a/packages/spec/src/data/data-engine.zod.ts b/packages/spec/src/data/data-engine.zod.ts index f388a53faae..1d95c4acd37 100644 --- a/packages/spec/src/data/data-engine.zod.ts +++ b/packages/spec/src/data/data-engine.zod.ts @@ -1191,10 +1191,13 @@ export const QueryWithTransportSchema = lazySchema( .extend({ ...(QueryTransportParamsSchema as unknown as z.ZodObject).shape, expand: describedCanonicalExpand(), + // The lowering sink the sentence below names is the one maintainer + // ruling C on #5158 fixed. The id lives in this comment and not in the + // prose, which is printed AT the customer, where it resolves to nothing. where: TransportFilterValueSchema.optional().describe( 'Filtering criteria (WHERE) — a filter condition, or the input-only `FilterArray` ' + "sugar (`['status', '=', 'open']`), which is lowered through `parseFilterAST` " - + 'before the query is produced (#5158 ruling C).' + + 'before the query is produced.' ), }) .transform((bag, ctx) => foldQueryTransportBag(bag as Record, ctx)), From b98eff5b01567f156bc198297a7a91dbf83cd734 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 18 Sep 2026 00:36:12 +0000 Subject: [PATCH 09/10] docs(changeset): the five POST-body shapes this narrows, with their replacements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The refusal paragraph claimed "each one the door could serve is still served". Measured false: `FindDataRequestSchema.query` was `QuerySchema` (strip mode), the route forwards the ORIGINAL body, and the door read and served five shapes the new slot now refuses at the ingress: {$orderby: 'name desc'} -> orderBy [{name, desc}] {sort: '-created_at'} -> orderBy [{created_at, desc}] {$orderby: ['name']} -> orderBy [{name, asc}] {$filter: '{"status":"open"}'} -> where {status: 'open'} the same JSON string on `filters` / `filter` A changeset becomes the CHANGELOG, so the sentence that ships to consumers now names each one with its FROM -> TO, keeps the true half (none of these value shapes was ever declared), and states that the GET querystring path still serves them -- a reader meeting the 400 otherwise reads it as the platform refusing the spelling everywhere. Prose only; no code, no test, no gate moves. The `$top: 'abc'` / `$top: ''` bullet is unchanged, and `Clause-②: yes (widening)` stands: the slot's declared contract was `QuerySchema`, which never declared these shapes. Claude-Session: https://claude.ai/code/session_01LvwGppdonww4zGLWZo5rho Co-authored-by: Claude --- .../16066-query-transport-dialect-declared.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.changeset/16066-query-transport-dialect-declared.md b/.changeset/16066-query-transport-dialect-declared.md index e07948906d9..6413f950fe5 100644 --- a/.changeset/16066-query-transport-dialect-declared.md +++ b/.changeset/16066-query-transport-dialect-declared.md @@ -28,7 +28,21 @@ The query TRANSPORT dialect is declared — keys AND values — and the `findDat - a `$count` that is neither the boolean nor `'true'` / `'false'`; - two spellings of one slot carrying different values — reported at the canonical path, quoting the spelling the caller actually wrote (`$orderby`, not `orderBy`). -These refusals narrow no DECLARED surface: none of these value shapes was ever declared, and each one the door could serve is still served. They narrow how far an unservable body travels before it is refused — from the engine, or from a wrong answer under a `200`, to the ingress that can name the parameter to fix. The GET querystring path is unchanged: it does not parse through this schema. +These refusals narrow no DECLARED surface: none of these value shapes was ever declared — `FindDataRequestSchema.query` was `QuerySchema`, which STRIPPED every one of these keys rather than declaring it. + +**Five of them were nonetheless SERVED, and now answer `400 VALIDATION_FAILED` at the ingress.** The route forwards the ORIGINAL body, not the parse output, so a key the old schema stripped still reached the door, which read it and answered `200`. A `POST /data/:object/query` body written one of these five ways stops working; each has a declared spelling that means the same thing: + +| body that now answers `400` | what the door served it as | write instead | +|---|---|---| +| `{ $orderby: 'name desc' }` | `orderBy: [{ field: 'name', order: 'desc' }]` | `{ $orderby: { name: 'desc' } }` — or `{ orderBy: [{ field: 'name', order: 'desc' }] }` | +| `{ sort: '-created_at' }` | `orderBy: [{ field: 'created_at', order: 'desc' }]` | `{ sort: { created_at: 'desc' } }` — or `{ sort: [{ field: 'created_at', order: 'desc' }] }` | +| `{ $orderby: ['name'] }` | `orderBy: [{ field: 'name', order: 'asc' }]` | `{ $orderby: { name: 'asc' } }` — or the `SortNode[]` form | +| `{ $filter: '{"status":"open"}' }` | `where: { status: 'open' }` | `{ $filter: { status: 'open' } }` | +| that same JSON string on `filters` or `filter` | `where: { status: 'open' }` | the object form on whichever of the two keys you write | + +**`GET /data/:object` still serves every one of those shapes.** The querystring path does not parse through this schema at all — `FindDataRequestSchema` is parsed at exactly one call site, the POST handler — so `?$orderby=name desc`, `?sort=-created_at` and `?$filter={"status":"open"}` answer exactly as before. What narrowed is the POST body alone — the platform has not stopped accepting these spellings everywhere. + +The remaining refusals in the list narrow nothing that was served correctly; they move an unservable body's refusal earlier — from the engine, or from a wrong answer under a `200`, to the ingress that can name the parameter to fix. **`@objectstack/metadata-protocol` folds by the spec export** instead of its own table, and both resolved tables — plus the `$`-parameter list its `UNSUPPORTED_QUERY_PARAM` refusal quotes — are pinned byte-equal to their pre-change values. An undeclared `$` spelling is still refused loudly with the same `400 UNSUPPORTED_QUERY_PARAM`; the sentence now quotes `QUERY_TRANSPORT_DOLLAR_PARAMS` rather than a hand-copied list, so a spelling added to the table cannot leave the refusal naming a set the door no longer has. From 53591e48be2af08a3e41c50d9f4b955102595200 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 18 Sep 2026 01:45:58 +0000 Subject: [PATCH 10/10] fix(spec): declare the two transport-aware schemas' dropped refinements and re-site FindDataRequest `@objectstack/spec#build` exits 1 on this branch: the dropped-refinement ratchet that landed on `main` sees two published schemas this PR introduces -- `data/QueryTransportParams` and `data/QueryWithTransport` -- dropping four refinement sites each with no ledger line, and sees `api/FindDataRequest`'s one recorded site replaced by four under the new `query` slot. Declares them. The refinements are untouched: this is a visibility ratchet and the published JSON Schema is WIDER than the Zod type either way. Every site path and both header totals are copied from the gate's own output at this head (`748 refinement site(s) across 245 published schema(s)` / `0 ... DID reach the file, 3 had no JSON form`), not computed from the file. Claude-Session: https://claude.ai/code/session_01LvwGppdonww4zGLWZo5rho Co-authored-by: Claude --- .../spec/dropped-refinements.baseline.json | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/packages/spec/dropped-refinements.baseline.json b/packages/spec/dropped-refinements.baseline.json index 682685d96ad..81f381c4125 100644 --- a/packages/spec/dropped-refinements.baseline.json +++ b/packages/spec/dropped-refinements.baseline.json @@ -2,8 +2,8 @@ "description": "Shrink-only ledger of every PUBLISHED JSON Schema that is WIDER than the Zod type it was generated from, because a rule written as `.refine()` reaches the runtime and not the file (#18670). `z.toJSONSchema()` has no arm for a `custom` check: a plain record, the same record with a `.refine()`, and the same record with an ABORTING `.refine()` all project byte-identically (measured on zod 4.4.3, the version packages/spec resolves). So a document one of these files ACCEPTS can still be refused at parse time, and an author -- or an AI -- validating against packages/spec/json-schema/** finds out a release later. Each `sites` path is a position under that schema at which a refinement is dropped; the same paths are written onto the artifact itself as `x-dropped-refinements`. Hand-edited on purpose and with no `gen:` script: a generator would let a new gap be admitted by running a command instead of by a decision, which is the silence this ledger exists to end. Adding, removing or moving a site fails packages/spec/scripts/build-schemas.ts until the line moves with it, and the failure prints the corrected entry in full. ⛔ Do not delete or weaken a refinement to shorten this file -- the runtime rule is correct; it is the projection that is silent. Narrowing the published shape to match the Zod type is a public contract change and is NOT what this ledger does.", "measured": { "zod": "4.4.3", - "publishedSchemasWithDroppedRefinements": 243, - "droppedRefinementSites": 737, + "publishedSchemasWithDroppedRefinements": 245, + "droppedRefinementSites": 748, "refinementSitesThatDidProject": 0, "refinementSitesWithNoJsonFormToCompare": 3 }, @@ -166,7 +166,10 @@ }, "api/FindDataRequest": { "sites": [ - "query.where.lazy" + "query.in.having.lazy", + "query.in.where.options[0].lazy.options[0].items[0]", + "query.in.where.options[0].lazy.options[0].items[1]", + "query.in.where.options[0].lazy.options[2].items[0]" ] }, "api/GetFlowResponse": { @@ -860,6 +863,22 @@ "where.lazy" ] }, + "data/QueryTransportParams": { + "sites": [ + "$filter.options[0].lazy.options[0].items[0]", + "$filter.options[0].lazy.options[0].items[1]", + "$filter.options[0].lazy.options[2].items[0]", + "$filter.options[1].options[1].lazy" + ] + }, + "data/QueryWithTransport": { + "sites": [ + "in.having.lazy", + "in.where.options[0].lazy.options[0].items[0]", + "in.where.options[0].lazy.options[0].items[1]", + "in.where.options[0].lazy.options[2].items[0]" + ] + }, "data/ReferenceIdValue": { "sites": [ ""