From 136551752abae45c5cff53f7aa9d711e5d735d7e Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 07:07:36 +0000 Subject: [PATCH 1/7] fix(spec)!: a dataset or measure filter refuses a list inside a nested relation on save (#20080) The analytics where door flattens a nested-relation condition to a dotted member and refuses an equality-slot list inside it; the shared FilterConditionSchema deliberately does not descend there (ruling A on #19889). The two analytics carriers now refine their filter with the analytics door's walk, in the face's words, so such a dataset no longer saves clean and fails every chart. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01QcAS3qiYYZNezaxZxaUdMV --- ...ataset-filter-nested-relation-list.test.ts | 228 ++++++++++++++++++ packages/spec/src/ui/dataset.zod.ts | 152 +++++++++++- 2 files changed, 376 insertions(+), 4 deletions(-) create mode 100644 packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts diff --git a/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts b/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts new file mode 100644 index 00000000000..b6f43769290 --- /dev/null +++ b/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts @@ -0,0 +1,228 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#20080] A list in the EQUALITY slot inside a NESTED-RELATION condition is + * refused when a dataset or measure filter is SAVED, in the words the analytics + * `where` door refuses it with when that filter is CHARTED. + * + * Measured before the change, on `origin/main` `9e7824a445`: + * `DatasetSchema.safeParse` with `filter: { account: { region: ['a'] } }`, and + * with a measure `filter` of `{ account: { region: { $eq: ['a'] } } }`, both + * answered `success: true`, while the analytics door flattens the relation to + * the dotted member `account.region` and refuses the list with + * `INVALID_FILTER` / 400. The dataset saved clean and every chart on it failed. + * + * Triage record 5825670610 (remedy A) routed the fix to the two analytics + * carriers and left the shared `FilterConditionSchema` as ruling A on #19889 + * (record 5805248669) put it: the engine reads such a spec as a deep-equality + * comparand, so the shared door does not descend it. + * + * ## What would make these pins worthless, and what stops it + * + * - A refinement that refuses everything passes every REFUSE row. §4 runs each + * control through both carriers and requires the parse to succeed AND keep + * the filter, so an over-wide walk goes red there. + * - "Same words" read as "similar words". §2 builds the analytics door's own + * refusal — the one-entry node it hands the shared face, with the path of the + * node that holds the entry — and compares the carrier's issue message to it + * for EQUALITY, after removing only the location clause the face appends. §2 + * also proves that clause is really there, so the removal is not vacuous. + * - A second report of a refusal the shared schema already makes. §3 requires + * EXACTLY ONE issue at a top-level list's path, so a walk that judged + * outside a nested relation as well goes red there. + */ + +import { describe, expect, it } from 'vitest'; + +import { StandardErrorCode } from '../api/errors.zod'; +import { assertListComparandShapes } from '../data/filter-comparand-shape'; +import { FilterConditionSchema } from '../data/filter.zod'; +import { DatasetMeasureSchema, DatasetSchema } from './dataset.zod'; + +type Issue = { path: PropertyKey[]; message: string }; +type Parsed = { success: boolean; error?: { issues: readonly Issue[] } }; + +/** The one issue at `path` (dot-joined), failing loudly on none or several. */ +function issueAt(result: Parsed, path: string): Issue { + expect(result.success, `expected a refusal at ${path}`).toBe(false); + const issues = (result.error?.issues ?? []).filter((i) => i.path.join('.') === path); + expect( + issues, + `issues raised: ${JSON.stringify((result.error?.issues ?? []).map((i) => i.path))}`, + ).toHaveLength(1); + return issues[0]!; +} + +/** + * The analytics door's refusal of one field entry: `assertNoListInEqualitySlot` + * hands `{ [field]: spec }` to the shared face with the path of the node that + * holds the entry (`where.account` for `{ account: { region: [...] } }`). + */ +function analyticsDoorRefusal(field: string, spec: unknown, holder: string): Error & { code?: string; status?: number } { + try { + assertListComparandShapes({ [field]: spec }, undefined, holder); + } catch (error) { + return error as Error & { code?: string; status?: number }; + } + throw new Error(`the face accepted ${JSON.stringify({ [field]: spec })} at ${holder}`); +} + +const dataset = (extra: Record) => ({ + name: 'deals_ds', + label: 'Deals', + object: 'deal', + include: ['account', 'account.owner'], + dimensions: [{ name: 'stage', field: 'stage', type: 'string' }], + measures: [{ name: 'deal_count', aggregate: 'count' }], + ...extra, +}); + +const withScope = (filter: unknown) => dataset({ filter }); +const withMeasureFilter = (filter: unknown) => + dataset({ measures: [{ name: 'deal_count', aggregate: 'count', filter }] }); + +/** + * Each nested-relation shape: the filter, its path under `filter`, the field + * the door names, the list, whether it is the `$eq` spelling, and the path of + * the node the analytics door hands to the face. + */ +const REFUSED: ReadonlyArray = [ + ['implicit, two members', { account: { region: ['a', 'b'] } }, 'account.region', 'region', ['a', 'b'], false, 'where.account'], + ['implicit, one member', { account: { region: ['a'] } }, 'account.region', 'region', ['a'], false, 'where.account'], + ['implicit, EMPTY — still an array in this slot', { account: { region: [] } }, 'account.region', 'region', [], false, 'where.account'], + ['$eq, two members', { account: { region: { $eq: ['a', 'b'] } } }, 'account.region.$eq', 'region', ['a', 'b'], true, 'where.account'], + ['$eq, EMPTY', { account: { region: { $eq: [] } } }, 'account.region.$eq', 'region', [], true, 'where.account'], + ['$eq beside another operator', { account: { region: { $ne: 'x', $eq: ['a'] } } }, 'account.region.$eq', 'region', ['a'], true, 'where.account'], + ['two relation hops deep', { account: { owner: { region: ['a'] } } }, 'account.owner.region', 'region', ['a'], false, 'where.account.owner'], + ['beside a scalar sibling in the relation', { account: { tier: 'gold', region: ['a'] } }, 'account.region', 'region', ['a'], false, 'where.account'], + ['under $and', { $and: [{ stage: 'won' }, { account: { region: ['a'] } }] }, '$and.1.account.region', 'region', ['a'], false, 'where.$and[1].account'], + ['under $or', { $or: [{ account: { region: { $eq: ['a'] } } }] }, '$or.0.account.region.$eq', 'region', ['a'], true, 'where.$or[0].account'], + ['under $not', { $not: { account: { region: ['a'] } } }, '$not.account.region', 'region', ['a'], false, 'where.$not.account'], + ['under $not over $or', { $not: { $or: [{ stage: 'won' }, { account: { region: ['a'] } }] } }, '$not.$or.1.account.region', 'region', ['a'], false, 'where.$not.$or[1].account'], +]; + +// --------------------------------------------------------------------------- +// §1 Both carriers refuse, at the list's own path +// --------------------------------------------------------------------------- + +describe('#20080 §1 — both analytics carriers refuse a list inside a nested relation', () => { + it.each(REFUSED)('DatasetSchema.filter refuses %s', (_label, filter, issuePath) => { + const issue = issueAt(DatasetSchema.safeParse(withScope(filter)), `filter.${issuePath}`); + expect(issue.message).toMatch(/requires a single comparable value, but received an array/); + }); + + it.each(REFUSED)('a measure filter refuses %s, through DatasetSchema', (_label, filter, issuePath) => { + issueAt(DatasetSchema.safeParse(withMeasureFilter(filter)), `measures.0.filter.${issuePath}`); + }); + + it.each(REFUSED)('DatasetMeasureSchema refuses %s on its own', (_label, filter, issuePath) => { + issueAt(DatasetMeasureSchema.safeParse({ name: 'deal_count', aggregate: 'count', filter }), `filter.${issuePath}`); + }); + + it('refuses every such list in one document, on both carriers at once', () => { + const result = DatasetSchema.safeParse(dataset({ + filter: { account: { region: ['a'], tier: { $eq: ['gold'] } } }, + measures: [{ name: 'deal_count', aggregate: 'count', filter: { account: { owner: { region: [] } } } }], + })); + expect(result.success).toBe(false); + expect(result.error!.issues.map((i) => i.path.join('.')).sort()).toEqual([ + 'filter.account.region', + 'filter.account.tier.$eq', + 'measures.0.filter.account.owner.region', + ]); + }); +}); + +// --------------------------------------------------------------------------- +// §2 The carrier prints the analytics door's sentence +// --------------------------------------------------------------------------- + +describe('#20080 §2 — the carrier refusal is the analytics door\'s sentence', () => { + it.each(REFUSED)('%s', (_label, filter, issuePath, field, list, eq, holder) => { + const door = analyticsDoorRefusal(field, eq ? { $eq: list } : list, holder); + // The door's refusal is the ADR-0112 class-1 envelope the face throws. + expect(door.code).toBe(StandardErrorCode.enum.INVALID_FILTER); + expect(door.status).toBe(400); + // The one clause only the door can write is really there, exactly once… + const location = ` at ${holder}.${field}${eq ? '.$eq' : ''}.`; + expect(door.message.split(location)).toHaveLength(2); + // …and removing it leaves the carrier's message, character for character. + const carrier = issueAt(DatasetSchema.safeParse(withScope(filter)), `filter.${issuePath}`); + expect(carrier.message).toBe(door.message.replace(location, '.')); + }); + + it('names the leaf field and both remedies, as the door does', () => { + const message = issueAt( + DatasetSchema.safeParse(withScope({ account: { region: ['a', 'b'] } })), + 'filter.account.region', + ).message; + expect(message).toMatch(/^The implicit-equality comparand on field "region" requires a single comparable value/); + expect(message).toContain('For "one of these values" use {"$in": […]} (authoring: in)'); + expect(message).toContain('{"$contains": "…"} (authoring: contains), an $or of those for any-of.'); + expect(message).toMatch(/The filter was NOT applied, .*UNFILTERED result set\.$/); + }); +}); + +// --------------------------------------------------------------------------- +// §3 A list the shared schema already refuses is refused once, not twice +// --------------------------------------------------------------------------- + +describe('#20080 §3 — outside a nested relation the shared schema answers, once', () => { + it.each([ + ['top level, implicit', { region: ['a'] }, 'region', /^The implicit-equality comparand on field "region"/], + ['top level, $eq', { region: { $eq: ['a'] } }, 'region.$eq', /^Operator "\$eq" on field "region"/], + ['under $and', { $and: [{ region: ['a'] }] }, '$and.0.region', /^The implicit-equality comparand on field "region"/], + ['under $not', { $not: { region: { $eq: [] } } }, '$not.region.$eq', /^Operator "\$eq" on field "region"/], + ] as const)('%s', (_label, filter, issuePath, opening) => { + const issue = issueAt(DatasetSchema.safeParse(withScope(filter)), `filter.${issuePath}`); + expect(issue.message).toMatch(opening); + issueAt(DatasetMeasureSchema.safeParse({ name: 'deal_count', aggregate: 'count', filter }), `filter.${issuePath}`); + }); +}); + +// --------------------------------------------------------------------------- +// §4 CONTROLS — accepted on both carriers, and kept +// --------------------------------------------------------------------------- + +describe('#20080 §4 — what a nested relation may still carry', () => { + const day = new Date('2026-07-01T00:00:00.000Z'); + it.each([ + ['a scalar', { account: { region: 'a' } }], + ['a scalar, two hops deep', { account: { owner: { region: 'a' } } }], + ['null — the has-no-value predicate', { account: { region: null } }], + ['a Date', { account: { closed_at: day } }], + ['$eq: a scalar', { account: { region: { $eq: 'a' } } }], + ['$eq: null', { account: { region: { $eq: null } } }], + ['$eq: a { $field } reference', { account: { amount: { $eq: { $field: 'budget' } } } }], + ['$in keeps its list', { account: { region: { $in: ['a', 'b'] } } }], + ['$in: [] stays the declared predicate', { account: { region: { $in: [] } } }], + ['$nin keeps its list', { account: { region: { $nin: ['a'] } } }], + ['$between keeps its pair', { account: { score: { $between: [1, 9] } } }], + ['$ne carrying a list — no ruling decides it', { account: { region: { $ne: ['a'] } } }], + ['a scalar under $and', { $and: [{ account: { region: 'a' } }] }], + ])('%s', (_label, filter) => { + const scoped = DatasetSchema.safeParse(withScope(filter)); + expect(scoped.success, JSON.stringify(scoped.error?.issues)).toBe(true); + // Accepted means KEPT: the carrier returns the filter it was given. + expect(scoped.data!.filter).toEqual(filter); + const measure = DatasetMeasureSchema.safeParse({ name: 'deal_count', aggregate: 'count', filter }); + expect(measure.success, JSON.stringify(measure.error?.issues)).toBe(true); + expect(measure.data!.filter).toEqual(filter); + }); + + it('the shared FilterConditionSchema keeps its own reach — ruling A is untouched', () => { + for (const filter of [{ account: { region: ['a'] } }, { account: { region: { $eq: ['a'] } } }]) { + const result = FilterConditionSchema.safeParse(filter); + expect(result.success, JSON.stringify(result.error?.issues)).toBe(true); + expect(result.data).toEqual(filter); + } + }); +}); diff --git a/packages/spec/src/ui/dataset.zod.ts b/packages/spec/src/ui/dataset.zod.ts index 3e9fb04033f..781a5b6df01 100644 --- a/packages/spec/src/ui/dataset.zod.ts +++ b/packages/spec/src/ui/dataset.zod.ts @@ -6,6 +6,7 @@ import { strictObject } from '../shared/strict-object'; import { ProtectionSchema } from '../shared/protection.zod'; import { MetadataProtectionFields } from '../kernel/metadata-protection.zod'; import { FilterConditionSchema } from '../data/filter.zod'; +import { arrayEqualityComparandMessage } from '../data/filter-comparand-refusal-text'; import { SnakeCaseIdentifierSchema } from '../shared/identifiers.zod'; import { I18nLabelSchema } from './i18n.zod'; import { AggregationFunction, DateGranularity } from '../data/query.zod'; @@ -79,6 +80,139 @@ const DATASET_NO_SQL = + '`derived: { op, of: [...] }`, which combines OTHER measures in this dataset by name. ' + 'Joins are compiled from `Dataset.include` — you never write an ON clause.'; +// ── [#20080] The analytics door's reach, on the two analytics carriers ────── + +/** + * A node the analytics `where` door walks: a plain object, not `null`, not an + * array and not a `Date` — that door's own `isFilterObject` + * (`service-analytics` `strategies/filter-normalizer.ts`), mirrored here. + */ +function isAnalyticsFilterObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) && !(value instanceof Date); +} + +/** + * A NESTED-RELATION field spec, as the analytics door decides it + * (`isNestedRelationSpec` in the same file): a plain object — prototype + * `Object.prototype` or `null`, so a `Map` or a class instance stays a + * comparand — carrying no `$` key. + */ +function isAnalyticsNestedRelationSpec(spec: unknown): spec is Record { + if (!isAnalyticsFilterObject(spec)) return false; + const proto = Object.getPrototypeOf(spec); + if (proto !== Object.prototype && proto !== null) return false; + return !Object.keys(spec).some((key) => key.startsWith('$')); +} + +/** + * [#20080] Refuse, when a dataset or measure filter is SAVED, the list the + * analytics `where` door refuses when that filter is CHARTED: an ARRAY in the + * EQUALITY slot of a field inside a NESTED-RELATION condition — + * `{ account: { region: ['a'] } }` and `{ account: { region: { $eq: ['a'] } } }`. + * + * ## Why this is a carrier refinement and not the shared schema's + * + * `FilterConditionSchema` refuses an equality-slot list on the field entries + * of a condition and of every `$and` / `$or` / `$not` member, and deliberately + * NOT inside a field spec with no `$` key: that is the shared comparand face's + * reach, and the engine reads such a spec as a deep-equality comparand (ruling + * A on #19889, record 5805248669). That ruling stands, and this function does + * not touch the shared schema. + * + * The analytics door reads the same spec differently. Its `fieldLeaves` + * flattens a nested relation to the dotted member `account.region`, so the + * entries inside are comparisons in their own right, and + * `assertNoListInEqualitySlot` hands each equality-slot list among them to the + * shared face, which refuses it with `INVALID_FILTER` / 400. A dataset `filter` + * and a measure `filter` are charted through that door on every path: the + * dataset executor hands them to the analytics query, and the native-SQL and + * ObjectQL strategies and the draft preview each lower them through it. So + * until this refinement such a dataset saved clean and every chart built on it + * failed, for somebody else, later. Triage routed the fix to these two + * carriers (record 5825670610, remedy A): the refusal moves to save, and no + * filter changes meaning. + * + * ## The reach is the analytics door's, and only the part the schema lacks + * + * The walk is `mapWhereFieldEntries`' traversal: `$and` / `$or` arrays and + * `$not` are descended; every other `$` key at node level is skipped; a + * nested-relation spec is descended, at any depth; every other field entry is + * judged exactly as `assertNoListInEqualitySlot` judges it — the implicit form + * when the spec is an array, the explicit form when the spec is an object whose + * `$eq` is an array. No depth bound: that door has none. + * + * Only an entry INSIDE a nested relation is reported. Every other entry the + * walk visits is a field entry of the condition or of a combinator member, + * which `FilterConditionSchema`'s own refinement already refuses with the same + * words — reporting it here as well would raise the one refusal twice. + * + * ## The words are the face's + * + * `arrayEqualityComparandMessage` is the one builder the shared face and the + * schema door both print from. The analytics door names the field it hands to + * the face — the leaf key (`region`), with the location `at + * where.account.region` appended. This refinement prints the same sentence and + * leaves the location out, as the schema door does, because the issue's own + * `path` carries it (`filter.account.region`, + * `measures.0.filter.account.region.$eq`). + * + * ⛔ Not judged here: `$ne` carrying a list (no ruling decides it); the list + * operators, which keep their lists (`$in: []` / `$nin: []` included); every + * scalar, `null` included; and the face's OTHER arms inside a nested relation + * (a malformed `$in`, a `null` ordering comparand, a `$between` endpoint), + * which this card does not move. Nothing is stripped: the parse fails, and a + * filter that is refused is never a filter that is dropped. + */ +function refuseNestedRelationEqualityLists( + node: unknown, + ctx: z.RefinementCtx, + path: (string | number)[] = [], + insideRelation = false, +): void { + if (!isAnalyticsFilterObject(node)) return; + for (const [key, spec] of Object.entries(node)) { + if (key === '$and' || key === '$or') { + if (Array.isArray(spec)) { + spec.forEach((member, index) => + refuseNestedRelationEqualityLists(member, ctx, [...path, key, index], insideRelation)); + } + continue; + } + if (key === '$not') { + refuseNestedRelationEqualityLists(spec, ctx, [...path, key], insideRelation); + continue; + } + if (key.startsWith('$')) continue; + if (isAnalyticsNestedRelationSpec(spec)) { + refuseNestedRelationEqualityLists(spec, ctx, [...path, key], true); + continue; + } + if (!insideRelation) continue; // `FilterConditionSchema` judges this entry itself + if (Array.isArray(spec)) { + ctx.addIssue({ + code: 'custom', + path: [...path, key], + message: arrayEqualityComparandMessage(spec, { field: key }), + }); + } else if (isAnalyticsFilterObject(spec) && Array.isArray(spec.$eq)) { + ctx.addIssue({ + code: 'custom', + path: [...path, key, '$eq'], + message: arrayEqualityComparandMessage(spec.$eq, { op: '$eq', field: key }), + }); + } + } +} + +/** + * The filter both analytics carriers declare — `DatasetSchema.filter` and + * `DatasetMeasureSchema.filter` — which is `FilterConditionSchema` plus + * {@link refuseNestedRelationEqualityLists}. Every other schema that carries a + * `FilterCondition` keeps the shared schema's reach. + */ +const AnalyticsCarrierFilterSchema = lazySchema(() => + FilterConditionSchema.superRefine((filter, ctx) => refuseNestedRelationEqualityLists(filter, ctx))); + /** * Dimension — a groupable axis (e.g. "region", "close_date by quarter"). */ @@ -187,8 +321,13 @@ export const DatasetMeasureSchema = lazySchema(() => strictObject({ .meta({ title: 'Aggregate' }), /** Base field, or `relationship[.relationship].field` path. Optional for `count` (count(*)). */ field: z.string().optional().describe('Aggregated field; optional for count(*)').meta({ title: 'Field' }), - /** Measure-scoped filter (e.g. only won deals for "won_amount"). */ - filter: FilterConditionSchema.optional().meta({ title: 'Filter' }), + /** + * Measure-scoped filter (e.g. only won deals for "won_amount"). [#20080] A + * list in the equality slot inside a nested relation is refused on save, as + * the analytics door refuses it on chart — see + * {@link refuseNestedRelationEqualityLists}. + */ + filter: AnalyticsCarrierFilterSchema.optional().meta({ title: 'Filter' }), /** * Display format — a NUMERAL pattern controlling grouping, decimals and * percent: `"0,0.00"`, `"0.0%"`. A `$` in the pattern is still honoured as a @@ -345,8 +484,13 @@ export const DatasetSchema = lazySchema(() => strictObject({ .optional() .describe('Relationship names/paths to join (derived from object graph; max 3 hops)'), - /** Definition-level filter (the dataset's intrinsic scope, e.g. non-deleted). */ - filter: FilterConditionSchema.optional().describe('Intrinsic dataset scope filter'), + /** + * Definition-level filter (the dataset's intrinsic scope, e.g. non-deleted). + * [#20080] A list in the equality slot inside a nested relation is refused + * on save, as the analytics door refuses it on chart — see + * {@link refuseNestedRelationEqualityLists}. + */ + filter: AnalyticsCarrierFilterSchema.optional().describe('Intrinsic dataset scope filter'), /** The semantic contract presentations bind to. */ dimensions: z.array(DatasetDimensionSchema).describe('Groupable axes'), From 232c0482d7f62bd329f156ea1e93a79e4c326cc4 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 07:10:15 +0000 Subject: [PATCH 2/7] fix(spec): register the nested-relation list refusal under ADR-0087; correct the pending exception sentences Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01QcAS3qiYYZNezaxZxaUdMV --- .changeset/19888-analytics-implicit-array.md | 2 +- ...19889-filter-schema-door-array-equality.md | 2 +- ...080-dataset-filter-nested-relation-list.md | 39 +++++++++++ ...relation-equality-array-refused-at-save.ts | 70 +++++++++++++++++++ ...quality-array-comparand-refused-at-save.ts | 10 ++- ...filter-equality-array-comparand-refused.ts | 12 ++-- 6 files changed, 125 insertions(+), 10 deletions(-) create mode 100644 .changeset/20080-dataset-filter-nested-relation-list.md create mode 100644 packages/spec/src/migrations/entries/semantic/18.dataset-filter-nested-relation-equality-array-refused-at-save.ts diff --git a/.changeset/19888-analytics-implicit-array.md b/.changeset/19888-analytics-implicit-array.md index d819b051478..ea75a785994 100644 --- a/.changeset/19888-analytics-implicit-array.md +++ b/.changeset/19888-analytics-implicit-array.md @@ -27,4 +27,4 @@ Ruling 乙 of #19757 refuses a list in the equality slot at the shared comparand Each list is now handed to the shared face's equality arm before any node is built. Both spellings of one condition therefore get the same refusal, with the same wording, path and `$in` prescription. The draft-data preview runs the same gate, so a drafted chart refuses what the published chart refuses. Before, it compared each row against the list's string form. -Who is affected: nothing in this repository's examples, seeds or docs authors the shape (measured over `examples/**`, `packages/**` and the fenced code in the docs). Stored datasets, dashboard widget filters, report runtime filters and measure filters in a deployment were NOT measured. In this release the authoring schema refuses the shape too, when such a document is saved (a separate change in `@objectstack/spec`, ADR-0087 entry `filter-equality-array-comparand-refused-at-save`). One position is judged only here: a list inside a nested relation, which this door flattens to a dotted member. A document carrying that still publishes, and it is refused when it is charted. The refusal names the field and the path. `$ne` with a list is not part of the ruling and is not judged here. The list operators (`$in`, `$nin`, `$between`) keep their lists, and every scalar, `null` included, compiles as before. +Who is affected: nothing in this repository's examples, seeds or docs authors the shape (measured over `examples/**`, `packages/**` and the fenced code in the docs). Stored datasets, dashboard widget filters, report runtime filters and measure filters in a deployment were NOT measured. In this release the authoring schema refuses the shape too, when such a document is saved (a separate change in `@objectstack/spec`, ADR-0087 entry `filter-equality-array-comparand-refused-at-save`). One position is judged here and not by the shared authoring schema: a list inside a nested relation, which this door flattens to a dotted member. A dataset `filter` or measure `filter` carrying one is refused when it is saved, in the same words (a separate change in `@objectstack/spec`, ADR-0087 entry `dataset-filter-nested-relation-equality-array-refused-at-save`). Any other `where` that reaches this door with one, such as a caller `where` or a dataset selection's `runtimeFilter`, is refused only when it is charted. The refusal names the field and the path. `$ne` with a list is not part of the ruling and is not judged here. The list operators (`$in`, `$nin`, `$between`) keep their lists, and every scalar, `null` included, compiles as before. diff --git a/.changeset/19889-filter-schema-door-array-equality.md b/.changeset/19889-filter-schema-door-array-equality.md index d27f04252d3..fe9c0979a87 100644 --- a/.changeset/19889-filter-schema-door-array-equality.md +++ b/.changeset/19889-filter-schema-door-array-equality.md @@ -21,7 +21,7 @@ Two request doors parse these carriers, and they now answer before the analytics ## What does NOT change - **Nothing stored is rewritten, and nothing is dropped.** The parse fails and strips nothing. The read path does not re-validate stored rows, so a stored document keeps loading, and its next save is refused. Such a filter has failed every query since the equality-slot change, so the refusal is a repair. -- **The reach is the face's, and no wider.** A field spec with no `$` key, such as the nested-relation condition `{ account: { region: ['a'] } }`, is not judged, because the face does not judge it either. The analytics `where` door does refuse that shape when a dataset or measure filter is charted. +- **The reach is the face's, and no wider.** A field spec with no `$` key, such as the nested-relation condition `{ account: { region: ['a'] } }`, is not judged by `FilterConditionSchema`, because the face does not judge it either. The analytics `where` door does refuse that shape, because it flattens the relation to a dotted member. So the two carriers that door charts, the dataset `filter` and the measure `filter`, refuse it on save as well, in the same words. That is a separate change in this release, ADR-0087 entry `dataset-filter-nested-relation-equality-array-refused-at-save`. - **The data-engine calls' `where` option still parses.** Its type is a union whose first arm is an open record. The face refuses the shape when the call runs. - **`$ne` carrying an array is not judged.** - The list operators keep their arrays, `$in: []` and `$nin: []` included. Every scalar, `null`, a `Date` and a `{ $field }` reference pass as before. diff --git a/.changeset/20080-dataset-filter-nested-relation-list.md b/.changeset/20080-dataset-filter-nested-relation-list.md new file mode 100644 index 00000000000..571b0d03fe4 --- /dev/null +++ b/.changeset/20080-dataset-filter-nested-relation-list.md @@ -0,0 +1,39 @@ +--- +"@objectstack/spec": minor +--- + +fix(spec)!: a dataset or measure filter with a list inside a nested relation is refused when it is saved, not when it is charted (#20080) + +**BREAKING**: an accept-set narrowing of a published authoring schema, shipped as `minor` under the repo's launch-window convention for accept-set narrowings. The hand-migration prescription is registered under protocol major 18 as `dataset-filter-nested-relation-equality-array-refused-at-save`. + +## What changes + +`DatasetSchema.filter` and `DatasetMeasureSchema.filter` now refuse an ARRAY in the equality slot of a field inside a nested-relation condition. That covers the implicit form `{ account: { region: ['a'] } }` and the explicit form `{ account: { region: { $eq: ['a'] } } }`, the empty array included, at any relation depth and under `$and` / `$or` / `$not`. So `defineStack`, `os validate` and a save through the metadata protocol (`422 INVALID_METADATA`) refuse such a dataset at the filter's path, for example `filter.account.region` or `measures.0.filter.account.region.$eq`. + +Before this change the dataset saved clean. The analytics `where` door, which charts both filters on every path, flattens the relation to the dotted member `account.region` and refuses the list with `INVALID_FILTER` / 400. So every chart built on the dataset failed. Measured on `origin/main` `9e7824a445`: `DatasetSchema.safeParse` answered `success: true` for both forms. + +The refusal is the analytics door's sentence: the field, the received list, and the two operators a list in that slot stood in for. The door adds `at where.account.region`. The schema leaves that out, because the issue's `path` already says where it is. + +## What does NOT change + +- **The shared `FilterConditionSchema` keeps its reach.** Every other schema that carries a `FilterCondition` still accepts a list inside a nested relation, because the engine reads that spec as a deep-equality comparand. +- **Nothing stored is rewritten, and nothing is dropped.** The parse fails and strips nothing. The read path does not re-validate stored rows, so a stored dataset keeps loading, and its next save is refused. Such a filter has failed every chart, so the refusal is a repair. +- A list outside a nested relation is refused as before, once, by `FilterConditionSchema`. +- `$ne` carrying an array is not judged. The list operators keep their arrays, `$in: []` and `$nin: []` included. Every scalar, `null`, a `Date` and a `{ $field }` reference inside a nested relation pass as before. + +## FROM → TO + +| you wrote | write instead | +|:--|:--| +| `{ account: { region: ['a', 'b'] } }`, meaning "one of these values" | `{ account: { region: { $in: ['a', 'b'] } } }` | +| `{ account: { tags: ['a'] } }`, meaning "the stored list holds `a`" on a multi-value field | `{ account: { tags: { $contains: 'a' } } }` | +| `{ account: { region: ['a'] } }`, meaning one value | `{ account: { region: 'a' } }` | +| `{ account: { region: { $eq: [...] } } }` | any of the rows above | + +## Who is affected, measured + +Nothing shipped in this repository carries the shape. At `9e7824a445`, the dataset and measure filters of `examples/app-crm`, `examples/app-showcase`, `examples/app-todo` and the platform objects, plus the dataset examples in `content/docs` and `skills`, carry no list inside a nested relation. Deployed datasets were NOT measured. Validating each stack, or re-saving each dataset, finds every instance. + +Clause-②: no (narrowing) — nothing is widened. No key is added, removed or renamed, no exported symbol moves, and the operator vocabulary is unchanged. One comparand shape in one position of two carriers, which the analytics door already refused on every chart, is now refused on save as well. + + diff --git a/packages/spec/src/migrations/entries/semantic/18.dataset-filter-nested-relation-equality-array-refused-at-save.ts b/packages/spec/src/migrations/entries/semantic/18.dataset-filter-nested-relation-equality-array-refused-at-save.ts new file mode 100644 index 00000000000..4ae56aba97a --- /dev/null +++ b/packages/spec/src/migrations/entries/semantic/18.dataset-filter-nested-relation-equality-array-refused-at-save.ts @@ -0,0 +1,70 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import type { SemanticMigration } from '../../types.js'; + +// The analytics carriers' half of filter-equality-array-comparand-refused-at-save. +// That entry refuses an equality-slot list on save wherever the shared face +// refuses it, and names a list inside a nested-relation condition as a position +// it deliberately leaves to execution. This one closes that position on the two +// carriers the analytics where door charts — a dataset filter and a measure +// filter — because that door flattens a nested relation to a dotted member and +// refuses the list there. Recorded as its own entry because the surface is +// narrower (two carriers, one position) and the shared schema does not move. +export const entry: SemanticMigration = { + id: 'dataset-filter-nested-relation-equality-array-refused-at-save', + // No backticks in `surface` — build-upgrade-guide renders it inside a code + // span already, and a nested backtick would close it. + surface: + 'ui.Dataset filter and ui.DatasetMeasure filter — an ARRAY as an EQUALITY comparand on a ' + + 'field INSIDE a nested-relation condition, now refused when the dataset is PARSED: the ' + + 'implicit form { account: { region: [...] } } and the explicit form ' + + '{ account: { region: { $eq: [...] } } }, the empty array included, at any relation depth ' + + 'and under $and / $or / $not. Every other schema that carries a FilterCondition keeps the ' + + 'shared schema\'s reach', + replacement: + 'the operator the list was standing in for, on the same field inside the same relation, ' + + 'exactly as in filter-equality-array-comparand-refused-at-save. "One of these values" is ' + + '$in: { account: { region: { $in: ["a", "b"] } } } (authoring spelling "in"). "The stored ' + + 'multi-value field holds this value" is $contains with ONE member (authoring spelling ' + + '"contains"), and an $or of those for any-of. A filter that meant a single value writes that ' + + 'value: { account: { region: "a" } }. The list operators keep their arrays, empty lists ' + + 'included; every scalar, null above all, is untouched; and $ne is NOT judged by this entry', + reason: + 'Measured on origin/main 9e7824a445 before the change: DatasetSchema parsed a dataset whose ' + + 'filter was { account: { region: ["a"] } }, and one whose measure filter was ' + + '{ account: { region: { $eq: ["a"] } } }, GREEN — while the analytics where door, which ' + + 'charts both carriers on every path (the native-SQL and ObjectQL strategies and the draft ' + + 'preview), flattens the relation to the dotted member account.region and hands the list to ' + + 'the shared comparand-shape face, which refuses it with INVALID_FILTER / 400. So such a ' + + 'dataset saved clean and every chart built on it failed, for a different person, later. ' + + 'The shared FilterConditionSchema does not descend a field spec with no $ key, because the ' + + 'engine reads one as a deep-equality comparand; ruling A on #19889 (record 5805248669) put ' + + 'it there and it stays there. Triage on #20080 (record 5825670610) routed the fix to the two ' + + 'analytics carriers instead: they refine their filter with the analytics door\'s own walk ' + + '($and / $or arrays and $not descended, other $ keys skipped, a plain object with no $ key ' + + 'descended as a nested relation at any depth) and refuse, inside a nested relation only, ' + + 'exactly what that door refuses there, in the face\'s words from the one builder both doors ' + + 'import. The one difference is that the door appends the location (at ' + + 'where.account.region) and the carrier does not, because its issue carries the location as ' + + 'its path (filter.account.region, measures.0.filter.account.region.$eq). A list outside a ' + + 'nested relation is the shared schema\'s refusal and is reported once. No filter changes ' + + 'meaning: the refusal moves from chart time to save. Metadata AT REST is not rewritten and ' + + 'this entry adds no D2 conversion, for the reason the runtime entry gives: an array on ' + + 'equality has no single honest value. The read path does not re-validate stored rows, so a ' + + 'stored dataset keeps loading; what changes is that re-saving it through the metadata ' + + 'protocol (422 INVALID_METADATA), defineStack or os validate is refused at the filter\'s ' + + 'path. Such a filter has failed every chart since the analytics door began refusing it, so ' + + 'the refusal is a repair and not a loss. In-repo census at 9e7824a445: no dataset or ' + + 'measure filter in examples, platform objects, docs or skills carries the shape; deployed ' + + 'datasets were NOT measured. ADR-0021 / ADR-0087.', + acceptanceCriteria: + 'Validate every stack and re-save every stored dataset: os validate or defineStack, and a ' + + 'save through the metadata protocol, report each list in an equality slot inside a nested ' + + 'relation by path, with the field, the received list and both remedies, so the sweep of the ' + + 'two carriers is mechanical. Decide per filter what it meant — one of these values ($in), ' + + 'the stored list holds a value ($contains, an $or of them for several), or one value — and ' + + 're-check what each chart is supposed to show: the filter had been failing every chart. A ' + + 'filter that reaches the analytics door by any other route, such as a caller where or a ' + + 'dataset selection runtimeFilter, is still refused only when it is charted, with ' + + 'INVALID_FILTER / 400 naming the field and the path.', +}; diff --git a/packages/spec/src/migrations/entries/semantic/18.filter-equality-array-comparand-refused-at-save.ts b/packages/spec/src/migrations/entries/semantic/18.filter-equality-array-comparand-refused-at-save.ts index b8d4e7610fe..3779e34194f 100644 --- a/packages/spec/src/migrations/entries/semantic/18.filter-equality-array-comparand-refused-at-save.ts +++ b/packages/spec/src/migrations/entries/semantic/18.filter-equality-array-comparand-refused-at-save.ts @@ -53,8 +53,10 @@ export const entry: SemanticMigration = { + 'with no $ key (a nested-relation or deep-equality condition), which the face never ' + 'descends either. ⚠️ Three positions therefore still refuse only at execution. (1) A list ' + 'inside a nested-relation condition, { account: { region: ["a"] } }: the analytics where ' - + 'door flattens that to the dotted member account.region and refuses it when a dataset or ' - + 'measure filter is charted. (2) The where option of the data-engine calls (find, count, ' + + 'door flattens that to the dotted member account.region and refuses it when the filter is ' + + 'charted; a dataset filter and a measure filter refuse it on save as well, which is the ' + + 'sibling entry dataset-filter-nested-relation-equality-array-refused-at-save. (2) The where ' + + 'option of the data-engine calls (find, count, ' + 'update, delete, aggregate, vector find): its type is a union whose first arm is an open ' + 'record, so it parses and the face refuses it when the call runs. (3) $ne carrying a ' + 'list, which no ruling has decided. Two request doors parse these carriers and now answer ' @@ -79,5 +81,7 @@ export const entry: SemanticMigration = { + 'query. ⛔ A clean re-save is NOT a complete sweep for the three positions the reason ' + 'names: grep nested-relation conditions and data-engine where options for a field whose ' + 'value is a list, and exercise them, where the runtime doors refuse with INVALID_FILTER ' - + '/ 400 naming the field and the path.', + + '/ 400 naming the field and the path. A dataset filter or a measure filter is the ' + + 'exception: its nested-relation lists are refused on save too ' + + '(dataset-filter-nested-relation-equality-array-refused-at-save).', }; diff --git a/packages/spec/src/migrations/entries/semantic/18.filter-equality-array-comparand-refused.ts b/packages/spec/src/migrations/entries/semantic/18.filter-equality-array-comparand-refused.ts index eac947138cb..337de0dc988 100644 --- a/packages/spec/src/migrations/entries/semantic/18.filter-equality-array-comparand-refused.ts +++ b/packages/spec/src/migrations/entries/semantic/18.filter-equality-array-comparand-refused.ts @@ -55,11 +55,13 @@ export const entry: SemanticMigration = { + 'parseFilterAST and at the engine lowering seam, with INVALID_FILTER / 400; the analytics ' + 'where door in BOTH spellings, the FilterArray form through parseFilterAST and the OBJECT ' + 'form because that door hands each equality-slot list to the shared face before it builds ' - + 'a node, with the same INVALID_FILTER / 400 and the same sentence (that door alone also ' - + 'refuses a list inside a nested-relation condition, which it flattens to a dotted member); ' - + 'and, on SAVE, the schema door (FilterConditionSchema and the $eq operator slot), with the ' - + 'same sentence as a parse issue at the filter\'s own path, which is the sibling entry ' - + 'filter-equality-array-comparand-refused-at-save. ' + + 'a node, with the same INVALID_FILTER / 400 and the same sentence (that door alone, among ' + + 'the runtime doors, also refuses a list inside a nested-relation condition, which it ' + + 'flattens to a dotted member); and, on SAVE, the schema door (FilterConditionSchema and the ' + + '$eq operator slot), with the same sentence as a parse issue at the filter\'s own path, ' + + 'which is the sibling entry filter-equality-array-comparand-refused-at-save, plus the two ' + + 'carriers that analytics door charts (a dataset filter and a measure filter) inside a ' + + 'nested relation too, which is dataset-filter-nested-relation-equality-array-refused-at-save. ' + 'The ruling records the hosted product as running on the SQL family, where the top-level ' + 'shape was already a 400, so the population that can observe a change is self-hosted ' + 'driver-mongodb, plus any filter nested under a combinator on the SQL family (a 500 ' From 2c060e6da0ed4ac7aa6840f82998532feec60be9 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 07:10:33 +0000 Subject: [PATCH 3/7] chore(spec): regenerate the migration registry region Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01QcAS3qiYYZNezaxZxaUdMV --- packages/spec/src/migrations/registry.ts | 88 +++++++++++++++++++++--- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/packages/spec/src/migrations/registry.ts b/packages/spec/src/migrations/registry.ts index 0c2dd0ff6c4..98236c29c3c 100644 --- a/packages/spec/src/migrations/registry.ts +++ b/packages/spec/src/migrations/registry.ts @@ -7193,6 +7193,72 @@ const step18: MigrationStep = { + 'and a sweep that renamed either has over-applied the rule: batchSize is a COUNT of ' + 'documents, not a duration, and consistency / projection / hint are not numbers at all.', }, + // The analytics carriers' half of filter-equality-array-comparand-refused-at-save. + // That entry refuses an equality-slot list on save wherever the shared face + // refuses it, and names a list inside a nested-relation condition as a position + // it deliberately leaves to execution. This one closes that position on the two + // carriers the analytics where door charts — a dataset filter and a measure + // filter — because that door flattens a nested relation to a dotted member and + // refuses the list there. Recorded as its own entry because the surface is + // narrower (two carriers, one position) and the shared schema does not move. + { + id: 'dataset-filter-nested-relation-equality-array-refused-at-save', + // No backticks in `surface` — build-upgrade-guide renders it inside a code + // span already, and a nested backtick would close it. + surface: + 'ui.Dataset filter and ui.DatasetMeasure filter — an ARRAY as an EQUALITY comparand on a ' + + 'field INSIDE a nested-relation condition, now refused when the dataset is PARSED: the ' + + 'implicit form { account: { region: [...] } } and the explicit form ' + + '{ account: { region: { $eq: [...] } } }, the empty array included, at any relation depth ' + + 'and under $and / $or / $not. Every other schema that carries a FilterCondition keeps the ' + + 'shared schema\'s reach', + replacement: + 'the operator the list was standing in for, on the same field inside the same relation, ' + + 'exactly as in filter-equality-array-comparand-refused-at-save. "One of these values" is ' + + '$in: { account: { region: { $in: ["a", "b"] } } } (authoring spelling "in"). "The stored ' + + 'multi-value field holds this value" is $contains with ONE member (authoring spelling ' + + '"contains"), and an $or of those for any-of. A filter that meant a single value writes that ' + + 'value: { account: { region: "a" } }. The list operators keep their arrays, empty lists ' + + 'included; every scalar, null above all, is untouched; and $ne is NOT judged by this entry', + reason: + 'Measured on origin/main 9e7824a445 before the change: DatasetSchema parsed a dataset whose ' + + 'filter was { account: { region: ["a"] } }, and one whose measure filter was ' + + '{ account: { region: { $eq: ["a"] } } }, GREEN — while the analytics where door, which ' + + 'charts both carriers on every path (the native-SQL and ObjectQL strategies and the draft ' + + 'preview), flattens the relation to the dotted member account.region and hands the list to ' + + 'the shared comparand-shape face, which refuses it with INVALID_FILTER / 400. So such a ' + + 'dataset saved clean and every chart built on it failed, for a different person, later. ' + + 'The shared FilterConditionSchema does not descend a field spec with no $ key, because the ' + + 'engine reads one as a deep-equality comparand; ruling A on #19889 (record 5805248669) put ' + + 'it there and it stays there. Triage on #20080 (record 5825670610) routed the fix to the two ' + + 'analytics carriers instead: they refine their filter with the analytics door\'s own walk ' + + '($and / $or arrays and $not descended, other $ keys skipped, a plain object with no $ key ' + + 'descended as a nested relation at any depth) and refuse, inside a nested relation only, ' + + 'exactly what that door refuses there, in the face\'s words from the one builder both doors ' + + 'import. The one difference is that the door appends the location (at ' + + 'where.account.region) and the carrier does not, because its issue carries the location as ' + + 'its path (filter.account.region, measures.0.filter.account.region.$eq). A list outside a ' + + 'nested relation is the shared schema\'s refusal and is reported once. No filter changes ' + + 'meaning: the refusal moves from chart time to save. Metadata AT REST is not rewritten and ' + + 'this entry adds no D2 conversion, for the reason the runtime entry gives: an array on ' + + 'equality has no single honest value. The read path does not re-validate stored rows, so a ' + + 'stored dataset keeps loading; what changes is that re-saving it through the metadata ' + + 'protocol (422 INVALID_METADATA), defineStack or os validate is refused at the filter\'s ' + + 'path. Such a filter has failed every chart since the analytics door began refusing it, so ' + + 'the refusal is a repair and not a loss. In-repo census at 9e7824a445: no dataset or ' + + 'measure filter in examples, platform objects, docs or skills carries the shape; deployed ' + + 'datasets were NOT measured. ADR-0021 / ADR-0087.', + acceptanceCriteria: + 'Validate every stack and re-save every stored dataset: os validate or defineStack, and a ' + + 'save through the metadata protocol, report each list in an equality slot inside a nested ' + + 'relation by path, with the field, the received list and both remedies, so the sweep of the ' + + 'two carriers is mechanical. Decide per filter what it meant — one of these values ($in), ' + + 'the stored list holds a value ($contains, an $or of them for several), or one value — and ' + + 're-check what each chart is supposed to show: the filter had been failing every chart. A ' + + 'filter that reaches the analytics door by any other route, such as a caller where or a ' + + 'dataset selection runtimeFilter, is still refused only when it is charted, with ' + + 'INVALID_FILTER / 400 naming the field and the path.', + }, { id: 'dataset-measure-aggregate-field-type-refused', surface: 'dataset measure `aggregate` × `field` pairs (`DatasetMeasureSchema`, the rows ' @@ -8949,11 +9015,13 @@ const step18: MigrationStep = { + 'parseFilterAST and at the engine lowering seam, with INVALID_FILTER / 400; the analytics ' + 'where door in BOTH spellings, the FilterArray form through parseFilterAST and the OBJECT ' + 'form because that door hands each equality-slot list to the shared face before it builds ' - + 'a node, with the same INVALID_FILTER / 400 and the same sentence (that door alone also ' - + 'refuses a list inside a nested-relation condition, which it flattens to a dotted member); ' - + 'and, on SAVE, the schema door (FilterConditionSchema and the $eq operator slot), with the ' - + 'same sentence as a parse issue at the filter\'s own path, which is the sibling entry ' - + 'filter-equality-array-comparand-refused-at-save. ' + + 'a node, with the same INVALID_FILTER / 400 and the same sentence (that door alone, among ' + + 'the runtime doors, also refuses a list inside a nested-relation condition, which it ' + + 'flattens to a dotted member); and, on SAVE, the schema door (FilterConditionSchema and the ' + + '$eq operator slot), with the same sentence as a parse issue at the filter\'s own path, ' + + 'which is the sibling entry filter-equality-array-comparand-refused-at-save, plus the two ' + + 'carriers that analytics door charts (a dataset filter and a measure filter) inside a ' + + 'nested relation too, which is dataset-filter-nested-relation-equality-array-refused-at-save. ' + 'The ruling records the hosted product as running on the SQL family, where the top-level ' + 'shape was already a 400, so the population that can observe a change is self-hosted ' + 'driver-mongodb, plus any filter nested under a combinator on the SQL family (a 500 ' @@ -9028,8 +9096,10 @@ const step18: MigrationStep = { + 'with no $ key (a nested-relation or deep-equality condition), which the face never ' + 'descends either. ⚠️ Three positions therefore still refuse only at execution. (1) A list ' + 'inside a nested-relation condition, { account: { region: ["a"] } }: the analytics where ' - + 'door flattens that to the dotted member account.region and refuses it when a dataset or ' - + 'measure filter is charted. (2) The where option of the data-engine calls (find, count, ' + + 'door flattens that to the dotted member account.region and refuses it when the filter is ' + + 'charted; a dataset filter and a measure filter refuse it on save as well, which is the ' + + 'sibling entry dataset-filter-nested-relation-equality-array-refused-at-save. (2) The where ' + + 'option of the data-engine calls (find, count, ' + 'update, delete, aggregate, vector find): its type is a union whose first arm is an open ' + 'record, so it parses and the face refuses it when the call runs. (3) $ne carrying a ' + 'list, which no ruling has decided. Two request doors parse these carriers and now answer ' @@ -9054,7 +9124,9 @@ const step18: MigrationStep = { + 'query. ⛔ A clean re-save is NOT a complete sweep for the three positions the reason ' + 'names: grep nested-relation conditions and data-engine where options for a field whose ' + 'value is a list, and exercise them, where the runtime doors refuse with INVALID_FILTER ' - + '/ 400 naming the field and the path.', + + '/ 400 naming the field and the path. A dataset filter or a measure filter is the ' + + 'exception: its nested-relation lists are refused on save too ' + + '(dataset-filter-nested-relation-equality-array-refused-at-save).', }, // One entry for two doors on purpose: the two vocabularies spell one operator // and the rows being answered are one pair. Splitting it would put half the From 43df1403dd33a895a836f19861f06929028d6b35 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 07:27:31 +0000 Subject: [PATCH 4/7] fix(spec): refine the carriers' optional filter so the published schema keeps its shape; declare the new dropped-refinement sites Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01QcAS3qiYYZNezaxZxaUdMV --- .../spec/dropped-refinements.baseline.json | 13 +++++++++- packages/spec/src/ui/dataset.zod.ts | 24 ++++++++++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/packages/spec/dropped-refinements.baseline.json b/packages/spec/dropped-refinements.baseline.json index 77a87e632cc..fbe23a7d7a0 100644 --- a/packages/spec/dropped-refinements.baseline.json +++ b/packages/spec/dropped-refinements.baseline.json @@ -56,7 +56,9 @@ "manifest.dashboards.element.globalFilters.element", "manifest.dashboards.element.widgets.element", "manifest.datasets.element", + "manifest.datasets.element.filter", "manifest.datasets.element.include.element", + "manifest.datasets.element.measures.element.filter", "manifest.datasources.element", "manifest.flows.element", "manifest.flows.element.errorHandling", @@ -154,7 +156,9 @@ "data.options[1].manifest.dashboards.element.globalFilters.element", "data.options[1].manifest.dashboards.element.widgets.element", "data.options[1].manifest.datasets.element", + "data.options[1].manifest.datasets.element.filter", "data.options[1].manifest.datasets.element.include.element", + "data.options[1].manifest.datasets.element.measures.element.filter", "data.options[1].manifest.datasources.element", "data.options[1].manifest.flows.element", "data.options[1].manifest.flows.element.errorHandling", @@ -236,7 +240,9 @@ "options[1].manifest.dashboards.element.globalFilters.element", "options[1].manifest.dashboards.element.widgets.element", "options[1].manifest.datasets.element", + "options[1].manifest.datasets.element.filter", "options[1].manifest.datasets.element.include.element", + "options[1].manifest.datasets.element.measures.element.filter", "options[1].manifest.datasources.element", "options[1].manifest.flows.element", "options[1].manifest.flows.element.errorHandling", @@ -279,7 +285,9 @@ "data.packages.element.options[1].manifest.dashboards.element.globalFilters.element", "data.packages.element.options[1].manifest.dashboards.element.widgets.element", "data.packages.element.options[1].manifest.datasets.element", + "data.packages.element.options[1].manifest.datasets.element.filter", "data.packages.element.options[1].manifest.datasets.element.include.element", + "data.packages.element.options[1].manifest.datasets.element.measures.element.filter", "data.packages.element.options[1].manifest.datasources.element", "data.packages.element.options[1].manifest.flows.element", "data.packages.element.options[1].manifest.flows.element.errorHandling", @@ -1131,12 +1139,15 @@ "ui/Dataset": { "sites": [ "", + "filter", "filter.lazy", - "include.element" + "include.element", + "measures.element.filter" ] }, "ui/DatasetMeasure": { "sites": [ + "filter", "filter.lazy" ] }, diff --git a/packages/spec/src/ui/dataset.zod.ts b/packages/spec/src/ui/dataset.zod.ts index 781a5b6df01..f62ad5f1d9c 100644 --- a/packages/spec/src/ui/dataset.zod.ts +++ b/packages/spec/src/ui/dataset.zod.ts @@ -205,13 +205,25 @@ function refuseNestedRelationEqualityLists( } /** - * The filter both analytics carriers declare — `DatasetSchema.filter` and - * `DatasetMeasureSchema.filter` — which is `FilterConditionSchema` plus + * The optional filter both analytics carriers declare — `DatasetSchema.filter` + * and `DatasetMeasureSchema.filter` — which is `FilterConditionSchema` plus * {@link refuseNestedRelationEqualityLists}. Every other schema that carries a * `FilterCondition` keeps the shared schema's reach. + * + * The check sits on the OPTIONAL wrapper, not on `FilterConditionSchema` + * itself: refining the recursive schema would clone it, and the published JSON + * Schema would then inline a second copy of the condition beside the `$ref` it + * carries today. On the wrapper the condition keeps its identity, so the + * published body of `ui/Dataset` and `ui/DatasetMeasure` is unchanged, and the + * new rule is recorded as a dropped refinement at each carrier's `filter` in + * `dropped-refinements.baseline.json` (`z.toJSONSchema()` has no projection for + * it). An absent filter reaches the check as `undefined`, which the walk + * passes. */ -const AnalyticsCarrierFilterSchema = lazySchema(() => - FilterConditionSchema.superRefine((filter, ctx) => refuseNestedRelationEqualityLists(filter, ctx))); +function analyticsCarrierFilter() { + return FilterConditionSchema.optional().superRefine((filter, ctx) => + refuseNestedRelationEqualityLists(filter, ctx)); +} /** * Dimension — a groupable axis (e.g. "region", "close_date by quarter"). @@ -327,7 +339,7 @@ export const DatasetMeasureSchema = lazySchema(() => strictObject({ * the analytics door refuses it on chart — see * {@link refuseNestedRelationEqualityLists}. */ - filter: AnalyticsCarrierFilterSchema.optional().meta({ title: 'Filter' }), + filter: analyticsCarrierFilter().meta({ title: 'Filter' }), /** * Display format — a NUMERAL pattern controlling grouping, decimals and * percent: `"0,0.00"`, `"0.0%"`. A `$` in the pattern is still honoured as a @@ -490,7 +502,7 @@ export const DatasetSchema = lazySchema(() => strictObject({ * on save, as the analytics door refuses it on chart — see * {@link refuseNestedRelationEqualityLists}. */ - filter: AnalyticsCarrierFilterSchema.optional().describe('Intrinsic dataset scope filter'), + filter: analyticsCarrierFilter().describe('Intrinsic dataset scope filter'), /** The semantic contract presentations bind to. */ dimensions: z.array(DatasetDimensionSchema).describe('Groupable axes'), From 3ed391abf9f717f1be723d9064d8bb19ba9c1c74 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 08:27:13 +0000 Subject: [PATCH 5/7] chore(spec): the dropped-refinements ledger header counts the eleven new sites Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01QcAS3qiYYZNezaxZxaUdMV --- packages/spec/dropped-refinements.baseline.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/spec/dropped-refinements.baseline.json b/packages/spec/dropped-refinements.baseline.json index fbe23a7d7a0..463b9242eae 100644 --- a/packages/spec/dropped-refinements.baseline.json +++ b/packages/spec/dropped-refinements.baseline.json @@ -3,7 +3,7 @@ "measured": { "zod": "4.4.3", "publishedSchemasWithDroppedRefinements": 207, - "droppedRefinementSites": 574, + "droppedRefinementSites": 585, "refinementSitesThatDidProject": 369, "refinementSitesWithNoJsonFormToCompare": 0 }, From a053bfcc2a1520bae57e4c44f6c908add62f0297 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 10:16:00 +0000 Subject: [PATCH 6/7] chore(spec): re-derive this branch's eleven dropped-refinement sites on the merged ledger Applied from the schema build's printed corrected entries (six entries, eleven added sites, none removed); header droppedRefinementSites 588 as the build measures it: main's 577 plus this branch's 11, across 207 schemas. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01QcAS3qiYYZNezaxZxaUdMV --- packages/spec/dropped-refinements.baseline.json | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/packages/spec/dropped-refinements.baseline.json b/packages/spec/dropped-refinements.baseline.json index 2c71a486713..d1b2fa2af31 100644 --- a/packages/spec/dropped-refinements.baseline.json +++ b/packages/spec/dropped-refinements.baseline.json @@ -3,7 +3,7 @@ "measured": { "zod": "4.4.3", "publishedSchemasWithDroppedRefinements": 207, - "droppedRefinementSites": 577, + "droppedRefinementSites": 588, "refinementSitesThatDidProject": 369, "refinementSitesWithNoJsonFormToCompare": 0 }, @@ -56,7 +56,9 @@ "manifest.dashboards.element.globalFilters.element", "manifest.dashboards.element.widgets.element", "manifest.datasets.element", + "manifest.datasets.element.filter", "manifest.datasets.element.include.element", + "manifest.datasets.element.measures.element.filter", "manifest.datasources.element", "manifest.flows.element", "manifest.flows.element.errorHandling", @@ -154,7 +156,9 @@ "data.options[1].manifest.dashboards.element.globalFilters.element", "data.options[1].manifest.dashboards.element.widgets.element", "data.options[1].manifest.datasets.element", + "data.options[1].manifest.datasets.element.filter", "data.options[1].manifest.datasets.element.include.element", + "data.options[1].manifest.datasets.element.measures.element.filter", "data.options[1].manifest.datasources.element", "data.options[1].manifest.flows.element", "data.options[1].manifest.flows.element.errorHandling", @@ -236,7 +240,9 @@ "options[1].manifest.dashboards.element.globalFilters.element", "options[1].manifest.dashboards.element.widgets.element", "options[1].manifest.datasets.element", + "options[1].manifest.datasets.element.filter", "options[1].manifest.datasets.element.include.element", + "options[1].manifest.datasets.element.measures.element.filter", "options[1].manifest.datasources.element", "options[1].manifest.flows.element", "options[1].manifest.flows.element.errorHandling", @@ -279,7 +285,9 @@ "data.packages.element.options[1].manifest.dashboards.element.globalFilters.element", "data.packages.element.options[1].manifest.dashboards.element.widgets.element", "data.packages.element.options[1].manifest.datasets.element", + "data.packages.element.options[1].manifest.datasets.element.filter", "data.packages.element.options[1].manifest.datasets.element.include.element", + "data.packages.element.options[1].manifest.datasets.element.measures.element.filter", "data.packages.element.options[1].manifest.datasources.element", "data.packages.element.options[1].manifest.flows.element", "data.packages.element.options[1].manifest.flows.element.errorHandling", @@ -1134,12 +1142,15 @@ "ui/Dataset": { "sites": [ "", + "filter", "filter.lazy", - "include.element" + "include.element", + "measures.element.filter" ] }, "ui/DatasetMeasure": { "sites": [ + "filter", "filter.lazy" ] }, From 9425caec3933f8a37ea843f7a8faca5665799eed Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 27 Sep 2026 10:27:20 +0000 Subject: [PATCH 7/7] docs(spec): the dataset carrier's $ne note states what ruling A on #19886 now refuses #20204 refuses a list under $ne at the shared face and at FieldOperatorsSchema.$ne; the save door does not judge it yet (#20116). The docblock and one test label said no ruling decided it. Wording only. Co-authored-by: Claude Claude-Session: https://claude.ai/code/session_01QcAS3qiYYZNezaxZxaUdMV --- .../spec/src/ui/dataset-filter-nested-relation-list.test.ts | 2 +- packages/spec/src/ui/dataset.zod.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts b/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts index b6f43769290..28e5f89d8ce 100644 --- a/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts +++ b/packages/spec/src/ui/dataset-filter-nested-relation-list.test.ts @@ -206,7 +206,7 @@ describe('#20080 §4 — what a nested relation may still carry', () => { ['$in: [] stays the declared predicate', { account: { region: { $in: [] } } }], ['$nin keeps its list', { account: { region: { $nin: ['a'] } } }], ['$between keeps its pair', { account: { score: { $between: [1, 9] } } }], - ['$ne carrying a list — no ruling decides it', { account: { region: { $ne: ['a'] } } }], + ['$ne carrying a list — refused at the shared face (ruling A, #19886), not yet at this save door (#20116)', { account: { region: { $ne: ['a'] } } }], ['a scalar under $and', { $and: [{ account: { region: 'a' } }] }], ])('%s', (_label, filter) => { const scoped = DatasetSchema.safeParse(withScope(filter)); diff --git a/packages/spec/src/ui/dataset.zod.ts b/packages/spec/src/ui/dataset.zod.ts index f62ad5f1d9c..1f5a2c9700f 100644 --- a/packages/spec/src/ui/dataset.zod.ts +++ b/packages/spec/src/ui/dataset.zod.ts @@ -156,7 +156,11 @@ function isAnalyticsNestedRelationSpec(spec: unknown): spec is Record