From 2d3751109dcc93402101bdd95643d1388e93740e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 11:36:48 +0000 Subject: [PATCH 1/6] test(service-analytics,driver-memory): pin the shared filter lowering at the analytics seams and the cube face's door (red) Pins first, before the fix. Three seams, one file each: - read scope (compileScopedFilterToSql entry): a bare-day upper bound on a declared datetime answers SqlDriver.find's rows, for a caller filter and for an RLS using bound in both spellings; the last supported day; a declared date control; the typed scope as SQL. - analytics where / preview door: the where -> tree face compiles the lowered condition (typed on datetime members), the ObjectQL hand-off and echo carry it, and the draft preview evaluates it (with $null). - the cube face's new door: the two shared comparand doors refuse an undefined comparand on both exits; the face compiles the lowered condition and the $or / $null vocabulary, and still refuses $not. Red on this commit for exactly those rows. Claude-Session: https://claude.ai/code/session_01XY5uCwTjZj7884yYtyur4H Co-authored-by: Claude --- ...ory-analytics-shared-lowering-door.test.ts | 207 ++++++++++++++++ .../read-scope-shared-lowering-seam.test.ts | 219 +++++++++++++++++ .../where-door-shared-lowering-seam.test.ts | 226 ++++++++++++++++++ 3 files changed, 652 insertions(+) create mode 100644 packages/drivers/driver-memory/src/memory-analytics-shared-lowering-door.test.ts create mode 100644 packages/services/service-analytics/src/__tests__/read-scope-shared-lowering-seam.test.ts create mode 100644 packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts diff --git a/packages/drivers/driver-memory/src/memory-analytics-shared-lowering-door.test.ts b/packages/drivers/driver-memory/src/memory-analytics-shared-lowering-door.test.ts new file mode 100644 index 00000000000..bfee5c22aab --- /dev/null +++ b/packages/drivers/driver-memory/src/memory-analytics-shared-lowering-door.test.ts @@ -0,0 +1,207 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3] The analytics (cube) face's + * NEW door, at the entry of `normalizeFilters` — the one seam of the amendment's + * item 2 that did not exist before it: the two shared comparand doors + * (`assertListComparandShapes`, `normalizeFilterComparandTypes`) this face ran + * without, then the shared `FilterCondition → FilterCondition` lowering + * (`lowerFilterCondition`), then the face's own vocabulary gate and lowering. + * + * ## The doors (the face ran neither) + * + * Measured before this door, on the fixture below (`d` holds a value on rows 1 + * and 2, is null on row 3, and is absent on row 4), every other face behind its + * production seam refused these shapes with `INVALID_FILTER` / 400, while this + * one answered them: + * + * | cube `where` | rows | + * |----------------------------------|--------| + * | `{d: undefined}` | 3, 4 (as `{d: null}`) | + * | `{d: {$eq: undefined}}` | 3, 4 | + * | `{d: {$ne: undefined}}` | 1, 2 | + * | `{d: {$in: ['v1', undefined]}}` | 1 | + * + * ## The lowering's column-type scope (item 7) + * + * This face cannot read declared types — the driver's declared temporal kinds + * are private to it, and the face reaches them only as a storage-form + * conversion — so the lowering applies type-blind, as the item rules for such a + * seam. That is also the reading this face's own interim copy (`lteUpperBound`) + * and the driver's `find()` give, so no answer moves by it. + * + * ## The vocabulary (design §3.4) + * + * The lowering emits `$and`, `$or`, `$lt`, `$gte`, `$lte` and `$null`. This face + * compiled all of them but `$or` and `$null`; it now compiles those two, on both + * exits, and nothing wider — `$not` is still refused. Its vocabulary gate judges + * what the face compiles, the lowered condition, so a `$between` reaches it as + * the two bounds it lowers to. + */ + +import { describe, it, expect } from 'vitest'; +import { FILTER_LOGIC_CASES, FILTER_LOGIC_ROWS, type Cube, type FilterCondition } from '@objectstack/spec/data'; +import { InMemoryDriver } from './memory-driver.js'; +import { MemoryAnalyticsService } from './memory-analytics.js'; + +const ids = (rows: ReadonlyArray>) => rows.map((r) => String(r.id)).sort(); + +/** The echo's WHERE clause alone. */ +const whereOf = (sql: string) => /WHERE (.*?)(?: GROUP BY|$)/.exec(sql)?.[1]; + +/** One cube over the shared filter-logic fixture's columns. */ +const LOGIC_CUBE = { + name: 'logic', + title: 'Logic', + sql: 'logic_row', + measures: { count: { label: 'Count', type: 'count', sql: 'id' } }, + dimensions: Object.fromEntries( + ['id', 'a', 'b', 'c', 'd', 'owner', 'status', 'parent_object', 'parent_id', 'created_at'].map((n) => [ + n, + { label: n, type: n === 'created_at' ? 'time' : 'string', sql: n }, + ]), + ), +} as unknown as Cube; + +/** `d`: a value on 1 and 2, null on 3, absent on 4 — the fixture the door's table was measured on. */ +const DOOR_ROWS = [ + { id: '1', d: 'v1' }, + { id: '2', d: 'v2' }, + { id: '3', d: null }, + { id: '4' }, +]; + +/** Consecutive 10:00Z instants on a column declared `datetime`. */ +const DAY_ROWS = [ + { id: 'r27', created_at: '2026-07-27T10:00:00.000Z' }, + { id: 'r28', created_at: '2026-07-28T10:00:00.000Z' }, + { id: 'r29', created_at: '2026-07-29T10:00:00.000Z' }, +]; + +async function setup(rows: ReadonlyArray>) { + const driver = new InMemoryDriver({}); + await driver.connect(); + await driver.syncSchema('logic_row', { name: 'logic_row', fields: { created_at: { type: 'datetime' } } }); + for (const row of rows) await driver.create('logic_row', { ...row }); + const service = new MemoryAnalyticsService({ driver, cubes: [LOGIC_CUBE] }); + return { driver, service }; +} + +const cubeQuery = (where?: FilterCondition) => + ({ cube: 'logic', measures: ['count'], dimensions: ['id'], ...(where === undefined ? {} : { where }) }) as any; + +/** The live query path, the cube's rows and the echo's WHERE, for one `where`. */ +async function answer(rows: ReadonlyArray>, where?: FilterCondition) { + const { driver, service } = await setup(rows); + return { + find: ids(await driver.find('logic_row', where === undefined ? {} : { where })), + cube: ids((await service.query(cubeQuery(where))).rows), + echo: whereOf((await service.generateSql(cubeQuery(where))).sql), + }; +} + +/** Both exits refuse `where` in the ADR-0112 `INVALID_FILTER` / 400 envelope. */ +async function expectRefusedOnBothExits(where: FilterCondition) { + const { service } = await setup(DOOR_ROWS); + for (const [exit, run] of [ + ['query', () => service.query(cubeQuery(where))], + ['generateSql', () => service.generateSql(cubeQuery(where))], + ] as const) { + await expect(run(), `${exit}: ${JSON.stringify(where)} must be refused`).rejects.toMatchObject({ + code: 'INVALID_FILTER', + status: 400, + }); + } +} + +describe('[ADR-0053 D-D1 amended — #5930 step 3] the cube face runs the two shared comparand doors', () => { + it('CONTROLS: no where, a value and a null comparand answer as the live query path does', async () => { + expect((await answer(DOOR_ROWS)).cube).toEqual(['1', '2', '3', '4']); + const value = await answer(DOOR_ROWS, { d: 'v1' }); + expect(value.cube).toEqual(['1']); + expect(value.cube).toEqual(value.find); + const none = await answer(DOOR_ROWS, { d: null }); + expect(none.cube).toEqual(['3', '4']); + expect(none.cube).toEqual(none.find); + }); + + const UNDEFINED_COMPARANDS: Array<[string, FilterCondition]> = [ + ['an implicit undefined', { d: undefined } as FilterCondition], + ['undefined under $eq', { d: { $eq: undefined } } as FilterCondition], + ['undefined under $ne', { d: { $ne: undefined } } as FilterCondition], + ['an undefined $in member', { d: { $in: ['v1', undefined] } } as FilterCondition], + ]; + for (const [label, where] of UNDEFINED_COMPARANDS) { + it(`${label} is refused on both exits, as every other face refuses it`, async () => { + await expectRefusedOnBothExits(where); + }); + } +}); + +describe('[ADR-0053 D-D1 amended — #5930 step 3] the cube face compiles the lowered condition', () => { + it('a $between reaches the face as its two bounds, and answers the live query path\'s rows', async () => { + const got = await answer(DAY_ROWS, { created_at: { $between: ['2026-07-28', '2026-07-28'] } }); + expect(got.find).toEqual(['r28']); + expect(got.cube).toEqual(['r28']); + expect(got.echo).toBe("created_at >= '2026-07-28T00:00:00.000Z' AND created_at < '2026-07-29T00:00:00.000Z'"); + }); + + it('a bare-day $lte keeps the whole named day, and the last supported day keeps every row with a value', async () => { + const day = await answer(DAY_ROWS, { created_at: { $lte: '2026-07-28' } }); + expect(day.cube).toEqual(['r27', 'r28']); + expect(day.cube).toEqual(day.find); + expect(day.echo).toBe("created_at < '2026-07-29T00:00:00.000Z'"); + const last = await answer(DAY_ROWS, { created_at: { $lte: '9999-12-31' } }); + expect(last.cube).toEqual(['r27', 'r28', 'r29']); + expect(last.echo).toBe('created_at IS NOT NULL'); + }); + + it('a negative-polarity operator is compiled inside the NULL escape the lowering emits', async () => { + const got = await answer(DOOR_ROWS, { d: { $ne: 'v1' } }); + expect(got.cube).toEqual(['2', '3', '4']); + expect(got.cube).toEqual(got.find); + // The outer disjunction is the lowering's; the inner guard is this face's + // own interim copy (`notEquals`), idempotent in rows until its deletion card. + expect(got.echo).toBe("(d IS NULL OR (d IS NULL OR d != 'v1'))"); + }); +}); + +describe('[ADR-0053 D-D1 amended — #5930 step 3] the cube face compiles $or and $null, and nothing wider', () => { + it('$null: true selects the rows with no value, false the rows with one, on both exits', async () => { + const isNull = await answer(DOOR_ROWS, { d: { $null: true } }); + expect(isNull.cube).toEqual(['3', '4']); + expect(isNull.cube).toEqual(isNull.find); + expect(isNull.echo).toBe('d IS NULL'); + const hasValue = await answer(DOOR_ROWS, { d: { $null: false } }); + expect(hasValue.cube).toEqual(['1', '2']); + expect(hasValue.echo).toBe('d IS NOT NULL'); + }); + + it('$or: the boolean identities, on both exits', async () => { + const none = await answer(FILTER_LOGIC_ROWS, { $or: [] }); + expect(none.cube).toEqual([]); + expect(none.echo).toBe('1 = 0'); + const all = await answer(FILTER_LOGIC_ROWS, { $or: [{ a: 'x' }, {}] }); + expect(all.cube).toEqual(['1', '2', '3', '4']); + }); + + it('$or: a disjunction of conjunctions renders each branch in its own parentheses', async () => { + const got = await answer(FILTER_LOGIC_ROWS, { $or: [{ a: 'x', b: 'y' }, { a: 'qq', b: 'zz' }] }); + expect(got.cube).toEqual(['1', '4']); + expect(got.echo).toBe("((a = 'x' AND b = 'y') OR (a = 'qq' AND b = 'zz'))"); + }); + + it('FILTER_LOGIC_CASES: every $or case is answered with the live query path\'s rows', async () => { + const withOr = FILTER_LOGIC_CASES.filter((c) => /"\$or"/.test(JSON.stringify(c.filter)) && !/"\$(not|empty)"/.test(JSON.stringify(c.filter))); + expect(withOr.length).toBeGreaterThan(0); + for (const c of withOr) { + const got = await answer(FILTER_LOGIC_ROWS, c.filter); + expect(got.cube, `${c.name}: ${c.note ?? ''}`).toEqual(c.expected); + expect(got.cube, `${c.name}: the cube face and find() disagree`).toEqual(got.find); + } + }); + + it('$not is still refused on both exits', async () => { + await expectRefusedOnBothExits({ $not: { d: 'v1' } }); + }); +}); diff --git a/packages/services/service-analytics/src/__tests__/read-scope-shared-lowering-seam.test.ts b/packages/services/service-analytics/src/__tests__/read-scope-shared-lowering-seam.test.ts new file mode 100644 index 00000000000..54bc86b34ab --- /dev/null +++ b/packages/services/service-analytics/src/__tests__/read-scope-shared-lowering-seam.test.ts @@ -0,0 +1,219 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3] The analytics READ SCOPE's + * placement of the shared `FilterCondition → FilterCondition` lowering + * (`lowerFilterCondition`, `@objectstack/spec/data`): run once at the entry of + * `compileScopedFilterToSql`, right after the scope's placeholders resolve and + * before a single clause is compiled — the amendment's item 2 names this + * position. + * + * The table this file holds is the one filed as the read scope's divergence: + * one bare-day upper bound, on a column declared `datetime`, answered by the + * read scope with the named day's rows dropped, while `SqlDriver.find` on the + * same column kept them. Measured before this seam existed, on the fixture + * below (ISO-text rows at 10:00Z on 07-27, 07-28 and 07-29, and one row with + * no value): + * + * | face | rows kept for `$lte: '2026-07-28'` | + * |---------------------------------------------|-----------------------------------| + * | read scope (`compileScopedFilterToSql`) | c27 | + * | `SqlDriver.find`, column declared `datetime` | c27, c28 | + * + * Every row below asks each face that binds a read scope into SQL — the + * published export executed on the same database, and `NativeSQLStrategy`'s + * `applyReadScope` — for the rows `SqlDriver.find` answers on the same filter. + * + * ## The two halves of a read scope's filter + * + * - **A caller filter** — whatever a `getReadScope` producer hands the + * compiler, or a direct caller of the export. It reaches this seam as + * written, so this seam is the only place its bound is lowered. + * - **An RLS `using` bound** — the RLS compile seam (`judgeCompiledComparands`, + * step 2) lowers every compiled policy filter before `getReadFilter` returns + * it. The half is pinned in BOTH spellings the read scope can be handed: the + * policy as that seam lowers it on a typed guard, and the policy as written + * (what a guard without types, or any producer that skipped that seam, + * hands over). Both answer `find()`'s rows, so this face's answer does not + * depend on which one reached it. + * + * ## Column-type scope (the amendment's item 7) + * + * The read scope can read declared types — both of its consumers hand it + * `declaredValueShape` — so the whole-day rule rewrites a declared `datetime` + * column only, the scope `SqlDriver` holds. A `date` column compiles + * byte-identical to before (`<= day` orders exactly as `< next-day` on date + * text). A caller that hands no declarations reads NO column as `datetime`: + * the step-2 RLS seam's choice for a guard without types, for the same reason + * — it moves no answer. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { SqliteWasmDriver } from '@objectstack/driver-sqlite-wasm'; +import type { Cube, FilterCondition } from '@objectstack/spec/data'; +import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; + +import { compileScopedFilterToSql } from '../read-scope-sql.js'; +import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; + +const OBJECT = 'contract'; +const ALIAS = 't'; + +const FIELDS: Record> = { + id: { type: 'text', name: 'id' }, + signed_at: { type: 'datetime', name: 'signed_at' }, + due_on: { type: 'date', name: 'due_on' }, + stage: { type: 'text', name: 'stage' }, +}; + +/** Three ISO instants at 10:00Z on consecutive days, and one row with no value. */ +const ROWS = [ + { id: 'c27', signed_at: '2026-07-27T10:00:00.000Z', due_on: '2026-07-27', stage: 'won' }, + { id: 'c28', signed_at: '2026-07-28T10:00:00.000Z', due_on: '2026-07-28', stage: 'lost' }, + { id: 'c29', signed_at: '2026-07-29T10:00:00.000Z', due_on: '2026-07-29', stage: null }, + { id: 'cnull', signed_at: null, due_on: null, stage: 'won' }, +]; + +const CUBE: Cube = { + name: 'contracts', + sql: OBJECT, + measures: { n: { sql: '*', type: 'count', title: 'n' } }, + dimensions: { id: { label: 'id', type: 'string', sql: 'id' } }, + public: true, +} as unknown as Cube; + +const QUERY = { cube: 'contracts', dimensions: ['id'], measures: ['n'] } as AnalyticsQuery; + +/** What both production consumers hand the compiler: the object's declared value shapes. */ +const declaredValueShape = (field: string) => { + const def = FIELDS[field]; + return def ? { type: String(def.type) } : undefined; +}; + +const quiet = { + debug() {}, + info() {}, + warn() {}, + error() {}, + child() { + return quiet; + }, +}; + +describe('[ADR-0053 D-D1 amended — #5930 step 3] the read scope lowers its filter at the entry of compileScopedFilterToSql', () => { + let driver: SqliteWasmDriver; + let readScope: FilterCondition | undefined; + + const runRawSql = async (sql: string, params: unknown[]): Promise[]> => { + const result = await driver.execute(sql.replace(/\$\d+/g, '?'), params); + if (Array.isArray(result)) return result as Record[]; + if (result && typeof result === 'object' && 'rows' in (result as Record)) { + return (result as { rows: Record[] }).rows; + } + return []; + }; + + const ids = (rows: ReadonlyArray>): string[] => rows.map((r) => String(r.id)).sort(); + + beforeAll(async () => { + driver = new SqliteWasmDriver({ filename: ':memory:' }); + (driver as unknown as { logger: unknown }).logger = quiet; + await driver.initObjects([{ name: OBJECT, fields: FIELDS }] as never); + for (const row of ROWS) await driver.create(OBJECT, { ...row }); + }); + + afterAll(async () => { + await driver?.disconnect?.(); + }); + + /** `SqlDriver.find` — the answer every other face is held to. */ + const findFace = async (scope: FilterCondition) => ids((await driver.find(OBJECT, { where: scope } as never)) as never); + + /** The published export, its SQL executed on the same database. */ + const compiledFace = async (scope: FilterCondition, withTypes = true) => { + const { sql, params } = compileScopedFilterToSql(scope, ALIAS, withTypes ? { declaredValueShape } : {}); + return ids(await runRawSql(`SELECT "${ALIAS}"."id" AS "id" FROM "${OBJECT}" AS "${ALIAS}" WHERE ${sql}`, params)); + }; + + /** `NativeSQLStrategy.applyReadScope` — the merge site that turns a scope into an executed statement. */ + const nativeFace = async (scope: FilterCondition) => { + readScope = scope; + const ctx = { + getCube: (name: string) => (name === 'contracts' ? CUBE : undefined), + queryCapabilities: () => ({ nativeSql: true, objectqlAggregate: false, inMemory: false }), + getReadScope: () => readScope, + declaredValueShape: (object: string, field: string) => (object === OBJECT ? declaredValueShape(field) : undefined), + executeRawSql: (_object: string, sql: string, params: unknown[]) => runRawSql(sql, params), + } as unknown as StrategyContext; + return ids((await new NativeSQLStrategy().execute(QUERY, ctx)).rows); + }; + + /** Every read-scope face against `find()`, and against the literal list, so the two cannot be wrong together. */ + const expectSameRows = async (scope: FilterCondition, rows: string[]) => { + expect(await findFace(scope), 'find() — the fixture is not what this table assumes').toEqual(rows); + expect(await compiledFace(scope), 'compileScopedFilterToSql — the published export').toEqual(rows); + expect(await nativeFace(scope), 'NativeSQLStrategy.applyReadScope').toEqual(rows); + }; + + it('CONTROL: an unconstrained scope admits every row on every face', async () => { + await expectSameRows({ id: { $in: ['c27', 'c28', 'c29', 'cnull'] } }, ['c27', 'c28', 'c29', 'cnull']); + }); + + // ── The caller-filter half: this seam is the only place its bound is lowered ─ + + it('caller filter: a bare-day $lte on a declared datetime keeps the whole named day, as find() does', async () => { + await expectSameRows({ signed_at: { $lte: '2026-07-28' } }, ['c27', 'c28']); + }); + + it('caller filter: a bare-day $between on a declared datetime keeps the whole named day', async () => { + await expectSameRows({ signed_at: { $between: ['2026-07-28', '2026-07-28'] } }, ['c28']); + }); + + it('caller filter: the last supported day bounds nothing and keeps every row with a value', async () => { + await expectSameRows({ signed_at: { $lte: '9999-12-31' } }, ['c27', 'c28', 'c29']); + }); + + // ── The RLS `using` half, in both spellings the read scope can be handed ───── + + it('RLS using, as the RLS seam lowers it on a typed guard: the whole named day', async () => { + // `record.signed_at <= '2026-07-28'` compiled and lowered by + // `judgeCompiledComparands` with `signed_at` in the guard's datetime set. + await expectSameRows({ signed_at: { $lt: '2026-07-29' } }, ['c27', 'c28']); + }); + + it('RLS using, as written (a guard without types): the whole named day all the same', async () => { + await expectSameRows({ signed_at: { $lte: '2026-07-28' } }, ['c27', 'c28']); + }); + + it('RLS using on the last supported day, as the RLS seam lowers it: every row with a value', async () => { + await expectSameRows({ signed_at: { $null: false } }, ['c27', 'c28', 'c29']); + }); + + // ── The declared `date` control ──────────────────────────────────────────── + + it('a declared date column keeps its bound as written, and answers the same rows as find()', async () => { + await expectSameRows({ due_on: { $lte: '2026-07-28' } }, ['c27', 'c28']); + const { sql, params } = compileScopedFilterToSql({ due_on: { $lte: '2026-07-28' } }, ALIAS, { declaredValueShape }); + expect(sql).toBe('"t"."due_on" <= ?'); + expect(params).toEqual(['2026-07-28']); + }); + + // ── The typed scope, as SQL ──────────────────────────────────────────────── + + it('a declared datetime compiles half-open against the next day, in the calendar-string domain', () => { + const { sql, params } = compileScopedFilterToSql({ signed_at: { $lte: '2026-07-28' } }, ALIAS, { declaredValueShape }); + expect(sql).toBe('"t"."signed_at" < ?'); + expect(params).toEqual(['2026-07-29']); + }); + + it('with no declarations handed in, no column reads as datetime and the bound compiles as written', () => { + const { sql, params } = compileScopedFilterToSql({ signed_at: { $lte: '2026-07-28' } }, ALIAS); + expect(sql).toBe('"t"."signed_at" <= ?'); + expect(params).toEqual(['2026-07-28']); + }); + + it('the NULL-polarity guards apply whatever the type, and the rows stay find()\'s', async () => { + await expectSameRows({ stage: { $ne: 'won' } }, ['c28', 'c29']); + await expectSameRows({ $not: { stage: { $eq: 'lost' } } }, ['c27', 'c29', 'cnull']); + }); +}); diff --git a/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts b/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts new file mode 100644 index 00000000000..3b9e17af555 --- /dev/null +++ b/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts @@ -0,0 +1,226 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3] The analytics `where` and + * draft-preview door's placement of the shared `FilterCondition → + * FilterCondition` lowering (`lowerFilterCondition`, `@objectstack/spec/data`): + * run on what the door admitted — after both shared comparand faces + * (`normalizeWhereComparands`, or `parseFilterAST` for the array spelling) — + * before any face reads the condition. Filter tokens resolve upstream of this + * door on every path that reaches it (`AnalyticsService.resolveQueryTokens`, + * the per-request dataset-scope getter, `DatasetExecutor`'s selection tokens), + * so the lowering reads the comparand the comparison will run with (the + * amendment's item 3). + * + * Two faces consume the door, and each is pinned at its own entry: + * + * - **F10**, the `where` → tree face (`normalizeAnalyticsFilterTree`), which + * both strategies compile: `NativeSQLStrategy`'s SQL, `ObjectQLStrategy`'s + * `/analytics/sql` echo, and `ObjectQLStrategy`'s hand-off to the engine. + * - **F11**, the draft-preview evaluator (`evaluateAnalyticsQueryOverRows`). + * + * ## Column-type scope (item 7) + * + * F10 can read declared types through the strategy context's + * `declaredFieldType` hook, so it rewrites the whole-day bound on a member + * whose column is declared `datetime` and nowhere else — the engine seam's + * scope, which is what the ObjectQL hand-off meets next. A context with no hook + * reads no member as `datetime`. + * + * F11 evaluates drafted rows with no schema. Its lowering reads no member as + * `datetime`: its own bound copy (`lteBound`) keeps answering the whole-day + * rule until its deletion card, and a type-blind rewrite here would move one + * cell — `$lte` on the last supported day over a non-temporal value that sorts + * above it — away from the typed drivers' answer. The NULL-polarity guards + * apply on both faces whatever the type. + */ + +import { describe, it, expect } from 'vitest'; +import { FILTER_LOGIC_CASES, FILTER_LOGIC_ROWS, type Cube, type FilterCondition } from '@objectstack/spec/data'; +import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; + +import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; +import { evaluateAnalyticsQueryOverRows } from '../preview-evaluator.js'; + +const OBJECT = 'contract'; + +/** Each member names a column whose DECLARED type differs from its member name's hint. */ +const DECLARED_TYPES: Record = { + signed_at: 'datetime', + due_on: 'date', + stage: 'text', +}; + +const CUBE: Cube = { + name: 'contracts', + sql: OBJECT, + measures: { n: { sql: '*', type: 'count', title: 'n' } }, + dimensions: { + signed: { label: 'Signed', type: 'time', sql: 'signed_at' }, + due: { label: 'Due', type: 'time', sql: 'due_on' }, + stage: { label: 'Stage', type: 'string', sql: 'stage' }, + }, + public: true, +} as unknown as Cube; + +/** The strategies' reader, by hand: `signed` is the one member whose column is declared `datetime`. */ +const TYPED = { isDatetimeColumn: (member: string) => member === 'signed' }; +const UNTYPED = { isDatetimeColumn: () => false }; + +const leaf = (member: string, operator: string, values: unknown[]) => ({ kind: 'leaf', member, operator, values }); + +describe('[ADR-0053 D-D1 amended — #5930 step 3] F10: the where → tree face compiles the lowered condition', () => { + const tree = (where: FilterCondition, lowering = TYPED) => + (normalizeAnalyticsFilterTree as (q: unknown, l: unknown) => unknown)({ where }, lowering); + + it('a bare-day $lte on a datetime member becomes lt the next day', () => { + expect(tree({ signed: { $lte: '2026-07-28' } })).toEqual(leaf('signed', 'lt', ['2026-07-29'])); + }); + + it('the last supported day keeps only "has a value"', () => { + expect(tree({ signed: { $lte: '9999-12-31' } })).toEqual(leaf('signed', 'set', [])); + }); + + it('a bare-day $between on a datetime member becomes gte its minimum and lt the next day', () => { + expect(tree({ signed: { $between: ['2026-07-27', '2026-07-28'] } })).toEqual({ + kind: 'and', + children: [leaf('signed', 'gte', ['2026-07-27']), leaf('signed', 'lt', ['2026-07-29'])], + }); + }); + + it('a date member keeps its bound as written (item 7)', () => { + expect(tree({ due: { $lte: '2026-07-28' } })).toEqual(leaf('due', 'lte', ['2026-07-28'])); + }); + + it('with no declared-type reader, no member reads as datetime', () => { + expect(tree({ signed: { $lte: '2026-07-28' } }, UNTYPED)).toEqual(leaf('signed', 'lte', ['2026-07-28'])); + }); + + it('a negative-polarity leaf reaches the tree inside the NULL escape the seam emits, whatever the type', () => { + // The outer disjunction is the seam's `{ $or: [{ stage: { $null: true } }, + // { stage: { $ne: 'won' } }] }`. The inner one is this face's own interim + // copy of the same guard (`fieldLeaves`, #5298), which still wraps the + // `$ne` it meets: idempotent in rows (a guard of a guarded leaf admits the + // same rows), and removed by the face's deletion card — which updates this + // row to the single disjunction. + expect(tree({ stage: { $ne: 'won' } }, UNTYPED)).toEqual({ + kind: 'or', + children: [ + leaf('stage', 'notSet', []), + { kind: 'or', children: [leaf('stage', 'notSet', []), leaf('stage', 'notEquals', ['won'])] }, + ], + }); + }); + + it('an instant is never widened', () => { + expect(tree({ signed: { $lte: '2026-07-28T12:00:00.000Z' } })).toEqual( + leaf('signed', 'lte', ['2026-07-28T12:00:00.000Z']), + ); + }); +}); + +describe('[ADR-0053 D-D1 amended — #5930 step 3] F10 through ObjectQLStrategy: the engine hand-off and the echo', () => { + /** A context whose engine bridge records the filter the strategy hands it. */ + const ctxWith = (handed: Array>, typed = true): StrategyContext => + ({ + getCube: (name: string) => (name === 'contracts' ? CUBE : undefined), + queryCapabilities: () => ({ nativeSql: false, objectqlAggregate: true, inMemory: false }), + ...(typed ? { declaredFieldType: (object: string, field: string) => (object === OBJECT ? DECLARED_TYPES[field] : undefined) } : {}), + executeAggregate: async (_object: string, options: { filter?: Record }) => { + handed.push(options.filter ?? {}); + return []; + }, + }) as unknown as StrategyContext; + + const query = (where: FilterCondition): AnalyticsQuery => ({ cube: 'contracts', measures: ['n'], where }) as AnalyticsQuery; + + it('the engine receives the lowered bound for a datetime member', async () => { + const handed: Array> = []; + await new ObjectQLStrategy().execute(query({ signed: { $lte: '2026-07-28' } }), ctxWith(handed)); + expect(handed).toEqual([{ signed_at: { $lt: '2026-07-29' } }]); + }); + + it('the engine receives a date member\'s bound as written', async () => { + const handed: Array> = []; + await new ObjectQLStrategy().execute(query({ due: { $lte: '2026-07-28' } }), ctxWith(handed)); + expect(handed).toEqual([{ due_on: { $lte: '2026-07-28' } }]); + }); + + it('with no declared-type hook on the context, the engine receives the bound as written', async () => { + const handed: Array> = []; + await new ObjectQLStrategy().execute(query({ signed: { $lte: '2026-07-28' } }), ctxWith(handed, false)); + expect(handed).toEqual([{ signed_at: { $lte: '2026-07-28' } }]); + }); + + it('the /analytics/sql echo prints the half-open bound the engine executes', async () => { + const { sql, params } = await new ObjectQLStrategy().generateSql( + query({ signed: { $lte: '2026-07-28' } }), + ctxWith([]), + ); + expect(sql).toContain('signed_at < $1'); + expect(params).toEqual(['2026-07-29']); + }); +}); + +describe('[ADR-0053 D-D1 amended — #5930 step 3] F11: the draft preview evaluates the lowered condition', () => { + const PREVIEW_CUBE = { + name: 'rows', + sql: 'row', + measures: { n: { sql: '*', type: 'count', title: 'n' } }, + dimensions: { id: { label: 'id', type: 'string', sql: 'id' } }, + public: true, + } as unknown as Cube; + + const previewIds = (where: FilterCondition, rows: ReadonlyArray>): string[] => + evaluateAnalyticsQueryOverRows( + { cube: 'rows', dimensions: ['id'], measures: ['n'], where } as AnalyticsQuery, + PREVIEW_CUBE, + rows.map((r) => ({ ...r })), + ).rows.map((r) => String(r.id)).sort(); + + /** `stage` holds a value on p1, is null on p2, and is absent on p3. */ + const ROWS = [{ id: 'p1', stage: 'won' }, { id: 'p2', stage: null }, { id: 'p3' }]; + + it('a row with no value satisfies $ne, even against the text "null" (every driver\'s answer)', () => { + expect(previewIds({ stage: { $ne: 'null' } }, ROWS)).toEqual(['p1', 'p2', 'p3']); + }); + + it('a row with no value satisfies $nin, even when a member is the text "undefined"', () => { + expect(previewIds({ stage: { $nin: ['undefined'] } }, ROWS)).toEqual(['p1', 'p2', 'p3']); + }); + + it('a row with no value satisfies the negation of an equality it cannot hold', () => { + expect(previewIds({ $not: { stage: { $eq: 'null' } } }, ROWS)).toEqual(['p1', 'p2', 'p3']); + }); + + it('$null is evaluated: true selects the rows with no value, false the rows with one', () => { + expect(previewIds({ stage: { $null: true } }, ROWS)).toEqual(['p2', 'p3']); + expect(previewIds({ stage: { $null: false } }, ROWS)).toEqual(['p1']); + }); + + it('a bare-day bound is still answered through the whole named day (the face\'s own copy)', () => { + const rows = [{ id: 'd27', at: '2026-07-27T10:00:00.000Z' }, { id: 'd28', at: '2026-07-28T10:00:00.000Z' }, { id: 'd29', at: '2026-07-29T10:00:00.000Z' }]; + expect(previewIds({ at: { $lte: '2026-07-28' } }, rows)).toEqual(['d27', 'd28']); + expect(previewIds({ at: { $between: ['2026-07-28', '2026-07-28'] } }, rows)).toEqual(['d28']); + }); + + it('FILTER_LOGIC_CASES: the preview agrees with the shared table on every case it evaluates, and refuses the rest', () => { + const answered: string[] = []; + for (const c of FILTER_LOGIC_CASES) { + let ids: string[] | undefined; + try { + ids = previewIds(c.filter, FILTER_LOGIC_ROWS); + } catch (e) { + expect((e as { code?: unknown }).code, `${c.name}: a refusal must be the INVALID_FILTER envelope`).toBe('INVALID_FILTER'); + expect((e as { status?: unknown }).status).toBe(400); + continue; + } + expect(ids, `${c.name}: ${c.note ?? ''}`).toEqual(c.expected); + answered.push(c.name); + } + // The two `$null` rows are answered now, not refused. + expect(answered).toContain('$null true selects exactly the no-value rows'); + expect(answered).toContain('$null false selects exactly the valued rows'); + }); +}); From c9d2c46aa17e423558ae939302bec6f79dd3cc76 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 11:42:52 +0000 Subject: [PATCH 2/6] feat(service-analytics,driver-memory): run the shared filter lowering at the analytics seams and the cube face's new door ADR-0053 D-D1 as amended: one shared FilterCondition -> FilterCondition lowering (lowerFilterCondition, @objectstack/spec/data), run at each analytics seam after the comparand doors and after filter-token resolution. - analytics where / preview door: normalizeAnalyticsFilterTree lowers the condition the door admitted before buildNode reads it (F10), with a REQUIRED column-type reader: both strategies pass declaredDatetimeLowering (a member is datetime when its column is declared so); member readers and a context with no hook pass NO_DATETIME_COLUMNS. The draft preview lowers after normalizeWhereComparands with NO_DATETIME_COLUMNS (F11) and evaluates $null. - read scope: compileScopedFilterToSql lowers at its entry, after the placeholders resolve, typed by the declaredValueShape both consumers already hand it. - the cube face (F5): normalizeFilters runs assertListComparandShapes and normalizeFilterComparandTypes, then the lowering (type-blind: the face cannot read declarations), then its own gate on the lowered condition. It compiles $or (as a disjunction entry both exits render) and $null, and still refuses $not. Claude-Session: https://claude.ai/code/session_01XY5uCwTjZj7884yYtyur4H Co-authored-by: Claude --- .../driver-memory/src/memory-analytics.ts | 333 +++++++++++++----- .../src/analytics-service.ts | 8 +- .../src/preview-evaluator.ts | 31 +- .../service-analytics/src/read-scope-sql.ts | 43 ++- .../src/strategies/filter-normalizer.ts | 79 ++++- .../src/strategies/native-sql-strategy.ts | 16 +- .../src/strategies/objectql-strategy.ts | 34 +- 7 files changed, 436 insertions(+), 108 deletions(-) diff --git a/packages/drivers/driver-memory/src/memory-analytics.ts b/packages/drivers/driver-memory/src/memory-analytics.ts index 376acdfa4f6..a653beb9356 100644 --- a/packages/drivers/driver-memory/src/memory-analytics.ts +++ b/packages/drivers/driver-memory/src/memory-analytics.ts @@ -6,6 +6,15 @@ import type { Cube, AnalyticsQuery } from '@objectstack/spec/data'; // [#7117] `likePatternToGlobPattern` — the spec's one LIKE→GLOB translation, so // the SQL exit's wildcard rendering is not a third hand-copy of that escape. import { asciiCaseInsensitiveRegexSource, likePatternToGlobPattern } from '@objectstack/spec/data'; +// [ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3] This face's NEW door, run at +// the entry of {@link MemoryAnalyticsService.normalizeFilters}: the two shared +// comparand faces every other analytics face runs (ruling #7872; #6050 B), then +// the shared `FilterCondition → FilterCondition` lowering. +import { + assertListComparandShapes, + lowerFilterCondition, + normalizeFilterComparandTypes, +} from '@objectstack/spec/data'; import type { InMemoryDriver } from './memory-driver.js'; import { Logger, @@ -95,6 +104,14 @@ const MONGO_TO_CUBE_OPERATOR = Object.freeze({ // entry point is the divergence class #5374 closed for `$contains`. $icontains: 'icontains', $exists: 'set', + // [ADR-0053 D-D1, amended — #5930 step 3] `$null`, one of the two operators + // the shared lowering emits that this face did not compile (the other is the + // `$or` combinator, {@link ANALYTICS_FILTER_CAPABILITIES}): the NULL escape + // it puts around a negative-polarity leaf, and the "has a value" it leaves of + // a bound on the last supported day. Design §3.4's output vocabulary, and + // nothing wider — `$not`, `$between` (lowered away before this table is + // asked), `$startsWith`, `$endsWith` and `$empty` stay outside it. + $null: 'isNull', } as const); /** @@ -107,16 +124,21 @@ type CubeOperator = (typeof MONGO_TO_CUBE_OPERATOR)[keyof typeof MONGO_TO_CUBE_O /** * [#5345] What the analytics (cube) face compiles, for the shared filter walk. * - * `$and` is the one combinator: {@link MemoryAnalyticsService.flattenFilterCondition} - * folds its branches into the same implicit-AND list the top level already is. - * `$or` and `$not` have no expression in a flat `{member, operator, values}` - * pipeline at all — which is why they were being skipped, and why refusing is - * the answer here rather than a lowering nobody can write. + * `$and` folds its branches into the same implicit-AND list the top level + * already is ({@link MemoryAnalyticsService.flattenFilterCondition}). `$not` has + * no expression in this face's `{member, operator, values}` pipeline, which is + * why refusing it is the answer here rather than a lowering nobody can write. + * + * [ADR-0053 D-D1, amended — #5930 step 3] `$or` joins: the shared lowering emits + * it (the NULL escape around a negative-polarity leaf), so this face has to + * compile it, and it does — as a {@link NormalizedCubeDisjunction} each exit + * renders natively (mingo's `$or`, SQL's `OR`), with the boolean identities + * every other face gives it (`$or: []` is FALSE, a `{}` branch is TRUE). */ export const ANALYTICS_FILTER_CAPABILITIES: FilterFaceCapabilities = Object.freeze({ face: "driver-memory's analytics (cube) face", fieldOperators: new Set(Object.keys(MONGO_TO_CUBE_OPERATOR)), - combinators: new Set(['$and']), + combinators: new Set(['$and', '$or']), }); /** @@ -154,6 +176,7 @@ export const ANALYTICS_FILTER_CAPABILITIES: FilterFaceCapabilities = Object.free * caller, no spec schema and no serialized form observes this triple's shape. */ interface NormalizedCubeFilter { + readonly anyOf?: never; member: string; operator: CubeOperator; /** @@ -164,6 +187,29 @@ interface NormalizedCubeFilter { values: unknown[]; } +/** + * [ADR-0053 D-D1, amended — #5930 step 3] A disjunction of conjunctions: the + * `$or` this face now compiles. Each branch is an entry list read exactly as + * the top level is — implicitly ANDed, contested members kept apart — so a + * branch is whatever a `$or` element flattens to, nested `$or`s included. + * + * The boolean identities are the ones every other face gives (#5322): a branch + * with NO entries (a `{}` element) is TRUE and makes the whole disjunction TRUE; + * a disjunction with NO branches (`$or: []`) is FALSE. Both exits render those + * two cases as constants rather than leaving them to a backend's reading of an + * empty operand. + */ +interface NormalizedCubeDisjunction { + readonly anyOf: ReadonlyArray>; +} + +/** One entry of the implicitly-ANDed list `normalizeFilters` returns. */ +type NormalizedCubeEntry = NormalizedCubeFilter | NormalizedCubeDisjunction; + +function isDisjunction(entry: NormalizedCubeEntry): entry is NormalizedCubeDisjunction { + return entry.anyOf !== undefined; +} + /** * [#5374] What one lowered entry gives its predicate builder. * @@ -366,6 +412,12 @@ const CUBE_OPERATOR_TO_MONGO_PREDICATE: Readonly ((raw.length === 0 || Boolean(raw[0])) ? { $ne: null } : { $eq: null }), + // [ADR-0053 D-D1, amended — #5930 step 3] `$null`, the mirror of `set`: a + // presence flag, not a comparand, and "no value" is null or an absent key — + // mingo's `$eq: null` reads both, as MongoDB does. The flag is a boolean by + // the time it gets here (`assertFilterConditionShape` refuses anything else, + // #5347), so the `=== true` below is the whole choice. + isNull: ({ raw }) => (raw[0] === true ? { $eq: null } : { $ne: null }), }); /** @@ -569,6 +621,10 @@ const CUBE_OPERATOR_TO_SQL_PREDICATE: Readonly `${column} IS ${raw.length === 0 || Boolean(raw[0]) ? 'NOT NULL' : 'NULL'}`, + // [ADR-0053 D-D1, amended — #5930 step 3] `$null`, the mirror of `set` — the + // mingo row's `$eq: null` / `$ne: null`, spelled the way `read-scope-sql.ts` + // and `driver-sql` spell a null predicate. + isNull: ({ column, raw }) => `${column} IS ${raw[0] === true ? 'NULL' : 'NOT NULL'}`, }); /** @@ -1014,61 +1070,7 @@ export class MemoryAnalyticsService implements IAnalyticsService { // dropped (#5345). const normalizedFilters = this.normalizeFilters(query); if (normalizedFilters.length > 0) { - const matchStage: Record = {}; - /** - * [#13524] Predicates for a member that already has one — see the - * promotion below the loop for why they cannot be assigned. - */ - const contested: Record[] = []; - for (const filter of normalizedFilters) { - const fieldPath = this.resolveFieldPath(cube, filter.member); - // [#5374] The operator decides the WHOLE predicate, not just its name — - // so `notContains` can say `{$not: {$regex: …}}` instead of being forced - // into `{$not: }`, which mingo reads as no constraint at all. - // - // [#5373] `comparands` are the values as authored, in the storage form - // of the field they are compared against. There is no type recovery step - // any more, because there is no longer a stringification to recover - // FROM: a boolean reaches mingo as a boolean and `null` as `null`, so a - // predicate over `is_active` or `closed_at` selects the same rows - // `find()` selects instead of none / all of them. - const storageForm = this.storageFormFor(cube, filter.member); - const predicate = this.mongoPredicateBuilder(filter.operator)({ - comparands: filter.values.map(storageForm), - raw: filter.values, - storageForm, - substring: (value) => this.driver.filterSubstringPattern(value), - // [#6520] `$icontains`' fold, from the spec's shared definition rather - // than from the driver's Unicode-folding `filterSubstringPattern`. - asciiSubstring: (value) => new RegExp(asciiCaseInsensitiveRegexSource(String(value))), - }); - // [#13524] This was `matchStage[fieldPath] = …`, and the assignment was - // a WHOLESALE clobber — the widest member of this card's class. The two - // document-shaped translators lose a constraint only when two operators - // happen to lower onto the SAME key; here the stage is keyed by field - // path alone, so the second predicate on a member replaced the first - // ENTIRELY, for every operator pair. And `flattenFilterCondition` folds - // `$and` into this same flat list, so `{$and: [{name: {$contains:'a'}}, - // {name: {$ne:'b'}}]}` — two separate nodes, not one operator map — - // lost a constraint too. Measured on a three-row fixture: - // `{name: {$contains:'a', $ne:'b'}}` aggregated ['1','3'] and its - // key-swapped twin ['1'], while the reference matcher said ['1']. - // - // Same rule as the translators: free member merges inline, a taken one - // becomes its own `$and` branch of the SAME `$match`, where both - // predicates survive. No ranking is needed here — unlike a contested - // operator key, nothing is overwritten, so which predicate sits inline - // changes the document's shape but never its answer. - if (Object.prototype.hasOwnProperty.call(matchStage, fieldPath)) { - contested.push({ [fieldPath]: predicate }); - } else { - matchStage[fieldPath] = predicate; - } - } - // A field path can never BE `$and` — `resolveFieldPath` resolves cube - // members, and `flattenFilterCondition` refuses `$or` / `$not` and folds - // `$and` away before this runs — so this cannot collide with a member. - if (contested.length > 0) matchStage.$and = contested; + const matchStage = this.mongoConjunction(cube, normalizedFilters); if (Object.keys(matchStage).length > 0) { pipeline.push({ $match: matchStage }); } @@ -1510,20 +1512,7 @@ export class MemoryAnalyticsService implements IAnalyticsService { // construction for the same reason. There is deliberately no // `values.length > 0` guard any more: an empty list IS a predicate, and // skipping the clause described the whole table (see the `in` row). - const whereClauses: string[] = []; - const normalizedFilters = this.normalizeFilters(query); - for (const filter of normalizedFilters) { - const fieldPath = this.resolveFieldPath(cube, filter.member); - const storageForm = this.storageFormFor(cube, filter.member); - whereClauses.push(this.sqlPredicateBuilder(filter.operator)({ - column: fieldPath, - comparands: filter.values.map(storageForm), - raw: filter.values, - storageForm, - literal: (value) => this.toSqlLiteral(value), - globSubstring: (value) => this.toSqlLiteral(globSubstringPattern(value)), - })); - } + const whereClauses = this.sqlConjunction(cube, this.normalizeFilters(query)); let sql = `SELECT ${selectClauses.join(', ')} FROM ${tableName}`; if (whereClauses.length > 0) { @@ -1555,6 +1544,129 @@ export class MemoryAnalyticsService implements IAnalyticsService { // Helper Methods // =================================== + /** + * The mingo `$match` document one implicitly-ANDed entry list compiles to — + * the top-level list `normalizeFilters` returns, and (since #5930 step 3) + * each branch of a {@link NormalizedCubeDisjunction}, which is rendered by + * the same rule so a branch means exactly what a top-level list means. + */ + private mongoConjunction(cube: Cube, entries: ReadonlyArray): Record { + const matchStage: Record = {}; + /** + * [#13524] Predicates for a member that already has one — see the + * promotion below the loop for why they cannot be assigned. + */ + const contested: Record[] = []; + for (const entry of entries) { + // [ADR-0053 D-D1, amended — #5930 step 3] A disjunction. A branch with no + // entries is TRUE and absorbs it: no constraint at all. A disjunction + // with no branches is FALSE: the explicit constant, not an empty `$or` + // left to the matcher's reading. Otherwise each branch compiles by this + // same rule and mingo's `$or` joins them — joined through `$and` beside + // the member predicates, since one `$match` holds a single `$or` key. + if (isDisjunction(entry)) { + if (entry.anyOf.some((branch) => branch.length === 0)) continue; + contested.push( + entry.anyOf.length === 0 + ? { $expr: false } + : { $or: entry.anyOf.map((branch) => this.mongoConjunction(cube, branch)) }, + ); + continue; + } + const filter = entry; + const fieldPath = this.resolveFieldPath(cube, filter.member); + // [#5374] The operator decides the WHOLE predicate, not just its name — + // so `notContains` can say `{$not: {$regex: …}}` instead of being forced + // into `{$not: }`, which mingo reads as no constraint at all. + // + // [#5373] `comparands` are the values as authored, in the storage form + // of the field they are compared against. There is no type recovery step + // any more, because there is no longer a stringification to recover + // FROM: a boolean reaches mingo as a boolean and `null` as `null`, so a + // predicate over `is_active` or `closed_at` selects the same rows + // `find()` selects instead of none / all of them. + const storageForm = this.storageFormFor(cube, filter.member); + const predicate = this.mongoPredicateBuilder(filter.operator)({ + comparands: filter.values.map(storageForm), + raw: filter.values, + storageForm, + substring: (value) => this.driver.filterSubstringPattern(value), + // [#6520] `$icontains`' fold, from the spec's shared definition rather + // than from the driver's Unicode-folding `filterSubstringPattern`. + asciiSubstring: (value) => new RegExp(asciiCaseInsensitiveRegexSource(String(value))), + }); + // [#13524] This was `matchStage[fieldPath] = …`, and the assignment was + // a WHOLESALE clobber — the widest member of this card's class. The two + // document-shaped translators lose a constraint only when two operators + // happen to lower onto the SAME key; here the stage is keyed by field + // path alone, so the second predicate on a member replaced the first + // ENTIRELY, for every operator pair. And `flattenFilterCondition` folds + // `$and` into this same flat list, so `{$and: [{name: {$contains:'a'}}, + // {name: {$ne:'b'}}]}` — two separate nodes, not one operator map — + // lost a constraint too. Measured on a three-row fixture: + // `{name: {$contains:'a', $ne:'b'}}` aggregated ['1','3'] and its + // key-swapped twin ['1'], while the reference matcher said ['1']. + // + // Same rule as the translators: free member merges inline, a taken one + // becomes its own `$and` branch of the SAME `$match`, where both + // predicates survive. No ranking is needed here — unlike a contested + // operator key, nothing is overwritten, so which predicate sits inline + // changes the document's shape but never its answer. + if (Object.prototype.hasOwnProperty.call(matchStage, fieldPath)) { + contested.push({ [fieldPath]: predicate }); + } else { + matchStage[fieldPath] = predicate; + } + } + // A field path can never BE `$and` — `resolveFieldPath` resolves cube + // members, and `flattenFilterCondition` folds `$and` away and turns `$or` + // into a disjunction entry before this runs — so this cannot collide with a + // member. + if (contested.length > 0) matchStage.$and = contested; + return matchStage; + } + + /** + * The SQL clauses one implicitly-ANDed entry list compiles to, joined with + * `AND` by the caller — the twin of {@link mongoConjunction}, by the same + * rule on the same entries, so the echo describes the `$match` that ran. + * + * [ADR-0053 D-D1, amended — #5930 step 3] A disjunction renders as + * `(b1 OR b2 …)`, each branch parenthesised when it holds more than one + * clause, so no reading of it leans on `AND` binding tighter than `OR`. A + * branch that compiles to no clause is TRUE and drops the whole disjunction; + * a disjunction with no branches is `1 = 0`, the FALSE constant this exit + * already spells an empty `$in` with. + */ + private sqlConjunction(cube: Cube, entries: ReadonlyArray): string[] { + const clauses: string[] = []; + for (const entry of entries) { + if (isDisjunction(entry)) { + if (entry.anyOf.length === 0) { + clauses.push('1 = 0'); + continue; + } + const branches = entry.anyOf.map((branch) => this.sqlConjunction(cube, branch)); + if (branches.some((branch) => branch.length === 0)) continue; + clauses.push( + `(${branches.map((branch) => (branch.length === 1 ? branch[0] : `(${branch.join(' AND ')})`)).join(' OR ')})`, + ); + continue; + } + const fieldPath = this.resolveFieldPath(cube, entry.member); + const storageForm = this.storageFormFor(cube, entry.member); + clauses.push(this.sqlPredicateBuilder(entry.operator)({ + column: fieldPath, + comparands: entry.values.map(storageForm), + raw: entry.values, + storageForm, + literal: (value) => this.toSqlLiteral(value), + globSubstring: (value) => this.toSqlLiteral(globSubstringPattern(value)), + })); + } + return clauses; + } + /** * Normalize a query's `where` into the cube-style array the pipeline consumes. * @@ -1581,16 +1693,47 @@ export class MemoryAnalyticsService implements IAnalyticsService { * through a lowering fires or does not fire depending on key order and on * which sibling branch was walked first. Both public entry points (`query()` * and `generateSql()`) go through this method, so both refuse identically. + * + * ## The door in front of the gate (ADR-0053 D-D1, amended — #5930 step 3) + * + * This method is also the one seam of the amendment's item 2 that did not + * exist before it — the face's NEW door — and it runs, in order: + * + * 1. **The two shared comparand faces** this face ran without: + * `assertListComparandShapes`, then `normalizeFilterComparandTypes`, whose + * RETURN is the condition read from here on (a bigint within 2^53 + * narrowed to its number, copy-on-write). So an `undefined` comparand — + * implicit, under an operator, or as a list member — is refused + * `INVALID_FILTER` / 400 as it is on every other face (ruling #7872, + * #6050 B), where this face used to read it as `null`. + * 2. **The shared lowering**, `lowerFilterCondition`. Nothing resolves filter + * tokens on this face, so it reads the comparands as authored (item 3). + * It applies type-blind (item 7): the declared temporal kinds live inside + * the driver and reach this face only as a storage-form conversion, and + * type-blind is the reading this face's own bound copy + * (`lteUpperBound`) and the driver's `find()` already give, so it moves no + * answer. The lowering emits calendar strings; each exit converts them to + * the field's storage form as it converts any comparand (item 6). + * 3. **This face's own vocabulary gate**, on the LOWERED condition — what the + * face actually compiles, as a driver behind the engine seam judges the + * lowered filter it receives. So a `$between` reaches the gate as the two + * bounds it lowers to, and answers `find()`'s rows here too. + * + * Not an array: `where` is declared a `FilterCondition`, and an array here is + * left to the reading this method has always given it. */ - private normalizeFilters(query: unknown): NormalizedCubeFilter[] { + private normalizeFilters(query: unknown): NormalizedCubeEntry[] { if (!query || typeof query !== 'object') return []; - const out: NormalizedCubeFilter[] = []; + const out: NormalizedCubeEntry[] = []; const where = (query as { where?: unknown }).where; if (where && typeof where === 'object' && !Array.isArray(where)) { - assertFilterConditionShape(where, 'where', ANALYTICS_FILTER_CAPABILITIES); - this.flattenFilterCondition(where as Record, out, 'where'); + assertListComparandShapes(where); + const admitted = normalizeFilterComparandTypes(where); + const lowered = lowerFilterCondition(admitted); + assertFilterConditionShape(lowered, 'where', ANALYTICS_FILTER_CAPABILITIES); + this.flattenFilterCondition(lowered as Record, out, 'where'); } return out; @@ -1598,7 +1741,7 @@ export class MemoryAnalyticsService implements IAnalyticsService { private flattenFilterCondition( cond: Record, - out: NormalizedCubeFilter[], + out: NormalizedCubeEntry[], path: string, ): void { for (const [key, raw] of Object.entries(cond)) { @@ -1615,11 +1758,10 @@ export class MemoryAnalyticsService implements IAnalyticsService { // is the #3948 direction, and on an RLS read scope it is an unauthorized // read rather than a wrong number. // - // `undefined` falls through with it, matching the live query path, which - // has never distinguished the two (`normalizeFilterCondition` sends both - // to `toStorageForm` and lets mingo's null-equality rule decide). Agreeing - // with that path is the invariant (#5240); inventing a third reading of - // `{field: undefined}` here would break it in the other direction. + // [ADR-0053 D-D1, amended — #5930 step 3] `undefined` no longer reaches + // this walk: the shared comparand faces at the door refuse it in every + // comparand position, as every other face does (ruling #7872). It used to + // fall through here and read as `null`. // Logical combinators. `$and` folds into the same implicit-AND list; the // gate above has already proven it is an array of filter nodes. @@ -1629,12 +1771,27 @@ export class MemoryAnalyticsService implements IAnalyticsService { } continue; } - // [#5345] Unreachable via normalizeFilters — the gate refuses these for - // this face before the lowering starts. Kept as a throw rather than left + // [ADR-0053 D-D1, amended — #5930 step 3] `$or` becomes ONE entry holding + // each branch flattened on its own — a branch is an implicitly-ANDed list + // exactly like the top level — so the disjunction survives to both exits + // instead of being folded into the conjunction around it. The gate above + // has proven the operand an array of filter nodes. See + // {@link NormalizedCubeDisjunction} for the identities. + if (key === '$or') { + const anyOf = (raw as unknown[]).map((sub, index) => { + const branch: NormalizedCubeEntry[] = []; + this.flattenFilterCondition(sub as Record, branch, `${here}[${index}]`); + return branch; + }); + out.push({ anyOf }); + continue; + } + // [#5345] Unreachable via normalizeFilters — the gate refuses it for this + // face before the lowering starts. Kept as a throw rather than left // implicit so that the `continue` which caused #5345 cannot come back, and // so a future caller that lowers a condition without gating it first fails // loudly instead of silently widening the result set. - if (key === '$or' || key === '$not') { + if (key === '$not') { throw uncompilableCombinatorError(key, here, ANALYTICS_FILTER_CAPABILITIES); } diff --git a/packages/services/service-analytics/src/analytics-service.ts b/packages/services/service-analytics/src/analytics-service.ts index d22f648ef52..ce1c9d64876 100644 --- a/packages/services/service-analytics/src/analytics-service.ts +++ b/packages/services/service-analytics/src/analytics-service.ts @@ -61,6 +61,7 @@ import { collectFilterLeaves, lowerAnalyticsWhere, conjunctFieldKeys, + NO_DATETIME_COLUMNS, } from './strategies/filter-normalizer.js'; import { findCrossFieldComparand } from './comparand-shape.js'; import { compileDataset, type CompiledDataset, type RelationshipResolver } from './dataset-compiler.js'; @@ -2785,7 +2786,12 @@ export class AnalyticsService implements IAnalyticsService { /** Every member the compiled predicate will bind against, structure discarded. */ let members: string[]; try { - members = collectFilterLeaves(normalizeAnalyticsFilterTree(query)).map((leaf) => leaf.member); + // [ADR-0053 D-D1, amended — #5930 step 3] The tree the strategies compile + // is LOWERED; this gate reads the same lowering, so a member the + // lowering's NULL guard names is a member it judges. The column-type + // reader is immaterial here: the whole-day rule and the `$between` split + // never change which member a leaf names. + members = collectFilterLeaves(normalizeAnalyticsFilterTree(query, NO_DATETIME_COLUMNS)).map((leaf) => leaf.member); } catch { // A `where` this layer refuses outright — see the stand-down note above. return; diff --git a/packages/services/service-analytics/src/preview-evaluator.ts b/packages/services/service-analytics/src/preview-evaluator.ts index 27fb9a7a707..522ef30f91f 100644 --- a/packages/services/service-analytics/src/preview-evaluator.ts +++ b/packages/services/service-analytics/src/preview-evaluator.ts @@ -9,7 +9,7 @@ // // Scope (deliberately the dataset-query subset, not a general engine): // • Mongo-style `where` filters ($eq implicit, $ne/$gt/$gte/$lt/$lte/ -// $between/$in/$nin/$contains, $and/$or/$not) +// $between/$in/$nin/$contains/$null, $and/$or/$not) // • timeDimensions date-range filtering + granularity bucketing // (day/week/month/quarter/year) // • group-by dimensions; count / count_distinct / sum / avg / min / max @@ -48,9 +48,9 @@ import { explicitDateRangeWindow } from './date-range-array-arm.js'; // same reason: one rule, one spelling, on both faces. [#20010] And, through // the same gate, every other arm of the shared comparand-shape face. [#20035] // And the comparand-TYPE face, through the same gate again. -import { invalidFilterError, normalizeWhereComparands } from './strategies/filter-normalizer.js'; +import { invalidFilterError, normalizeWhereComparands, NO_DATETIME_COLUMNS } from './strategies/filter-normalizer.js'; import type { AnalyticsQuery, AnalyticsResult } from '@objectstack/spec/contracts'; -import { emptyGroupValueFor, type Cube } from '@objectstack/spec/data'; +import { emptyGroupValueFor, lowerFilterCondition, type Cube } from '@objectstack/spec/data'; type Row = Record; @@ -176,6 +176,16 @@ const PREVIEW_FIELD_OPERATORS = new Map([ if (min == null || max == null) return false; return compare(value, min) >= 0 && lteBound(value, max); }], + // [ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3] `$null` — the one + // operator the shared lowering emits that this face did not evaluate: the + // NULL escape it puts around a negative-polarity leaf (`$ne` of a value, + // `$nin`, `$notContains`), and the guard it puts on each leaf of a `$not` + // operand. The flag is a boolean by the time it reaches this table (the + // `where` door's `assertBooleanNullFlags` refused anything else), and "no + // value" is `null` or an absent key — the reading `formula` and + // `driver-memory` give it, measured against `FILTER_LOGIC_CASES`' two `$null` + // rows. Nothing wider joins with it: `$exists` and `$empty` stay refused. + ['$null', (value, expected) => (expected === true ? value == null : value != null)], ['$in', (value, expected) => Array.isArray(expected) && expected.some((e) => value === e || String(value) === String(e))], ['$nin', (value, expected) => Array.isArray(expected) && !expected.some((e) => value === e || String(value) === String(e))], ['$contains', (value, expected) => String(value ?? '').toLowerCase().includes(String(expected ?? '').toLowerCase())], @@ -672,7 +682,20 @@ export function evaluateAnalyticsQueryOverRows( // answered EVERY row; and a bigint within 2^53 was ordered as text // (`{ amt: { $gt: 2n } }` lost `amt = 10`), where publish narrows it to its // number and serves the right rows. - const where = normalizeWhereComparands(query.where); + // [ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3] Then the shared + // `FilterCondition → FilterCondition` lowering, on what the door admitted + // and before the vocabulary gate reads it — the `where` door's seam for this + // face (the amendment's item 2), with filter tokens already resolved by the + // `DatasetExecutor` that calls this evaluator (item 3). Its column-type + // reader is {@link NO_DATETIME_COLUMNS} (item 7): drafted rows carry no + // schema, so no member is read as `datetime` and {@link lteBound} keeps + // answering the whole-day rule as this face's own copy. A type-blind rewrite + // here would move one cell away from the typed drivers — `$lte` on the last + // supported day over a non-temporal value sorting above it. The NULL guards + // apply whatever the type; measured, they move only the rows this face read + // through `String()` — a row with no value against the text `'null'` or + // `'undefined'` — onto every driver's answer. + const where = lowerFilterCondition(normalizeWhereComparands(query.where), NO_DATETIME_COLUMNS); assertPreviewCanEvaluate(where); let filtered = rows.filter((r) => matchesWhere(r, where)); const timeDims = query.timeDimensions ?? []; diff --git a/packages/services/service-analytics/src/read-scope-sql.ts b/packages/services/service-analytics/src/read-scope-sql.ts index 425cb8e1a52..1549b1a601f 100644 --- a/packages/services/service-analytics/src/read-scope-sql.ts +++ b/packages/services/service-analytics/src/read-scope-sql.ts @@ -5,6 +5,10 @@ import type { FilterCondition } from '@objectstack/spec/data'; // at the ObjectQL merge sites by {@link assertReadScopeComparandsRunnable}, and // [#20018] at the end of {@link compileScopedFilterToSql} by the same function. import { assertListComparandShapes, normalizeFilterComparandTypes } from '@objectstack/spec/data'; +// [ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3] The shared lowering, run +// at the entry of {@link compileScopedFilterToSql}, after the placeholders +// resolve and before any clause is compiled. +import { lowerFilterCondition, type FilterLoweringOptions } from '@objectstack/spec/data'; // [#20068] The `$icontains` text-comparand door: the table's discrimination and // its reason half, asked at {@link compileOperator}'s `$icontains` arm and at // the engine-bound merges through {@link assertReadScopeComparandsRunnable}. @@ -737,6 +741,22 @@ function isFilterNode(v: unknown): v is Record { return v !== null && typeof v === 'object' && !Array.isArray(v); } +/** + * [ADR-0053 D-D1 item 7, amended 2026-09-30 — #5930 step 3] The read scope's + * column-type reader for the shared lowering: a column is `datetime` when the + * caller's declared value shape says so — the declaration both of this + * compiler's consumers already hand it for `$empty` (#20445), and the test + * `SqlDriver` indexes `datetimeFields` by. So the whole-day rule rewrites a + * declared `datetime` column and nothing else, and a `date`, `time` or + * non-temporal column compiles byte-identical to before. A caller that hands no + * declarations reads NO column as `datetime` — the step-2 RLS seam's reading of + * a guard without types, for the same reason: it moves no answer. + */ +function readScopeLowering(options: ReadScopeCompileOptions): FilterLoweringOptions { + const shape = options.declaredValueShape; + return { isDatetimeColumn: (field) => shape?.(field)?.type === 'datetime' }; +} + function quoteIdent(name: string, kind: string): string { if (typeof name !== 'string' || !IDENT.test(name)) { throw readScopeCompileError(`[read-scope-sql] unsafe ${kind} identifier "${String(name)}" — refusing to build read scope (fail-closed).`); @@ -758,13 +778,32 @@ export function compileScopedFilterToSql( // the sentence those gates give it. Held rather than raised early, the tree // lowered on that path is the unresolved one, and its SQL is discarded with // the throw. See the module header's #20075 section. - let lowered = filter; + let resolved = filter; let unresolvable: Error | undefined; try { - lowered = resolveReadScopePlaceholders(filter, alias, options.context); + resolved = resolveReadScopePlaceholders(filter, alias, options.context); } catch (e) { unresolvable = e as Error; } + // [ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3] The shared + // `FilterCondition → FilterCondition` lowering, at this compiler's ENTRY — + // the read-scope seam the amendment's item 2 names — right after the scope's + // placeholders resolve (item 3: a `{today}` bound is widened as the day it + // resolves to) and before a single clause is compiled. It is what gives this + // face the whole-day upper bound it never applied: a bare-day `$lte` (or a + // `$between` maximum) on a declared `datetime` column compiles `< next-day`, + // in the calendar-string domain, and answers the rows `SqlDriver.find` does + // on the same filter. It also lays the NULL-polarity guards on as structure, + // which this compiler's own copies (`nullSafeNegative`, + // `nullSafeNegationOperand`) already answered: idempotent in rows, and + // removed by this face's deletion card. + // + // The shared comparand faces below still judge the scope AS WRITTEN, after + // compilation (#20018's order). The lowering never refuses and never turns a + // scope those faces refuse into one they admit, so the order moves no + // verdict. An RLS `using` reaches here already lowered by the RLS compile + // seam; lowering it again is a no-op. + const lowered = lowerFilterCondition(resolved, readScopeLowering(options)); const params: unknown[] = []; const sql = compileNode(lowered, quotedAlias, params, options); // [#20018] The shared comparand faces, on the scope ALONE: the judgement the diff --git a/packages/services/service-analytics/src/strategies/filter-normalizer.ts b/packages/services/service-analytics/src/strategies/filter-normalizer.ts index 0718fc83eff..b424c0cb29b 100644 --- a/packages/services/service-analytics/src/strategies/filter-normalizer.ts +++ b/packages/services/service-analytics/src/strategies/filter-normalizer.ts @@ -476,6 +476,12 @@ import { textComparandRefusalReason, VALID_AST_OPERATORS, } from '@objectstack/spec/data'; +// [ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3] The shared lowering, run +// on what this door admitted, before any face reads the condition — see +// {@link normalizeAnalyticsFilterTree} (F10) and the draft preview (F11). +import { lowerFilterCondition, type FilterLoweringOptions } from '@objectstack/spec/data'; +import type { StrategyContext } from '@objectstack/spec/contracts'; +import type { DatasetScopedStrategyContext } from './types.js'; import { StandardErrorCode } from '@objectstack/spec/api'; import { CROSS_FIELD_COMPARISON_OPERATORS, @@ -2352,13 +2358,84 @@ export function conjunctFieldKeys(condition: Record): string[] * metadata meaning one thing: the same `where` on a plain `find()` already * lowers at the engine door (#5329), so refusing it here would have forked the * product by which face read the metadata. + * + * ## The shared lowering (ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3) + * + * This is the analytics `where` door's seam for F10, the amendment's item 2: + * the condition {@link lowerAnalyticsWhere} admitted — after both shared + * comparand faces, on either spelling — is lowered by `lowerFilterCondition` + * before {@link buildNode} reads it, so every compiler of the tree (the native + * SQL, the `/analytics/sql` echo, and the ObjectQL hand-off to the engine) + * reads one lowered condition. Filter tokens resolve upstream of every path + * that reaches here, so the lowering meets a resolved date macro (item 3). + * + * `lowering` is the caller's column-type reader (item 7), and it is REQUIRED, + * so no compile site can reach the tree without deciding it: a strategy passes + * the member's declared type through its context's `declaredFieldType` hook + * ({@link declaredDatetimeLowering}), and a position that cannot or need not + * read types passes {@link NO_DATETIME_COLUMNS}. {@link lowerAnalyticsWhere} + * itself stays un-lowered: its other readers ask about the AUTHORED condition + * (the keys an ad-hoc cube is minted from, the routing detectors), not about + * the predicate that runs. + * + * Every member reader downstream (`collectFilterLeaves`, `assertWhereFields`) + * reads this same lowered tree, so "the member the gate saw" stays "the column + * that reached SQL" — the lowering's NULL guards name the member they guard. */ export function normalizeAnalyticsFilterTree( query: { where?: unknown } | unknown, + lowering: FilterLoweringOptions, ): NormalizedFilterNode | null { const condition = lowerAnalyticsWhere(query); if (!condition) return null; - return buildNode(condition); + return buildNode(lowerFilterCondition(condition, lowering)); +} + +/** + * [ADR-0053 D-D1, amended — #5930 step 3] The shared lowering's reader for a + * position that cannot, or need not, read declared types: no member is a + * `datetime` column, so the whole-day rule and the `$between` split rewrite + * nothing and every bound reaches its face as written; the NULL-polarity guards + * apply all the same (they do not depend on the type). + * + * Three kinds of position use it. The draft preview, which evaluates drafted + * rows with no schema (its own bound copy keeps the whole-day rule until its + * deletion card). A strategy context the host wired no `declaredFieldType` hook + * into — the step-2 RLS seam's reading of a guard without types, for the same + * reason: the rule stays with each face's own copy and no answer moves. And a + * reader that collects MEMBERS rather than compiling (`assertWhereFields`, the + * cross-object envelope check): the two type-scoped rules never change which + * member a leaf names. + */ +export const NO_DATETIME_COLUMNS: FilterLoweringOptions = Object.freeze({ + isDatetimeColumn: () => false, +}); + +/** + * [ADR-0053 D-D1, amended — #5930 step 3] A strategy's column-type reader for + * the shared lowering (item 7): a `where` member is a `datetime` column when + * the host's declared-type hook says the column it binds against is one — + * `type === 'datetime'`, the test `SqlDriver` indexes `datetimeFields` by and + * the engine seam reads. `target` resolves a member to its (object, column) + * exactly as the strategy's own compile path resolves it, so the question is + * asked of the column the predicate will read. + * + * The hook is the context's optional `declaredFieldType` — the one + * `nonTextColumnResolver` asks — and a context without one gets + * {@link NO_DATETIME_COLUMNS}. + */ +export function declaredDatetimeLowering( + ctx: StrategyContext, + target: (member: string) => { object: string; field: string }, +): FilterLoweringOptions { + const declared = (ctx as DatasetScopedStrategyContext).declaredFieldType; + if (typeof declared !== 'function') return NO_DATETIME_COLUMNS; + return { + isDatetimeColumn: (member) => { + const { object, field } = target(member); + return declared.call(ctx, object, field) === 'datetime'; + }, + }; } /** diff --git a/packages/services/service-analytics/src/strategies/native-sql-strategy.ts b/packages/services/service-analytics/src/strategies/native-sql-strategy.ts index 333054d5cbb..c3132edd036 100644 --- a/packages/services/service-analytics/src/strategies/native-sql-strategy.ts +++ b/packages/services/service-analytics/src/strategies/native-sql-strategy.ts @@ -4,6 +4,7 @@ import type { AnalyticsQuery, AnalyticsResult } from '@objectstack/spec/contract import type { Cube } from '@objectstack/spec/data'; import type { AnalyticsStrategy, StrategyContext, DatasetScopedStrategyContext } from './types.js'; import { + declaredDatetimeLowering, lowerAnalyticsWhere, normalizeAnalyticsFilterTree, toSqlBindValue, @@ -437,6 +438,15 @@ export class NativeSQLStrategy implements AnalyticsStrategy { // dataset, which is why an inferred or manifest cube compiles unchanged. const datasetScope = (ctx as DatasetScopedStrategyContext).getDatasetScope?.(query.cube!); + // [ADR-0053 D-D1, amended — #5930 step 3] The column-type reader the `where` + // door's shared lowering applies at every filter position below — the + // measure filters, the `where` and the dataset's own scope (item 7): a + // member is `datetime` when the column it binds against is declared so, + // asked of the SAME target `compileFilterNode` coerces for. This face's own + // bare-day copy (`buildFilterClause`'s `lte` arm) stays until its deletion + // card, and is idempotent on the lowered bound. + const lowering = declaredDatetimeLowering(ctx, (member) => this.resolveStorageTarget(cube, member, tableName)); + // Build SELECT for measures if (query.measures && query.measures.length > 0) { for (const measure of query.measures) { @@ -449,7 +459,7 @@ export class NativeSQLStrategy implements AnalyticsStrategy { const measureFilter = datasetScope?.measureFilters?.[measure]; const predicate = measureFilter ? this.compileFilterNode( - normalizeAnalyticsFilterTree({ where: measureFilter }), + normalizeAnalyticsFilterTree({ where: measureFilter }, lowering), cube, tableName, joins, @@ -467,7 +477,7 @@ export class NativeSQLStrategy implements AnalyticsStrategy { // used to be dropped instead of compiled. const whereClauses: string[] = []; const filterSql = this.compileFilterNode( - normalizeAnalyticsFilterTree(query), + normalizeAnalyticsFilterTree(query, lowering), cube, tableName, joins, @@ -484,7 +494,7 @@ export class NativeSQLStrategy implements AnalyticsStrategy { // predicate with itself selects the same rows. if (datasetScope?.filter) { const scopeSql = this.compileFilterNode( - normalizeAnalyticsFilterTree({ where: datasetScope.filter }), + normalizeAnalyticsFilterTree({ where: datasetScope.filter }, lowering), cube, tableName, joins, diff --git a/packages/services/service-analytics/src/strategies/objectql-strategy.ts b/packages/services/service-analytics/src/strategies/objectql-strategy.ts index a400976a40c..4dcf27e647d 100644 --- a/packages/services/service-analytics/src/strategies/objectql-strategy.ts +++ b/packages/services/service-analytics/src/strategies/objectql-strategy.ts @@ -7,8 +7,10 @@ import type { AggregationFunction, Cube } from '@objectstack/spec/data'; import { markFilterSubtreeProvenance } from '@objectstack/spec/data'; import type { AnalyticsStrategy, StrategyContext, DatasetScopedStrategyContext } from './types.js'; import { + declaredDatetimeLowering, invalidFilterError, lowerAnalyticsWhere, + NO_DATETIME_COLUMNS, normalizeAnalyticsFilterTree, collectFilterLeaves, SQL_CONST_FALSE, @@ -181,6 +183,13 @@ export class ObjectQLStrategy implements AnalyticsStrategy { // inferred or manifest cube compiles unchanged. const datasetScope = (ctx as DatasetScopedStrategyContext).getDatasetScope?.(query.cube!); + // [ADR-0053 D-D1, amended — #5930 step 3] The column-type reader the `where` + // door's shared lowering applies at the three filter positions this path + // hands the engine (item 7): a member is `datetime` when the column it binds + // against is declared so. The engine seam lowers the same filter again with + // the same scope, and the lowering is idempotent. + const lowering = declaredDatetimeLowering(ctx, (member) => this.resolveStorageTarget(cube, member, objectName)); + // Build aggregations from measures. // // [#10413 phase 2] A measure's own `filter` (`stage: 'closed_won'`) lowers @@ -198,7 +207,7 @@ export class ObjectQLStrategy implements AnalyticsStrategy { const { field, method } = this.resolveMeasureAggregation(cube, measure); const measureFilter = datasetScope?.measureFilters?.[measure]; const filterCondition = measureFilter - ? this.filterNodeToCondition(normalizeAnalyticsFilterTree({ where: measureFilter }), cube) + ? this.filterNodeToCondition(normalizeAnalyticsFilterTree({ where: measureFilter }, lowering), cube) : null; aggregations.push( filterCondition @@ -217,7 +226,7 @@ export class ObjectQLStrategy implements AnalyticsStrategy { // Operands that cannot merge into their field's entry without one silently // replacing the other; ANDed in below so the engine intersects them. const conjuncts: Record[] = []; - this.applyFilterNode(normalizeAnalyticsFilterTree(query), cube, filter, conjuncts); + this.applyFilterNode(normalizeAnalyticsFilterTree(query, lowering), cube, filter, conjuncts); // #3650 — and the time-dimension WINDOWS, through the SAME merge, so a // `dateRange` and a caller `where` bound on one field compose instead of // clobbering each other. @@ -253,7 +262,7 @@ export class ObjectQLStrategy implements AnalyticsStrategy { // `null` = constrains nothing, which is the AND identity — nothing to add, // and nothing invented for a filter that says nothing. const scopeCondition = this.filterNodeToCondition( - normalizeAnalyticsFilterTree({ where: datasetScope.filter }), + normalizeAnalyticsFilterTree({ where: datasetScope.filter }, lowering), cube, ); if (scopeCondition) conjuncts.push(scopeCondition); @@ -439,6 +448,13 @@ export class ObjectQLStrategy implements AnalyticsStrategy { // the same channel `execute()` reads it from, so the echo cannot drift // from what actually ran. const datasetScope = (ctx as DatasetScopedStrategyContext).getDatasetScope?.(query.cube!); + // [ADR-0053 D-D1, amended — #5930 step 3] The same column-type reader + // `execute()` hands the `where` door's shared lowering, so the echo prints + // the lowered bound the engine receives — a bare-day `$lte` on a `datetime` + // member reads `< next-day` here because that is what runs. + const echoLowering = declaredDatetimeLowering(ctx, (member) => + this.resolveStorageTarget(cube, member, this.extractObjectName(cube)), + ); const crossByDim = new Map((plan?.crossDims ?? []).map((cd) => [cd.outputName, cd])); const joinClauses: string[] = []; const dimExpr = (dim: string): string => { @@ -479,7 +495,7 @@ export class ObjectQLStrategy implements AnalyticsStrategy { const { field, method } = this.resolveMeasureAggregation(cube, m); const measureFilter = datasetScope?.measureFilters?.[m]; const predicate = measureFilter - ? this.renderFilterNodeSql(normalizeAnalyticsFilterTree({ where: measureFilter }), cube, params, ctx) + ? this.renderFilterNodeSql(normalizeAnalyticsFilterTree({ where: measureFilter }, echoLowering), cube, params, ctx) : null; const aggSql = predicate ? this.conditionalAggregateSql(method, field, predicate) @@ -518,7 +534,7 @@ export class ObjectQLStrategy implements AnalyticsStrategy { // `$or` rendered as a conjunction (or dropped) is exactly the lie this // block's comment above warns about, in the other direction. const filterClause = this.renderFilterNodeSql( - normalizeAnalyticsFilterTree(query), + normalizeAnalyticsFilterTree(query, echoLowering), cube, params, ctx, @@ -531,7 +547,7 @@ export class ObjectQLStrategy implements AnalyticsStrategy { // same lowering `execute()` uses, so the two cannot drift. if (datasetScope?.filter) { const scopeSql = this.renderFilterNodeSql( - normalizeAnalyticsFilterTree({ where: datasetScope.filter }), + normalizeAnalyticsFilterTree({ where: datasetScope.filter }, echoLowering), cube, params, ctx, @@ -815,7 +831,7 @@ export class ObjectQLStrategy implements AnalyticsStrategy { const measureFilter = datasetScope?.measureFilters?.[m]; return measureFilter ? leaves( - normalizeAnalyticsFilterTree({ where: measureFilter }), + normalizeAnalyticsFilterTree({ where: measureFilter }, NO_DATETIME_COLUMNS), { kind: 'measure-filter', measure: m }, ) : []; @@ -823,9 +839,9 @@ export class ObjectQLStrategy implements AnalyticsStrategy { return Object.fromEntries([ ...measureLeaves, ...(datasetScope?.filter - ? leaves(normalizeAnalyticsFilterTree({ where: datasetScope.filter }), { kind: 'dataset-filter' }) + ? leaves(normalizeAnalyticsFilterTree({ where: datasetScope.filter }, NO_DATETIME_COLUMNS), { kind: 'dataset-filter' }) : []), - ...leaves(normalizeAnalyticsFilterTree(query), { kind: 'where' }), + ...leaves(normalizeAnalyticsFilterTree(query, NO_DATETIME_COLUMNS), { kind: 'where' }), ]); } From ab8a992423cb6b247a1642523ed4ccf52a887936 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 11:59:44 +0000 Subject: [PATCH 3/6] test(service-analytics,driver-memory): move the face pins to the lowered condition; spell nested relations dotted at the where door The analytics where door now spells a nested-relation field spec as the dotted members fieldLeaves compiles, before the shared lowering reads the condition, so a guard the lowering lays inside a $not names the member the leaf binds and never the relation key itself. Existing pins updated to the lowered structure, rows unchanged: - SQL-text / tree / engine-where shape pins for $ne, $nin, $notContains and $not now show the lowering's NULL guard around each face's own interim copy of it (idempotent in rows until the copies' deletion cards); every row-count assertion beside them is unchanged. - the where-door tests pass the required column-type reader (NO_DATETIME_COLUMNS). - the cube face's refusal and vocabulary pins: $or and $null are compiled, a $between reaches the face as its two bounds, and each is held to find()'s rows; $not, $startsWith, $endsWith and $empty stay refused. Claude-Session: https://claude.ai/code/session_01XY5uCwTjZj7884yYtyur4H Co-authored-by: Claude --- ...nalytics-20661-lte-whole-day-first.test.ts | 26 +++--- ...y-analytics-echo-operator-coverage.test.ts | 27 ++++-- .../memory-analytics-filter-refusal.test.ts | 63 ++++++++++---- .../comparand-door-single-source.test.ts | 4 +- .../__tests__/comparand-shape-refusal.test.ts | 4 +- .../cross-field-reference-refusal.test.ts | 18 +++- .../empty-combinator-identity.test.ts | 3 +- .../filter-normalizer-mixed-wrapper.test.ts | 6 +- .../filter-normalizer-not-null-safe.test.ts | 43 ++++++---- ...ter-normalizer-undefined-comparand.test.ts | 8 +- .../filter-operator-coverage.test.ts | 6 +- .../__tests__/filter-refusal-envelope.test.ts | 8 +- .../filter-value-type-fidelity.test.ts | 11 +-- .../icontains-text-comparand-refusal.test.ts | 13 +-- ...jectql-contains-canonical-operator.test.ts | 5 +- .../src/__tests__/objectql-daterange.test.ts | 5 +- .../__tests__/objectql-icontains-arm.test.ts | 4 +- .../read-scope-boolean-flag-comparand.test.ts | 10 ++- .../read-scope-not-null-safe.test.ts | 10 ++- ...read-scope-placeholder-three-faces.test.ts | 7 +- .../read-scope-shared-lowering-seam.test.ts | 20 ++++- .../__tests__/text-match-sqlite-nul.test.ts | 6 +- .../text-operator-case-exactness.test.ts | 4 +- .../text-operator-non-text-column.test.ts | 15 +++- .../where-boolean-flag-refusal.test.ts | 21 +++-- .../where-door-shared-lowering-seam.test.ts | 12 ++- .../__tests__/where-empty-operator.test.ts | 10 +-- .../where-equality-slot-list-refusal.test.ts | 8 +- .../__tests__/where-face-arms-refusal.test.ts | 8 +- .../__tests__/where-type-face-refusal.test.ts | 4 +- .../src/strategies/filter-normalizer.ts | 83 ++++++++++++++++++- 31 files changed, 347 insertions(+), 125 deletions(-) diff --git a/packages/drivers/driver-memory/src/memory-analytics-20661-lte-whole-day-first.test.ts b/packages/drivers/driver-memory/src/memory-analytics-20661-lte-whole-day-first.test.ts index 86b7b989278..7e230281f3e 100644 --- a/packages/drivers/driver-memory/src/memory-analytics-20661-lte-whole-day-first.test.ts +++ b/packages/drivers/driver-memory/src/memory-analytics-20661-lte-whole-day-first.test.ts @@ -148,18 +148,20 @@ describe('[#20661] the siblings on this face', () => { expect(ids(result.rows)).toEqual(['r27', 'r28']); }); - it('`$between` is refused on both exits, so its maximum never reaches a bound here', async () => { - // Not a lowering this face owns: widening `MONGO_TO_CUBE_OPERATOR` to take - // `$between` turns this red, and its maximum then owes the same whole-day - // order as the `lte` rows (`lteUpperBound`). - const { service } = await setup('declared'); - const between = cubeQuery({ created_at: { $between: ['2026-07-01', '2026-07-28'] } }); - for (const exit of [() => service.query(between), () => service.generateSql(between)]) { - const err = await exit().then( - () => undefined, - (e: unknown) => e as { code?: unknown; status?: unknown }, - ); - expect(err).toMatchObject({ code: 'INVALID_FILTER', status: 400 }); + it('`$between` reaches the face as its two bounds, and its maximum is widened BEFORE it is converted', async () => { + // [ADR-0053 D-D1, amended — #5930 step 3] This row used to pin `$between` + // refused on both exits, and said what widening it would owe: the same + // whole-day order as the `lte` rows. The face's door now runs the shared + // lowering first, which splits the range and widens the AUTHORED maximum + // in the calendar-string domain (`$lt: '2026-07-29'`); the exit converts + // that bound to the storage form like any comparand. D-E3's order holds by + // construction, declared or not. + for (const declaration of ['undeclared', 'declared'] as const) { + const got = await answer(declaration, { created_at: { $between: ['2026-07-01', '2026-07-28'] } }); + expect(got.cube, declaration).toEqual(got.find); + expect(got.cube, declaration).toEqual(['r27', 'r28']); } + expect((await answer('declared', { created_at: { $between: ['2026-07-01', '2026-07-28'] } })).echo) + .toBe("created_at >= '2026-07-01T00:00:00.000Z' AND created_at < '2026-07-29T00:00:00.000Z'"); }); }); diff --git a/packages/drivers/driver-memory/src/memory-analytics-echo-operator-coverage.test.ts b/packages/drivers/driver-memory/src/memory-analytics-echo-operator-coverage.test.ts index 0808dfe1f46..c8e1827930b 100644 --- a/packages/drivers/driver-memory/src/memory-analytics-echo-operator-coverage.test.ts +++ b/packages/drivers/driver-memory/src/memory-analytics-echo-operator-coverage.test.ts @@ -177,14 +177,27 @@ const ACCEPTED_CASES: Record = { $icontains: { name: { $icontains: 'industries' } }, $notContains: { name: { $notContains: 'Industries' } }, $exists: { name: { $exists: true } }, + // [ADR-0053 D-D1, amended — #5930 step 3] `$null` joined the face's table: + // the shared lowering emits it. Row 6's NULL `name` makes it a real predicate. + $null: { name: { $null: true } }, }; /** - * The spellings this face REFUSES — the complement, so the two sets are total. - * [#20446] `$empty` joined `FILTER_OPERATORS`, and this cube face refuses it as - * it refuses `$null`: a declared operator it has no lowering for. + * [ADR-0053 D-D1, amended — #5930 step 3] The spellings the face's door LOWERS + * before its vocabulary is asked: a `$between` reaches the face as the two + * bounds the shared lowering splits it into. Not in the face's table and not + * refused — so a third set, held to the same executed-echo invariant. */ -const REFUSED_OPERATORS = ['$between', '$startsWith', '$endsWith', '$null', '$empty'] as const; +const LOWERED_CASES: Record = { + $between: { amount: { $between: [20, 40] } }, +}; + +/** + * The spellings this face REFUSES — the complement, so the three sets are total. + * [#20446] `$empty` joined `FILTER_OPERATORS`, and this cube face refuses it: a + * declared operator it has no lowering for. + */ +const REFUSED_OPERATORS = ['$startsWith', '$endsWith', '$empty'] as const; describe('[#7117] the analytics echo renders the query it describes', () => { let db: any; @@ -441,8 +454,8 @@ describe('[#7117] the analytics echo renders the query it describes', () => { // ── The vocabulary, enumerated so the two tables cannot drift apart ──────── describe('the closed vocabulary, enumerated', () => { - it('the accepted and refused sets together are the whole Filter Protocol', () => { - expect(sortIds([...Object.keys(ACCEPTED_CASES), ...REFUSED_OPERATORS])) + it('the accepted, lowered and refused sets together are the whole Filter Protocol', () => { + expect(sortIds([...Object.keys(ACCEPTED_CASES), ...Object.keys(LOWERED_CASES), ...REFUSED_OPERATORS])) .toEqual(sortIds([...FILTER_OPERATORS])); }); @@ -451,7 +464,7 @@ describe('[#7117] the analytics echo renders the query it describes', () => { .toEqual(sortIds([...ANALYTICS_FILTER_CAPABILITIES.fieldOperators])); }); - for (const [op, where] of Object.entries(ACCEPTED_CASES)) { + for (const [op, where] of Object.entries({ ...ACCEPTED_CASES, ...LOWERED_CASES })) { it(`${op}: running the echo returns exactly the rows the query returns`, async () => { const executed = await executedIds(where); const echoed = await echoIds(where); diff --git a/packages/drivers/driver-memory/src/memory-analytics-filter-refusal.test.ts b/packages/drivers/driver-memory/src/memory-analytics-filter-refusal.test.ts index 0d5cedfaa44..cf85be73620 100644 --- a/packages/drivers/driver-memory/src/memory-analytics-filter-refusal.test.ts +++ b/packages/drivers/driver-memory/src/memory-analytics-filter-refusal.test.ts @@ -24,6 +24,14 @@ * Both public entry points are covered here — `query()` and `generateSql()` — * because both call `normalizeFilters` and a refusal on one only would leave the * other silently answering the old way. + * + * [ADR-0053 D-D1, amended 2026-09-30 — #5930 step 3] Three of those shapes are + * no longer refusals: the face now COMPILES `$or` and `$null` (the shared + * lowering emits both), and a `$between` reaches its vocabulary gate as the two + * bounds the lowering splits it into. Each is pinned below as answered with the + * rows the live query path gives it — the invariant this file has always held, + * refuse-or-agree, never a different row set. `$not`, `$startsWith` and + * `$endsWith` stay refused. */ import { describe, it, expect, beforeEach } from 'vitest'; @@ -104,23 +112,30 @@ describe('[#5345] MemoryAnalyticsService — filters it cannot compile are refus // ── The combinators (`continue` #1) ──────────────────────────────────────── - it('refuses $or instead of aggregating the whole table', async () => { + // [#5930 step 3] `$or` is compiled now, so the regression this section was + // written against — three rows asked for, five returned — is pinned as its + // answer: the rows the live query path returns, never the whole table. + const findCount = async (where: FilterCondition): Promise => (await driver.find('deals', { where })).length; + + it('compiles $or instead of aggregating the whole table', async () => { const where: FilterCondition = { $or: [{ stage: 'won' }, { stage: 'lost' }] }; - const err = await expectRefusal(() => count(where), '$or', 'where.$or'); - // The regression in one line: three rows were asked for, five were returned. - expect(err.message).toContain('WIDENS'); + expect(await count(where)).toBe(3); + expect(await count(where)).toBe(await findCount(where)); }); - it('refuses a $or nested inside a $and it CAN compile', async () => { + it('compiles a $or nested inside a $and', async () => { const where: FilterCondition = { $and: [{ owner: 'u1' }, { $or: [{ stage: 'won' }, { stage: 'open' }] }] }; - await expectRefusal(() => count(where), '$or', 'where.$and[1].$or'); + expect(await count(where)).toBe(2); + expect(await count(where)).toBe(await findCount(where)); }); - it('refuses a $or that sits beside a compilable sibling key', async () => { - // The sharpest shape: the sibling `owner` lowered fine, so the query ran and - // returned a plausible-looking number computed without the $or at all. + it('compiles a $or that sits beside a sibling key, ANDed with it', async () => { + // The sharpest shape when `$or` was dropped: the sibling `owner` lowered + // fine, so the query ran and returned a plausible-looking number computed + // without the $or at all. Compiled, it narrows. const where: FilterCondition = { owner: 'u1', $or: [{ stage: 'won' }, { stage: 'lost' }] }; - await expectRefusal(() => count(where), '$or'); + expect(await count(where)).toBe(2); + expect(await count(where)).toBe(await findCount(where)); }); it('refuses $not — the shape an RLS read scope compiles to', async () => { @@ -130,19 +145,31 @@ describe('[#5345] MemoryAnalyticsService — filters it cannot compile are refus }); it('names the combinators it CAN compile, so the refusal is actionable', async () => { - const err = await expectRefusal(() => count({ $or: [{ stage: 'won' }] })); - expect(err.message).toContain('Supported combinators on this surface: $and'); + const err = await expectRefusal(() => count({ $not: { stage: 'won' } })); + expect(err.message).toContain('Supported combinators on this surface: $and, $or'); }); // ── The unmapped operators (`continue` #2) ───────────────────────────────── const UNCOMPILABLE: Array<{ op: string; where: FilterCondition }> = [ - { op: '$between', where: { amount: { $between: [100, 200] } } }, { op: '$startsWith', where: { name: { $startsWith: 'al' } } }, { op: '$endsWith', where: { name: { $endsWith: 'ta' } } }, - { op: '$null', where: { closed_at: { $null: true } } }, ]; + // [#5930 step 3] Answered, not refused: `$null` joined this face's table, and + // a `$between` reaches it as its two bounds. Each agrees with the live path. + const NOW_COMPILED: Array<{ op: string; where: FilterCondition; rows: number }> = [ + { op: '$between', where: { amount: { $between: [100, 200] } }, rows: 2 }, + { op: '$null', where: { closed_at: { $null: true } }, rows: 2 }, + ]; + + for (const { op, where, rows } of NOW_COMPILED) { + it(`compiles ${op}, with the rows the live query path returns`, async () => { + expect(await count(where)).toBe(rows); + expect(await findCount(where)).toBe(rows); + }); + } + for (const { op, where } of UNCOMPILABLE) { it(`refuses ${op} — declared by the Filter Protocol, not compilable by this face`, async () => { const err = await expectRefusal(() => count(where), op); @@ -221,17 +248,17 @@ describe('[#5345] MemoryAnalyticsService — filters it cannot compile are refus () => service.generateSql(asQuery({ cube: 'deals', measures: ['deals.count'], - where: { $or: [{ stage: 'won' }, { stage: 'lost' }] }, + where: { $not: { stage: 'lost' } }, })), - '$or', + '$not', ); await expectRefusal( () => service.generateSql(asQuery({ cube: 'deals', measures: ['deals.count'], - where: { amount: { $between: [1, 2] } }, + where: { name: { $startsWith: 'al' } }, })), - '$between', + '$startsWith', ); const ok = await service.generateSql(asQuery({ cube: 'deals', diff --git a/packages/services/service-analytics/src/__tests__/comparand-door-single-source.test.ts b/packages/services/service-analytics/src/__tests__/comparand-door-single-source.test.ts index e66aede4131..f90f0cd2372 100644 --- a/packages/services/service-analytics/src/__tests__/comparand-door-single-source.test.ts +++ b/packages/services/service-analytics/src/__tests__/comparand-door-single-source.test.ts @@ -68,7 +68,7 @@ import { ACCEPTED_FILTER_COMPARAND_TYPES_SENTENCE, } from '@objectstack/spec/data'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { compileScopedFilterToSql } from '../read-scope-sql.js'; import { isBindableComparand, @@ -77,7 +77,7 @@ import { unrenderableTextComparandMessage, } from '../comparand-shape.js'; -const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as any); +const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as any, NO_DATETIME_COLUMNS); const scope = (where: unknown) => compileScopedFilterToSql(where as FilterCondition, 'person'); /** `'accept'`, or the refusal's ADR-0112 envelope as `CODE/status`. */ diff --git a/packages/services/service-analytics/src/__tests__/comparand-shape-refusal.test.ts b/packages/services/service-analytics/src/__tests__/comparand-shape-refusal.test.ts index 659cdda2574..fc4d846dfde 100644 --- a/packages/services/service-analytics/src/__tests__/comparand-shape-refusal.test.ts +++ b/packages/services/service-analytics/src/__tests__/comparand-shape-refusal.test.ts @@ -87,7 +87,7 @@ import { describe, it, expect } from 'vitest'; import type { FilterCondition } from '@objectstack/spec/data'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { compileScopedFilterToSql } from '../read-scope-sql.js'; import { isBindableComparand, @@ -108,7 +108,7 @@ function refusalOf(run: () => unknown): WireBearingError { throw new Error('expected the compiler to refuse this filter, but it returned'); } -const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as any); +const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as any, NO_DATETIME_COLUMNS); const scope = (where: unknown) => compileScopedFilterToSql(where as FilterCondition, 'person'); // ── The analytics `where` door ─────────────────────────────────────────────── diff --git a/packages/services/service-analytics/src/__tests__/cross-field-reference-refusal.test.ts b/packages/services/service-analytics/src/__tests__/cross-field-reference-refusal.test.ts index 6034e969226..057f4d26d14 100644 --- a/packages/services/service-analytics/src/__tests__/cross-field-reference-refusal.test.ts +++ b/packages/services/service-analytics/src/__tests__/cross-field-reference-refusal.test.ts @@ -97,7 +97,7 @@ import { import type { Cube, FilterCondition } from '@objectstack/spec/data'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { compileScopedFilterToSql } from '../read-scope-sql.js'; import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; import { @@ -129,7 +129,7 @@ function refusalOf(run: () => unknown): WireBearingError { ); } -const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as any); +const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as any, NO_DATETIME_COLUMNS); const scope = (where: unknown) => compileScopedFilterToSql(where as FilterCondition, 'deal'); /** Every leaf comparand the normalizer produced, structure discarded. */ @@ -214,12 +214,22 @@ describe("[#7598] the #5222 corpus's SUPPORTED arm is ROUTED by the `where` door expect(tree({ amount: { $ne: { $field: 'budget' } } })).toEqual({ kind: 'leaf', member: 'amount', operator: 'notEquals', values: [{ $field: 'budget' }], }); - // …and the literal keeps its guard, unchanged. This pair is the whole claim. + // …and the literal keeps its guard. This pair is the whole claim. + // [ADR-0053 D-D1, amended — #5930 step 3] The guard now arrives twice: the + // shared lowering's NULL escape (outer), around this face's own interim + // copy of it (inner) — the same rows, until the copy's deletion card. The + // reference above gets neither, from either. expect(tree({ amount: { $ne: 5 } })).toEqual({ kind: 'or', children: [ { kind: 'leaf', member: 'amount', operator: 'notSet', values: [] }, - { kind: 'leaf', member: 'amount', operator: 'notEquals', values: [5] }, + { + kind: 'or', + children: [ + { kind: 'leaf', member: 'amount', operator: 'notSet', values: [] }, + { kind: 'leaf', member: 'amount', operator: 'notEquals', values: [5] }, + ], + }, ], }); }); diff --git a/packages/services/service-analytics/src/__tests__/empty-combinator-identity.test.ts b/packages/services/service-analytics/src/__tests__/empty-combinator-identity.test.ts index 3d4802d4dcf..914494bbd0d 100644 --- a/packages/services/service-analytics/src/__tests__/empty-combinator-identity.test.ts +++ b/packages/services/service-analytics/src/__tests__/empty-combinator-identity.test.ts @@ -51,10 +51,11 @@ import { DatasetSchema } from '@objectstack/spec/ui'; import { normalizeAnalyticsFilterTree, collectFilterLeaves, + NO_DATETIME_COLUMNS, } from '../strategies/filter-normalizer.js'; import { AnalyticsService } from '../analytics-service.js'; -const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where }); +const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where }, NO_DATETIME_COLUMNS); const FALSE_NODE = { kind: 'const', value: false }; const TRUE_NODE = { kind: 'const', value: true }; diff --git a/packages/services/service-analytics/src/__tests__/filter-normalizer-mixed-wrapper.test.ts b/packages/services/service-analytics/src/__tests__/filter-normalizer-mixed-wrapper.test.ts index efbdc65f017..e04b656816c 100644 --- a/packages/services/service-analytics/src/__tests__/filter-normalizer-mixed-wrapper.test.ts +++ b/packages/services/service-analytics/src/__tests__/filter-normalizer-mixed-wrapper.test.ts @@ -86,7 +86,7 @@ */ import { describe, it, expect } from 'vitest'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; /** The ADR-0112 fields a refusal must carry. */ interface FilterRefusal extends Error { @@ -96,7 +96,7 @@ interface FilterRefusal extends Error { function refusalFor(where: unknown): FilterRefusal | undefined { try { - normalizeAnalyticsFilterTree({ where }); + normalizeAnalyticsFilterTree({ where }, NO_DATETIME_COLUMNS); return undefined; } catch (e) { return e as FilterRefusal; @@ -104,7 +104,7 @@ function refusalFor(where: unknown): FilterRefusal | undefined { } function treeFor(where: unknown): unknown { - return normalizeAnalyticsFilterTree({ where }); + return normalizeAnalyticsFilterTree({ where }, NO_DATETIME_COLUMNS); } /** diff --git a/packages/services/service-analytics/src/__tests__/filter-normalizer-not-null-safe.test.ts b/packages/services/service-analytics/src/__tests__/filter-normalizer-not-null-safe.test.ts index d9321c9d09b..d2f10d4aa7c 100644 --- a/packages/services/service-analytics/src/__tests__/filter-normalizer-not-null-safe.test.ts +++ b/packages/services/service-analytics/src/__tests__/filter-normalizer-not-null-safe.test.ts @@ -73,7 +73,7 @@ import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contract import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { compileScopedFilterToSql } from '../read-scope-sql.js'; /** @@ -322,9 +322,14 @@ describe('[#5325] analytics `where` — NULL-safe `$not` and the boolean identit describe('a NULL column does not satisfy the negated condition', () => { it('the guard rides the LEAF, so the emitted SQL negates a TOTAL predicate', async () => { const { sql } = await sqlFor({ $not: { stage: 'won' } }); + // [ADR-0053 D-D1, amended — #5930 step 3] The shared lowering totalises + // the `$not` operand first (`{ stage: { $null: false } }` beside the + // leaf); this face's own copy then guards the leaf again. `X AND (X AND + // Y)` ≡ `X AND Y`: the predicate, and the ids above, are unchanged until + // the copy's deletion card. Asserted as emitted, for this file's reason. expect(sql).toBe( 'SELECT id AS "id", COUNT(*) AS "total" FROM "deal" ' + - 'WHERE NOT ((stage IS NOT NULL AND stage = $1)) GROUP BY id', + 'WHERE NOT ((stage IS NOT NULL AND (stage IS NOT NULL AND stage = $1))) GROUP BY id', ); }); @@ -362,8 +367,10 @@ describe('[#5325] analytics `where` — NULL-safe `$not` and the boolean identit // and they are unchanged. Asserted as it is actually emitted rather than // trimmed to the prettier form: a pin that describes SQL the compiler does // not produce is how the next reader learns to distrust this file. + // [ADR-0053 D-D1, amended — #5930 step 3] …and a third time: the shared + // lowering's own `allowNull` escape on the `$not` operand, outermost. const { sql } = await sqlFor({ $not: { stage: { $ne: 'won' } } }); - expect(sql).toContain('NOT ((stage IS NULL OR (stage IS NULL OR stage != $1)))'); + expect(sql).toContain('NOT ((stage IS NULL OR (stage IS NULL OR (stage IS NULL OR stage != $1))))'); }); it('`$not` of an ordering comparison returns the NULL rows', async () => { @@ -441,8 +448,13 @@ describe('[#5325] analytics `where` — NULL-safe `$not` and the boolean identit // leaf reads. Asserted on the generated SQL only — `region` is not a column // of this fixture, which is the point: both halves resolve to ONE member. const { sql } = await sqlFor({ $not: { account: { region: 'NA' } } }); - expect(sql).toContain('NOT (("account"."region" IS NOT NULL AND "account"."region" = $1))'); + // [ADR-0053 D-D1, amended — #5930 step 3] The door spells the nested + // relation dotted before the shared lowering reads it, so the lowering's + // guard lands on the dotted member too (outer), and this face's own copy + // adds its own (inner) — never a guard on `account` itself. + expect(sql).toContain('NOT (("account"."region" IS NOT NULL AND ("account"."region" IS NOT NULL AND "account"."region" = $1)))'); expect(sql).not.toContain('"deal"."account" IS NOT NULL'); + expect(sql).not.toMatch(/(^|[^."])account IS NOT NULL/); }); }); @@ -489,12 +501,15 @@ describe('[#5325] analytics `where` — NULL-safe `$not` and the boolean identit // Not a dialect equivalent (`IS DISTINCT FROM` / `<=>`): `NOT LIKE` has no // such form, so the family would have needed two shapes. The cost-list // measurement (#5298 §2/§3) found the query plans identical either way. + // [ADR-0053 D-D1, amended — #5930 step 3] The shared lowering's NULL + // escape (outer) now arrives around this face's own copy (inner): the + // same OR expansion, twice, the same rows, until the copy's deletion card. expect((await sqlFor({ stage: { $ne: 'won' } })).sql) - .toContain('WHERE (stage IS NULL OR stage != $1)'); + .toContain('WHERE (stage IS NULL OR (stage IS NULL OR stage != $1))'); expect((await sqlFor({ stage: { $nin: ['won'] } })).sql) - .toContain('WHERE (stage IS NULL OR stage NOT IN ($1))'); + .toContain('WHERE (stage IS NULL OR (stage IS NULL OR stage NOT IN ($1)))'); expect((await sqlFor({ stage: { $notContains: 'wo' } })).sql) - .toContain('WHERE (stage IS NULL OR stage NOT LIKE $1 ESCAPE $2)'); + .toContain('WHERE (stage IS NULL OR (stage IS NULL OR stage NOT LIKE $1 ESCAPE $2))'); }); it('the ObjectQL path and the display SQL agree with the raw-SQL path', async () => { @@ -835,16 +850,16 @@ describe('[#5325] analytics `where` — NULL-safe `$not` and the boolean identit // The emitter, without a database in the way: `notSet` / `set` with EMPTY // `values`, which is what makes every compiler of this tree — both // strategies AND the display-SQL echo — answer alike without a third arm. - expect(normalizeAnalyticsFilterTree({ where: { stage: { $eq: null } } })).toEqual({ + expect(normalizeAnalyticsFilterTree({ where: { stage: { $eq: null } } }, NO_DATETIME_COLUMNS)).toEqual({ kind: 'leaf', member: 'stage', operator: 'notSet', values: [], }); - expect(normalizeAnalyticsFilterTree({ where: { stage: { $ne: null } } })).toEqual({ + expect(normalizeAnalyticsFilterTree({ where: { stage: { $ne: null } } }, NO_DATETIME_COLUMNS)).toEqual({ kind: 'leaf', member: 'stage', operator: 'set', values: [], }); - expect(normalizeAnalyticsFilterTree({ where: { stage: { $eq: null } } })) - .toEqual(normalizeAnalyticsFilterTree({ where: { stage: null } })); - expect(normalizeAnalyticsFilterTree({ where: { stage: { $ne: null } } })) - .toEqual(normalizeAnalyticsFilterTree({ where: { stage: { $null: false } } })); + expect(normalizeAnalyticsFilterTree({ where: { stage: { $eq: null } } }, NO_DATETIME_COLUMNS)) + .toEqual(normalizeAnalyticsFilterTree({ where: { stage: null } }, NO_DATETIME_COLUMNS)); + expect(normalizeAnalyticsFilterTree({ where: { stage: { $ne: null } } }, NO_DATETIME_COLUMNS)) + .toEqual(normalizeAnalyticsFilterTree({ where: { stage: { $null: false } } }, NO_DATETIME_COLUMNS)); }); it('an EMPTY STRING comparand is still a value comparison — the fix does not over-reach', async () => { @@ -854,7 +869,7 @@ describe('[#5325] analytics `where` — NULL-safe `$not` and the boolean identit const { sql, params } = await sqlFor({ stage: { $eq: '' } }); expect(sql).toContain('WHERE stage = $1'); expect(params).toEqual(['']); - expect(normalizeAnalyticsFilterTree({ where: { stage: { $eq: '' } } })).toEqual({ + expect(normalizeAnalyticsFilterTree({ where: { stage: { $eq: '' } } }, NO_DATETIME_COLUMNS)).toEqual({ kind: 'leaf', member: 'stage', operator: 'equals', values: [''], }); // And a non-null comparand of the same operators is still a VALUE diff --git a/packages/services/service-analytics/src/__tests__/filter-normalizer-undefined-comparand.test.ts b/packages/services/service-analytics/src/__tests__/filter-normalizer-undefined-comparand.test.ts index 30737f36a11..c28f74f5f5a 100644 --- a/packages/services/service-analytics/src/__tests__/filter-normalizer-undefined-comparand.test.ts +++ b/packages/services/service-analytics/src/__tests__/filter-normalizer-undefined-comparand.test.ts @@ -123,7 +123,7 @@ */ import { describe, it, expect } from 'vitest'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; /** The ADR-0112 fields a refusal must carry. */ interface FilterRefusal extends Error { @@ -133,7 +133,7 @@ interface FilterRefusal extends Error { function refusalFor(where: unknown): FilterRefusal | undefined { try { - normalizeAnalyticsFilterTree({ where }); + normalizeAnalyticsFilterTree({ where }, NO_DATETIME_COLUMNS); return undefined; } catch (e) { return e as FilterRefusal; @@ -141,7 +141,7 @@ function refusalFor(where: unknown): FilterRefusal | undefined { } function treeFor(where: unknown): unknown { - return normalizeAnalyticsFilterTree({ where }); + return normalizeAnalyticsFilterTree({ where }, NO_DATETIME_COLUMNS); } /** @@ -407,7 +407,7 @@ describe('[#6386] the `null` control group does not move', () => { // The refusal is about a KEY INSIDE a `where`; a missing `where` is the // legitimate way to say "no filter" and must stay silent. expect(treeFor(undefined)).toBeNull(); - expect(normalizeAnalyticsFilterTree({})).toBeNull(); + expect(normalizeAnalyticsFilterTree({}, NO_DATETIME_COLUMNS)).toBeNull(); expect(treeFor({})).toBeNull(); expect(treeFor([])).toBeNull(); }); diff --git a/packages/services/service-analytics/src/__tests__/filter-operator-coverage.test.ts b/packages/services/service-analytics/src/__tests__/filter-operator-coverage.test.ts index a1be118fb20..f024d108d74 100644 --- a/packages/services/service-analytics/src/__tests__/filter-operator-coverage.test.ts +++ b/packages/services/service-analytics/src/__tests__/filter-operator-coverage.test.ts @@ -28,7 +28,7 @@ import type { AnalyticsQuery, FilterCondition } from '@objectstack/spec/data'; import type { StrategyContext } from '@objectstack/spec/contracts'; import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; interface Row { id: string; @@ -191,7 +191,7 @@ describe('analytics filters — every authorable operator reaches the query (#41 // rows the filter excludes. A typo'd or non-spec operator is a caller // error, and a loud one — the same call driver-memory made in #3948. expect(() => - normalizeAnalyticsFilterTree({ where: { name: { $sortOf: 'alpha' } } }), + normalizeAnalyticsFilterTree({ where: { name: { $sortOf: 'alpha' } } }, NO_DATETIME_COLUMNS), ).toThrow(/Unsupported filter operator "\$sortOf"/); }); @@ -199,7 +199,7 @@ describe('analytics filters — every authorable operator reaches the query (#41 // [#20010] The shared comparand-shape face answers this one first now, in // its own words (the words the FilterArray spelling gets); this door's // "two-element" sentence is no longer reached from `normalizeAnalyticsFilterTree`. - expect(() => normalizeAnalyticsFilterTree({ where: { score: { $between: [10] } } })).toThrow( + expect(() => normalizeAnalyticsFilterTree({ where: { score: { $between: [10] } } }, NO_DATETIME_COLUMNS)).toThrow( /requires a \[min, max\] value array/, ); }); diff --git a/packages/services/service-analytics/src/__tests__/filter-refusal-envelope.test.ts b/packages/services/service-analytics/src/__tests__/filter-refusal-envelope.test.ts index 5a4b432f14e..afb9af6ee30 100644 --- a/packages/services/service-analytics/src/__tests__/filter-refusal-envelope.test.ts +++ b/packages/services/service-analytics/src/__tests__/filter-refusal-envelope.test.ts @@ -40,7 +40,7 @@ */ import { describe, it, expect } from 'vitest'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; /** The ADR-0112 fields a refusal must carry. */ interface FilterRefusal extends Error { @@ -51,7 +51,7 @@ interface FilterRefusal extends Error { /** Run the normalizer over a `where` and return the error it threw, if any. */ function refusalFor(where: unknown): FilterRefusal | undefined { try { - normalizeAnalyticsFilterTree({ where }); + normalizeAnalyticsFilterTree({ where }, NO_DATETIME_COLUMNS); return undefined; } catch (e) { return e as FilterRefusal; @@ -282,12 +282,12 @@ describe('[#5352] the refusal SET is unchanged — only the error shape moved', for (const c of ACCEPTED) { it(`still ACCEPTS (and compiles identically): ${c.name}`, () => { expect(refusalFor(c.where), `${c.name} was refused — the refusal set moved`).toBeUndefined(); - expect(normalizeAnalyticsFilterTree({ where: c.where })).toEqual(c.tree); + expect(normalizeAnalyticsFilterTree({ where: c.where }, NO_DATETIME_COLUMNS)).toEqual(c.tree); }); } it('carries no `where` at all → no constraint, no error', () => { - expect(normalizeAnalyticsFilterTree({})).toBeNull(); + expect(normalizeAnalyticsFilterTree({}, NO_DATETIME_COLUMNS)).toBeNull(); }); }); diff --git a/packages/services/service-analytics/src/__tests__/filter-value-type-fidelity.test.ts b/packages/services/service-analytics/src/__tests__/filter-value-type-fidelity.test.ts index 159881c4b21..520d09f12b5 100644 --- a/packages/services/service-analytics/src/__tests__/filter-value-type-fidelity.test.ts +++ b/packages/services/service-analytics/src/__tests__/filter-value-type-fidelity.test.ts @@ -55,6 +55,7 @@ import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; import { normalizeAnalyticsFilterTree, toSqlBindValue, + NO_DATETIME_COLUMNS, } from '../strategies/filter-normalizer.js'; import { AnalyticsService } from '../analytics-service.js'; @@ -163,7 +164,7 @@ const CASES: Array<{ describe('[#5526] a leaf carries the author comparand ITSELF', () => { for (const c of CASES) { it(`${c.name} → leaf ${JSON.stringify(c.leaf)}`, () => { - const node = normalizeAnalyticsFilterTree({ where: { code: { $eq: c.value } } }); + const node = normalizeAnalyticsFilterTree({ where: { code: { $eq: c.value } } }, NO_DATETIME_COLUMNS); expect(node?.kind).toBe('leaf'); // Exact set, not "contains": the leaf carries one comparand and it is the // author's. @@ -173,7 +174,7 @@ describe('[#5526] a leaf carries the author comparand ITSELF', () => { it('the leaf value is the SAME REFERENCE for a non-primitive, i.e. nothing re-encoded it', () => { const when = new Date('2026-03-04T05:06:07.000Z'); - const node = normalizeAnalyticsFilterTree({ where: { closed: { $gte: when } } }) as + const node = normalizeAnalyticsFilterTree({ where: { closed: { $gte: when } } }, NO_DATETIME_COLUMNS) as | { kind: 'leaf'; values: unknown[] } | null; expect(node?.values[0]).toBe(when); @@ -239,7 +240,7 @@ describe('[#5526] the SQL bind form converts only what a driver cannot bind', () // shared comparand-TYPE face, which the door runs before any leaf is built // (#7872, 2026-08-12: 「refuses everything else loudly at the compile // face」), in its sentence and at its path — the FilterArray spelling's. - const refusal = (): unknown => normalizeAnalyticsFilterTree({ where: { code: { $eq: undefined } } }); + const refusal = (): unknown => normalizeAnalyticsFilterTree({ where: { code: { $eq: undefined } } }, NO_DATETIME_COLUMNS); expect(refusal).toThrowError(/^Filter comparand at where\.code\.\$eq is undefined\./); // Still the module's one envelope (#5352), so the REST face answers 400. try { @@ -252,7 +253,7 @@ describe('[#5526] the SQL bind form converts only what a driver cannot bind', () // The neighbouring `null` comparand keeps compiling, and to the null // PREDICATE rather than a value comparison (#5332) — the row that proves the // refusal did not widen to `== null`. - expect(normalizeAnalyticsFilterTree({ where: { code: { $eq: null } } })).toEqual({ + expect(normalizeAnalyticsFilterTree({ where: { code: { $eq: null } } }, NO_DATETIME_COLUMNS)).toEqual({ kind: 'leaf', member: 'code', operator: 'notSet', values: [], }); }); @@ -523,7 +524,7 @@ describe('[#5526] the LIKE family stringifies at the emitter, on all three emitt }; /** The operand the engine receives, via the private converter. */ const engineOperand = (where: FilterCondition): unknown => { - const node = normalizeAnalyticsFilterTree({ where }) as { + const node = normalizeAnalyticsFilterTree({ where }, NO_DATETIME_COLUMNS) as { operator: string; values: unknown[]; }; diff --git a/packages/services/service-analytics/src/__tests__/icontains-text-comparand-refusal.test.ts b/packages/services/service-analytics/src/__tests__/icontains-text-comparand-refusal.test.ts index 47d639ea2b3..5ecc8e4c6a1 100644 --- a/packages/services/service-analytics/src/__tests__/icontains-text-comparand-refusal.test.ts +++ b/packages/services/service-analytics/src/__tests__/icontains-text-comparand-refusal.test.ts @@ -41,7 +41,7 @@ import { DatasetSchema, type Dataset } from '@objectstack/spec/ui'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; import { declaredRefusalMessage, resolveThrownHttpError, serverFaultProvenance } from '@objectstack/types'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; import { evaluateAnalyticsQueryOverRows } from '../preview-evaluator.js'; @@ -88,7 +88,7 @@ const REFUSED_COMPARANDS: Array<[string, unknown]> = [ ['a Date', new Date('2026-01-01T00:00:00.000Z')], ]; -const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as never); +const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as never, NO_DATETIME_COLUMNS); function refusalOf(run: () => unknown): Refusal { let out: unknown; @@ -448,9 +448,12 @@ describe('[#20068] ⛔ no widening by analogy — the case-exact family keeps it it('the `where` door still compiles each of them with an empty comparand', () => { for (const op of SIBLINGS) { - const node = tree({ name: { [op]: '' } }) as { kind: string; children?: unknown[]; operator?: string }; - // `$notContains` is NULL-safe (#5298), so it wraps its leaf in an `or`. - const leaf = node.kind === 'or' ? (node.children as Array<{ operator: string }>)[1] : node; + type Node = { kind: string; children?: Node[]; operator?: string }; + // `$notContains` is NULL-safe (#5298), so it wraps its leaf in an `or` — + // [ADR-0053 D-D1, amended — #5930 step 3] twice now, the shared + // lowering's escape around this face's own: descend to the operator leaf. + let leaf = tree({ name: { [op]: '' } }) as Node; + while (leaf.kind === 'or') leaf = leaf.children![1]; expect(leaf, op).toMatchObject({ operator: CUBE_OP[op], values: [''] }); } }); diff --git a/packages/services/service-analytics/src/__tests__/objectql-contains-canonical-operator.test.ts b/packages/services/service-analytics/src/__tests__/objectql-contains-canonical-operator.test.ts index dcb1a443b6c..bd4c7f105e3 100644 --- a/packages/services/service-analytics/src/__tests__/objectql-contains-canonical-operator.test.ts +++ b/packages/services/service-analytics/src/__tests__/objectql-contains-canonical-operator.test.ts @@ -255,8 +255,11 @@ describe('[#5557] `contains` reaches the engine as `$contains`, comparand taken // the null-predicate disjunct. What THIS case asserts is unaffected and // still exact: the operator key is the declared `$notContains` and the // comparand is the author's literal `'a.b'`, not a `$regex` pattern. + // [ADR-0053 D-D1, amended — #5930 step 3] …twice now: the shared + // lowering's escape around this face's own copy. Same operator key, same + // literal comparand, same rows. expect(await engineFilter({ stage: { $notContains: 'a.b' } })).toEqual({ - $and: [{ $or: [{ stage: null }, { stage: { $notContains: 'a.b' } }] }], + $and: [{ $or: [{ stage: null }, { $or: [{ stage: null }, { stage: { $notContains: 'a.b' } }] }] }], }); expect(await engineFilter({ stage: { $startsWith: 'a.b' } })).toEqual({ stage: { $startsWith: 'a.b' }, diff --git a/packages/services/service-analytics/src/__tests__/objectql-daterange.test.ts b/packages/services/service-analytics/src/__tests__/objectql-daterange.test.ts index d3c83231dd6..7afe125cc3f 100644 --- a/packages/services/service-analytics/src/__tests__/objectql-daterange.test.ts +++ b/packages/services/service-analytics/src/__tests__/objectql-daterange.test.ts @@ -348,9 +348,12 @@ describe('ObjectQLStrategy — window ∧ where on one field (#3650)', () => { // What this case is about is unchanged and still visible: BOTH operands // survive, the second as its own `$and` conjunct rather than overwriting the // bare equality. + // [ADR-0053 D-D1, amended — #5930 step 3] The null predicate now arrives + // twice, the shared lowering's escape around this face's own copy — the + // same rows; both operands still survive. expect(seen[0].filter).toEqual({ stage: 'won', - $and: [{ $or: [{ stage: null }, { stage: { $ne: 'lost' } }] }], + $and: [{ $or: [{ stage: null }, { $or: [{ stage: null }, { stage: { $ne: 'lost' } }] }] }], }); }); }); diff --git a/packages/services/service-analytics/src/__tests__/objectql-icontains-arm.test.ts b/packages/services/service-analytics/src/__tests__/objectql-icontains-arm.test.ts index 8183520321c..675b38e2b46 100644 --- a/packages/services/service-analytics/src/__tests__/objectql-icontains-arm.test.ts +++ b/packages/services/service-analytics/src/__tests__/objectql-icontains-arm.test.ts @@ -41,7 +41,7 @@ import { ObjectQL } from '@objectstack/objectql'; import { FILTER_OPERATORS, LOGICAL_OPERATORS, type Cube, type FilterCondition } from '@objectstack/spec/data'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; -import { collectFilterLeaves, normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { collectFilterLeaves, normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; import type { DatasetScope } from '../strategies/types.js'; @@ -330,7 +330,7 @@ describe('[#20098] the catch-all is untouched', () => { { f: { $exists: true } }, { f: { $exists: false } }, { f: null } as unknown as FilterCondition, { f: 'x' }, ]; for (const where of samples) { - for (const leaf of collectFilterLeaves(normalizeAnalyticsFilterTree({ where } as never))) { + for (const leaf of collectFilterLeaves(normalizeAnalyticsFilterTree({ where } as never, NO_DATETIME_COLUMNS))) { emitted.add(leaf.operator); expect(() => convert(leaf.operator, leaf.values), `${JSON.stringify(where)} → ${leaf.operator}`).not.toThrow(); } diff --git a/packages/services/service-analytics/src/__tests__/read-scope-boolean-flag-comparand.test.ts b/packages/services/service-analytics/src/__tests__/read-scope-boolean-flag-comparand.test.ts index 8955f4debec..b0508d18e35 100644 --- a/packages/services/service-analytics/src/__tests__/read-scope-boolean-flag-comparand.test.ts +++ b/packages/services/service-analytics/src/__tests__/read-scope-boolean-flag-comparand.test.ts @@ -281,14 +281,20 @@ describe('[#6387] the polarity table moved WITH the emitter (#5146 / #5298)', () // `$nin` makes the constraint non-total, so `nullGuardForFieldSpec` has to // consult the table; `$null: true` says a NULL row DOES satisfy it, so the // leaf is guarded with `IS NULL OR (…)`. + // [ADR-0053 D-D1, amended — #5930 step 3] The shared lowering, at this + // compiler's entry, reads the same table and lays the same `allowNull` + // guard on first (outer); this compiler's own copy then adds its own + // (inner). Same predicate, until the copy's deletion card. expect(sql({ $not: { d: { $null: true, $nin: ['x'] } } })).toBe( - 'NOT ((("t"."d" IS NULL OR ("t"."d" IS NULL AND ("t"."d" IS NULL OR "t"."d" NOT IN (?))))))', + 'NOT ((("t"."d" IS NULL OR (("t"."d" IS NULL OR ("t"."d" IS NULL AND ("t"."d" IS NULL OR "t"."d" NOT IN (?))))))))', ); }); it('requireValue polarity: a NULL column does NOT satisfy $null: false', () => { + // [ADR-0053 D-D1, amended — #5930 step 3] The shared lowering's + // `requireValue` guard, then this compiler's own — see the row above. expect(sql({ $not: { d: { $null: false, $nin: ['x'] } } })).toBe( - 'NOT (("t"."d" IS NOT NULL AND ("t"."d" IS NOT NULL AND ("t"."d" IS NULL OR "t"."d" NOT IN (?)))))', + 'NOT (("t"."d" IS NOT NULL AND ("t"."d" IS NOT NULL AND ("t"."d" IS NOT NULL AND ("t"."d" IS NULL OR "t"."d" NOT IN (?))))))', ); }); diff --git a/packages/services/service-analytics/src/__tests__/read-scope-not-null-safe.test.ts b/packages/services/service-analytics/src/__tests__/read-scope-not-null-safe.test.ts index 258e2af79d3..749d08e7b0a 100644 --- a/packages/services/service-analytics/src/__tests__/read-scope-not-null-safe.test.ts +++ b/packages/services/service-analytics/src/__tests__/read-scope-not-null-safe.test.ts @@ -240,7 +240,11 @@ describe('[#5297] read-scope `$not` — boolean identities and NULL safety', () it('the guard rides the leaf, so the emitted SQL negates a TOTAL predicate', () => { const { sql } = compileScopedFilterToSql({ $not: { stage: 'won' } } as FilterCondition, ALIAS); - expect(sql).toBe('NOT (("t"."stage" IS NOT NULL AND "t"."stage" = ?))'); + // [ADR-0053 D-D1, amended — #5930 step 3] The shared lowering at this + // compiler's entry totalises the operand first; this compiler's own copy + // guards the leaf again. `X AND (X AND Y)` ≡ `X AND Y` until the copy's + // deletion card; the id sets in this block are the guarantee. + expect(sql).toBe('NOT (("t"."stage" IS NOT NULL AND ("t"."stage" IS NOT NULL AND "t"."stage" = ?)))'); }); it('`$not` over MULTIPLE columns admits a row that is NULL in EITHER', () => { @@ -399,8 +403,10 @@ describe('[#5297] read-scope `$not` — boolean identities and NULL safety', () * different row sets — the defect, not a smaller version of the fix. */ it('$ne / $nin / $notContains are NULL-safe outside a $not too (#5298)', () => { + // [ADR-0053 D-D1, amended — #5930 step 3] The shared lowering's NULL + // escape (outer) around this compiler's own (inner): the same rows. expect(compileScopedFilterToSql({ stage: { $ne: 'won' } } as FilterCondition, ALIAS).sql) - .toBe('("t"."stage" IS NULL OR "t"."stage" <> ?)'); + .toBe('(("t"."stage" IS NULL OR ("t"."stage" IS NULL OR "t"."stage" <> ?)))'); expect(ids({ stage: { $ne: 'won' } })).toEqual(['2', '3', '4']); expect(ids({ stage: { $nin: ['won'] } })).toEqual(['2', '3', '4']); expect(ids({ stage: { $notContains: 'wo' } })).toEqual(['2', '3', '4']); diff --git a/packages/services/service-analytics/src/__tests__/read-scope-placeholder-three-faces.test.ts b/packages/services/service-analytics/src/__tests__/read-scope-placeholder-three-faces.test.ts index 8b0a2436376..6b3bc33f509 100644 --- a/packages/services/service-analytics/src/__tests__/read-scope-placeholder-three-faces.test.ts +++ b/packages/services/service-analytics/src/__tests__/read-scope-placeholder-three-faces.test.ts @@ -482,8 +482,11 @@ describe('[#20075] `compileScopedFilterToSql` — the public export', () => { const UNCHANGED: Array<{ scope: unknown; sql: string; params: unknown[] }> = [ { scope: { owner: 'u_me' }, sql: '"deal"."owner" = ?', params: ['u_me'] }, { + // [ADR-0053 D-D1, amended — #5930 step 3] The shared lowering's NULL + // escape now wraps this compiler's own: the same rows, with or without + // a context — which is what this row is about. scope: { owner: { $ne: 'u_me' } }, - sql: '("deal"."owner" IS NULL OR "deal"."owner" <> ?)', + sql: '(("deal"."owner" IS NULL OR ("deal"."owner" IS NULL OR "deal"."owner" <> ?)))', params: ['u_me'], }, { scope: { region: { $in: ['emea', 'amer'] } }, sql: '"deal"."region" IN (?, ?)', params: ['emea', 'amer'] }, @@ -505,7 +508,7 @@ describe('[#20075] `compileScopedFilterToSql` — the public export', () => { it('with a context, a placeholder binds its resolved value', () => { expect(compile({ owner: { $ne: '{current_user_id}' } }, { context: MEMBER })).toEqual({ - sql: '("deal"."owner" IS NULL OR "deal"."owner" <> ?)', + sql: '(("deal"."owner" IS NULL OR ("deal"."owner" IS NULL OR "deal"."owner" <> ?)))', params: ['u_me'], }); }); diff --git a/packages/services/service-analytics/src/__tests__/read-scope-shared-lowering-seam.test.ts b/packages/services/service-analytics/src/__tests__/read-scope-shared-lowering-seam.test.ts index 54bc86b34ab..464ad64dce1 100644 --- a/packages/services/service-analytics/src/__tests__/read-scope-shared-lowering-seam.test.ts +++ b/packages/services/service-analytics/src/__tests__/read-scope-shared-lowering-seam.test.ts @@ -48,8 +48,9 @@ * — it moves no answer. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll, vi } from 'vitest'; import { SqliteWasmDriver } from '@objectstack/driver-sqlite-wasm'; +import type { ExecutionContext } from '@objectstack/spec/kernel'; import type { Cube, FilterCondition } from '@objectstack/spec/data'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; @@ -206,6 +207,23 @@ describe('[ADR-0053 D-D1 amended — #5930 step 3] the read scope lowers its fil expect(params).toEqual(['2026-07-29']); }); + it('a date macro resolves first, and the lowering widens the day it resolved to (item 3)', () => { + vi.useFakeTimers({ toFake: ['Date'] }); + try { + vi.setSystemTime(new Date('2026-03-15T23:30:00.000Z')); + const scope = { signed_at: { $lte: '{today}' } } as FilterCondition; + expect(compileScopedFilterToSql(scope, ALIAS, { declaredValueShape })).toEqual({ + sql: '"t"."signed_at" < ?', + params: ['2026-03-16'], + }); + // The caller's calendar decides the day, and the bound is the day after it. + const shanghai = { timezone: 'Asia/Shanghai' } as ExecutionContext; + expect(compileScopedFilterToSql(scope, ALIAS, { declaredValueShape, context: shanghai }).params).toEqual(['2026-03-17']); + } finally { + vi.useRealTimers(); + } + }); + it('with no declarations handed in, no column reads as datetime and the bound compiles as written', () => { const { sql, params } = compileScopedFilterToSql({ signed_at: { $lte: '2026-07-28' } }, ALIAS); expect(sql).toBe('"t"."signed_at" <= ?'); diff --git a/packages/services/service-analytics/src/__tests__/text-match-sqlite-nul.test.ts b/packages/services/service-analytics/src/__tests__/text-match-sqlite-nul.test.ts index 45554823a81..b07fcce130a 100644 --- a/packages/services/service-analytics/src/__tests__/text-match-sqlite-nul.test.ts +++ b/packages/services/service-analytics/src/__tests__/text-match-sqlite-nul.test.ts @@ -305,9 +305,11 @@ describe('[#20025] the compiled constructs, per shape', () => { it('`$contains` / `$notContains` / `$icontains` take instr() and bind the comparand raw', async () => { expect(scope({ v: { $contains: 'a*b' } } as FilterCondition)) .toEqual({ sql: 'instr("t"."v", ?) > 0', params: ['a*b'] }); - // The #5298 NULL-safe wrapper composes around the negated construct unchanged. + // The #5298 NULL-safe wrapper composes around the negated construct unchanged + // — [ADR-0053 D-D1, amended — #5930 step 3] inside the shared lowering's own + // escape, which now reaches this compiler first. expect(scope({ v: { $notContains: 'a*b' } } as FilterCondition)) - .toEqual({ sql: '("t"."v" IS NULL OR NOT (instr("t"."v", ?) > 0))', params: ['a*b'] }); + .toEqual({ sql: '(("t"."v" IS NULL OR ("t"."v" IS NULL OR NOT (instr("t"."v", ?) > 0))))', params: ['a*b'] }); expect(scope({ v: { $icontains: 'A*b' } } as FilterCondition)) .toEqual({ sql: 'instr(lower("t"."v"), lower(?)) > 0', params: ['A*b'] }); const n = await native({ v: { $contains: 'a*b' } } as FilterCondition); diff --git a/packages/services/service-analytics/src/__tests__/text-operator-case-exactness.test.ts b/packages/services/service-analytics/src/__tests__/text-operator-case-exactness.test.ts index 8060e65067a..b1ac0d138d4 100644 --- a/packages/services/service-analytics/src/__tests__/text-operator-case-exactness.test.ts +++ b/packages/services/service-analytics/src/__tests__/text-operator-case-exactness.test.ts @@ -293,8 +293,10 @@ describe('[#15684] the compiled TEXT, per dialect', () => { .toEqual({ sql: 'instr("t"."name", ?) > 0', params: ['acme'] }); // `$notContains` keeps the read scope's NULL-safe wrapper around the // negated construct — the polarity moved, the #5298 rule did not. + // [ADR-0053 D-D1, amended — #5930 step 3] …inside the shared lowering's own + // escape, which now reaches this compiler first. expect(compileScopedFilterToSql({ name: { $notContains: 'acme' } } as FilterCondition, 't', { dialect: 'sqlite' })) - .toEqual({ sql: '("t"."name" IS NULL OR NOT (instr("t"."name", ?) > 0))', params: ['acme'] }); + .toEqual({ sql: '(("t"."name" IS NULL OR ("t"."name" IS NULL OR NOT (instr("t"."name", ?) > 0))))', params: ['acme'] }); const starts = await nativeSql({ name: { $startsWith: 'ACME' } }, 'sqlite'); expect(starts.sql).toContain('WHERE name GLOB $1'); expect(starts.params).toEqual(['ACME*']); diff --git a/packages/services/service-analytics/src/__tests__/text-operator-non-text-column.test.ts b/packages/services/service-analytics/src/__tests__/text-operator-non-text-column.test.ts index 36fc80eab13..e09c4359d66 100644 --- a/packages/services/service-analytics/src/__tests__/text-operator-non-text-column.test.ts +++ b/packages/services/service-analytics/src/__tests__/text-operator-non-text-column.test.ts @@ -172,17 +172,22 @@ describe('[#14079] read-scope-sql compiles the contract\'s constant for a declar }); it('$notContains compiles to 1 = 1 — the exact complement, NULL rows included', () => { - expect(compile({ score: { $notContains: '5' } } as FilterCondition)).toEqual({ sql: '1 = 1', params: [] }); + // [ADR-0053 D-D1, amended — #5930 step 3] Inside the shared lowering's + // NULL escape, which reaches this compiler first: `x IS NULL OR 1 = 1` is + // still TRUE for every row. + expect(compile({ score: { $notContains: '5' } } as FilterCondition)).toEqual({ sql: '(("t"."score" IS NULL OR 1 = 1))', params: [] }); }); it('composes with the NULL-safe $not rewrite: the negation of the constant is total', () => { // `nullSafeNegationOperand` guards the leaf first, then the constant // replaces the LIKE: TRUE for every row, what the JS faces answer for // `!contains` on a number; its mirror is FALSE for every row. + // [ADR-0053 D-D1, amended — #5930 step 3] …after the shared lowering's own + // guard on the operand, which reaches this compiler first. expect(compile({ $not: { score: { $contains: '5' } } } as FilterCondition).sql) - .toBe('NOT (("t"."score" IS NOT NULL AND 1 = 0))'); + .toBe('NOT (("t"."score" IS NOT NULL AND ("t"."score" IS NOT NULL AND 1 = 0)))'); expect(compile({ $not: { score: { $notContains: '5' } } } as FilterCondition).sql) - .toBe('NOT ((("t"."score" IS NULL OR 1 = 1)))'); + .toBe('NOT ((("t"."score" IS NULL OR (("t"."score" IS NULL OR 1 = 1)))))'); }); it('a text column beside it is untouched, and params stay aligned with the LIKE that IS bound', () => { @@ -194,8 +199,10 @@ describe('[#14079] read-scope-sql compiles the contract\'s constant for a declar it('without the option, or when the column is text, the LIKE is byte-identical to before', () => { expect(compileScopedFilterToSql({ score: { $contains: '5' } } as FilterCondition, ALIAS)) .toEqual({ sql: '"t"."score" LIKE ? ESCAPE ?', params: ['%5%', '\\'] }); + // [ADR-0053 D-D1, amended — #5930 step 3] The shared lowering's NULL escape + // around this compiler's own; the LIKE inside is the same bytes. expect(compile({ name: { $notContains: '5' } } as FilterCondition).sql) - .toBe('("t"."name" IS NULL OR "t"."name" NOT LIKE ? ESCAPE ?)'); + .toBe('(("t"."name" IS NULL OR ("t"."name" IS NULL OR "t"."name" NOT LIKE ? ESCAPE ?)))'); }); it('a comparand the contract refuses is refused AHEAD of the constant', () => { diff --git a/packages/services/service-analytics/src/__tests__/where-boolean-flag-refusal.test.ts b/packages/services/service-analytics/src/__tests__/where-boolean-flag-refusal.test.ts index 67ecb853cf6..6c3041ed7f2 100644 --- a/packages/services/service-analytics/src/__tests__/where-boolean-flag-refusal.test.ts +++ b/packages/services/service-analytics/src/__tests__/where-boolean-flag-refusal.test.ts @@ -44,7 +44,7 @@ import { DatasetSchema, type Dataset } from '@objectstack/spec/ui'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; import { declaredRefusalMessage, resolveThrownHttpError, serverFaultProvenance } from '@objectstack/types'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; import { evaluateAnalyticsQueryOverRows } from '../preview-evaluator.js'; @@ -94,7 +94,7 @@ const NON_BOOLEAN: Array<[string, unknown]> = [ ['2n', 2n], ]; -const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as never); +const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as never, NO_DATETIME_COLUMNS); function refusalOf(run: () => unknown): Refusal { let out: unknown; @@ -246,11 +246,14 @@ const IS_NULL: Record<'top' | 'not' | 'notNe', ControlFamily> = { engine: { $and: [{ $not: { stage: null } }] }, rows: ['r1', 'r3'], }, + // [ADR-0053 D-D1, amended — #5930 step 3] The shared lowering's `allowNull` + // escape on the `$not` operand now wraps this face's own copy of it: one more + // `stage IS NULL OR …` outermost. The rows are the family's, unchanged. notNe: { - tree: { kind: 'not', child: { kind: 'or', children: [notSet, { kind: 'and', children: [notSet, neLost] }] } }, - sql: `${SELECT}NOT ((stage IS NULL OR (stage IS NULL AND (stage IS NULL OR stage != $1))))${TAIL}`, + tree: { kind: 'not', child: { kind: 'or', children: [notSet, { kind: 'or', children: [notSet, { kind: 'and', children: [notSet, neLost] }] }] } }, + sql: `${SELECT}NOT ((stage IS NULL OR (stage IS NULL OR (stage IS NULL AND (stage IS NULL OR stage != $1)))))${TAIL}`, params: ['lost'], - engine: { $and: [{ $not: { $or: [{ stage: null }, { stage: null, $and: [{ $or: [{ stage: null }, { stage: { $ne: 'lost' } }] }] }] } }] }, + engine: { $and: [{ $not: { $or: [{ stage: null }, { $or: [{ stage: null }, { stage: null, $and: [{ $or: [{ stage: null }, { stage: { $ne: 'lost' } }] }] }] }] } }] }, rows: ['r1', 'r3'], }, }; @@ -263,11 +266,13 @@ const IS_NOT_NULL: Record<'top' | 'not' | 'notNe', ControlFamily> = { engine: { $and: [{ $not: { stage: { $ne: null } } }] }, rows: ['r2'], }, + // [ADR-0053 D-D1, amended — #5930 step 3] …and the `requireValue` guard the + // same way: one more `stage IS NOT NULL AND …`. Rows unchanged. notNe: { - tree: { kind: 'not', child: { kind: 'and', children: [set, { kind: 'and', children: [set, neLost] }] } }, - sql: `${SELECT}NOT ((stage IS NOT NULL AND (stage IS NOT NULL AND (stage IS NULL OR stage != $1))))${TAIL}`, + tree: { kind: 'not', child: { kind: 'and', children: [set, { kind: 'and', children: [set, { kind: 'and', children: [set, neLost] }] }] } }, + sql: `${SELECT}NOT ((stage IS NOT NULL AND (stage IS NOT NULL AND (stage IS NOT NULL AND (stage IS NULL OR stage != $1)))))${TAIL}`, params: ['lost'], - engine: { $and: [{ $not: { stage: { $ne: null }, $and: [{ stage: { $ne: null } }, { $or: [{ stage: null }, { stage: { $ne: 'lost' } }] }] } }] }, + engine: { $and: [{ $not: { stage: { $ne: null }, $and: [{ stage: { $ne: null } }, { stage: { $ne: null } }, { $or: [{ stage: null }, { stage: { $ne: 'lost' } }] }] } }] }, rows: ['r2', 'r3'], }, }; diff --git a/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts b/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts index 3b9e17af555..260a294355b 100644 --- a/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts +++ b/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts @@ -39,7 +39,7 @@ import { describe, it, expect } from 'vitest'; import { FILTER_LOGIC_CASES, FILTER_LOGIC_ROWS, type Cube, type FilterCondition } from '@objectstack/spec/data'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { collectFilterLeaves, normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; import { evaluateAnalyticsQueryOverRows } from '../preview-evaluator.js'; @@ -113,6 +113,16 @@ describe('[ADR-0053 D-D1 amended — #5930 step 3] F10: the where → tree face }); }); + it('a nested relation under $not is guarded on the dotted member its leaf reads, never on the relation key', () => { + // The nested spelling is this door's sugar (the engine refuses it); the + // door spells it dotted before the lowering reads it, so the lowering's + // guard lands on `account.region` — not on whatever `account` resolves to. + const members = (where: FilterCondition) => + [...new Set(collectFilterLeaves(tree(where) as never).map((l) => l.member))].sort(); + expect(members({ $not: { account: { region: 'NA' } } })).toEqual(['account.region']); + expect(members({ $not: { account: { owner: { name: 'x' } } } })).toEqual(['account.owner.name']); + }); + it('an instant is never widened', () => { expect(tree({ signed: { $lte: '2026-07-28T12:00:00.000Z' } })).toEqual( leaf('signed', 'lte', ['2026-07-28T12:00:00.000Z']), diff --git a/packages/services/service-analytics/src/__tests__/where-empty-operator.test.ts b/packages/services/service-analytics/src/__tests__/where-empty-operator.test.ts index 6b6e0ac591b..d35a48ae62c 100644 --- a/packages/services/service-analytics/src/__tests__/where-empty-operator.test.ts +++ b/packages/services/service-analytics/src/__tests__/where-empty-operator.test.ts @@ -26,7 +26,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import type { Cube, ValueShapeFieldDef } from '@objectstack/spec/data'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; import type { DatasetScopedStrategyContext } from '../strategies/types.js'; @@ -104,23 +104,23 @@ const query = (where: unknown): AnalyticsQuery => describe('[#20445] normalizeAnalyticsFilterTree — `$empty` lowers to its own valueless leaf', () => { it('$empty: true → an `empty` leaf, $empty: false → `notEmpty`', () => { - expect(normalizeAnalyticsFilterTree({ where: { name: { $empty: true } } })).toEqual({ + expect(normalizeAnalyticsFilterTree({ where: { name: { $empty: true } } }, NO_DATETIME_COLUMNS)).toEqual({ kind: 'leaf', member: 'name', operator: 'empty', values: [], }); - expect(normalizeAnalyticsFilterTree({ where: { name: { $empty: false } } })).toEqual({ + expect(normalizeAnalyticsFilterTree({ where: { name: { $empty: false } } }, NO_DATETIME_COLUMNS)).toEqual({ kind: 'leaf', member: 'name', operator: 'notEmpty', values: [], }); }); it('under $not the leaf takes no NULL guard: the arm is total on every consumer', () => { - expect(normalizeAnalyticsFilterTree({ where: { $not: { name: { $empty: true } } } })).toEqual({ + expect(normalizeAnalyticsFilterTree({ where: { $not: { name: { $empty: true } } } }, NO_DATETIME_COLUMNS)).toEqual({ kind: 'not', child: { kind: 'leaf', member: 'name', operator: 'empty', values: [] }, }); }); for (const flag of ['true', 0, null, [true], new Date(0)]) { it(`a non-boolean flag (${String(flag)}) is refused INVALID_FILTER / 400, with the null flags`, async () => { - const err = await refusalOf(() => normalizeAnalyticsFilterTree({ where: { name: { $empty: flag } } })); + const err = await refusalOf(() => normalizeAnalyticsFilterTree({ where: { name: { $empty: flag } } }, NO_DATETIME_COLUMNS)); expect(err.code).toBe('INVALID_FILTER'); expect(err.status).toBe(400); expect(err.message).toContain('Operator "$empty" on field "name" requires a boolean comparand'); diff --git a/packages/services/service-analytics/src/__tests__/where-equality-slot-list-refusal.test.ts b/packages/services/service-analytics/src/__tests__/where-equality-slot-list-refusal.test.ts index 0222851971d..526ee755385 100644 --- a/packages/services/service-analytics/src/__tests__/where-equality-slot-list-refusal.test.ts +++ b/packages/services/service-analytics/src/__tests__/where-equality-slot-list-refusal.test.ts @@ -38,7 +38,7 @@ import { assertListComparandShapes, type Cube } from '@objectstack/spec/data'; import { DatasetSchema, type Dataset } from '@objectstack/spec/ui'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; import { evaluateAnalyticsQueryOverRows } from '../preview-evaluator.js'; @@ -49,7 +49,7 @@ interface Refusal extends Error { status?: unknown; } -const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as never); +const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as never, NO_DATETIME_COLUMNS); function refusalOf(run: () => unknown): Refusal { let out: unknown; @@ -183,7 +183,9 @@ describe('[#19888] the neighbouring shapes compile exactly as before', () => { ['null under $eq — the same predicate', { stage: { $eq: null } }, leaf('stage', 'notSet', [])], ['a list under $in — the prescribed remedy', { stage: { $in: ['won', 'lost'] } }, leaf('stage', 'in', ['won', 'lost'])], ['the empty $in — the FALSE constant', { stage: { $in: [] } }, { kind: 'const', value: false }], - ['the empty $nin — the TRUE constant', { stage: { $nin: [] } }, { kind: 'const', value: true }], + // [ADR-0053 D-D1, amended — #5930 step 3] The TRUE constant, inside the + // shared lowering's NULL escape (`$nin` is negative-polarity): TRUE still. + ['the empty $nin — the TRUE constant', { stage: { $nin: [] } }, { kind: 'or', children: [leaf('stage', 'notSet', []), { kind: 'const', value: true }] }], ['a nested-relation scalar', { acct: { region: 'NA' } }, leaf('acct.region', 'equals', ['NA'])], ['a field reference under $eq (served on the engine path)', { amount: { $eq: { $field: 'budget' } } }, leaf('amount', 'equals', [{ $field: 'budget' }])], ]; diff --git a/packages/services/service-analytics/src/__tests__/where-face-arms-refusal.test.ts b/packages/services/service-analytics/src/__tests__/where-face-arms-refusal.test.ts index 874cbf5a798..01dc8867234 100644 --- a/packages/services/service-analytics/src/__tests__/where-face-arms-refusal.test.ts +++ b/packages/services/service-analytics/src/__tests__/where-face-arms-refusal.test.ts @@ -49,7 +49,7 @@ import { assertListComparandShapes, type Cube } from '@objectstack/spec/data'; import { DatasetSchema, type Dataset } from '@objectstack/spec/ui'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; -import { normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; import { evaluateAnalyticsQueryOverRows } from '../preview-evaluator.js'; @@ -60,7 +60,7 @@ interface Refusal extends Error { status?: unknown; } -const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as never); +const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as never, NO_DATETIME_COLUMNS); function refusalOf(run: () => unknown): Refusal { let out: unknown; @@ -255,7 +255,9 @@ describe('[#20010] the neighbouring shapes compile exactly as before', () => { ['a list under $in', { stage: { $in: ['won', 'lost'] } }, leaf('stage', 'in', ['won', 'lost'])], ['falsy $in members are values, not blanks', { stage: { $in: [0, '', false] } }, leaf('stage', 'in', [0, '', false])], ['the empty $in — the FALSE constant', { stage: { $in: [] } }, { kind: 'const', value: false }], - ['the empty $nin — the TRUE constant', { stage: { $nin: [] } }, { kind: 'const', value: true }], + // [ADR-0053 D-D1, amended — #5930 step 3] The TRUE constant, inside the + // shared lowering's NULL escape (`$nin` is negative-polarity): TRUE still. + ['the empty $nin — the TRUE constant', { stage: { $nin: [] } }, { kind: 'or', children: [leaf('stage', 'notSet', []), { kind: 'const', value: true }] }], ['a scalar ordering comparand', { amt: { $gt: 5 } }, leaf('amt', 'gt', [5])], ['a { $field } in an ordering slot (served on the engine path)', { amt: { $gt: { $field: 'id' } } }, leaf('amt', 'gt', [{ $field: 'id' }])], ['the implicit null — the has-no-value predicate', { stage: null }, leaf('stage', 'notSet', [])], diff --git a/packages/services/service-analytics/src/__tests__/where-type-face-refusal.test.ts b/packages/services/service-analytics/src/__tests__/where-type-face-refusal.test.ts index fb7ac9c9b45..5a585491cf0 100644 --- a/packages/services/service-analytics/src/__tests__/where-type-face-refusal.test.ts +++ b/packages/services/service-analytics/src/__tests__/where-type-face-refusal.test.ts @@ -46,7 +46,7 @@ import { normalizeFilterComparandTypes, type Cube } from '@objectstack/spec/data import { DatasetSchema, type Dataset } from '@objectstack/spec/ui'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; -import { lowerAnalyticsWhere, normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; +import { lowerAnalyticsWhere, normalizeAnalyticsFilterTree, NO_DATETIME_COLUMNS } from '../strategies/filter-normalizer.js'; import { NativeSQLStrategy } from '../strategies/native-sql-strategy.js'; import { ObjectQLStrategy } from '../strategies/objectql-strategy.js'; import { evaluateAnalyticsQueryOverRows } from '../preview-evaluator.js'; @@ -57,7 +57,7 @@ interface Refusal extends Error { status?: unknown; } -const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as never); +const tree = (where: unknown) => normalizeAnalyticsFilterTree({ where } as never, NO_DATETIME_COLUMNS); function refusalOf(run: () => unknown): Refusal { let out: unknown; diff --git a/packages/services/service-analytics/src/strategies/filter-normalizer.ts b/packages/services/service-analytics/src/strategies/filter-normalizer.ts index b424c0cb29b..bfa50896183 100644 --- a/packages/services/service-analytics/src/strategies/filter-normalizer.ts +++ b/packages/services/service-analytics/src/strategies/filter-normalizer.ts @@ -2388,7 +2388,88 @@ export function normalizeAnalyticsFilterTree( ): NormalizedFilterNode | null { const condition = lowerAnalyticsWhere(query); if (!condition) return null; - return buildNode(lowerFilterCondition(condition, lowering)); + return buildNode(lowerFilterCondition(spellNestedRelationsDotted(condition), lowering)); +} + +/** + * [ADR-0053 D-D1, amended — #5930 step 3] Spell every nested-relation field + * spec (`{ account: { region: 'NA' } }`) as the DOTTED members it names + * (`{ 'account.region': 'NA' }`), before the shared lowering reads the + * condition. + * + * The nested spelling is this door's own sugar: the schema accepts it, the + * engine refuses it on every driver, and {@link fieldLeaves} is what gives it a + * meaning here — it compiles each nested key as the dotted member. The shared + * lowering has no such reading: it takes `account` for a column and, inside a + * `$not`, guards it — `account IS NOT NULL`, a predicate on whatever the member + * `account` resolves to, which is not the member the leaf reads (the reason + * {@link guardFieldEntry} flattens before it guards). Spelled dotted first, + * the lowering guards `account.region`, the member the leaf binds. + * + * It rewrites only the spelling: every dotted member is the one + * {@link fieldLeaves} would have produced, in the same order, and a member that + * already has an entry keeps both, the second as an `$and` conjunct. An EMPTY + * spec is left as it is, for {@link fieldLeaves}' zero-operator refusal — + * flattening `{}` would make the constraint vanish. Copy-on-write: a condition + * with no nested relation comes back as the same object. + */ +function spellNestedRelationsDotted(node: Record): Record { + const isNonEmptyRelation = (spec: unknown): spec is Record => + isNestedRelationSpec(spec) && Object.keys(spec).length > 0; + const dottedPairs = (prefix: string, spec: Record): Array<[string, unknown]> => + Object.entries(spec).flatMap(([key, value]): Array<[string, unknown]> => { + const dotted = `${prefix}.${key}`; + return isNonEmptyRelation(value) ? dottedPairs(dotted, value) : [[dotted, value]]; + }); + + let changed = false; + const entries: Array<[string, unknown]> = []; + const conjuncts: Record[] = []; + const seen = new Set(); + const put = (key: string, value: unknown): void => { + if (seen.has(key)) conjuncts.push({ [key]: value }); + else { + seen.add(key); + entries.push([key, value]); + } + }; + for (const [key, spec] of Object.entries(node)) { + if ((key === '$and' || key === '$or') && Array.isArray(spec)) { + let copy: unknown[] | undefined; + spec.forEach((child, index) => { + if (!isFilterObject(child)) return; + const spelled = spellNestedRelationsDotted(child); + if (spelled !== child) { + copy ??= [...spec]; + copy[index] = spelled; + } + }); + if (copy) changed = true; + put(key, copy ?? spec); + continue; + } + if (key === '$not' && isFilterObject(spec)) { + const spelled = spellNestedRelationsDotted(spec); + if (spelled !== spec) changed = true; + put(key, spelled); + continue; + } + if (!key.startsWith('$') && isNonEmptyRelation(spec)) { + changed = true; + for (const [dotted, value] of dottedPairs(key, spec)) put(dotted, value); + continue; + } + put(key, spec); + } + if (!changed) return node; + const out: Record = Object.fromEntries(entries); + if (conjuncts.length > 0) { + // A malformed `$and` is {@link buildNode}'s to refuse, in its own words; + // folding a conjunct into it would replace the shape it refuses. + if ('$and' in out && !Array.isArray(out.$and)) return node; + out.$and = [...((out.$and as unknown[] | undefined) ?? []), ...conjuncts]; + } + return out; } /** From b9c340f55cce9372ad2bbf0db7f3bcb71d1285e0 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 12:01:31 +0000 Subject: [PATCH 4/6] test(service-analytics,driver-memory): type the seam pins for tsc Claude-Session: https://claude.ai/code/session_01XY5uCwTjZj7884yYtyur4H Co-authored-by: Claude --- .../src/memory-analytics-shared-lowering-door.test.ts | 6 +++--- .../__tests__/where-door-shared-lowering-seam.test.ts | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/drivers/driver-memory/src/memory-analytics-shared-lowering-door.test.ts b/packages/drivers/driver-memory/src/memory-analytics-shared-lowering-door.test.ts index bfee5c22aab..b1a29d325a9 100644 --- a/packages/drivers/driver-memory/src/memory-analytics-shared-lowering-door.test.ts +++ b/packages/drivers/driver-memory/src/memory-analytics-shared-lowering-door.test.ts @@ -78,11 +78,11 @@ const DAY_ROWS = [ { id: 'r29', created_at: '2026-07-29T10:00:00.000Z' }, ]; -async function setup(rows: ReadonlyArray>) { +async function setup(rows: ReadonlyArray) { const driver = new InMemoryDriver({}); await driver.connect(); await driver.syncSchema('logic_row', { name: 'logic_row', fields: { created_at: { type: 'datetime' } } }); - for (const row of rows) await driver.create('logic_row', { ...row }); + for (const row of rows) await driver.create('logic_row', { ...(row as Record) }); const service = new MemoryAnalyticsService({ driver, cubes: [LOGIC_CUBE] }); return { driver, service }; } @@ -91,7 +91,7 @@ const cubeQuery = (where?: FilterCondition) => ({ cube: 'logic', measures: ['count'], dimensions: ['id'], ...(where === undefined ? {} : { where }) }) as any; /** The live query path, the cube's rows and the echo's WHERE, for one `where`. */ -async function answer(rows: ReadonlyArray>, where?: FilterCondition) { +async function answer(rows: ReadonlyArray, where?: FilterCondition) { const { driver, service } = await setup(rows); return { find: ids(await driver.find('logic_row', where === undefined ? {} : { where })), diff --git a/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts b/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts index 260a294355b..78f95d461e6 100644 --- a/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts +++ b/packages/services/service-analytics/src/__tests__/where-door-shared-lowering-seam.test.ts @@ -36,7 +36,7 @@ */ import { describe, it, expect } from 'vitest'; -import { FILTER_LOGIC_CASES, FILTER_LOGIC_ROWS, type Cube, type FilterCondition } from '@objectstack/spec/data'; +import { FILTER_LOGIC_CASES, FILTER_LOGIC_ROWS, type Cube, type FilterCondition, type FilterLoweringOptions } from '@objectstack/spec/data'; import type { AnalyticsQuery, StrategyContext } from '@objectstack/spec/contracts'; import { collectFilterLeaves, normalizeAnalyticsFilterTree } from '../strategies/filter-normalizer.js'; @@ -65,8 +65,8 @@ const CUBE: Cube = { } as unknown as Cube; /** The strategies' reader, by hand: `signed` is the one member whose column is declared `datetime`. */ -const TYPED = { isDatetimeColumn: (member: string) => member === 'signed' }; -const UNTYPED = { isDatetimeColumn: () => false }; +const TYPED: FilterLoweringOptions = { isDatetimeColumn: (member: string) => member === 'signed' }; +const UNTYPED: FilterLoweringOptions = { isDatetimeColumn: () => false }; const leaf = (member: string, operator: string, values: unknown[]) => ({ kind: 'leaf', member, operator, values }); @@ -182,11 +182,11 @@ describe('[ADR-0053 D-D1 amended — #5930 step 3] F11: the draft preview evalua public: true, } as unknown as Cube; - const previewIds = (where: FilterCondition, rows: ReadonlyArray>): string[] => + const previewIds = (where: FilterCondition, rows: ReadonlyArray): string[] => evaluateAnalyticsQueryOverRows( { cube: 'rows', dimensions: ['id'], measures: ['n'], where } as AnalyticsQuery, PREVIEW_CUBE, - rows.map((r) => ({ ...r })), + rows.map((r) => ({ ...(r as Record) })), ).rows.map((r) => String(r.id)).sort(); /** `stage` holds a value on p1, is null on p2, and is absent on p3. */ From 4f4a2ada717e1073a2d7b1aeba3422abc242acd9 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 12:11:03 +0000 Subject: [PATCH 5/6] chore(changeset): the analytics seams' shared lowering, and the cube face's new door Claude-Session: https://claude.ai/code/session_01XY5uCwTjZj7884yYtyur4H Co-authored-by: Claude --- .changeset/20810-analytics-seam-lowering.md | 21 +++++++++++++++++ .changeset/20810-memory-cube-face-door.md | 26 +++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 .changeset/20810-analytics-seam-lowering.md create mode 100644 .changeset/20810-memory-cube-face-door.md diff --git a/.changeset/20810-analytics-seam-lowering.md b/.changeset/20810-analytics-seam-lowering.md new file mode 100644 index 00000000000..9ad1394e3c3 --- /dev/null +++ b/.changeset/20810-analytics-seam-lowering.md @@ -0,0 +1,21 @@ +--- +'@objectstack/service-analytics': minor +--- + +The analytics seams now run the one shared filter lowering (`lowerFilterCondition`, `@objectstack/spec/data`) that ADR-0053 D-D1, as amended, places at every seam that runs the shared comparand doors: after the doors and after filter-token resolution, so each face compiles one lowered condition — the `$between` split, the whole-day upper bound on a bare `YYYY-MM-DD`, the last supported day, and the NULL-polarity guards. + +Clause-②: yes + +**The read scope (`compileScopedFilterToSql`).** The scope is lowered at the compiler's entry, right after its placeholders resolve. It reads each column's declared type from the `declaredValueShape` option both of its consumers already pass, so the whole-day rule rewrites a declared `datetime` column and nothing else; with no declarations handed in, no column is read as `datetime`. Corrected answers, each now the rows `SqlDriver.find` returns for the same filter: + +- a bare-day `$lte` on a declared `datetime` column kept only the rows before that day and dropped the day itself. Rows at 10:00Z on 07-27, 07-28 and 07-29 under `{ signed_at: { $lte: '2026-07-28' } }` answered 07-27 alone; they now answer 07-27 and 07-28, compiled as `< '2026-07-29'`. This is the NativeSQL statement's read scope and the `/analytics/sql` echo's. +- a bare-day `$between` on a declared `datetime` column answered no row for a one-day range, and now answers that day's rows. +- a `{today}` (or any date-macro) upper bound is widened as the day it resolves to. + +An RLS `using` bound already reached the read scope lowered by the RLS compile seam, and answers as before. A declared `date` column compiles byte-identical to before. + +**The analytics `where` and draft-preview door.** The condition the door admits is lowered before either face reads it. The `where` → tree face (both strategies) reads each member's declared column type through the host's declared-type hook: a bare-day `$lte` on a `datetime` member now reaches the engine as `$lt` the next day, and the `/analytics/sql` echo prints that half-open bound — the statement the engine runs, where it used to print `<=` the named day. Rows are unchanged on every strategy. A nested-relation filter (`{ account: { region: 'NA' } }`) is spelled as the dotted member it has always compiled to before the lowering reads it, so a guard it adds under `$not` names that member. + +The draft preview (`queryDataset` with `previewDrafts`) now evaluates `$null` — the one operator the lowering emits that it did not — so a drafted chart filtered on `{ field: { $null: true } }` is answered instead of refused `INVALID_FILTER` / 400; `$exists` and `$empty` stay refused. Corrected answers: a row with no value now satisfies `$ne`, `$nin` and the negation of an equality even when the comparand is the text `"null"` or `"undefined"`, which this face used to compare as text against the missing value — the answer every data driver gives. Its bare-day bounds answer as before. + +Compiled SQL for `$ne`, `$nin`, `$notContains` and a `$not` operand now carries the lowering's NULL guard around each face's own copy of it: the same rows, a longer statement, until those copies are deleted. diff --git a/.changeset/20810-memory-cube-face-door.md b/.changeset/20810-memory-cube-face-door.md new file mode 100644 index 00000000000..68eec306f9e --- /dev/null +++ b/.changeset/20810-memory-cube-face-door.md @@ -0,0 +1,26 @@ +--- +'@objectstack/driver-memory': minor +--- + +`MemoryAnalyticsService` (the in-memory analytics cube face) now runs the two shared filter comparand doors every other analytics face runs, then the shared filter lowering of ADR-0053 D-D1 (as amended), before it compiles a query's `where`. It also compiles `$or` and `$null`, the two parts of the lowering's output it could not. + +Clause-②: yes (narrowing) + + + +**BREAKING**: `query()` and `generateSql()` now refuse, with `INVALID_FILTER` / 400, filter comparands they used to answer. Each is refused the same way by every other analytics face and by the query engine, so a filter that worked here worked nowhere else. Measured on a fixture where `d` is `'v1'`, `'v2'`, `null` and absent: + +- `undefined` in any comparand position — `{ d: undefined }` and `{ d: { $eq: undefined } }` answered the no-value rows, `{ d: { $ne: undefined } }` the valued ones, `{ d: { $in: ['v1', undefined] } }` row `v1`; +- a `null` member of `$in` / `$nin` (`{ d: { $in: ['v1', null] } }` answered `v1` and both no-value rows), and a `null` under an ordering operator (`{ d: { $gt: null } }` answered no row); +- a scalar where `$in` / `$nin` takes a list (`{ d: { $in: 'v1' } }`); +- a plain object, a `Map` or a binary value as a comparand — `{ d: { $ne: { a: 1 } } }` and `{ d: new Map() }` answered EVERY row. + +The fix is to write the comparand you mean: `null` or `{ $null: true }` for "has no value", a list for `$in` / `$nin` (and `{ $null: true }` in a `$or` for "one of these, or no value"), a scalar for an ordering operator. + +Corrected answers, each now what the live query path (`find()`) returns: + +- a bigint comparand within 2^53 (`{ n: { $gt: 2n } }`) is read as its number and answered; beyond 2^53 it is refused `INVALID_FILTER` / 400. Both used to fail with an uncoded error. +- `$between` is answered as its two bounds, with a bare-day maximum widened to the whole day before it is converted to the field's storage form; it was refused. +- `$null` (true: no value; false: has a value) and `$or` (a `{}` branch is TRUE, `$or: []` is FALSE) are compiled on both exits; they were refused. `$not`, `$startsWith`, `$endsWith` and `$empty` stay refused. `ANALYTICS_FILTER_CAPABILITIES` names `$null` and the `$or` combinator accordingly. + +The `generateSql()` echo and the `query()` pipeline dump for `$ne`, `$nin` and `$notContains` now show the lowering's NULL escape around this face's own guard — `(d IS NULL OR (d IS NULL OR d != 'v1'))` — the same rows as before. From edb3e2de482cf74a6d1c438b4ecd354fec64e799 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 30 Sep 2026 12:56:12 +0000 Subject: [PATCH 6/6] test(driver-memory): move the cube face's conformance pins to its widened vocabulary The claim's surface gained this file (amendment 5911719808) for exactly this change. Three pins encoded the cube face's vocabulary and shape from before the ruled widening; each is re-spelled, none removed: - every $not case must still refuse; the $or cases moved to the answered column, which the file's case-by-case invariant holds to find()'s rows; - $null joins the declared-operator probe roster; - the $notContains pipeline dump shows the lowering's NULL escape. Claude-Session: https://claude.ai/code/session_01XY5uCwTjZj7884yYtyur4H Co-authored-by: Claude --- ...memory-driver-filter-logic-conformance.test.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/drivers/driver-memory/src/memory-driver-filter-logic-conformance.test.ts b/packages/drivers/driver-memory/src/memory-driver-filter-logic-conformance.test.ts index 79369e9fcd0..9a14facc55e 100644 --- a/packages/drivers/driver-memory/src/memory-driver-filter-logic-conformance.test.ts +++ b/packages/drivers/driver-memory/src/memory-driver-filter-logic-conformance.test.ts @@ -264,7 +264,7 @@ describe('[#5345] MemoryAnalyticsService — the same table, through the THIRD f * ends: at least one case must be genuinely answered, and the shapes the cube * pipeline cannot express must be the ones refused. */ - it('at least one case is answered, and every combinator case is refused rather than dropped', async () => { + it('at least one case is answered, and every $not case is refused rather than dropped', async () => { const answered: string[] = []; const refused: string[] = []; for (const c of FILTER_LOGIC_CASES) { @@ -274,10 +274,13 @@ describe('[#5345] MemoryAnalyticsService — the same table, through the THIRD f expect(refused.length, 'nothing was refused — the silent drop is back').toBeGreaterThan(0); // Every case whose filter mentions a combinator this face cannot lower must // be in the refused column, by name — not merely "some things were refused". - const uncompilable = FILTER_LOGIC_CASES.filter((c) => /"\$(or|not)"/.test(JSON.stringify(c.filter))); + // [ADR-0053 D-D1, amended — #5930 step 3] `$or` left this set: the face + // compiles it now (the shared lowering emits it), so its cases moved to the + // agreeing column, which the case-by-case invariant above holds them to. + const uncompilable = FILTER_LOGIC_CASES.filter((c) => /"\$not"/.test(JSON.stringify(c.filter))); expect(uncompilable.length).toBeGreaterThan(0); for (const c of uncompilable) { - expect(refused, `${c.name}: a $or/$not case must refuse, never answer`).toContain(c.name); + expect(refused, `${c.name}: a $not case must refuse, never answer`).toContain(c.name); } }); }); @@ -646,6 +649,8 @@ const DECLARED_OPERATOR_PROBES: Record = { // this block's own header warns against. $icontains: { name: { $icontains: 'BET' } } as FilterCondition, $exists: { closed_at: { $exists: false } } as FilterCondition, + // [ADR-0053 D-D1, amended — #5930 step 3] `$null` joined the face's table. + $null: { closed_at: { $null: true } } as FilterCondition, }; describe('[#5374] operator semantics — the analytics face against the live query path', () => { @@ -815,7 +820,9 @@ describe('[#5374] operator semantics — the analytics face against the live que // classes rather than an `i` — this is what #6520 compiled, made visible. ['$icontains', { name: { $icontains: 'BET' } } as FilterCondition, '{"name":{"$regex":"/[Bb][Ee][Tt]/"}}'], // The negation still wraps a pattern, and now the pattern is legible. - ['$notContains', { name: { $notContains: 'et' } } as FilterCondition, '{"name":{"$not":{"$regex":"/et/"}}}'], + // [ADR-0053 D-D1, amended — #5930 step 3] …inside the shared lowering's NULL + // escape, which the face's door now applies before it compiles. + ['$notContains', { name: { $notContains: 'et' } } as FilterCondition, '{"$and":[{"$or":[{"name":{"$eq":null}},{"name":{"$not":{"$regex":"/et/"}}}]}]}'], // A comparand carrying a regex metacharacter shows its ESCAPE. `a.p` is a // literal here, not "any character between a and p" (#5567's direction), and // the dump is the only place an author can see which of the two ran.