From 6ed215cbc2cfb392f4e4e239c7c9d19f7c045452 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 15:47:34 +0000 Subject: [PATCH 1/8] =?UTF-8?q?fix(driver-sqlite-wasm):=20a=20text=20value?= =?UTF-8?q?=20crosses=20sql.js=20byte-for-byte=20=E2=80=94=20no=20truncati?= =?UTF-8?q?on=20at=20U+0000,=20no=20leading=20U+FEFF=20dropped=20on=20read?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- .../src/knex-wasm-dialect.ts | 19 +- .../src/sqljs-exact-text.ts | 233 ++++++++++++++++++ 2 files changed, 247 insertions(+), 5 deletions(-) create mode 100644 packages/drivers/driver-sqlite-wasm/src/sqljs-exact-text.ts diff --git a/packages/drivers/driver-sqlite-wasm/src/knex-wasm-dialect.ts b/packages/drivers/driver-sqlite-wasm/src/knex-wasm-dialect.ts index 3563fcf5680..7b91865c73f 100644 --- a/packages/drivers/driver-sqlite-wasm/src/knex-wasm-dialect.ts +++ b/packages/drivers/driver-sqlite-wasm/src/knex-wasm-dialect.ts @@ -26,6 +26,7 @@ import { createRequire } from 'node:module'; import type { SqlJsStatic } from 'sql.js'; +import { exactTextBindings, readExactRow } from './sqljs-exact-text.js'; import { WasmSqliteConnection, type PersistMode, @@ -229,7 +230,11 @@ export function getClient_WasmSqlite(): any { if (!connection) throw new Error('No connection provided'); const db = connection.raw; - const bindings = formatBindings(obj.bindings); + // A text binding holding U+0000 is rebound so sql.js's text bind cannot + // cut it at the NUL; `sql` is then the statement actually run. Every + // other statement comes back unchanged. The classification below keeps + // reading `obj.sql` — the rewrite never touches a statement's keyword. + const { sql, bindings } = exactTextBindings(obj.sql, formatBindings(obj.bindings)); // ── 1. EXECUTE ──────────────────────────────────────────────────────── // Three execution shapes. None of them decides persistence: that is @@ -244,7 +249,7 @@ export function getClient_WasmSqlite(): any { // return rows used by Knex's schema introspection/columnInfo, and // `db.run` discards those rows. if (DDL_RE.test(obj.sql)) { - db.run(obj.sql, bindings as any); + db.run(sql, bindings as any); obj.response = []; } else if ( isRowReturningExecution(obj.method, obj.returning) || @@ -252,12 +257,16 @@ export function getClient_WasmSqlite(): any { ) { // Row-returning branch. NOTE this is also where `INSERT … RETURNING *` // and `UPDATE … RETURNING *` land — statements that very much write. - const stmt = db.prepare(obj.sql); + const stmt = db.prepare(sql); try { if (bindings.length) stmt.bind(bindings as any); const rows: Record[] = []; + let columns: string[] | undefined; while (stmt.step()) { - rows.push(stmt.getAsObject()); + // Not `getAsObject()`: its text decode stops at U+0000 and drops a + // leading U+FEFF (see `sqljs-exact-text.ts`). + columns ??= stmt.getColumnNames(); + rows.push(readExactRow(stmt, columns)); } obj.response = rows; } finally { @@ -266,7 +275,7 @@ export function getClient_WasmSqlite(): any { } else { // Row-less write path: execute via `run` and capture SQLite's // per-connection lastID / changes counters. - db.run(obj.sql, bindings as any); + db.run(sql, bindings as any); const changes = db.getRowsModified(); let lastID: number | bigint = 0; if (obj.method === 'insert') { diff --git a/packages/drivers/driver-sqlite-wasm/src/sqljs-exact-text.ts b/packages/drivers/driver-sqlite-wasm/src/sqljs-exact-text.ts new file mode 100644 index 00000000000..e403643eb27 --- /dev/null +++ b/packages/drivers/driver-sqlite-wasm/src/sqljs-exact-text.ts @@ -0,0 +1,233 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Exact text transport for sql.js: a text value crosses this driver's engine + * boundary byte-for-byte, the way it crosses better-sqlite3's. + * + * ## What sql.js does to a text value, and where + * + * Measured on sql.js 1.14.1, the version the lockfile installs, against + * better-sqlite3 as the control (which round-trips every case below): + * + * - **WRITE — the bind.** `Statement.bindString` copies the JS string to the + * WASM heap and calls `sqlite3_bind_text(stmt, i, ptr, -1, …)`. A length of + * `-1` tells SQLite to read up to the first NUL byte, so `'a'` + U+0000 + + * `'b'` is stored as the one byte `61`, and `'ab'` + U+0000 as `6162`. A + * U+FEFF survives the bind: its bytes are not NUL. + * - **READ — the column decode.** `Statement.getString` returns + * `sqlite3_column_text` through `UTF8ToString`, which stops at the first NUL + * byte and decodes with a `TextDecoder` built with the default + * `ignoreBOM: false`. So a stored `610062` reads `'a'`, and a stored text + * that begins `EFBBBF` reads without its U+FEFF. `getAsObject` and `exec` + * both go through it. + * + * Neither raises. Both are silent changes to a value the caller wrote, which is + * why the answer here is a round trip and never a quieter variant of the loss. + * + * ## How each half is made exact + * + * sql.js has no length-aware text bind and no length-aware text read, so each + * half goes through the one sql.js path that does carry a byte length: + * + * - **Read** ({@link readExactRow}): a cell `get()` returned as a string is a + * TEXT cell, and `Statement.getBlob` on it answers `sqlite3_column_bytes` + * bytes of `sqlite3_column_blob` — the stored bytes, NUL and BOM included + * (SQLite hands a UTF-8 TEXT value to `column_blob` unconverted). They are + * decoded with `ignoreBOM: true`. `getBlob` is not in sql.js's typings, but + * it is the method sql.js's own `get()` calls for a BLOB column, and it keeps + * its name in all three 1.14.1 builds this package can load (`sql-wasm.js`, + * `sql-wasm-browser.js`, `sql-wasm-debug.js`); `getString` does not. Its + * absence is refused loudly rather than read around. + * - **Write** ({@link exactTextBindings}): a string that holds U+0000 — the + * only text the `-1` bind truncates — is bound as its UTF-8 bytes, which + * sql.js binds as a BLOB with an explicit length, and its placeholder is + * rewritten to `+CAST(? AS TEXT)`, so SQLite turns those bytes back into a + * TEXT value unchanged. The unary `+` matters: `CAST(… AS TEXT)` alone + * carries TEXT affinity and changes a comparison a bare bound text does not + * (`5 < ' x'` is 1 bound, 0 through a bare CAST); `+CAST(…)` has no affinity, + * like the bound text it stands in for. Every other binding, and every + * statement that binds no such string, is left exactly as it was. + * + * The rewrite needs the positional `?` a binding belongs to, so it reads the + * statement with SQLite's own tokenizer rules (quoted strings and identifiers, + * comments). When it cannot match placeholders to bindings one-for-one, it + * refuses the statement ({@link cannotPlaceExactTextError}) instead of guessing + * — a value bound to the wrong placeholder would be a worse defect than the + * truncation it replaces. + */ + +import type { Statement } from 'sql.js'; + +const NUL = String.fromCharCode(0x00); + +/** UTF-8 in; the leading U+FEFF, when present, is content — never a signature. */ +const EXACT_UTF8_DECODER = new TextDecoder('utf-8', { ignoreBOM: true }); +const UTF8_ENCODER = new TextEncoder(); + +/** The placeholder a U+0000-bearing text binding is moved into. */ +const EXACT_TEXT_PLACEHOLDER = '+CAST(? AS TEXT)'; + +/** Decode stored UTF-8 bytes exactly as better-sqlite3 hands them back. */ +export function decodeExactText(bytes: Uint8Array): string { + return EXACT_UTF8_DECODER.decode(bytes); +} + +/** The slice of a sql.js `Statement` the exact read needs. */ +type ExactReadStatement = Pick & { + getBlob?: (index: number) => Uint8Array; +}; + +/** + * Read the current row of a stepped statement into an object keyed by column + * name — `getAsObject()`'s shape — with every TEXT cell read from its stored + * bytes rather than through sql.js's NUL-terminated, BOM-stripping decode. + * + * `columns` is `stmt.getColumnNames()`, taken once per statement by the caller. + */ +export function readExactRow( + stmt: ExactReadStatement, + columns: readonly string[], +): Record { + const values = stmt.get(); + const row: Record = {}; + for (let i = 0; i < columns.length; i += 1) { + const value = values[i]; + row[columns[i]] = typeof value === 'string' ? decodeExactText(storedBytes(stmt, i)) : value; + } + return row; +} + +function storedBytes(stmt: ExactReadStatement, index: number): Uint8Array { + if (typeof stmt.getBlob !== 'function') { + throw new Error( + 'driver-sqlite-wasm cannot read a text value exactly: this sql.js build has no ' + + '`Statement.getBlob`, the only sql.js read that carries a byte length. Reading ' + + 'through `getString` instead would cut the value at an embedded U+0000 and drop a ' + + 'leading U+FEFF, so the read is refused. Install the sql.js version this package ' + + 'declares.', + ); + } + return stmt.getBlob(index); +} + +/** The positional parameters of a statement, located by SQLite's tokenizer rules. */ +interface ParameterScan { + /** Offsets of every bare `?` outside quotes and comments, in order. */ + readonly positional: readonly number[]; + /** A `?NNN`, `:name`, `@name` or `$name` parameter was seen as well. */ + readonly otherForms: boolean; +} + +const DIGIT_RE = /[0-9]/; +const IDENT_CHAR_RE = /[A-Za-z0-9_$]/; + +function scanParameters(sql: string): ParameterScan { + const positional: number[] = []; + let otherForms = false; + const n = sql.length; + let i = 0; + while (i < n) { + const ch = sql[i]; + if (ch === "'" || ch === '"' || ch === '`') { + // A quoted string or identifier; a doubled quote is an escaped quote. + i += 1; + while (i < n) { + if (sql[i] === ch) { + if (sql[i + 1] !== ch) break; + i += 1; + } + i += 1; + } + i += 1; + continue; + } + if (ch === '[') { + const close = sql.indexOf(']', i + 1); + i = close === -1 ? n : close + 1; + continue; + } + if (ch === '-' && sql[i + 1] === '-') { + const eol = sql.indexOf('\n', i + 2); + i = eol === -1 ? n : eol + 1; + continue; + } + if (ch === '/' && sql[i + 1] === '*') { + const end = sql.indexOf('*/', i + 2); + i = end === -1 ? n : end + 2; + continue; + } + if (ch === '?') { + if (DIGIT_RE.test(sql[i + 1] ?? '')) otherForms = true; + else positional.push(i); + } else if ( + (ch === ':' || ch === '@' || ch === '$') && + IDENT_CHAR_RE.test(sql[i + 1] ?? '') && + !IDENT_CHAR_RE.test(sql[i - 1] ?? '') + ) { + otherForms = true; + } + i += 1; + } + return { positional, otherForms }; +} + +/** + * The located refusal for a statement whose placeholders cannot be matched to + * its bindings while one of them is a text holding U+0000. ADR-0112 envelope: + * `code` + `status` at the throw site. + */ +export function cannotPlaceExactTextError( + positional: number, + bindings: number, + otherForms: boolean, +): Error & { code: string; status: number } { + const found = + `${positional} positional "?" parameter(s) for ${bindings} binding(s)` + + (otherForms ? ', plus a numbered or named parameter' : ''); + return Object.assign( + new Error( + `driver-sqlite-wasm cannot write a text value holding U+0000 through this statement: ` + + `it binds such a value exactly only through positional "?" parameters matched ` + + `one-for-one to the bindings, and this statement has ${found}. The statement was not ` + + `run, because sql.js's own text bind would have cut the value at the U+0000.`, + ), + { code: 'NOT_IMPLEMENTED', status: 501 }, + ); +} + +/** + * Rebind every text binding that holds U+0000 so it reaches SQLite whole. + * + * Returns `sql` and `bindings` unchanged — the same array — when no binding is + * such a string, which is every statement that does not carry one. Otherwise + * each such binding becomes its UTF-8 bytes and its `?` becomes + * `+CAST(? AS TEXT)`; see the module docblock for why that pair is exact. + */ +export function exactTextBindings( + sql: string, + bindings: unknown[], +): { sql: string; bindings: unknown[] } { + const truncatable: number[] = []; + for (let i = 0; i < bindings.length; i += 1) { + const b = bindings[i]; + if (typeof b === 'string' && b.includes(NUL)) truncatable.push(i); + } + if (truncatable.length === 0) return { sql, bindings }; + + const scan = scanParameters(sql); + if (scan.otherForms || scan.positional.length !== bindings.length) { + throw cannotPlaceExactTextError(scan.positional.length, bindings.length, scan.otherForms); + } + + const rebound = bindings.slice(); + let out = ''; + let from = 0; + for (const i of truncatable) { + const at = scan.positional[i]; + out += sql.slice(from, at) + EXACT_TEXT_PLACEHOLDER; + from = at + 1; + rebound[i] = UTF8_ENCODER.encode(bindings[i] as string); + } + out += sql.slice(from); + return { sql: out, bindings: rebound }; +} From dbf70091ea78663fe77cd8f250826f575a0d6040 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 15:50:06 +0000 Subject: [PATCH 2/8] test(driver-sqlite-wasm): pin the text write seam, the read seam and the round trip separately, with filters and refusals Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- .../sqlite-wasm-text-bytes-roundtrip.test.ts | 244 ++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts diff --git a/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts b/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts new file mode 100644 index 00000000000..988688889ac --- /dev/null +++ b/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts @@ -0,0 +1,244 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [#19978] A text value round-trips through `SqliteWasmDriver` byte-for-byte — + * an embedded U+0000 and a leading U+FEFF included — the way it does through + * `SqlDriver` on better-sqlite3. + * + * Before the fix, sql.js lost bytes at two separate seams (measured, sql.js + * 1.14.1): its text BIND cut a value at its first U+0000 (`'a'` + U+0000 + + * `'b'` stored as the one byte `61`), and its text READ both stopped at a + * stored NUL byte and dropped a leading U+FEFF. Neither raised. The mechanism + * and the fix are documented in `sqljs-exact-text.ts`. + * + * Each seam is pinned on its own, because one can hide the other: + * + * - the WRITE is judged on `hex(v)` — ASCII, so the read decode under test + * cannot repair or mask what was stored; + * - the READ is judged on cells planted by a SQL literal (`cast(x'…' as text)`), + * so no bind is involved — the shape of a database file a native SQLite wrote + * and this driver then opens; + * - the round trip and the filters are judged through the public driver doors. + * + * Every expected value is the value written, and every expected hex is the + * written string's own UTF-8 — the standard better-sqlite3 was measured to meet + * on every case below. + */ + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import type { DriverOptions } from '@objectstack/spec/data'; +import { SqliteWasmDriver } from './index.js'; +import { exactTextBindings, readExactRow } from './sqljs-exact-text.js'; + +const NUL = String.fromCharCode(0x00); +const BOM = String.fromCharCode(0xfeff); + +const BYPASS: DriverOptions = { bypassTenantAudit: true }; +const TABLE = 'text_bytes_roundtrip'; + +/** label → the exact string written. `plain` is the control that always worked. */ +const CASES: Readonly> = { + nul_mid: 'a' + NUL + 'b', + nul_trail: 'ab' + NUL, + bom_lead: BOM + 'hello', + bom_mid: 'he' + BOM + 'llo', + // Longer than 16 bytes: sql.js's debug build decodes short strings by hand + // and only longer ones through its BOM-stripping TextDecoder. + bom_lead_long: BOM + 'a value longer than sixteen bytes', + plain: 'plain', +}; + +const utf8Hex = (s: string) => Buffer.from(s, 'utf8').toString('hex').toUpperCase(); + +interface WireBearingError extends Error { + code?: string; + status?: number; +} + +describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', () => { + let driver: SqliteWasmDriver; + + const storedHex = async (label: string) => { + const rows = (await driver.execute(`select hex(v) as h from ${TABLE} where label = ?`, [ + label, + ])) as Array<{ h: string }>; + expect(rows, `one stored row for ${label}`).toHaveLength(1); + return rows[0].h; + }; + + const labelsWhere = async (where: Record) => { + const rows = (await driver.find(TABLE, { where } as any, BYPASS)) as Array<{ label: string }>; + return rows.map((r) => r.label).sort(); + }; + + beforeAll(async () => { + driver = new SqliteWasmDriver({ filename: ':memory:' }); + await driver.initObjects([ + { name: TABLE, fields: { label: { type: 'string' }, v: { type: 'text' } } }, + ]); + for (const [label, v] of Object.entries(CASES)) { + await driver.create(TABLE, { label, v }, BYPASS); + } + }, 60_000); + + afterAll(async () => { + await driver.disconnect(); + }); + + describe('create → findOne', () => { + for (const [label, wrote] of Object.entries(CASES)) { + it(`${label}: the stored bytes are the written string's UTF-8`, async () => { + expect(await storedHex(label)).toBe(utf8Hex(wrote)); + }); + + it(`${label}: reads back as the string written`, async () => { + const row = (await driver.findOne(TABLE, { where: { label } } as any, BYPASS)) as { + v: unknown; + } | null; + expect(typeof row?.v).toBe('string'); + expect(row?.v).toStrictEqual(wrote); + }); + } + }); + + describe('update', () => { + it('an update to a value holding U+0000 and a leading U+FEFF stores and reads it whole', async () => { + const wrote = BOM + 'x' + NUL + 'y' + NUL; + await driver.create(TABLE, { label: 'upd', v: 'before' }, BYPASS); + const [{ id }] = (await driver.find(TABLE, { where: { label: 'upd' } } as any, BYPASS)) as Array<{ + id: string; + }>; + const returned = (await driver.update(TABLE, id, { v: wrote }, BYPASS)) as { v: unknown }; + expect(returned.v).toStrictEqual(wrote); + expect(await storedHex('upd')).toBe(utf8Hex(wrote)); + const read = (await driver.findOne(TABLE, { where: { label: 'upd' } } as any, BYPASS)) as { + v: unknown; + }; + expect(read.v).toStrictEqual(wrote); + await driver.delete(TABLE, id, BYPASS); + }); + }); + + describe('filters on the value', () => { + for (const [label, wrote] of Object.entries(CASES)) { + it(`equality on ${label}'s exact value selects that row and no other`, async () => { + expect(await labelsWhere({ v: wrote })).toEqual([label]); + }); + } + + it('equality on a NUL-bearing value does not match its prefix — the stored value is whole', async () => { + // Before the fix the stored `nul_mid` was `a`, and a comparand bound + // through the same truncating bind was `a` too — so this read matched. + expect(await labelsWhere({ v: 'a' })).toEqual([]); + }); + + it('$in places each NUL-bearing comparand at its own placeholder', async () => { + expect(await labelsWhere({ v: { $in: ['plain', CASES.nul_trail, CASES.nul_mid] } })).toEqual([ + 'nul_mid', + 'nul_trail', + 'plain', + ]); + }); + + it('$contains U+FEFF selects every row holding one, and reads each back whole', async () => { + const rows = (await driver.find(TABLE, { where: { v: { $contains: BOM } } } as any, BYPASS)) as Array<{ + label: string; + v: string; + }>; + expect(rows.map((r) => r.label).sort()).toEqual(['bom_lead', 'bom_lead_long', 'bom_mid']); + for (const r of rows) expect(r.v).toStrictEqual(CASES[r.label]); + }); + + it('$startsWith U+FEFF selects exactly the rows that begin with one', async () => { + expect(await labelsWhere({ v: { $startsWith: BOM } })).toEqual(['bom_lead', 'bom_lead_long']); + }); + }); + + describe('the read seam alone — cells no bind ever touched', () => { + const PLANTED: Readonly> = { + planted_nul: { hex: '610062', reads: 'a' + NUL + 'b' }, + planted_bom: { hex: 'EFBBBF78', reads: BOM + 'x' }, + planted_bom_long: { + hex: 'EFBBBF' + utf8Hex('a value longer than sixteen bytes'), + reads: BOM + 'a value longer than sixteen bytes', + }, + planted_plain: { hex: utf8Hex('plain'), reads: 'plain' }, + }; + + beforeAll(async () => { + for (const [label, { hex }] of Object.entries(PLANTED)) { + await driver.execute( + `insert into ${TABLE} (id, label, v) values ('${label}', '${label}', cast(x'${hex}' as text))`, + ); + } + }); + + for (const [label, { hex, reads }] of Object.entries(PLANTED)) { + it(`a stored ${hex} reads back as its exact text`, async () => { + expect(await storedHex(label)).toBe(hex); + const row = (await driver.findOne(TABLE, { where: { label } } as any, BYPASS)) as { v: unknown }; + expect(row.v).toStrictEqual(reads); + }); + } + }); + + describe('the rewritten placeholder behaves as the bound text it replaces', () => { + it('it carries no affinity: an integer still sorts below a text, as against a bound text', async () => { + // SQLite orders INTEGER below TEXT when neither operand has affinity, so + // `5 < ` is 1 for a bound text. A bare `CAST(? AS TEXT)` has + // TEXT affinity, turns 5 into '5', and answers 0 for a text starting + // below '5' — which is why the placeholder is `+CAST(? AS TEXT)`. + const rows = (await driver.execute('select 5 < ? as r', [' ' + NUL])) as Array<{ r: number }>; + expect(rows).toEqual([{ r: 1 }]); + }); + + it('a statement with no NUL-bearing text binding is run exactly as compiled', () => { + const sql = 'select * from t where a = ? and b = ?'; + const bindings = ['x', BOM + 'y']; + const out = exactTextBindings(sql, bindings); + expect(out.sql).toBe(sql); + expect(out.bindings).toBe(bindings); + }); + + it('only the NUL-bearing binding is moved, into its own placeholder, past quoted and commented ?', () => { + const sql = + "select '?' as q, `a?` from t /* ? */ where a = ? -- ?\n and b = ? and c = \"?\" and d = ?"; + const out = exactTextBindings(sql, ['x', 'y' + NUL, 'z']); + expect(out.sql).toBe( + "select '?' as q, `a?` from t /* ? */ where a = ? -- ?\n and b = +CAST(? AS TEXT) and c = \"?\" and d = ?", + ); + expect(out.bindings[0]).toBe('x'); + expect(Array.from(out.bindings[1] as Uint8Array)).toEqual([0x79, 0x00]); + expect(out.bindings[2]).toBe('z'); + }); + }); + + describe('refusals — a NUL-bearing text the rewrite cannot place is never bound truncated', () => { + it('a numbered parameter is refused with NOT_IMPLEMENTED / 501, through the driver', async () => { + let caught: WireBearingError | undefined; + try { + await driver.execute(`select ?1 as v`, ['a' + NUL + 'b']); + } catch (e) { + caught = e as WireBearingError; + } + expect(caught?.code).toBe('NOT_IMPLEMENTED'); + expect(caught?.status).toBe(501); + }); + + it('placeholders that do not match the bindings one-for-one are refused with NOT_IMPLEMENTED / 501', () => { + let caught: WireBearingError | undefined; + try { + exactTextBindings('select ? as a', ['a' + NUL, 'extra']); + } catch (e) { + caught = e as WireBearingError; + } + expect(caught?.code).toBe('NOT_IMPLEMENTED'); + expect(caught?.status).toBe(501); + }); + + it('a sql.js build without Statement.getBlob refuses the read instead of decoding through getString', () => { + const withoutGetBlob = { get: () => ['a'] }; + expect(() => readExactRow(withoutGetBlob as any, ['v'])).toThrow(/getBlob/); + }); + }); +}); From 88c69c0706a1d9fbaffa8c244db67df35c51f6ab Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 15:51:42 +0000 Subject: [PATCH 3/8] test(driver-sqlite-wasm): the update pin removes its row whatever its verdict Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- .../sqlite-wasm-text-bytes-roundtrip.test.ts | 21 ++++++++++++------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts b/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts index 988688889ac..57557226033 100644 --- a/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts +++ b/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts @@ -108,14 +108,19 @@ describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', const [{ id }] = (await driver.find(TABLE, { where: { label: 'upd' } } as any, BYPASS)) as Array<{ id: string; }>; - const returned = (await driver.update(TABLE, id, { v: wrote }, BYPASS)) as { v: unknown }; - expect(returned.v).toStrictEqual(wrote); - expect(await storedHex('upd')).toBe(utf8Hex(wrote)); - const read = (await driver.findOne(TABLE, { where: { label: 'upd' } } as any, BYPASS)) as { - v: unknown; - }; - expect(read.v).toStrictEqual(wrote); - await driver.delete(TABLE, id, BYPASS); + // Removed whatever the verdict: a row left behind would answer the + // filter pins below and fail them for a reason that is not theirs. + try { + const returned = (await driver.update(TABLE, id, { v: wrote }, BYPASS)) as { v: unknown }; + expect(returned.v).toStrictEqual(wrote); + expect(await storedHex('upd')).toBe(utf8Hex(wrote)); + const read = (await driver.findOne(TABLE, { where: { label: 'upd' } } as any, BYPASS)) as { + v: unknown; + }; + expect(read.v).toStrictEqual(wrote); + } finally { + await driver.delete(TABLE, id, BYPASS); + } }); }); From d64527eb02a36e62cc6bc3a189e7d16f2842d8eb Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 15:56:39 +0000 Subject: [PATCH 4/8] =?UTF-8?q?fix(driver-sqlite-wasm):=20number=20paramet?= =?UTF-8?q?ers=20as=20SQLite=20does,=20so=20every=20parameter=20form=20car?= =?UTF-8?q?ries=20a=20NUL-bearing=20text=20whole=20=E2=80=94=20no=20statem?= =?UTF-8?q?ent=20is=20refused?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- .../sqlite-wasm-text-bytes-roundtrip.test.ts | 68 ++++--- .../src/sqljs-exact-text.ts | 180 ++++++++++-------- 2 files changed, 147 insertions(+), 101 deletions(-) diff --git a/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts b/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts index 57557226033..c7e7492a526 100644 --- a/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts +++ b/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts @@ -50,11 +50,6 @@ const CASES: Readonly> = { const utf8Hex = (s: string) => Buffer.from(s, 'utf8').toString('hex').toUpperCase(); -interface WireBearingError extends Error { - code?: string; - status?: number; -} - describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', () => { let driver: SqliteWasmDriver; @@ -206,11 +201,13 @@ describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', }); it('only the NUL-bearing binding is moved, into its own placeholder, past quoted and commented ?', () => { + // None of `'it''s ?'`, `` `a?` ``, `[b?]`, `"?"`, the two comments or the + // identifier `c$d` holds a parameter, as SQLite tokenizes them. const sql = - "select '?' as q, `a?` from t /* ? */ where a = ? -- ?\n and b = ? and c = \"?\" and d = ?"; + "select 'it''s ?' as q, `a?`, [b?], c$d from t /* ? :x */ where a = ? -- ? @y\n and b = ? and c = \"?\" and d = ?"; const out = exactTextBindings(sql, ['x', 'y' + NUL, 'z']); expect(out.sql).toBe( - "select '?' as q, `a?` from t /* ? */ where a = ? -- ?\n and b = +CAST(? AS TEXT) and c = \"?\" and d = ?", + "select 'it''s ?' as q, `a?`, [b?], c$d from t /* ? :x */ where a = ? -- ? @y\n and b = +CAST(? AS TEXT) and c = \"?\" and d = ?", ); expect(out.bindings[0]).toBe('x'); expect(Array.from(out.bindings[1] as Uint8Array)).toEqual([0x79, 0x00]); @@ -218,29 +215,48 @@ describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', }); }); - describe('refusals — a NUL-bearing text the rewrite cannot place is never bound truncated', () => { - it('a numbered parameter is refused with NOT_IMPLEMENTED / 501, through the driver', async () => { - let caught: WireBearingError | undefined; - try { - await driver.execute(`select ?1 as v`, ['a' + NUL + 'b']); - } catch (e) { - caught = e as WireBearingError; - } - expect(caught?.code).toBe('NOT_IMPLEMENTED'); - expect(caught?.status).toBe(501); + describe('parameters numbered or named — the binding lands where SQLite numbers it', () => { + it('?NNN: the value bound to index 2 reaches ?2, whatever its position in the text', async () => { + const wrote = 'a' + NUL + 'b'; + const rows = (await driver.execute('select ?2 as first, ?1 as second', ['plain', wrote])) as Array<{ + first: unknown; + second: unknown; + }>; + expect(rows).toEqual([{ first: wrote, second: 'plain' }]); }); - it('placeholders that do not match the bindings one-for-one are refused with NOT_IMPLEMENTED / 501', () => { - let caught: WireBearingError | undefined; - try { - exactTextBindings('select ? as a', ['a' + NUL, 'extra']); - } catch (e) { - caught = e as WireBearingError; - } - expect(caught?.code).toBe('NOT_IMPLEMENTED'); - expect(caught?.status).toBe(501); + it(':name: the value bound to a named parameter reaches it', async () => { + const wrote = BOM + 'x' + NUL; + // knex counts `?` against the array, so the named parameter takes index + // 1 and the trailing `?` index 2, which nothing binds (SQL NULL). + const rows = (await driver.execute('select :v as named, ? as unbound', [wrote])) as Array<{ + named: unknown; + unbound: unknown; + }>; + expect(rows).toEqual([{ named: wrote, unbound: null }]); }); + it('every form is numbered as SQLite numbers it, and each occurrence of a NUL-bearing index is wrapped', () => { + // SQLite: ?2 → 2 (largest 2), ? → 3, :n → 4, ?1 → 1, :n → 4 again. + const sql = 'select ?2 as a, ? as b, :n as c, ?1 as d, :n as e'; + const out = exactTextBindings(sql, ['p', 'q' + NUL, 'r', 's' + NUL]); + expect(out.sql).toBe( + 'select +CAST(?2 AS TEXT) as a, ? as b, +CAST(:n AS TEXT) as c, ?1 as d, +CAST(:n AS TEXT) as e', + ); + expect(out.bindings[0]).toBe('p'); + expect(Array.from(out.bindings[1] as Uint8Array)).toEqual([0x71, 0x00]); + expect(out.bindings[2]).toBe('r'); + expect(Array.from(out.bindings[3] as Uint8Array)).toEqual([0x73, 0x00]); + }); + + it('a NUL-bearing binding no parameter receives is left for sql.js to answer as before', () => { + const out = exactTextBindings('select ? as a', ['x', 'y' + NUL]); + expect(out.sql).toBe('select ? as a'); + expect(out.bindings).toEqual(['x', 'y' + NUL]); + }); + }); + + describe('the read refuses rather than falls back', () => { it('a sql.js build without Statement.getBlob refuses the read instead of decoding through getString', () => { const withoutGetBlob = { get: () => ['a'] }; expect(() => readExactRow(withoutGetBlob as any, ['v'])).toThrow(/getBlob/); diff --git a/packages/drivers/driver-sqlite-wasm/src/sqljs-exact-text.ts b/packages/drivers/driver-sqlite-wasm/src/sqljs-exact-text.ts index e403643eb27..6f2b9e0405a 100644 --- a/packages/drivers/driver-sqlite-wasm/src/sqljs-exact-text.ts +++ b/packages/drivers/driver-sqlite-wasm/src/sqljs-exact-text.ts @@ -40,20 +40,26 @@ * absence is refused loudly rather than read around. * - **Write** ({@link exactTextBindings}): a string that holds U+0000 — the * only text the `-1` bind truncates — is bound as its UTF-8 bytes, which - * sql.js binds as a BLOB with an explicit length, and its placeholder is - * rewritten to `+CAST(? AS TEXT)`, so SQLite turns those bytes back into a - * TEXT value unchanged. The unary `+` matters: `CAST(… AS TEXT)` alone - * carries TEXT affinity and changes a comparison a bare bound text does not - * (`5 < ' x'` is 1 bound, 0 through a bare CAST); `+CAST(…)` has no affinity, - * like the bound text it stands in for. Every other binding, and every - * statement that binds no such string, is left exactly as it was. + * sql.js binds as a BLOB with an explicit length, and every parameter that + * receives it is wrapped as `+CAST( AS TEXT)`, so SQLite turns + * those bytes back into a TEXT value unchanged. The unary `+` matters: + * `CAST(… AS TEXT)` alone carries TEXT affinity and changes a comparison a + * bare bound text does not (`5 < ' x'` is 1 bound, 0 through a bare CAST); + * `+CAST(…)` has no affinity, like the bound text it stands in for. Every + * other binding, and every statement that binds no such string, is left + * exactly as it was. * - * The rewrite needs the positional `?` a binding belongs to, so it reads the - * statement with SQLite's own tokenizer rules (quoted strings and identifiers, - * comments). When it cannot match placeholders to bindings one-for-one, it - * refuses the statement ({@link cannotPlaceExactTextError}) instead of guessing - * — a value bound to the wrong placeholder would be a worse defect than the - * truncation it replaces. + * ## Which parameter receives which binding + * + * sql.js binds an array positionally: `bindings[i]` goes to parameter index + * `i + 1`. Which tokens carry that index is SQLite's rule, applied here as + * SQLite applies it: a bare `?` takes the largest index so far plus one; `?NNN` + * takes `NNN` (and raises the largest to it); a named parameter (`:name`, + * `@name`, `#name`, `$name`) takes the index of its first occurrence, or the + * largest plus one when new. Tokens inside quoted strings and identifiers and + * inside comments are not parameters. Wrapping a parameter in `+CAST(… AS + * TEXT)` adds and removes no parameter token, so every index — the wrapped + * ones' and all the others' — is the one SQLite assigned before the rewrite. */ import type { Statement } from 'sql.js'; @@ -64,9 +70,6 @@ const NUL = String.fromCharCode(0x00); const EXACT_UTF8_DECODER = new TextDecoder('utf-8', { ignoreBOM: true }); const UTF8_ENCODER = new TextEncoder(); -/** The placeholder a U+0000-bearing text binding is moved into. */ -const EXACT_TEXT_PLACEHOLDER = '+CAST(? AS TEXT)'; - /** Decode stored UTF-8 bytes exactly as better-sqlite3 hands them back. */ export function decodeExactText(bytes: Uint8Array): string { return EXACT_UTF8_DECODER.decode(bytes); @@ -110,26 +113,38 @@ function storedBytes(stmt: ExactReadStatement, index: number): Uint8Array { return stmt.getBlob(index); } -/** The positional parameters of a statement, located by SQLite's tokenizer rules. */ -interface ParameterScan { - /** Offsets of every bare `?` outside quotes and comments, in order. */ - readonly positional: readonly number[]; - /** A `?NNN`, `:name`, `@name` or `$name` parameter was seen as well. */ - readonly otherForms: boolean; +/** One parameter token of a statement, with the index SQLite assigns it. */ +interface ParameterToken { + readonly start: number; + readonly end: number; + /** 1-based, as SQLite numbers parameters and sql.js binds an array. */ + readonly index: number; } +/** SQLite's `IdChar`: ASCII letters and digits, `_`, `$`, and every non-ASCII code unit. */ +const ID_CHAR_RE = /[A-Za-z0-9_$\u0080-￿]/; const DIGIT_RE = /[0-9]/; -const IDENT_CHAR_RE = /[A-Za-z0-9_$]/; +const WHITESPACE_RE = /\s/; + +const isIdChar = (c: string | undefined) => c !== undefined && ID_CHAR_RE.test(c); -function scanParameters(sql: string): ParameterScan { - const positional: number[] = []; - let otherForms = false; +/** + * Every parameter token in `sql`, numbered by SQLite's rule (see the module + * docblock). Mirrors SQLite's tokenizer for the three things that decide it: + * quoted strings and identifiers (`'…'`, `"…"`, `` `…` `` with a doubled quote + * as an escape, and `[…]`), comments (`--` to end of line, `/* … *\/`), and the + * variable forms, including the Tcl `::` and `(…)` suffixes a named parameter + * may carry. + */ +function scanParameters(sql: string): ParameterToken[] { + const tokens: ParameterToken[] = []; + const named = new Map(); + let largest = 0; const n = sql.length; let i = 0; while (i < n) { const ch = sql[i]; if (ch === "'" || ch === '"' || ch === '`') { - // A quoted string or identifier; a doubled quote is an escaped quote. i += 1; while (i < n) { if (sql[i] === ch) { @@ -157,42 +172,57 @@ function scanParameters(sql: string): ParameterScan { continue; } if (ch === '?') { - if (DIGIT_RE.test(sql[i + 1] ?? '')) otherForms = true; - else positional.push(i); - } else if ( - (ch === ':' || ch === '@' || ch === '$') && - IDENT_CHAR_RE.test(sql[i + 1] ?? '') && - !IDENT_CHAR_RE.test(sql[i - 1] ?? '') - ) { - otherForms = true; + let end = i + 1; + while (end < n && DIGIT_RE.test(sql[end])) end += 1; + let index: number; + if (end === i + 1) { + largest += 1; + index = largest; + } else { + index = Number(sql.slice(i + 1, end)); + if (index > largest) largest = index; + } + tokens.push({ start: i, end, index }); + i = end; + continue; + } + // `$` is itself an identifier character, so it opens a parameter only where + // a token starts; `:`, `@` and `#` never belong to an identifier. + if ((ch === ':' || ch === '@' || ch === '#' || ch === '$') && !(ch === '$' && isIdChar(sql[i - 1]))) { + let end = i + 1; + let nameChars = 0; + while (end < n) { + const c = sql[end]; + if (isIdChar(c)) { + nameChars += 1; + end += 1; + } else if (c === '(' && nameChars > 0) { + let close = end + 1; + while (close < n && !WHITESPACE_RE.test(sql[close]) && sql[close] !== ')') close += 1; + end = sql[close] === ')' ? close + 1 : close; + break; + } else if (c === ':' && sql[end + 1] === ':') { + end += 2; + } else { + break; + } + } + if (nameChars > 0) { + const name = sql.slice(i, end); + let index = named.get(name); + if (index === undefined) { + largest += 1; + index = largest; + named.set(name, index); + } + tokens.push({ start: i, end, index }); + i = end; + continue; + } } i += 1; } - return { positional, otherForms }; -} - -/** - * The located refusal for a statement whose placeholders cannot be matched to - * its bindings while one of them is a text holding U+0000. ADR-0112 envelope: - * `code` + `status` at the throw site. - */ -export function cannotPlaceExactTextError( - positional: number, - bindings: number, - otherForms: boolean, -): Error & { code: string; status: number } { - const found = - `${positional} positional "?" parameter(s) for ${bindings} binding(s)` + - (otherForms ? ', plus a numbered or named parameter' : ''); - return Object.assign( - new Error( - `driver-sqlite-wasm cannot write a text value holding U+0000 through this statement: ` + - `it binds such a value exactly only through positional "?" parameters matched ` + - `one-for-one to the bindings, and this statement has ${found}. The statement was not ` + - `run, because sql.js's own text bind would have cut the value at the U+0000.`, - ), - { code: 'NOT_IMPLEMENTED', status: 501 }, - ); + return tokens; } /** @@ -200,33 +230,33 @@ export function cannotPlaceExactTextError( * * Returns `sql` and `bindings` unchanged — the same array — when no binding is * such a string, which is every statement that does not carry one. Otherwise - * each such binding becomes its UTF-8 bytes and its `?` becomes - * `+CAST(? AS TEXT)`; see the module docblock for why that pair is exact. + * each such binding becomes its UTF-8 bytes and every parameter token that + * receives it becomes `+CAST( AS TEXT)`; see the module docblock for why + * that pair is exact and why no other index moves. + * + * A binding no parameter receives is left as the string it was: sql.js then + * answers exactly what it answered before — a range error for an index past + * the statement's parameters, nothing at all for an index no token reads. */ export function exactTextBindings( sql: string, bindings: unknown[], ): { sql: string; bindings: unknown[] } { - const truncatable: number[] = []; + const truncatable = new Set(); for (let i = 0; i < bindings.length; i += 1) { const b = bindings[i]; - if (typeof b === 'string' && b.includes(NUL)) truncatable.push(i); - } - if (truncatable.length === 0) return { sql, bindings }; - - const scan = scanParameters(sql); - if (scan.otherForms || scan.positional.length !== bindings.length) { - throw cannotPlaceExactTextError(scan.positional.length, bindings.length, scan.otherForms); + if (typeof b === 'string' && b.includes(NUL)) truncatable.add(i + 1); } + if (truncatable.size === 0) return { sql, bindings }; const rebound = bindings.slice(); let out = ''; let from = 0; - for (const i of truncatable) { - const at = scan.positional[i]; - out += sql.slice(from, at) + EXACT_TEXT_PLACEHOLDER; - from = at + 1; - rebound[i] = UTF8_ENCODER.encode(bindings[i] as string); + for (const token of scanParameters(sql)) { + if (!truncatable.has(token.index)) continue; + out += `${sql.slice(from, token.start)}+CAST(${sql.slice(token.start, token.end)} AS TEXT)`; + from = token.end; + rebound[token.index - 1] = UTF8_ENCODER.encode(bindings[token.index - 1] as string); } out += sql.slice(from); return { sql: out, bindings: rebound }; From dd9a202afd9dab0aff736e78dc4f127c19730792 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 15:59:35 +0000 Subject: [PATCH 5/8] chore(changeset): driver-sqlite-wasm text round trip Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- .../19978-sqlite-wasm-text-roundtrip.md | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .changeset/19978-sqlite-wasm-text-roundtrip.md diff --git a/.changeset/19978-sqlite-wasm-text-roundtrip.md b/.changeset/19978-sqlite-wasm-text-roundtrip.md new file mode 100644 index 00000000000..5c1ce6c68fb --- /dev/null +++ b/.changeset/19978-sqlite-wasm-text-roundtrip.md @@ -0,0 +1,21 @@ +--- +"@objectstack/driver-sqlite-wasm": patch +--- + +fix(driver-sqlite-wasm): a text value now round-trips byte-for-byte, as it does through `SqlDriver` on better-sqlite3 — an embedded U+0000 no longer cuts the stored value short, and a leading U+FEFF is no longer dropped when the value is read (#19978) + +Clause-②: no + +`SqliteWasmDriver` changed a text value without raising, at two points in sql.js (measured on sql.js 1.14.1, the version the lockfile installs, against better-sqlite3, which round-trips every value below): + +- **Write.** sql.js binds a string with a length of `-1`, so SQLite stores it only up to the first U+0000: `'a'` + U+0000 + `'b'` was stored as the single byte `61`, and `'ab'` + U+0000 as `6162`. +- **Read.** sql.js decodes a text cell up to the first NUL byte, through a decoder that drops a leading byte-order mark: a stored `610062` read back as `'a'`, and a stored text beginning with U+FEFF read back without it. A U+FEFF was always stored, because the write keeps it. + +What changes: + +- Every text cell the driver reads is decoded from its stored bytes, so a U+0000 anywhere in it and a U+FEFF at its start come back as stored. This includes values already on disk: a stored text that begins with U+FEFF now reads back with it. +- A string value holding U+0000 is bound as its UTF-8 bytes, and the parameter that receives it becomes `+CAST( AS TEXT)`, so SQLite stores the same TEXT value better-sqlite3 stores, and compares it the same way. Positional `?`, numbered `?NNN` and named parameters are all numbered as SQLite numbers them. A statement that binds no such string runs exactly as before. +- An equality filter on such a value now compares the whole value. Before, the comparand was cut at the same U+0000 as the stored value, so `{ v: 'a' }` also matched a row written as `'a'` + U+0000 + `'b'`. It no longer does. +- If the loaded sql.js has no `Statement.getBlob` (the one sql.js read that carries a byte length), reading a text cell throws instead of decoding through the lossy path. sql.js 1.14.1 has it in its Node, browser and debug builds. + +What does not change: a value already stored cut short stays cut short. The bytes after the U+0000 were never written, so nothing can restore them. From 58ee6c042d0d39ad651eab494ea04735710d0a48 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 16:19:01 +0000 Subject: [PATCH 6/8] test(driver-sqlite-wasm): type the text round-trip queries as DriverQuery instead of erasing them Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- .../src/sqlite-wasm-text-bytes-roundtrip.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts b/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts index c7e7492a526..bc53a42dc9f 100644 --- a/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts +++ b/packages/drivers/driver-sqlite-wasm/src/sqlite-wasm-text-bytes-roundtrip.test.ts @@ -27,6 +27,7 @@ import { describe, it, expect, beforeAll, afterAll } from 'vitest'; import type { DriverOptions } from '@objectstack/spec/data'; +import type { DriverQuery } from '@objectstack/spec/contracts'; import { SqliteWasmDriver } from './index.js'; import { exactTextBindings, readExactRow } from './sqljs-exact-text.js'; @@ -61,8 +62,8 @@ describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', return rows[0].h; }; - const labelsWhere = async (where: Record) => { - const rows = (await driver.find(TABLE, { where } as any, BYPASS)) as Array<{ label: string }>; + const labelsWhere = async (where: DriverQuery['where']) => { + const rows = (await driver.find(TABLE, { where }, BYPASS)) as Array<{ label: string }>; return rows.map((r) => r.label).sort(); }; @@ -87,7 +88,7 @@ describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', }); it(`${label}: reads back as the string written`, async () => { - const row = (await driver.findOne(TABLE, { where: { label } } as any, BYPASS)) as { + const row = (await driver.findOne(TABLE, { where: { label } }, BYPASS)) as { v: unknown; } | null; expect(typeof row?.v).toBe('string'); @@ -100,7 +101,7 @@ describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', it('an update to a value holding U+0000 and a leading U+FEFF stores and reads it whole', async () => { const wrote = BOM + 'x' + NUL + 'y' + NUL; await driver.create(TABLE, { label: 'upd', v: 'before' }, BYPASS); - const [{ id }] = (await driver.find(TABLE, { where: { label: 'upd' } } as any, BYPASS)) as Array<{ + const [{ id }] = (await driver.find(TABLE, { where: { label: 'upd' } }, BYPASS)) as Array<{ id: string; }>; // Removed whatever the verdict: a row left behind would answer the @@ -109,7 +110,7 @@ describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', const returned = (await driver.update(TABLE, id, { v: wrote }, BYPASS)) as { v: unknown }; expect(returned.v).toStrictEqual(wrote); expect(await storedHex('upd')).toBe(utf8Hex(wrote)); - const read = (await driver.findOne(TABLE, { where: { label: 'upd' } } as any, BYPASS)) as { + const read = (await driver.findOne(TABLE, { where: { label: 'upd' } }, BYPASS)) as { v: unknown; }; expect(read.v).toStrictEqual(wrote); @@ -141,7 +142,7 @@ describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', }); it('$contains U+FEFF selects every row holding one, and reads each back whole', async () => { - const rows = (await driver.find(TABLE, { where: { v: { $contains: BOM } } } as any, BYPASS)) as Array<{ + const rows = (await driver.find(TABLE, { where: { v: { $contains: BOM } } }, BYPASS)) as Array<{ label: string; v: string; }>; @@ -176,7 +177,7 @@ describe('[#19978] driver-sqlite-wasm — text values round-trip byte-for-byte', for (const [label, { hex, reads }] of Object.entries(PLANTED)) { it(`a stored ${hex} reads back as its exact text`, async () => { expect(await storedHex(label)).toBe(hex); - const row = (await driver.findOne(TABLE, { where: { label } } as any, BYPASS)) as { v: unknown }; + const row = (await driver.findOne(TABLE, { where: { label } }, BYPASS)) as { v: unknown }; expect(row.v).toStrictEqual(reads); }); } From 6a195b37c6af9fcba265588530716f93157033bd Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 16:53:46 +0000 Subject: [PATCH 7/8] =?UTF-8?q?docs(driver-sql):=20correct=20the=20two=20p?= =?UTF-8?q?ending=20json-backfill=20sentences=20this=20change=20made=20fal?= =?UTF-8?q?se=20=E2=80=94=20sql.js=20now=20reads=20a=20leading=20U+FEFF=20?= =?UTF-8?q?and=20an=20embedded=20NUL=20back=20verbatim?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- .changeset/19912-json-backfill-depth-limit.md | 2 +- .changeset/19978-sqlite-wasm-text-roundtrip.md | 1 + packages/drivers/driver-sql/src/sql-driver.ts | 4 ++-- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.changeset/19912-json-backfill-depth-limit.md b/.changeset/19912-json-backfill-depth-limit.md index 50a39cd6671..b7ab7ec8901 100644 --- a/.changeset/19912-json-backfill-depth-limit.md +++ b/.changeset/19912-json-backfill-depth-limit.md @@ -7,6 +7,6 @@ fix(driver-sql): the local SQLite `Field.json` storage backfill no longer turns The backfill that converges legacy json cells on their JSON-encoded form (it came with the change that made the SQLite write path JSON-encode every json value; the issue number that change cites no longer resolves, and its live record is the `SqlDriver.backfillCanonicalJsonEncoding` doc block and `sql-driver-12380-json-roundtrip.test.ts`) ran one `UPDATE … set col = json_quote(col)` over every TEXT cell SQLite's `json_valid()` rejects. `json_valid()` answers 0 for JSON nested more than 1000 levels deep (SQLite's JSON depth limit in every build this repository bundles), while the driver reads such a cell with `JSON.parse` without trouble. So a deep array written correctly through the driver was quoted into a JSON string by the next `syncSchema` / `initObjects`, and read back as a string from then on — silently, with no error. -SQL now only pre-selects the candidate cells, a page at a time. The driver's own codec decides each one: a cell `JSON.parse` reads is left exactly as stored; a cell it cannot read is a legacy plain string and is rewritten to `JSON.stringify` of that string, byte-for-byte what the old statement wrote for it wherever the engine reads the stored bytes back verbatim. A cell the engine does not read back verbatim is left as stored and keeps reading as it did, where the old statement rewrote it: on `SqliteWasmDriver`, a legacy text with a leading U+FEFF or an embedded NUL (sql.js drops both when it reads the text); on any engine, text holding invalid UTF-8. Each rewrite is a compare-and-set on the text it was decided from, so a value written concurrently is never overwritten, and a re-run over a converged table still writes nothing. This covers every local SQLite face that inherits the backfill: `SqlDriver` on every client it treats as SQLite (`better-sqlite3`, `sqlite3` and its alias `sqlite`), `SqliteWasmDriver`, and `TursoDriver` in local mode. +SQL now only pre-selects the candidate cells, a page at a time. The driver's own codec decides each one: a cell `JSON.parse` reads is left exactly as stored; a cell it cannot read is a legacy plain string and is rewritten to `JSON.stringify` of that string, byte-for-byte what the old statement wrote for it wherever the engine reads the stored bytes back verbatim. A cell the engine does not read back verbatim is left as stored and keeps reading as it did, where the old statement rewrote it: on any engine, text holding invalid UTF-8. `SqlDriver` on better-sqlite3, `SqliteWasmDriver` (sql.js) and `TursoDriver` in local mode (libsql) were each measured to read such a cell back with U+FFFD in place of the invalid bytes, so the text the rewrite would be decided from is not the stored text, and the cell is left alone. A legacy text with a leading U+FEFF or an embedded NUL is read back verbatim on all three, and is rewritten like any other plain string. Each rewrite is a compare-and-set on the text it was decided from, so a value written concurrently is never overwritten, and a re-run over a converged table still writes nothing. This covers every local SQLite face that inherits the backfill: `SqlDriver` on every client it treats as SQLite (`better-sqlite3`, `sqlite3` and its alias `sqlite`), `SqliteWasmDriver`, and `TursoDriver` in local mode. The decision rule is exported from `@objectstack/driver-sql` as `recoverUnencodedJsonText(stored)`, and `@objectstack/driver-turso`'s remote codec-residue backfill now imports it instead of carrying its own copy, so the local and remote backfills apply one rule. That is a new public export on `@objectstack/driver-sql`'s root entry, and the reason this package takes `minor`: the function returns `null` for text `JSON.parse` accepts and `JSON.stringify(stored)` for text it rejects, and it is exported so that `SqlDriver.backfillCanonicalJsonEncoding` and the remote backfill's `recoverResidueCell` decide each cell by one shared rule rather than by two copies that could drift apart. The remote backfill's behaviour is unchanged. diff --git a/.changeset/19978-sqlite-wasm-text-roundtrip.md b/.changeset/19978-sqlite-wasm-text-roundtrip.md index 5c1ce6c68fb..e9b12b95aac 100644 --- a/.changeset/19978-sqlite-wasm-text-roundtrip.md +++ b/.changeset/19978-sqlite-wasm-text-roundtrip.md @@ -16,6 +16,7 @@ What changes: - Every text cell the driver reads is decoded from its stored bytes, so a U+0000 anywhere in it and a U+FEFF at its start come back as stored. This includes values already on disk: a stored text that begins with U+FEFF now reads back with it. - A string value holding U+0000 is bound as its UTF-8 bytes, and the parameter that receives it becomes `+CAST( AS TEXT)`, so SQLite stores the same TEXT value better-sqlite3 stores, and compares it the same way. Positional `?`, numbered `?NNN` and named parameters are all numbered as SQLite numbers them. A statement that binds no such string runs exactly as before. - An equality filter on such a value now compares the whole value. Before, the comparand was cut at the same U+0000 as the stored value, so `{ v: 'a' }` also matched a row written as `'a'` + U+0000 + `'b'`. It no longer does. +- The local `Field.json` storage backfill (`SqlDriver.backfillCanonicalJsonEncoding`) now converges a legacy json text cell holding a leading U+FEFF or an embedded U+0000 on this driver too: measured on the next `initObjects`, a stored `EFBBBF78` becomes `22EFBBBF7822` and a stored `610062` becomes `22615C75303030306222`, the same bytes better-sqlite3 writes, where before this change both were left as stored because sql.js did not read them back verbatim. - If the loaded sql.js has no `Statement.getBlob` (the one sql.js read that carries a byte length), reading a text cell throws instead of decoding through the lossy path. sql.js 1.14.1 has it in its Node, browser and debug builds. What does not change: a value already stored cut short stays cut short. The bytes after the U+0000 were never written, so nothing can restore them. diff --git a/packages/drivers/driver-sql/src/sql-driver.ts b/packages/drivers/driver-sql/src/sql-driver.ts index 16a8b92491b..b60f5b11721 100644 --- a/packages/drivers/driver-sql/src/sql-driver.ts +++ b/packages/drivers/driver-sql/src/sql-driver.ts @@ -11308,8 +11308,8 @@ export class SqlDriver implements IDataDriver { * the page read and the write is never overwritten. A page's writes commit as * one transaction. Because the new text is a function of the old text alone, * the compare-and-set also withholds, in the safe direction, a cell whose - * stored bytes the engine does not read back verbatim (sql.js drops an - * embedded NUL and a leading U+FEFF; any engine replaces invalid UTF-8): its + * stored bytes the engine does not read back verbatim (any engine replaces + * invalid UTF-8 with U+FFFD; measured on better-sqlite3, sql.js, libsql): its * bytes are not what was read, so it is not written, and it keeps reading as * it did. On better-sqlite3 every other text round-trips byte-for-byte, and * `JSON.stringify` of it is byte-identical to the `json_quote()` this method From 98cb908738599a72538e5f32df4531dc357e61d2 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:46:21 +0000 Subject: [PATCH 8/8] =?UTF-8?q?docs(driver-sql):=20name=20only=20the=20eng?= =?UTF-8?q?ines=20measured=20in=20the=20json-backfill=20sentences=20?= =?UTF-8?q?=E2=80=94=20TursoDriver=20local=20runs=20on=20better-sqlite3,?= =?UTF-8?q?=20not=20libsql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_01Bvd69VPa6puiNzzPUroDBx Co-authored-by: Claude --- .changeset/19912-json-backfill-depth-limit.md | 2 +- packages/drivers/driver-sql/src/sql-driver.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.changeset/19912-json-backfill-depth-limit.md b/.changeset/19912-json-backfill-depth-limit.md index b7ab7ec8901..650dc9dce6e 100644 --- a/.changeset/19912-json-backfill-depth-limit.md +++ b/.changeset/19912-json-backfill-depth-limit.md @@ -7,6 +7,6 @@ fix(driver-sql): the local SQLite `Field.json` storage backfill no longer turns The backfill that converges legacy json cells on their JSON-encoded form (it came with the change that made the SQLite write path JSON-encode every json value; the issue number that change cites no longer resolves, and its live record is the `SqlDriver.backfillCanonicalJsonEncoding` doc block and `sql-driver-12380-json-roundtrip.test.ts`) ran one `UPDATE … set col = json_quote(col)` over every TEXT cell SQLite's `json_valid()` rejects. `json_valid()` answers 0 for JSON nested more than 1000 levels deep (SQLite's JSON depth limit in every build this repository bundles), while the driver reads such a cell with `JSON.parse` without trouble. So a deep array written correctly through the driver was quoted into a JSON string by the next `syncSchema` / `initObjects`, and read back as a string from then on — silently, with no error. -SQL now only pre-selects the candidate cells, a page at a time. The driver's own codec decides each one: a cell `JSON.parse` reads is left exactly as stored; a cell it cannot read is a legacy plain string and is rewritten to `JSON.stringify` of that string, byte-for-byte what the old statement wrote for it wherever the engine reads the stored bytes back verbatim. A cell the engine does not read back verbatim is left as stored and keeps reading as it did, where the old statement rewrote it: on any engine, text holding invalid UTF-8. `SqlDriver` on better-sqlite3, `SqliteWasmDriver` (sql.js) and `TursoDriver` in local mode (libsql) were each measured to read such a cell back with U+FFFD in place of the invalid bytes, so the text the rewrite would be decided from is not the stored text, and the cell is left alone. A legacy text with a leading U+FEFF or an embedded NUL is read back verbatim on all three, and is rewritten like any other plain string. Each rewrite is a compare-and-set on the text it was decided from, so a value written concurrently is never overwritten, and a re-run over a converged table still writes nothing. This covers every local SQLite face that inherits the backfill: `SqlDriver` on every client it treats as SQLite (`better-sqlite3`, `sqlite3` and its alias `sqlite`), `SqliteWasmDriver`, and `TursoDriver` in local mode. +SQL now only pre-selects the candidate cells, a page at a time. The driver's own codec decides each one: a cell `JSON.parse` reads is left exactly as stored; a cell it cannot read is a legacy plain string and is rewritten to `JSON.stringify` of that string, byte-for-byte what the old statement wrote for it wherever the engine reads the stored bytes back verbatim. A cell the engine does not read back verbatim is left as stored and keeps reading as it did, where the old statement rewrote it: text holding invalid UTF-8, measured on better-sqlite3 and sql.js. `SqlDriver` on better-sqlite3, `TursoDriver` in local mode (which runs on better-sqlite3 too) and `SqliteWasmDriver` (sql.js) were each measured to read such a cell back with U+FFFD in place of the invalid bytes, so the text the rewrite would be decided from is not the stored text, and the cell is left alone. A legacy text with a leading U+FEFF or an embedded NUL is read back verbatim on all three faces, and is rewritten like any other plain string. Each rewrite is a compare-and-set on the text it was decided from, so a value written concurrently is never overwritten, and a re-run over a converged table still writes nothing. This covers every local SQLite face that inherits the backfill: `SqlDriver` on every client it treats as SQLite (`better-sqlite3`, `sqlite3` and its alias `sqlite`), `SqliteWasmDriver`, and `TursoDriver` in local mode. The decision rule is exported from `@objectstack/driver-sql` as `recoverUnencodedJsonText(stored)`, and `@objectstack/driver-turso`'s remote codec-residue backfill now imports it instead of carrying its own copy, so the local and remote backfills apply one rule. That is a new public export on `@objectstack/driver-sql`'s root entry, and the reason this package takes `minor`: the function returns `null` for text `JSON.parse` accepts and `JSON.stringify(stored)` for text it rejects, and it is exported so that `SqlDriver.backfillCanonicalJsonEncoding` and the remote backfill's `recoverResidueCell` decide each cell by one shared rule rather than by two copies that could drift apart. The remote backfill's behaviour is unchanged. diff --git a/packages/drivers/driver-sql/src/sql-driver.ts b/packages/drivers/driver-sql/src/sql-driver.ts index b60f5b11721..0b2caa5d3e0 100644 --- a/packages/drivers/driver-sql/src/sql-driver.ts +++ b/packages/drivers/driver-sql/src/sql-driver.ts @@ -11308,8 +11308,8 @@ export class SqlDriver implements IDataDriver { * the page read and the write is never overwritten. A page's writes commit as * one transaction. Because the new text is a function of the old text alone, * the compare-and-set also withholds, in the safe direction, a cell whose - * stored bytes the engine does not read back verbatim (any engine replaces - * invalid UTF-8 with U+FFFD; measured on better-sqlite3, sql.js, libsql): its + * stored bytes the engine does not read back verbatim (better-sqlite3 and + * sql.js, the engines measured, read invalid UTF-8 back as U+FFFD): its * bytes are not what was read, so it is not written, and it keeps reading as * it did. On better-sqlite3 every other text round-trips byte-for-byte, and * `JSON.stringify` of it is byte-identical to the `json_quote()` this method