From 6f30bcc4a320fac3ebfda4f363a20e85ad5d69de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Bramer=20Schmidt?= Date: Fri, 17 Jul 2026 21:15:56 +0700 Subject: [PATCH 1/2] Fix MySQL introspection failing on MariaDB servers Studio's tables introspection aggregated column metadata with json_arrayagg, which does not exist before MariaDB 10.5 (prisma/studio#1511) and returns LONGTEXT rather than native JSON on any MariaDB version, while cast(... as json) is invalid syntax there (prisma/studio#1367). The MySQL adapter now detects the server flavor once per adapter via select version() and, on MariaDB, aggregates columns with coalesce(concat('[', group_concat(json_object(...) separator ','), ']'), '[]') instead. String-aggregated columns payloads are parsed back into arrays on the client, which also hardens introspection against MySQL transports that return json_arrayagg results as strings. Version detection failures fall back to the existing MySQL SQL. Verified end-to-end against MariaDB 10.4 and 10.11 containers and the Vitess-backed MySQL integration suite. Co-Authored-By: Claude Fable 5 --- .changeset/mariadb-introspection-compat.md | 5 + FEATURES.md | 1 + data/mysql-core/adapter.ts | 42 ++- data/mysql-core/introspection.mariadb.test.ts | 271 ++++++++++++++++++ data/mysql-core/introspection.ts | 130 +++++++-- 5 files changed, 426 insertions(+), 23 deletions(-) create mode 100644 .changeset/mariadb-introspection-compat.md create mode 100644 data/mysql-core/introspection.mariadb.test.ts diff --git a/.changeset/mariadb-introspection-compat.md b/.changeset/mariadb-introspection-compat.md new file mode 100644 index 00000000..942570ba --- /dev/null +++ b/.changeset/mariadb-introspection-compat.md @@ -0,0 +1,5 @@ +--- +"@prisma/studio-core": patch +--- + +Fix MySQL introspection failing on MariaDB. The adapter now detects MariaDB via `select version()` and aggregates column metadata with `group_concat(json_object(...))` instead of `json_arrayagg`, which does not exist before MariaDB 10.5 and cannot be cast to JSON on any MariaDB version. String-aggregated introspection payloads are parsed on the client, which also hardens introspection against transports that return `json_arrayagg` results as strings. diff --git a/FEATURES.md b/FEATURES.md index 2acaa70e..1da99049 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -10,6 +10,7 @@ Each adapter handles introspection, querying, inserts, updates, and deletes whil Studio introspects connected databases to build schemas, tables, columns, relationships, filter operators, and timezone metadata. This gives users an accurate live model of the database and keeps table navigation grounded in current structure. A fresh Studio mount performs this discovery once, while actual adapter or database-availability changes invalidate cached metadata and load it again. +The MySQL adapter detects MariaDB servers via `select version()` and switches to a MariaDB-compatible column aggregation (`group_concat` of `json_object` parsed on the client), so introspection works on all supported MariaDB versions where `json_arrayagg` or JSON casts are unavailable. ## Deployable Prisma Postgres Demo diff --git a/data/mysql-core/adapter.ts b/data/mysql-core/adapter.ts index 1f2906ec..b4930097 100644 --- a/data/mysql-core/adapter.ts +++ b/data/mysql-core/adapter.ts @@ -40,10 +40,14 @@ import { getUpdateRefetchQuery, } from "./dml"; import { + detectMySQLServerFlavor, + getServerVersionQuery, getTablesQuery, getTimezoneQuery, mockTablesQuery, mockTimezoneQuery, + type MySQLServerFlavor, + normalizeTablesQueryResult, } from "./introspection"; import { lintMySQLWithExplainFallback } from "./sql-lint"; @@ -148,11 +152,41 @@ export function createMySQLAdapter( } } + let cachedServerFlavor: MySQLServerFlavor | null = null; + + async function detectServerFlavor( + options: Parameters[0], + ): Promise { + if (cachedServerFlavor) { + return cachedServerFlavor; + } + + try { + const [error, versions] = await executor.execute( + getServerVersionQuery(otherRequirements), + options, + ); + + if (error) { + // fall back to the MySQL introspection SQL without caching, so the + // next introspection retries the detection. + return "mysql"; + } + + cachedServerFlavor = detectMySQLServerFlavor(versions[0]?.version); + + return cachedServerFlavor; + } catch { + return "mysql"; + } + } + async function introspectDatabase( options: Parameters[0], ): Promise> { try { - const tablesQuery = getTablesQuery(otherRequirements); + const serverFlavor = await detectServerFlavor(options); + const tablesQuery = getTablesQuery(otherRequirements, serverFlavor); const timezoneQuery = getTimezoneQuery(otherRequirements); const [[tablesError, tables], [timezoneError, timezones]] = @@ -174,7 +208,11 @@ export function createMySQLAdapter( return [ null, - createIntrospection({ query: tablesQuery, tables, timezone }), + createIntrospection({ + query: tablesQuery, + tables: normalizeTablesQueryResult(tables), + timezone, + }), ]; } catch (error: unknown) { return createMySQLAdapterError({ error: error as Error }); diff --git a/data/mysql-core/introspection.mariadb.test.ts b/data/mysql-core/introspection.mariadb.test.ts new file mode 100644 index 00000000..e5088eb5 --- /dev/null +++ b/data/mysql-core/introspection.mariadb.test.ts @@ -0,0 +1,271 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { SequenceExecutor } from "../executor"; +import type { Query } from "../query"; +import { createMySQLAdapter } from "./adapter"; +import { + detectMySQLServerFlavor, + getServerVersionQuery, + getTablesQuery, + mockTablesQuery, + normalizeTablesQueryResult, +} from "./introspection"; + +describe("mysql-core MariaDB introspection compatibility", () => { + describe("detectMySQLServerFlavor", () => { + it("detects MariaDB from a plain MariaDB version string", () => { + expect(detectMySQLServerFlavor("10.4.34-MariaDB")).toBe("mariadb"); + }); + + it("detects MariaDB from a distribution-suffixed version string", () => { + expect( + detectMySQLServerFlavor("10.11.6-MariaDB-1:10.11.6+maria~ubu2204"), + ).toBe("mariadb"); + }); + + it("detects MariaDB from a replication-prefixed version string", () => { + expect(detectMySQLServerFlavor("5.5.5-10.5.23-MariaDB-log")).toBe( + "mariadb", + ); + }); + + it("detects MySQL from a MySQL version string", () => { + expect(detectMySQLServerFlavor("8.0.40")).toBe("mysql"); + expect(detectMySQLServerFlavor("8.0.40-vitess")).toBe("mysql"); + }); + + it("falls back to MySQL when the version is missing", () => { + expect(detectMySQLServerFlavor(undefined)).toBe("mysql"); + expect(detectMySQLServerFlavor(null)).toBe("mysql"); + expect(detectMySQLServerFlavor("")).toBe("mysql"); + }); + }); + + describe("getServerVersionQuery", () => { + it("selects the server version", () => { + expect(getServerVersionQuery()).toMatchInlineSnapshot(` + { + "parameters": [], + "sql": "select version() as \`version\`", + "transformations": undefined, + } + `); + }); + }); + + describe("getTablesQuery", () => { + it("uses json_arrayagg for MySQL", () => { + const query = getTablesQuery(undefined, "mysql"); + + expect(query.sql).toContain("json_arrayagg(json_object("); + expect(query.sql).not.toContain("group_concat"); + }); + + it("uses json_arrayagg for the default flavor", () => { + expect(getTablesQuery().sql).toContain("json_arrayagg(json_object("); + }); + + it("avoids json_arrayagg and json casts for MariaDB", () => { + const query = getTablesQuery(undefined, "mariadb"); + + // json_arrayagg only exists on MariaDB >= 10.5 (#1511) and + // cast(... as json) is invalid syntax on MariaDB (#1367). + expect(query.sql).not.toContain("json_arrayagg"); + expect(query.sql.toLowerCase()).not.toContain("as json)"); + expect(query.sql).toContain( + "coalesce(concat('[', group_concat(json_object(", + ); + expect(query.sql).toContain("separator ','), ']'), '[]')"); + }); + + it("keeps everything but the columns aggregation identical across flavors", () => { + const mysqlQuery = getTablesQuery(undefined, "mysql"); + const mariadbQuery = getTablesQuery(undefined, "mariadb"); + + expect(mariadbQuery.parameters).toEqual(mysqlQuery.parameters); + expect(mariadbQuery.sql).toBe( + mysqlQuery.sql.replace( + /json_arrayagg\((.*)\) as `columns`/, + "coalesce(concat('[', group_concat($1 separator ','), ']'), '[]') as `columns`", + ), + ); + }); + }); + + describe("normalizeTablesQueryResult", () => { + it("keeps already-parsed columns as-is", () => { + const tables = mockTablesQuery(); + + expect(normalizeTablesQueryResult(tables)).toEqual(tables); + }); + + it("parses string-aggregated columns payloads", () => { + const tables = mockTablesQuery(); + const stringified = tables.map((table) => ({ + ...table, + columns: JSON.stringify(table.columns), + })); + + expect(normalizeTablesQueryResult(stringified as never)).toEqual(tables); + }); + + it("throws a descriptive error for invalid columns payloads", () => { + const [table] = mockTablesQuery(); + + expect(() => + normalizeTablesQueryResult([ + { ...table, columns: "{ not json" as never }, + ]), + ).toThrowError(/animals/); + + expect(() => + normalizeTablesQueryResult([ + { ...table, columns: '{"not":"an array"}' as never }, + ]), + ).toThrowError(/animals/); + }); + }); + + describe("createMySQLAdapter introspect", () => { + function createRecordingExecutor(args: { + version?: string; + versionError?: Error; + stringifyColumns?: boolean; + }): { executor: SequenceExecutor; queries: Query[] } { + const queries: Query[] = []; + + const execute: SequenceExecutor["execute"] = (query) => { + queries.push(query); + + const sql = query.sql.toLowerCase(); + + if (sql.includes("version()")) { + if (args.versionError) { + return Promise.resolve([args.versionError]); + } + + return Promise.resolve([null, [{ version: args.version }] as never]); + } + + if (sql.includes("timezone")) { + return Promise.resolve([null, [{ timezone: "UTC" }] as never]); + } + + const tables = mockTablesQuery(); + + return Promise.resolve([ + null, + (args.stringifyColumns + ? tables.map((table) => ({ + ...table, + columns: JSON.stringify(table.columns), + })) + : tables) as never, + ]); + }; + + return { + executor: { + execute, + executeSequence: vi.fn() as SequenceExecutor["executeSequence"], + }, + queries, + }; + } + + it("uses the MariaDB-compatible tables query against MariaDB", async () => { + const { executor, queries } = createRecordingExecutor({ + version: "10.4.34-MariaDB", + stringifyColumns: true, + }); + const adapter = createMySQLAdapter({ executor }); + + const [error, result] = await adapter.introspect({}); + + expect(error).toBeNull(); + expect(result?.schemas["studio"]?.tables["animals"]).toBeDefined(); + expect( + result?.schemas["studio"]?.tables["animals"]?.columns["id"] + ?.isAutoincrement, + ).toBe(true); + + const tablesQuery = queries.find((query) => + query.sql.includes("information_schema"), + ); + + expect(tablesQuery?.sql).not.toContain("json_arrayagg"); + expect(tablesQuery?.sql).toContain("group_concat(json_object("); + }); + + it("keeps the MySQL tables query against MySQL", async () => { + const { executor, queries } = createRecordingExecutor({ + version: "8.0.40", + }); + const adapter = createMySQLAdapter({ executor }); + + const [error, result] = await adapter.introspect({}); + + expect(error).toBeNull(); + expect(result?.schemas["studio"]?.tables["animals"]).toBeDefined(); + + const tablesQuery = queries.find((query) => + query.sql.includes("information_schema"), + ); + + expect(tablesQuery?.sql).toContain("json_arrayagg(json_object("); + expect(tablesQuery?.sql).not.toContain("group_concat"); + }); + + it("falls back to the MySQL tables query when version detection fails", async () => { + const { executor, queries } = createRecordingExecutor({ + versionError: new Error("version() unavailable"), + }); + const adapter = createMySQLAdapter({ executor }); + + const [error, result] = await adapter.introspect({}); + + expect(error).toBeNull(); + expect(result?.schemas["studio"]?.tables["animals"]).toBeDefined(); + + const tablesQuery = queries.find((query) => + query.sql.includes("information_schema"), + ); + + expect(tablesQuery?.sql).toContain("json_arrayagg(json_object("); + }); + + it("detects the server flavor once per adapter", async () => { + const { executor, queries } = createRecordingExecutor({ + version: "10.11.6-MariaDB", + stringifyColumns: true, + }); + const adapter = createMySQLAdapter({ executor }); + + await adapter.introspect({}); + await adapter.introspect({}); + + const versionQueries = queries.filter((query) => + query.sql.toLowerCase().includes("version()"), + ); + + expect(versionQueries).toHaveLength(1); + }); + + it("parses string-aggregated columns even on MySQL transports", async () => { + const { executor } = createRecordingExecutor({ + version: "8.0.40", + stringifyColumns: true, + }); + const adapter = createMySQLAdapter({ executor }); + + const [error, result] = await adapter.introspect({}); + + expect(error).toBeNull(); + expect( + Object.keys( + result?.schemas["studio"]?.tables["animals"]?.columns ?? {}, + ), + ).toContain("id"); + }); + }); +}); diff --git a/data/mysql-core/introspection.ts b/data/mysql-core/introspection.ts index aefa0705..1f59925b 100644 --- a/data/mysql-core/introspection.ts +++ b/data/mysql-core/introspection.ts @@ -48,8 +48,48 @@ interface Database { }; } +/** + * The flavor of the connected MySQL-compatible server. + * + * MariaDB requires a different columns aggregation: `json_arrayagg` only + * exists on MariaDB >= 10.5 and `cast(... as json)` is invalid syntax there + * because JSON is an alias for LONGTEXT. + */ +export type MySQLServerFlavor = "mariadb" | "mysql"; + +/** + * Detects the server flavor from a `select version()` result. + * + * MariaDB reports versions like `10.4.34-MariaDB`, + * `10.11.6-MariaDB-1:10.11.6+maria~ubu2204` or, behind replication-compatible + * setups, `5.5.5-10.5.23-MariaDB-log`. Anything else is treated as MySQL. + */ +export function detectMySQLServerFlavor( + version: string | null | undefined, +): MySQLServerFlavor { + return typeof version === "string" && + version.toLowerCase().includes("mariadb") + ? "mariadb" + : "mysql"; +} + +export function getServerVersionQuery( + requirements?: Omit, +) { + const builder = getMySQLBuilder(requirements); + + return compile(builder.selectNoFrom(sql`version()`.as("version"))); +} + +export function mockServerVersionQuery() { + return [{ version: "8.0.40" }] as const satisfies QueryResult< + typeof getServerVersionQuery + >; +} + export function getTablesQuery( requirements?: Omit, + flavor: MySQLServerFlavor = "mysql", ) { const database = sql`database()`; @@ -110,33 +150,81 @@ export function getTablesQuery( "t.TABLE_TYPE as type", ]) .$narrowType<{ type: "BASE TABLE" | "VIEW" }>() - .select((eb) => - eb - .fn[number], "TABLE_NAME">[]>( - "json_arrayagg", - [ - jsonBuildObject({ - autoincrement: eb.ref("c.autoincrement"), - computed: eb.ref("c.computed"), - datatype: eb.ref("c.datatype"), - default: eb.ref("c.default"), - fk_column: eb.ref("c.fk_column"), - fk_table: eb.ref("c.fk_table"), - name: eb.ref("c.name"), - position: eb.ref("c.position"), - pk: eb.ref("c.pk"), - nullable: eb.ref("c.nullable"), - }), - ], - ) - .as("columns"), - ) + .select((eb) => { + type Columns = Omit< + InferResult[number], + "TABLE_NAME" + >[]; + + const columnsJson = jsonBuildObject({ + autoincrement: eb.ref("c.autoincrement"), + computed: eb.ref("c.computed"), + datatype: eb.ref("c.datatype"), + default: eb.ref("c.default"), + fk_column: eb.ref("c.fk_column"), + fk_table: eb.ref("c.fk_table"), + name: eb.ref("c.name"), + position: eb.ref("c.position"), + pk: eb.ref("c.pk"), + nullable: eb.ref("c.nullable"), + }); + + // MariaDB has no `json_arrayagg` before 10.5 (#1511) and no JSON cast + // type at all (#1367), so aggregate with `group_concat` into a JSON + // array string instead. The string payload is parsed back into an + // array by `normalizeTablesQueryResult`. + const aggregated = + flavor === "mariadb" + ? sql`coalesce(concat('[', group_concat(${columnsJson} separator ','), ']'), '[]')` + : sql`json_arrayagg(${columnsJson})`; + + return aggregated.as("columns"); + }) .orderBy("t.TABLE_SCHEMA") .orderBy("t.TABLE_NAME") .orderBy("t.TABLE_TYPE"), ); } +/** + * Normalizes the `columns` payload of a tables query result. + * + * On MariaDB the columns are aggregated into a JSON array string (see + * {@link getTablesQuery}), and some transports also return `json_arrayagg` + * results as strings instead of parsed arrays. This parses those string + * payloads so downstream consumers always receive arrays. + */ +export function normalizeTablesQueryResult( + tables: QueryResult, +): QueryResult { + return tables.map((table) => { + const { columns } = table; + + if (typeof columns !== "string") { + return table; + } + + let parsed: unknown; + + try { + parsed = JSON.parse(columns); + } catch (error: unknown) { + throw new Error( + `Failed to parse introspected columns for table "${table.name}".`, + { cause: error }, + ); + } + + if (!Array.isArray(parsed)) { + throw new Error( + `Expected introspected columns for table "${table.name}" to be an array.`, + ); + } + + return { ...table, columns: parsed }; + }); +} + export function mockTablesQuery() { return [ { From 9eccabc9c9d78770e7268fb0fcc437c6fc58325a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Bramer=20Schmidt?= Date: Sat, 18 Jul 2026 15:10:48 +0700 Subject: [PATCH 2/2] Make MariaDB introspection independent of group_concat_max_len Addresses PR review feedback: the group_concat-based aggregation was capped by the session's group_concat_max_len (1MB default on MariaDB, sometimes configured much lower), which could silently truncate the JSON payload on very wide tables. The MariaDB flavor now uses a dedicated tables query (getMariaDBTablesQuery) that returns one row per column with no JSON functions and no aggregation at all, and groupMariaDBTablesQueryResult groups the rows into the aggregated shape on the client. This is immune to any aggregation size cap and works on every MariaDB version. The MySQL tables query is restored to its original json_arrayagg form. Includes a regression test grouping over 1MB of column metadata, and was verified against MariaDB 10.4/10.11 containers including with group_concat_max_len lowered to 128 bytes. Co-Authored-By: Claude Fable 5 --- .changeset/mariadb-introspection-compat.md | 2 +- FEATURES.md | 2 +- data/mysql-core/adapter.ts | 54 ++++- data/mysql-core/introspection.mariadb.test.ts | 207 ++++++++++++++---- data/mysql-core/introspection.ts | 165 ++++++++++---- 5 files changed, 331 insertions(+), 99 deletions(-) diff --git a/.changeset/mariadb-introspection-compat.md b/.changeset/mariadb-introspection-compat.md index 942570ba..b38a27af 100644 --- a/.changeset/mariadb-introspection-compat.md +++ b/.changeset/mariadb-introspection-compat.md @@ -2,4 +2,4 @@ "@prisma/studio-core": patch --- -Fix MySQL introspection failing on MariaDB. The adapter now detects MariaDB via `select version()` and aggregates column metadata with `group_concat(json_object(...))` instead of `json_arrayagg`, which does not exist before MariaDB 10.5 and cannot be cast to JSON on any MariaDB version. String-aggregated introspection payloads are parsed on the client, which also hardens introspection against transports that return `json_arrayagg` results as strings. +Fix MySQL introspection failing on MariaDB. The adapter now detects MariaDB via `select version()` and uses a dedicated tables query that returns one row per column and groups the result on the client, avoiding `json_arrayagg` (missing before MariaDB 10.5), JSON casts (invalid syntax on MariaDB), and any server-side string aggregation that would be truncated at `group_concat_max_len`. Introspection also parses string-encoded `json_arrayagg` payloads returned by some MySQL transports. diff --git a/FEATURES.md b/FEATURES.md index 1da99049..2a18b4e3 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -10,7 +10,7 @@ Each adapter handles introspection, querying, inserts, updates, and deletes whil Studio introspects connected databases to build schemas, tables, columns, relationships, filter operators, and timezone metadata. This gives users an accurate live model of the database and keeps table navigation grounded in current structure. A fresh Studio mount performs this discovery once, while actual adapter or database-availability changes invalidate cached metadata and load it again. -The MySQL adapter detects MariaDB servers via `select version()` and switches to a MariaDB-compatible column aggregation (`group_concat` of `json_object` parsed on the client), so introspection works on all supported MariaDB versions where `json_arrayagg` or JSON casts are unavailable. +The MySQL adapter detects MariaDB servers via `select version()` and switches to a MariaDB-compatible tables query that returns one row per column and groups the result on the client, so introspection works on all supported MariaDB versions where `json_arrayagg` or JSON casts are unavailable and cannot be truncated by `group_concat_max_len`. ## Deployable Prisma Postgres Demo diff --git a/data/mysql-core/adapter.ts b/data/mysql-core/adapter.ts index b4930097..2200bcbd 100644 --- a/data/mysql-core/adapter.ts +++ b/data/mysql-core/adapter.ts @@ -41,9 +41,11 @@ import { } from "./dml"; import { detectMySQLServerFlavor, + getMariaDBTablesQuery, getServerVersionQuery, getTablesQuery, getTimezoneQuery, + groupMariaDBTablesQueryResult, mockTablesQuery, mockTimezoneQuery, type MySQLServerFlavor, @@ -181,19 +183,53 @@ export function createMySQLAdapter( } } + /** + * Fetches table metadata using the tables query that matches the server + * flavor. MariaDB gets a one-row-per-column query grouped on the client, so + * results can never be truncated by `group_concat_max_len`. + */ + async function fetchTables( + serverFlavor: MySQLServerFlavor, + options: Parameters[0], + ): Promise<{ + query: Query; + result: Either>; + }> { + if (serverFlavor === "mariadb") { + const query = getMariaDBTablesQuery(otherRequirements); + const [error, rows] = await executor.execute(query, options); + + return { + query, + result: error ? [error] : [null, groupMariaDBTablesQueryResult(rows)], + }; + } + + const query = getTablesQuery(otherRequirements); + const [error, rows] = await executor.execute(query, options); + + return { + query, + result: error ? [error] : [null, normalizeTablesQueryResult(rows)], + }; + } + async function introspectDatabase( options: Parameters[0], ): Promise> { try { const serverFlavor = await detectServerFlavor(options); - const tablesQuery = getTablesQuery(otherRequirements, serverFlavor); const timezoneQuery = getTimezoneQuery(otherRequirements); - const [[tablesError, tables], [timezoneError, timezones]] = - await Promise.all([ - executor.execute(tablesQuery, options), - executor.execute(timezoneQuery, options), - ]); + const [tablesFetch, [timezoneError, timezones]] = await Promise.all([ + fetchTables(serverFlavor, options), + executor.execute(timezoneQuery, options), + ]); + + const { + query: tablesQuery, + result: [tablesError, tables], + } = tablesFetch; if (tablesError) { return createMySQLAdapterError({ @@ -208,11 +244,7 @@ export function createMySQLAdapter( return [ null, - createIntrospection({ - query: tablesQuery, - tables: normalizeTablesQueryResult(tables), - timezone, - }), + createIntrospection({ query: tablesQuery, tables, timezone }), ]; } catch (error: unknown) { return createMySQLAdapterError({ error: error as Error }); diff --git a/data/mysql-core/introspection.mariadb.test.ts b/data/mysql-core/introspection.mariadb.test.ts index e5088eb5..d5378c5a 100644 --- a/data/mysql-core/introspection.mariadb.test.ts +++ b/data/mysql-core/introspection.mariadb.test.ts @@ -1,16 +1,42 @@ import { describe, expect, it, vi } from "vitest"; import type { SequenceExecutor } from "../executor"; -import type { Query } from "../query"; +import type { Query, QueryResult } from "../query"; import { createMySQLAdapter } from "./adapter"; import { detectMySQLServerFlavor, + getMariaDBTablesQuery, getServerVersionQuery, getTablesQuery, + groupMariaDBTablesQueryResult, mockTablesQuery, normalizeTablesQueryResult, } from "./introspection"; +type MariaDBTablesRow = QueryResult[number]; + +function toMariaDBTablesRows( + tables: ReturnType = mockTablesQuery(), +): MariaDBTablesRow[] { + return tables.flatMap((table) => + table.columns.map((column) => ({ + column_autoincrement: column.autoincrement, + column_computed: column.computed, + column_datatype: column.datatype, + column_default: column.default, + column_fk_column: column.fk_column, + column_fk_table: column.fk_table, + column_name: column.name, + column_nullable: column.nullable, + column_pk: column.pk, + column_position: column.position, + name: table.name, + schema: table.schema, + type: table.type, + })), + ); +} + describe("mysql-core MariaDB introspection compatibility", () => { describe("detectMySQLServerFlavor", () => { it("detects MariaDB from a plain MariaDB version string", () => { @@ -53,42 +79,117 @@ describe("mysql-core MariaDB introspection compatibility", () => { }); }); - describe("getTablesQuery", () => { - it("uses json_arrayagg for MySQL", () => { - const query = getTablesQuery(undefined, "mysql"); + describe("getMariaDBTablesQuery", () => { + it("uses no JSON functions and no aggregation", () => { + const query = getMariaDBTablesQuery(); + const sql = query.sql.toLowerCase(); - expect(query.sql).toContain("json_arrayagg(json_object("); - expect(query.sql).not.toContain("group_concat"); + // json_arrayagg only exists on MariaDB >= 10.5 (#1511) and + // cast(... as json) is invalid syntax on MariaDB (#1367). + expect(sql).not.toContain("json_arrayagg"); + expect(sql).not.toContain("as json)"); + expect(sql).not.toContain("json_object"); + // string aggregation is silently truncated at group_concat_max_len, + // so the MariaDB query must not aggregate at all. + expect(sql).not.toContain("group_concat"); + expect(sql).not.toContain("group by"); + }); + + it("returns one row per column ordered by table and position", () => { + const query = getMariaDBTablesQuery(); + + expect(query.sql).toContain("`c`.`name` as `column_name`"); + expect(query.sql).toContain( + "order by `t`.`TABLE_SCHEMA`, `t`.`TABLE_NAME`, `t`.`TABLE_TYPE`, `c`.`position`", + ); }); - it("uses json_arrayagg for the default flavor", () => { - expect(getTablesQuery().sql).toContain("json_arrayagg(json_object("); + it("compiles to a stable query", () => { + expect(getMariaDBTablesQuery()).toMatchInlineSnapshot(` + { + "parameters": [ + "auto_increment", + "on update CURRENT_TIMESTAMP", + "STORED GENERATED", + "VIRTUAL GENERATED", + "YES", + "PRIMARY", + "BASE TABLE", + "VIEW", + ], + "sql": "with \`cols\` as (select \`c\`.\`COLUMN_DEFAULT\` as \`default\`, \`c\`.\`COLUMN_NAME\` as \`name\`, \`c\`.\`COLUMN_TYPE\` as \`datatype\`, \`c\`.\`ORDINAL_POSITION\` as \`position\`, \`c\`.\`TABLE_NAME\`, \`kcu\`.\`REFERENCED_TABLE_NAME\` as \`fk_table\`, \`kcu\`.\`REFERENCED_COLUMN_NAME\` as \`fk_column\`, \`pk_kcu\`.\`ORDINAL_POSITION\` as \`pk\`, \`c\`.\`EXTRA\` = ? as \`autoincrement\`, \`c\`.\`EXTRA\` in (?, ?, ?) as \`computed\`, \`c\`.\`IS_NULLABLE\` = ? as \`nullable\` from \`information_schema\`.\`columns\` as \`c\` left join \`information_schema\`.\`KEY_COLUMN_USAGE\` as \`kcu\` on \`kcu\`.\`TABLE_SCHEMA\` = database() and \`kcu\`.\`TABLE_NAME\` = \`c\`.\`TABLE_NAME\` and \`kcu\`.\`COLUMN_NAME\` = \`c\`.\`COLUMN_NAME\` and \`kcu\`.\`POSITION_IN_UNIQUE_CONSTRAINT\` is not null left join \`information_schema\`.\`KEY_COLUMN_USAGE\` as \`pk_kcu\` on \`pk_kcu\`.\`TABLE_SCHEMA\` = database() and \`pk_kcu\`.\`TABLE_NAME\` = \`c\`.\`TABLE_NAME\` and \`pk_kcu\`.\`COLUMN_NAME\` = \`c\`.\`COLUMN_NAME\` and \`pk_kcu\`.\`CONSTRAINT_NAME\` = ? where \`c\`.\`TABLE_SCHEMA\` = database()) select database() as \`schema\`, \`t\`.\`TABLE_NAME\` as \`name\`, \`t\`.\`TABLE_TYPE\` as \`type\`, \`c\`.\`autoincrement\` as \`column_autoincrement\`, \`c\`.\`computed\` as \`column_computed\`, \`c\`.\`datatype\` as \`column_datatype\`, \`c\`.\`default\` as \`column_default\`, \`c\`.\`fk_column\` as \`column_fk_column\`, \`c\`.\`fk_table\` as \`column_fk_table\`, \`c\`.\`name\` as \`column_name\`, \`c\`.\`nullable\` as \`column_nullable\`, \`c\`.\`pk\` as \`column_pk\`, \`c\`.\`position\` as \`column_position\` from \`information_schema\`.\`tables\` as \`t\` inner join \`cols\` as \`c\` on \`c\`.\`TABLE_NAME\` = \`t\`.\`TABLE_NAME\` where \`t\`.\`TABLE_SCHEMA\` = database() and \`t\`.\`TABLE_TYPE\` in (?, ?) order by \`t\`.\`TABLE_SCHEMA\`, \`t\`.\`TABLE_NAME\`, \`t\`.\`TABLE_TYPE\`, \`c\`.\`position\`", + "transformations": undefined, + } + `); }); - it("avoids json_arrayagg and json casts for MariaDB", () => { - const query = getTablesQuery(undefined, "mariadb"); + it("keeps the MySQL tables query on json_arrayagg", () => { + const query = getTablesQuery(); - // json_arrayagg only exists on MariaDB >= 10.5 (#1511) and - // cast(... as json) is invalid syntax on MariaDB (#1367). - expect(query.sql).not.toContain("json_arrayagg"); - expect(query.sql.toLowerCase()).not.toContain("as json)"); - expect(query.sql).toContain( - "coalesce(concat('[', group_concat(json_object(", + expect(query.sql).toContain("json_arrayagg(json_object("); + expect(query.sql).not.toContain("group_concat"); + expect(query.parameters).toEqual(getMariaDBTablesQuery().parameters); + }); + }); + + describe("groupMariaDBTablesQueryResult", () => { + it("groups one-row-per-column results into the aggregated shape", () => { + expect(groupMariaDBTablesQueryResult(toMariaDBTablesRows())).toEqual( + mockTablesQuery(), ); - expect(query.sql).toContain("separator ','), ']'), '[]')"); }); - it("keeps everything but the columns aggregation identical across flavors", () => { - const mysqlQuery = getTablesQuery(undefined, "mysql"); - const mariadbQuery = getTablesQuery(undefined, "mariadb"); + it("returns no tables for no rows", () => { + expect(groupMariaDBTablesQueryResult([])).toEqual([]); + }); - expect(mariadbQuery.parameters).toEqual(mysqlQuery.parameters); - expect(mariadbQuery.sql).toBe( - mysqlQuery.sql.replace( - /json_arrayagg\((.*)\) as `columns`/, - "coalesce(concat('[', group_concat($1 separator ','), ']'), '[]') as `columns`", - ), + it("keeps identically named tables in different schemas apart", () => { + const rows = toMariaDBTablesRows(); + const otherSchemaRows = rows.map((row) => ({ + ...row, + schema: "other", + })); + + const grouped = groupMariaDBTablesQueryResult([ + ...rows, + ...otherSchemaRows, + ]); + + expect(grouped).toHaveLength(mockTablesQuery().length * 2); + }); + + it("is not size-limited, unlike server-side string aggregation", () => { + // group_concat_max_len defaults to 1MB on MariaDB; build metadata well + // beyond that to prove client-side grouping cannot truncate columns. + const columnCount = 3000; + const longDefault = "x".repeat(512); + const rows: MariaDBTablesRow[] = Array.from( + { length: columnCount }, + (_, index) => ({ + column_autoincrement: 0, + column_computed: 0, + column_datatype: "varchar(1024)", + column_default: longDefault, + column_fk_column: null, + column_fk_table: null, + column_name: `column_${index}`, + column_nullable: 1, + column_pk: null, + column_position: index + 1, + name: "wide_table", + schema: "studio", + type: "BASE TABLE", + }), ); + + expect(JSON.stringify(rows).length).toBeGreaterThan(1024 * 1024); + + const [table] = groupMariaDBTablesQueryResult(rows); + + expect(table?.columns).toHaveLength(columnCount); + expect(table?.columns.at(0)?.name).toBe("column_0"); + expect(table?.columns.at(-1)?.name).toBe(`column_${columnCount - 1}`); + expect(table?.columns.at(-1)?.default).toBe(longDefault); }); }); @@ -130,7 +231,6 @@ describe("mysql-core MariaDB introspection compatibility", () => { function createRecordingExecutor(args: { version?: string; versionError?: Error; - stringifyColumns?: boolean; }): { executor: SequenceExecutor; queries: Query[] } { const queries: Query[] = []; @@ -151,17 +251,12 @@ describe("mysql-core MariaDB introspection compatibility", () => { return Promise.resolve([null, [{ timezone: "UTC" }] as never]); } - const tables = mockTablesQuery(); + if (sql.includes("column_autoincrement")) { + // MariaDB-flavored tables query - one row per column. + return Promise.resolve([null, toMariaDBTablesRows() as never]); + } - return Promise.resolve([ - null, - (args.stringifyColumns - ? tables.map((table) => ({ - ...table, - columns: JSON.stringify(table.columns), - })) - : tables) as never, - ]); + return Promise.resolve([null, mockTablesQuery() as never]); }; return { @@ -176,7 +271,6 @@ describe("mysql-core MariaDB introspection compatibility", () => { it("uses the MariaDB-compatible tables query against MariaDB", async () => { const { executor, queries } = createRecordingExecutor({ version: "10.4.34-MariaDB", - stringifyColumns: true, }); const adapter = createMySQLAdapter({ executor }); @@ -194,7 +288,8 @@ describe("mysql-core MariaDB introspection compatibility", () => { ); expect(tablesQuery?.sql).not.toContain("json_arrayagg"); - expect(tablesQuery?.sql).toContain("group_concat(json_object("); + expect(tablesQuery?.sql).not.toContain("group_concat"); + expect(tablesQuery?.sql).toContain("`column_autoincrement`"); }); it("keeps the MySQL tables query against MySQL", async () => { @@ -237,7 +332,6 @@ describe("mysql-core MariaDB introspection compatibility", () => { it("detects the server flavor once per adapter", async () => { const { executor, queries } = createRecordingExecutor({ version: "10.11.6-MariaDB", - stringifyColumns: true, }); const adapter = createMySQLAdapter({ executor }); @@ -251,12 +345,33 @@ describe("mysql-core MariaDB introspection compatibility", () => { expect(versionQueries).toHaveLength(1); }); - it("parses string-aggregated columns even on MySQL transports", async () => { - const { executor } = createRecordingExecutor({ - version: "8.0.40", - stringifyColumns: true, - }); - const adapter = createMySQLAdapter({ executor }); + it("parses string-aggregated columns on MySQL transports", async () => { + const { executor } = createRecordingExecutor({ version: "8.0.40" }); + const stringifyingExecutor: SequenceExecutor = { + ...executor, + execute: async (query, options) => { + const [error, rows] = await executor.execute(query, options); + + if (error) { + return [error]; + } + + if (!query.sql.includes("information_schema")) { + return [null, rows as never]; + } + + return [ + null, + (rows as unknown as ReturnType).map( + (table) => ({ + ...table, + columns: JSON.stringify(table.columns), + }), + ) as never, + ]; + }, + }; + const adapter = createMySQLAdapter({ executor: stringifyingExecutor }); const [error, result] = await adapter.introspect({}); diff --git a/data/mysql-core/introspection.ts b/data/mysql-core/introspection.ts index 1f59925b..06b32c37 100644 --- a/data/mysql-core/introspection.ts +++ b/data/mysql-core/introspection.ts @@ -51,9 +51,9 @@ interface Database { /** * The flavor of the connected MySQL-compatible server. * - * MariaDB requires a different columns aggregation: `json_arrayagg` only - * exists on MariaDB >= 10.5 and `cast(... as json)` is invalid syntax there - * because JSON is an alias for LONGTEXT. + * MariaDB requires a different tables query ({@link getMariaDBTablesQuery}): + * `json_arrayagg` only exists on MariaDB >= 10.5 and `cast(... as json)` is + * invalid syntax there because JSON is an alias for LONGTEXT. */ export type MySQLServerFlavor = "mariadb" | "mysql"; @@ -87,15 +87,14 @@ export function mockServerVersionQuery() { >; } -export function getTablesQuery( +function getColumnsQuery( requirements?: Omit, - flavor: MySQLServerFlavor = "mysql", ) { const database = sql`database()`; const builder = getMySQLBuilder(requirements); - const columnsQuery = builder + return builder .selectFrom("information_schema.columns as c") .leftJoin("information_schema.KEY_COLUMN_USAGE as kcu", (jb) => jb @@ -133,6 +132,14 @@ export function getTablesQuery( ]).as("computed"), eb("c.IS_NULLABLE", "=", "YES").as("nullable"), ]); +} + +export function getTablesQuery( + requirements?: Omit, +) { + const database = sql`database()`; + + const columnsQuery = getColumnsQuery(requirements); return compile( getMySQLBuilder(requirements) @@ -150,49 +157,127 @@ export function getTablesQuery( "t.TABLE_TYPE as type", ]) .$narrowType<{ type: "BASE TABLE" | "VIEW" }>() - .select((eb) => { - type Columns = Omit< - InferResult[number], - "TABLE_NAME" - >[]; - - const columnsJson = jsonBuildObject({ - autoincrement: eb.ref("c.autoincrement"), - computed: eb.ref("c.computed"), - datatype: eb.ref("c.datatype"), - default: eb.ref("c.default"), - fk_column: eb.ref("c.fk_column"), - fk_table: eb.ref("c.fk_table"), - name: eb.ref("c.name"), - position: eb.ref("c.position"), - pk: eb.ref("c.pk"), - nullable: eb.ref("c.nullable"), - }); - - // MariaDB has no `json_arrayagg` before 10.5 (#1511) and no JSON cast - // type at all (#1367), so aggregate with `group_concat` into a JSON - // array string instead. The string payload is parsed back into an - // array by `normalizeTablesQueryResult`. - const aggregated = - flavor === "mariadb" - ? sql`coalesce(concat('[', group_concat(${columnsJson} separator ','), ']'), '[]')` - : sql`json_arrayagg(${columnsJson})`; - - return aggregated.as("columns"); - }) + .select((eb) => + eb + .fn[number], "TABLE_NAME">[]>( + "json_arrayagg", + [ + jsonBuildObject({ + autoincrement: eb.ref("c.autoincrement"), + computed: eb.ref("c.computed"), + datatype: eb.ref("c.datatype"), + default: eb.ref("c.default"), + fk_column: eb.ref("c.fk_column"), + fk_table: eb.ref("c.fk_table"), + name: eb.ref("c.name"), + position: eb.ref("c.position"), + pk: eb.ref("c.pk"), + nullable: eb.ref("c.nullable"), + }), + ], + ) + .as("columns"), + ) .orderBy("t.TABLE_SCHEMA") .orderBy("t.TABLE_NAME") .orderBy("t.TABLE_TYPE"), ); } +/** + * MariaDB-compatible variant of {@link getTablesQuery}. + * + * MariaDB has no `json_arrayagg` before 10.5 (#1511) and no JSON cast type at + * all (#1367), and any server-side string aggregation (`group_concat`, + * MariaDB's own `json_arrayagg`) silently truncates at the session's + * `group_concat_max_len`. This query therefore avoids aggregation entirely: it + * returns one row per column, and {@link groupMariaDBTablesQueryResult} groups + * the rows into the {@link getTablesQuery} result shape on the client. + */ +export function getMariaDBTablesQuery( + requirements?: Omit, +) { + const database = sql`database()`; + + return compile( + getMySQLBuilder(requirements) + .with("cols", () => getColumnsQuery(requirements)) + .selectFrom("information_schema.tables as t") + .innerJoin("cols as c", (jb) => + jb.onRef("c.TABLE_NAME", "=", "t.TABLE_NAME"), + ) + .where("t.TABLE_SCHEMA", "=", database) + .where("t.TABLE_TYPE", "in", ["BASE TABLE", "VIEW"]) + .select([ + database.as("schema"), + "t.TABLE_NAME as name", + "t.TABLE_TYPE as type", + "c.autoincrement as column_autoincrement", + "c.computed as column_computed", + "c.datatype as column_datatype", + "c.default as column_default", + "c.fk_column as column_fk_column", + "c.fk_table as column_fk_table", + "c.name as column_name", + "c.nullable as column_nullable", + "c.pk as column_pk", + "c.position as column_position", + ]) + .$narrowType<{ type: "BASE TABLE" | "VIEW" }>() + .orderBy("t.TABLE_SCHEMA") + .orderBy("t.TABLE_NAME") + .orderBy("t.TABLE_TYPE") + .orderBy("c.position"), + ); +} + +/** + * Groups the one-row-per-column result of {@link getMariaDBTablesQuery} into + * the aggregated {@link getTablesQuery} result shape. + */ +export function groupMariaDBTablesQueryResult( + rows: QueryResult, +): QueryResult { + const tables = new Map[number]>(); + + for (const row of rows) { + const key = JSON.stringify([row.schema, row.name, row.type]); + + let table = tables.get(key); + + if (!table) { + table = { + columns: [], + name: row.name, + schema: row.schema, + type: row.type, + }; + tables.set(key, table); + } + + table.columns.push({ + autoincrement: row.column_autoincrement, + computed: row.column_computed, + datatype: row.column_datatype, + default: row.column_default, + fk_column: row.column_fk_column, + fk_table: row.column_fk_table, + name: row.column_name, + nullable: row.column_nullable, + pk: row.column_pk, + position: row.column_position, + }); + } + + return [...tables.values()]; +} + /** * Normalizes the `columns` payload of a tables query result. * - * On MariaDB the columns are aggregated into a JSON array string (see - * {@link getTablesQuery}), and some transports also return `json_arrayagg` - * results as strings instead of parsed arrays. This parses those string - * payloads so downstream consumers always receive arrays. + * Some transports return `json_arrayagg` results as strings instead of parsed + * arrays. This parses those string payloads so downstream consumers always + * receive arrays. */ export function normalizeTablesQueryResult( tables: QueryResult,