From 323ea1b07510d0d78913508362c19630bf29b281 Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Fri, 14 Aug 2026 02:22:14 +0200 Subject: [PATCH 01/16] feat(iceberg): add DuckDB SQL access for REST catalogs - upgrade the bundled DuckDB Node API to 1.5.5-r.1 - add trusted Iceberg catalog attachment and SQL execution lifecycle - resolve S3-compatible credentials through secure storage - create temporary catalog and storage secrets with guaranteed cleanup - support Generic REST, Polaris, Lakekeeper, and Nessie catalogs - handle Lakekeeper and Nessie OAuth scopes and Nessie warehouse routing - extend the Iceberg wizard with SQL storage configuration and verification - add SQL access testing and support information to Health Status - add typed IPC, capability reporting, cancellation, and bounded results - improve Iceberg deletion messaging and wizard validation - add focused runtime, cleanup, statement-policy, and credential tests --- release/app/package-lock.json | 184 ++++--- release/app/package.json | 2 +- .../icebergDatalake.ipcHandlers.ts | 16 + src/main/services/icebergDatalake.service.ts | 502 ++++++++++++++++++ .../dataLake/IcebergConnectionWizard.tsx | 404 +++++++++++--- .../dataLake/iceberg/IcebergDetail.tsx | 294 ++++++++-- .../controllers/icebergDatalake.controller.ts | 26 + src/renderer/screens/dataLake/index.tsx | 45 +- src/renderer/services/iceberg.service.ts | 21 + src/types/iceberg.ts | 43 ++ src/types/ipc.ts | 4 + .../services/icebergDatalake.service.test.ts | 86 +++ .../icebergSqlRuntime.service.test.ts | 258 +++++++++ 13 files changed, 1709 insertions(+), 176 deletions(-) create mode 100644 tests/unit/main/services/icebergSqlRuntime.service.test.ts diff --git a/release/app/package-lock.json b/release/app/package-lock.json index 2f57cabe..99edac9c 100644 --- a/release/app/package-lock.json +++ b/release/app/package-lock.json @@ -11,7 +11,7 @@ "license": "MIT", "dependencies": { "@databricks/sql": "^1.11.0", - "@duckdb/node-api": "^1.5.2-r.1", + "@duckdb/node-api": "1.5.5-r.1", "better-sqlite3": "^12.2.0", "drizzle-orm": "^0.44.4", "fs-extra": "^11.4.0", @@ -80,32 +80,37 @@ } }, "node_modules/@duckdb/node-api": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-api/-/node-api-1.5.2-r.1.tgz", - "integrity": "sha512-OzBBnS0JGXMoS5mzKNY/Ylr7SshcRQiLFIoxQ4AlePwJ2fNeDL/fbHu/knjxUrXwW1fJBTUgwWftmxDdnZZb3A==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-api/-/node-api-1.5.5-r.1.tgz", + "integrity": "sha512-1eF3lV9PaaWzA4f4Rbd3GTGXt3aOsoU5aOVi8RXcbbx07YuAs4aoICcrFDBdesYX4ZiDfWeqIfxIJHYGIiwfwg==", "license": "MIT", "dependencies": { - "@duckdb/node-bindings": "1.5.2-r.1" + "@duckdb/node-bindings": "1.5.5-r.1" } }, "node_modules/@duckdb/node-bindings": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings/-/node-bindings-1.5.2-r.1.tgz", - "integrity": "sha512-bUg3bLVj70YVku6fKyQJS8ASORl7kM7YFVFznsEB9pWbtazPj+ME2x2FUk0WiTzjJdutjzSSGXF066mB4bGGZA==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings/-/node-bindings-1.5.5-r.1.tgz", + "integrity": "sha512-5thbXfoBUcB5Wxou6DkXpnKmXTgj0IA6cLY0YFfsLI4VRdZV0gWsMMziXi26qWzgrn7H/HP35czkKVc+jvaT0g==", "license": "MIT", + "dependencies": { + "detect-libc": "^2.1.2" + }, "optionalDependencies": { - "@duckdb/node-bindings-darwin-arm64": "1.5.2-r.1", - "@duckdb/node-bindings-darwin-x64": "1.5.2-r.1", - "@duckdb/node-bindings-linux-arm64": "1.5.2-r.1", - "@duckdb/node-bindings-linux-x64": "1.5.2-r.1", - "@duckdb/node-bindings-win32-arm64": "1.5.2-r.1", - "@duckdb/node-bindings-win32-x64": "1.5.2-r.1" + "@duckdb/node-bindings-darwin-arm64": "1.5.5-r.1", + "@duckdb/node-bindings-darwin-x64": "1.5.5-r.1", + "@duckdb/node-bindings-linux-arm64": "1.5.5-r.1", + "@duckdb/node-bindings-linux-arm64-musl": "1.5.5-r.1", + "@duckdb/node-bindings-linux-x64": "1.5.5-r.1", + "@duckdb/node-bindings-linux-x64-musl": "1.5.5-r.1", + "@duckdb/node-bindings-win32-arm64": "1.5.5-r.1", + "@duckdb/node-bindings-win32-x64": "1.5.5-r.1" } }, "node_modules/@duckdb/node-bindings-darwin-arm64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-arm64/-/node-bindings-darwin-arm64-1.5.2-r.1.tgz", - "integrity": "sha512-v35FyKOb8EJCvaiPF7k0gvKiJTXR7PPQDNoWR0Gu+YSX5O9b+DIguzt1348Of3HebHy6ATSMzlUekaVA9YXu+g==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-arm64/-/node-bindings-darwin-arm64-1.5.5-r.1.tgz", + "integrity": "sha512-knLjoyxwt1aBpRqxrbIbVdDCI+uMhTe8C+2KVncz4OLgI3KsqgeYIYy8Uee+5dvSSs84BOHMIKLuBhzutBX1bw==", "cpu": [ "arm64" ], @@ -116,9 +121,9 @@ ] }, "node_modules/@duckdb/node-bindings-darwin-x64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-x64/-/node-bindings-darwin-x64-1.5.2-r.1.tgz", - "integrity": "sha512-SU9dIJ1BluKkkGxi4UsP4keqkkstB2YDySF9KcYu3EZKIVM3FTv2zc7XO38dXnHOq6+F3WqhWWZvD+XU945p7A==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-x64/-/node-bindings-darwin-x64-1.5.5-r.1.tgz", + "integrity": "sha512-FOSlP2K15L6xkzqjUlEeJLIhFqahm14zVQ5Lcf953ww1MpE/GNysvIC6DyJWlPI2rL9Oez0sIMpE3Uj6G7oXvw==", "cpu": [ "x64" ], @@ -129,9 +134,22 @@ ] }, "node_modules/@duckdb/node-bindings-linux-arm64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-arm64/-/node-bindings-linux-arm64-1.5.2-r.1.tgz", - "integrity": "sha512-3Tra9xM3aM3denaER4KhJ6//6PpmPbik9ECBQ+sh9PyKaEgHw/0kAcKnLm5EzWUnXF0qYmZlewvkCrse8KmOYw==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-arm64/-/node-bindings-linux-arm64-1.5.5-r.1.tgz", + "integrity": "sha512-ITkwQ/0SvyeOGdGMiMGKGiID8IgwWPLI0DDqcr1DND1QfiwP4OuGYxNHosWaNFS1z7LEbjN18GZtRcwmYCVH/w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@duckdb/node-bindings-linux-arm64-musl": { + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-arm64-musl/-/node-bindings-linux-arm64-musl-1.5.5-r.1.tgz", + "integrity": "sha512-6GwbW28aK9LQNOSn5S/NS79Ak8TXtAFVGLp1+0td2mt5l21ge6smDcjm9IOgi8O+aSZUHEDQhHvCdFYOjAFvSA==", "cpu": [ "arm64" ], @@ -142,9 +160,22 @@ ] }, "node_modules/@duckdb/node-bindings-linux-x64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-x64/-/node-bindings-linux-x64-1.5.2-r.1.tgz", - "integrity": "sha512-pcQvZRHiIfJ9cq8parkSQczQHEml/IeGfnDCMAbEgD6+jaV9Y9Y5Ph1kP9aR+bm6him1S5ZIEr3kZbihjKnWbA==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-x64/-/node-bindings-linux-x64-1.5.5-r.1.tgz", + "integrity": "sha512-zuhH2VduUs1N//Tch+M5Y3RYUd5rNiafnBJpsz0z7eQ/x/8WRDVuMleSRPYdkvHTbllAGc/8ZUjC4FvqScxwoA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@duckdb/node-bindings-linux-x64-musl": { + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-x64-musl/-/node-bindings-linux-x64-musl-1.5.5-r.1.tgz", + "integrity": "sha512-NI6wpYx+mzTwfBNGIrEBn04Je/m34SYLLrZy7yAg5U8W+9HZ2N7j2Rem64Bte4Nl7E93385Z22kHpbOExyYFRA==", "cpu": [ "x64" ], @@ -155,9 +186,9 @@ ] }, "node_modules/@duckdb/node-bindings-win32-arm64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-arm64/-/node-bindings-win32-arm64-1.5.2-r.1.tgz", - "integrity": "sha512-Ji8tym+N3LkrhVt0Up3bsacD/kpg4/JXFJQqxswiYvBaNCQOk+D+aiVS0GN5pcqvmnG7V7TpsDRzkLEFaWp1vw==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-arm64/-/node-bindings-win32-arm64-1.5.5-r.1.tgz", + "integrity": "sha512-+7w5Q24HZEqLMtVn+agNAxtzxo2gmaaBWK3RfwCgsQFfl00lP8D6hNnOJnUnHVObvqdWw3CzTkYsOQKDY8+a/A==", "cpu": [ "arm64" ], @@ -168,9 +199,9 @@ ] }, "node_modules/@duckdb/node-bindings-win32-x64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-x64/-/node-bindings-win32-x64-1.5.2-r.1.tgz", - "integrity": "sha512-5XqcqC+4R8ghBEEbnc2a0sqfz1zyPBRb9YcmIWfiuDoCYSYFbKhmHcEyNftZDHcwCoLOHXnUin45jraex4STqQ==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-x64/-/node-bindings-win32-x64-1.5.5-r.1.tgz", + "integrity": "sha512-7KAdShoWQz7YXKvUneIu9ujxIVCSSA6pJ5QSZBDkNUCW+7RBLV/aH2Uy3K0Vl04reaFedaS3aewfstX2SQS5XQ==", "cpu": [ "x64" ], @@ -589,9 +620,9 @@ } }, "node_modules/detect-libc": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", - "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==", + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", "license": "Apache-2.0", "engines": { "node": ">=8" @@ -1875,60 +1906,75 @@ } }, "@duckdb/node-api": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-api/-/node-api-1.5.2-r.1.tgz", - "integrity": "sha512-OzBBnS0JGXMoS5mzKNY/Ylr7SshcRQiLFIoxQ4AlePwJ2fNeDL/fbHu/knjxUrXwW1fJBTUgwWftmxDdnZZb3A==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-api/-/node-api-1.5.5-r.1.tgz", + "integrity": "sha512-1eF3lV9PaaWzA4f4Rbd3GTGXt3aOsoU5aOVi8RXcbbx07YuAs4aoICcrFDBdesYX4ZiDfWeqIfxIJHYGIiwfwg==", "requires": { - "@duckdb/node-bindings": "1.5.2-r.1" + "@duckdb/node-bindings": "1.5.5-r.1" } }, "@duckdb/node-bindings": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings/-/node-bindings-1.5.2-r.1.tgz", - "integrity": "sha512-bUg3bLVj70YVku6fKyQJS8ASORl7kM7YFVFznsEB9pWbtazPj+ME2x2FUk0WiTzjJdutjzSSGXF066mB4bGGZA==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings/-/node-bindings-1.5.5-r.1.tgz", + "integrity": "sha512-5thbXfoBUcB5Wxou6DkXpnKmXTgj0IA6cLY0YFfsLI4VRdZV0gWsMMziXi26qWzgrn7H/HP35czkKVc+jvaT0g==", "requires": { - "@duckdb/node-bindings-darwin-arm64": "1.5.2-r.1", - "@duckdb/node-bindings-darwin-x64": "1.5.2-r.1", - "@duckdb/node-bindings-linux-arm64": "1.5.2-r.1", - "@duckdb/node-bindings-linux-x64": "1.5.2-r.1", - "@duckdb/node-bindings-win32-arm64": "1.5.2-r.1", - "@duckdb/node-bindings-win32-x64": "1.5.2-r.1" + "@duckdb/node-bindings-darwin-arm64": "1.5.5-r.1", + "@duckdb/node-bindings-darwin-x64": "1.5.5-r.1", + "@duckdb/node-bindings-linux-arm64": "1.5.5-r.1", + "@duckdb/node-bindings-linux-arm64-musl": "1.5.5-r.1", + "@duckdb/node-bindings-linux-x64": "1.5.5-r.1", + "@duckdb/node-bindings-linux-x64-musl": "1.5.5-r.1", + "@duckdb/node-bindings-win32-arm64": "1.5.5-r.1", + "@duckdb/node-bindings-win32-x64": "1.5.5-r.1", + "detect-libc": "^2.1.2" } }, "@duckdb/node-bindings-darwin-arm64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-arm64/-/node-bindings-darwin-arm64-1.5.2-r.1.tgz", - "integrity": "sha512-v35FyKOb8EJCvaiPF7k0gvKiJTXR7PPQDNoWR0Gu+YSX5O9b+DIguzt1348Of3HebHy6ATSMzlUekaVA9YXu+g==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-arm64/-/node-bindings-darwin-arm64-1.5.5-r.1.tgz", + "integrity": "sha512-knLjoyxwt1aBpRqxrbIbVdDCI+uMhTe8C+2KVncz4OLgI3KsqgeYIYy8Uee+5dvSSs84BOHMIKLuBhzutBX1bw==", "optional": true }, "@duckdb/node-bindings-darwin-x64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-x64/-/node-bindings-darwin-x64-1.5.2-r.1.tgz", - "integrity": "sha512-SU9dIJ1BluKkkGxi4UsP4keqkkstB2YDySF9KcYu3EZKIVM3FTv2zc7XO38dXnHOq6+F3WqhWWZvD+XU945p7A==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-darwin-x64/-/node-bindings-darwin-x64-1.5.5-r.1.tgz", + "integrity": "sha512-FOSlP2K15L6xkzqjUlEeJLIhFqahm14zVQ5Lcf953ww1MpE/GNysvIC6DyJWlPI2rL9Oez0sIMpE3Uj6G7oXvw==", "optional": true }, "@duckdb/node-bindings-linux-arm64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-arm64/-/node-bindings-linux-arm64-1.5.2-r.1.tgz", - "integrity": "sha512-3Tra9xM3aM3denaER4KhJ6//6PpmPbik9ECBQ+sh9PyKaEgHw/0kAcKnLm5EzWUnXF0qYmZlewvkCrse8KmOYw==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-arm64/-/node-bindings-linux-arm64-1.5.5-r.1.tgz", + "integrity": "sha512-ITkwQ/0SvyeOGdGMiMGKGiID8IgwWPLI0DDqcr1DND1QfiwP4OuGYxNHosWaNFS1z7LEbjN18GZtRcwmYCVH/w==", + "optional": true + }, + "@duckdb/node-bindings-linux-arm64-musl": { + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-arm64-musl/-/node-bindings-linux-arm64-musl-1.5.5-r.1.tgz", + "integrity": "sha512-6GwbW28aK9LQNOSn5S/NS79Ak8TXtAFVGLp1+0td2mt5l21ge6smDcjm9IOgi8O+aSZUHEDQhHvCdFYOjAFvSA==", "optional": true }, "@duckdb/node-bindings-linux-x64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-x64/-/node-bindings-linux-x64-1.5.2-r.1.tgz", - "integrity": "sha512-pcQvZRHiIfJ9cq8parkSQczQHEml/IeGfnDCMAbEgD6+jaV9Y9Y5Ph1kP9aR+bm6him1S5ZIEr3kZbihjKnWbA==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-x64/-/node-bindings-linux-x64-1.5.5-r.1.tgz", + "integrity": "sha512-zuhH2VduUs1N//Tch+M5Y3RYUd5rNiafnBJpsz0z7eQ/x/8WRDVuMleSRPYdkvHTbllAGc/8ZUjC4FvqScxwoA==", + "optional": true + }, + "@duckdb/node-bindings-linux-x64-musl": { + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-linux-x64-musl/-/node-bindings-linux-x64-musl-1.5.5-r.1.tgz", + "integrity": "sha512-NI6wpYx+mzTwfBNGIrEBn04Je/m34SYLLrZy7yAg5U8W+9HZ2N7j2Rem64Bte4Nl7E93385Z22kHpbOExyYFRA==", "optional": true }, "@duckdb/node-bindings-win32-arm64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-arm64/-/node-bindings-win32-arm64-1.5.2-r.1.tgz", - "integrity": "sha512-Ji8tym+N3LkrhVt0Up3bsacD/kpg4/JXFJQqxswiYvBaNCQOk+D+aiVS0GN5pcqvmnG7V7TpsDRzkLEFaWp1vw==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-arm64/-/node-bindings-win32-arm64-1.5.5-r.1.tgz", + "integrity": "sha512-+7w5Q24HZEqLMtVn+agNAxtzxo2gmaaBWK3RfwCgsQFfl00lP8D6hNnOJnUnHVObvqdWw3CzTkYsOQKDY8+a/A==", "optional": true }, "@duckdb/node-bindings-win32-x64": { - "version": "1.5.2-r.1", - "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-x64/-/node-bindings-win32-x64-1.5.2-r.1.tgz", - "integrity": "sha512-5XqcqC+4R8ghBEEbnc2a0sqfz1zyPBRb9YcmIWfiuDoCYSYFbKhmHcEyNftZDHcwCoLOHXnUin45jraex4STqQ==", + "version": "1.5.5-r.1", + "resolved": "https://registry.npmjs.org/@duckdb/node-bindings-win32-x64/-/node-bindings-win32-x64-1.5.5-r.1.tgz", + "integrity": "sha512-7KAdShoWQz7YXKvUneIu9ujxIVCSSA6pJ5QSZBDkNUCW+7RBLV/aH2Uy3K0Vl04reaFedaS3aewfstX2SQS5XQ==", "optional": true }, "@tootallnate/quickjs-emscripten": { @@ -2229,9 +2275,9 @@ } }, "detect-libc": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.0.4.tgz", - "integrity": "sha512-3UDv+G9CsCKO1WKMGw9fwq/SWJYbI0c5Y7LU1AXYoDdbhE2AHQ6N6Nb34sG8Fj7T5APy8qXDCKuuIHd1BR0tVA==" + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==" }, "drizzle-orm": { "version": "0.44.4", diff --git a/release/app/package.json b/release/app/package.json index 5905b912..55a65ece 100644 --- a/release/app/package.json +++ b/release/app/package.json @@ -16,7 +16,7 @@ }, "dependencies": { "@databricks/sql": "^1.11.0", - "@duckdb/node-api": "^1.5.2-r.1", + "@duckdb/node-api": "1.5.5-r.1", "better-sqlite3": "^12.2.0", "drizzle-orm": "^0.44.4", "fs-extra": "^11.4.0", diff --git a/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts b/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts index a6ae3334..7ec92256 100644 --- a/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts +++ b/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts @@ -46,6 +46,22 @@ export const registerIcebergDatalakeHandlers = () => { IcebergDatalakeService.testInstanceConnection(id), ); + ipcMain.handle('iceberg:sqlCapability', (_e, id: string) => + IcebergDatalakeService.getSqlCapability(id), + ); + + ipcMain.handle('iceberg:verifySqlAccess', (_e, id: string) => + IcebergDatalakeService.verifySqlAccess(id), + ); + + ipcMain.handle('iceberg:executeSql', (_e, params) => + IcebergDatalakeService.executeSql(params), + ); + + ipcMain.handle('iceberg:cancelSql', (_e, executionId: string) => + IcebergDatalakeService.cancelSql(executionId), + ); + ipcMain.handle('iceberg:listNamespaces', (_e, id: string, parent?) => IcebergDatalakeService.listNamespaces(id, parent), ); diff --git a/src/main/services/icebergDatalake.service.ts b/src/main/services/icebergDatalake.service.ts index 66ff1118..a9f17da7 100644 --- a/src/main/services/icebergDatalake.service.ts +++ b/src/main/services/icebergDatalake.service.ts @@ -13,6 +13,7 @@ import * as path from 'path'; import { pathToFileURL } from 'url'; import { spawn } from 'child_process'; import { app } from 'electron'; +import { DuckDBInstance, StatementType } from '@duckdb/node-api'; import { loadDatabaseFile, updateDatabase } from '../utils/fileHelper'; import secureStorage from './secureStorage.service'; @@ -38,11 +39,28 @@ import type { IcebergImportFileFormat, IcebergTableOperationResult, IcebergNamespaceOperationResult, + IcebergSqlCapability, + IcebergSqlExecutionParams, + IcebergSqlExecutionResult, + IcebergSqlStatementClass, } from '../../types/iceberg'; import type { CloudConnection, CloudStorageConfig } from '../../types/frontend'; import type { PostgresConnection } from '../../types/backend'; export class IcebergDatalakeService { + private static readonly activeSqlExecutions = new Map(); + + private static readonly sqlStatementClasses: Partial< + Record + > = { + [StatementType.SELECT]: 'select', + [StatementType.CREATE]: 'create', + [StatementType.DROP]: 'drop', + [StatementType.INSERT]: 'insert', + [StatementType.UPDATE]: 'update', + [StatementType.DELETE]: 'delete', + }; + private static readonly cloudProviders = [ 'aws', 'azure', @@ -54,6 +72,15 @@ export class IcebergDatalakeService { 'garage', ] as const; + private static readonly duckdbIcebergStorageProviders = [ + 'aws', + 'minio', + 'cloudflare-r2', + 'backblaze-b2', + 'rustfs', + 'garage', + ] as const; + private static readonly catalogCapabilities: IcebergCatalogCapability[] = [ { type: 'sqlite', @@ -708,6 +735,207 @@ export class IcebergDatalakeService { } } + private static async validateSqlStorageBinding( + config: Pick< + IcebergInstanceConfig, + | 'catalogType' + | 'nessieWarehouse' + | 'sqlEnabled' + | 'sqlStorageConnectionId' + | 'sqlStorageProvider' + | 'sqlStorageBucket' + | 'sqlWarehouseMatchAcknowledged' + >, + ): Promise { + if (!config.sqlEnabled) return; + if ( + config.catalogType !== 'rest' && + config.catalogType !== 'polaris' && + config.catalogType !== 'lakekeeper' && + config.catalogType !== 'nessie' + ) { + throw new Error('ICEBERG_SQL_REST_CATALOG_REQUIRED'); + } + if (config.catalogType === 'nessie' && !config.nessieWarehouse?.trim()) { + throw new Error('ICEBERG_SQL_NESSIE_WAREHOUSE_REQUIRED'); + } + if ( + !config.sqlStorageConnectionId?.trim() || + !config.sqlStorageProvider || + !config.sqlStorageBucket?.trim() + ) { + throw new Error( + 'ICEBERG_REQUIRED_FIELD: sqlStorageConnectionId/sqlStorageProvider/sqlStorageBucket', + ); + } + if (!config.sqlWarehouseMatchAcknowledged) { + throw new Error('ICEBERG_SQL_WAREHOUSE_MATCH_REQUIRED'); + } + + const { connection } = + await IcebergDatalakeService.resolveCloudStorageConnection( + config.sqlStorageConnectionId, + ); + if ( + !IcebergDatalakeService.duckdbIcebergStorageProviders.includes( + connection.provider as (typeof IcebergDatalakeService.duckdbIcebergStorageProviders)[number], + ) + ) { + throw new Error('ICEBERG_SQL_STORAGE_PROVIDER_NOT_SUPPORTED'); + } + if (config.sqlStorageProvider !== connection.provider) { + throw new Error('ICEBERG_SQL_STORAGE_PROVIDER_MISMATCH'); + } + } + + private static quoteSqlLiteral(value: string): string { + return `'${value.replace(/'/g, "''")}'`; + } + + private static quoteSqlIdentifier(value: string): string { + return `"${value.replace(/"/g, '""')}"`; + } + + private static getSqlRuntimeFingerprint(): string { + try { + // eslint-disable-next-line global-require, @typescript-eslint/no-var-requires + const pkg = require('@duckdb/node-api/package.json'); + return `duckdb-node-api:${String(pkg.version)}`; + } catch { + return 'duckdb-node-api:unknown'; + } + } + + private static async buildDuckDbIcebergSql( + instance: IcebergInstanceConfig, + names: { catalogSecret: string; storageSecret: string; alias: string }, + ): Promise<{ + catalogSecretSql: string; + storageSecretSql: string; + attachSql: string; + }> { + await IcebergDatalakeService.validateSqlStorageBinding(instance); + const connectionId = instance.sqlStorageConnectionId!; + const { connection, config } = + await IcebergDatalakeService.resolveCloudStorageConnection(connectionId); + const cfg = config as Record; + const keyId = String(cfg.accessKeyId ?? cfg.applicationKeyId ?? '').trim(); + const secret = String( + cfg.secretAccessKey ?? cfg.applicationKey ?? '', + ).trim(); + if (!keyId || !secret) { + throw new Error('ICEBERG_SQL_STORAGE_CREDENTIALS_MISSING'); + } + + const storageOptions = [ + 'TYPE S3', + `KEY_ID ${IcebergDatalakeService.quoteSqlLiteral(keyId)}`, + `SECRET ${IcebergDatalakeService.quoteSqlLiteral(secret)}`, + `REGION ${IcebergDatalakeService.quoteSqlLiteral(String(cfg.region ?? 'us-east-1'))}`, + `SCOPE ${IcebergDatalakeService.quoteSqlLiteral( + `s3://${instance.sqlStorageBucket}/${instance.sqlStoragePrefix?.replace(/^\/+|\/+$/g, '') ?? ''}`, + )}`, + ]; + if (cfg.sessionToken) { + storageOptions.push( + `SESSION_TOKEN ${IcebergDatalakeService.quoteSqlLiteral(String(cfg.sessionToken))}`, + ); + } + if (cfg.endpoint) { + const endpoint = String(cfg.endpoint); + const parsed = new URL( + endpoint.includes('://') ? endpoint : `https://${endpoint}`, + ); + storageOptions.push( + `ENDPOINT ${IcebergDatalakeService.quoteSqlLiteral(parsed.host)}`, + `USE_SSL ${parsed.protocol === 'https:' ? 'true' : 'false'}`, + ); + } + if (connection.provider !== 'aws') { + storageOptions.push( + `URL_STYLE ${IcebergDatalakeService.quoteSqlLiteral(String(cfg.urlStyle ?? 'path'))}`, + ); + } + + const catalogOptions = ['TYPE ICEBERG']; + if (instance.catalogAuthMode === 'token') { + const token = instance.catalogAccessTokenKey + ? await secureStorage.getCredential(instance.catalogAccessTokenKey) + : undefined; + if (!token) throw new Error('ICEBERG_ACCESS_TOKEN_REQUIRED'); + catalogOptions.push( + `TOKEN ${IcebergDatalakeService.quoteSqlLiteral(token)}`, + ); + } else if (instance.catalogAuthMode === 'oauth-client-credentials') { + const clientSecret = instance.oauthClientSecretKey + ? await secureStorage.getCredential(instance.oauthClientSecretKey) + : undefined; + if ( + !instance.oauthClientId || + !clientSecret || + !instance.oauthServerUri + ) { + throw new Error('ICEBERG_OAUTH_CLIENT_CREDENTIALS_REQUIRED'); + } + catalogOptions.push( + `CLIENT_ID ${IcebergDatalakeService.quoteSqlLiteral(instance.oauthClientId)}`, + `CLIENT_SECRET ${IcebergDatalakeService.quoteSqlLiteral(clientSecret)}`, + `OAUTH2_SERVER_URI ${IcebergDatalakeService.quoteSqlLiteral(instance.oauthServerUri)}`, + ); + if (instance.oauthScope) { + catalogOptions.push( + `OAUTH2_SCOPE ${IcebergDatalakeService.quoteSqlLiteral(instance.oauthScope)}`, + ); + } else if ( + instance.catalogType === 'nessie' || + instance.catalogType === 'lakekeeper' + ) { + // DuckDB otherwise defaults to PRINCIPAL_ROLE:ALL, which standard + // Nessie and Lakekeeper OIDC clients commonly reject. Request the + // catalog scope assigned to their service clients. + catalogOptions.push( + `OAUTH2_SCOPE ${IcebergDatalakeService.quoteSqlLiteral('catalog')}`, + ); + } + } + + const configuredEndpoint = instance.endpoint?.trim(); + if (!configuredEndpoint) + throw new Error('ICEBERG_REQUIRED_FIELD: endpoint'); + const endpoint = + instance.catalogType === 'nessie' + ? IcebergDatalakeService.buildNessieRestUri({ + ...instance, + nessieWarehouse: undefined, + }) + : configuredEndpoint; + const warehouse = + instance.catalogType === 'nessie' + ? instance.nessieWarehouse?.trim() + : instance.catalogName; + if (!warehouse) throw new Error('ICEBERG_REQUIRED_FIELD: catalogName'); + + return { + storageSecretSql: `CREATE TEMPORARY SECRET ${IcebergDatalakeService.quoteSqlIdentifier( + names.storageSecret, + )} (${storageOptions.join(', ')})`, + catalogSecretSql: `CREATE TEMPORARY SECRET ${IcebergDatalakeService.quoteSqlIdentifier( + names.catalogSecret, + )} (${catalogOptions.join(', ')})`, + attachSql: `ATTACH ${IcebergDatalakeService.quoteSqlLiteral( + warehouse, + )} AS ${IcebergDatalakeService.quoteSqlIdentifier(names.alias)} (TYPE ICEBERG, SECRET ${IcebergDatalakeService.quoteSqlIdentifier( + names.catalogSecret, + )}, ENDPOINT ${IcebergDatalakeService.quoteSqlLiteral( + endpoint, + )}, ACCESS_DELEGATION_MODE ${IcebergDatalakeService.quoteSqlLiteral('none')}${ + (instance.catalogAuthMode ?? 'none') === 'none' + ? `, AUTHORIZATION_TYPE ${IcebergDatalakeService.quoteSqlLiteral('none')}` + : '' + })`, + }; + } + static getCapabilities(): IcebergCapabilities { return { catalogs: IcebergDatalakeService.catalogCapabilities.map((item) => ({ @@ -778,6 +1006,7 @@ export class IcebergDatalakeService { try { IcebergDatalakeService.validateCatalogWarehousePair(data); IcebergDatalakeService.validateCatalogAuthentication(data); + await IcebergDatalakeService.validateSqlStorageBinding(data); const id = uuidv4(); const now = new Date().toISOString(); @@ -843,8 +1072,34 @@ export class IcebergDatalakeService { ...instances[idx], ...data, }; + const sqlAttachmentChanged = + [ + 'catalogType', + 'endpoint', + 'catalogName', + 'catalogAuthMode', + 'oauthClientId', + 'oauthServerUri', + 'oauthScope', + 'nessieReference', + 'nessieWarehouse', + 'sqlEnabled', + 'sqlStorageConnectionId', + 'sqlStorageProvider', + 'sqlStorageBucket', + 'sqlStoragePrefix', + 'sqlWarehouseMatchAcknowledged', + ].some( + (field) => + data[field as keyof UpdateIcebergInstanceDTO] !== undefined && + data[field as keyof UpdateIcebergInstanceDTO] !== + instances[idx][field as keyof IcebergInstanceConfig], + ) || + !!data.accessToken || + !!data.oauthClientSecret; IcebergDatalakeService.validateCatalogWarehousePair(updatedConfig); IcebergDatalakeService.validateCatalogAuthentication(updatedConfig); + await IcebergDatalakeService.validateSqlStorageBinding(updatedConfig); // Handle access token update if (data.accessToken) { @@ -874,6 +1129,12 @@ export class IcebergDatalakeService { ...instances[idx], ...rest, id, + ...(sqlAttachmentChanged + ? { + sqlAccessVerifiedAt: undefined, + sqlRuntimeFingerprint: undefined, + } + : {}), updatedAt: new Date().toISOString(), }; @@ -1216,6 +1477,247 @@ export class IcebergDatalakeService { } } + static async getSqlCapability(id: string): Promise { + const instance = await IcebergDatalakeService.getInstance(id); + const runtimeFingerprint = + IcebergDatalakeService.getSqlRuntimeFingerprint(); + if (!instance.sqlEnabled) { + return { + available: false, + reason: 'ICEBERG_SQL_DISABLED', + canRead: false, + canWrite: false, + supportedStatements: [], + }; + } + if ( + !instance.sqlAccessVerifiedAt || + instance.sqlRuntimeFingerprint !== runtimeFingerprint + ) { + return { + available: false, + reason: 'ICEBERG_SQL_UNVERIFIED', + runtimeFingerprint, + canRead: false, + canWrite: false, + supportedStatements: [], + }; + } + return { + available: true, + runtimeFingerprint, + canRead: true, + canWrite: true, + supportedStatements: [ + 'select', + 'create', + 'drop', + 'insert', + 'update', + 'delete', + ], + }; + } + + static async verifySqlAccess(id: string): Promise { + try { + const executionId = `verify-${uuidv4()}`; + await IcebergDatalakeService.withAttachedSqlCatalog( + id, + executionId, + async (connection, alias) => { + await connection.runAndReadUntil( + 'SELECT table_schema, table_name FROM information_schema.tables WHERE table_catalog = ? LIMIT 1', + 1, + [alias], + ); + }, + ); + const runtimeFingerprint = + IcebergDatalakeService.getSqlRuntimeFingerprint(); + const instances = await IcebergDatalakeService.readInstances(); + const index = instances.findIndex((instance) => instance.id === id); + if (index < 0) throw new Error(`Iceberg instance not found: ${id}`); + const checkedAt = new Date().toISOString(); + instances[index] = { + ...instances[index], + sqlAccessVerifiedAt: checkedAt, + sqlRuntimeFingerprint: runtimeFingerprint, + updatedAt: checkedAt, + }; + await IcebergDatalakeService.writeInstances(instances); + return { + success: true, + catalogConnected: true, + warehouseConnected: true, + checkedAt, + }; + } catch (error) { + // eslint-disable-next-line no-console + console.error('[IcebergDatalakeService] verifySqlAccess error'); + return { + success: false, + catalogConnected: false, + warehouseConnected: false, + checkedAt: new Date().toISOString(), + error: error instanceof Error ? error.message : String(error), + }; + } + } + + private static async withAttachedSqlCatalog( + instanceId: string, + executionId: string, + callback: (connection: any, alias: string) => Promise, + ): Promise { + if (!executionId.trim() || executionId.length > 120) { + throw new Error('ICEBERG_SQL_EXECUTION_ID_INVALID'); + } + if (IcebergDatalakeService.activeSqlExecutions.has(executionId)) { + throw new Error('ICEBERG_SQL_EXECUTION_ID_DUPLICATE'); + } + const instance = await IcebergDatalakeService.getInstance(instanceId); + const suffix = uuidv4().replace(/-/g, ''); + const names = { + alias: `iceberg_${suffix}`, + catalogSecret: `iceberg_catalog_${suffix}`, + storageSecret: `iceberg_storage_${suffix}`, + }; + const sql = await IcebergDatalakeService.buildDuckDbIcebergSql( + instance, + names, + ); + let duckdbInstance: any; + let connection: any; + let attached = false; + let stage = 'initialize'; + try { + duckdbInstance = await DuckDBInstance.create(':memory:'); + connection = await duckdbInstance.connect(); + IcebergDatalakeService.activeSqlExecutions.set(executionId, connection); + stage = 'install-extensions'; + await connection.run('INSTALL httpfs'); + await connection.run('INSTALL iceberg'); + stage = 'load-extensions'; + await connection.run('LOAD httpfs'); + await connection.run('LOAD iceberg'); + stage = 'create-storage-secret'; + await connection.run(sql.storageSecretSql); + stage = 'create-catalog-secret'; + await connection.run(sql.catalogSecretSql); + stage = 'attach'; + await connection.run(sql.attachSql); + attached = true; + stage = 'execute'; + return await callback(connection, names.alias); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message.startsWith('ICEBERG_')) throw error; + throw new Error(`ICEBERG_SQL_RUNTIME_FAILED: ${stage}`); + } finally { + IcebergDatalakeService.activeSqlExecutions.delete(executionId); + if (connection) { + if (attached) { + try { + await connection.run( + `DETACH ${IcebergDatalakeService.quoteSqlIdentifier(names.alias)}`, + ); + } catch { + // Continue best-effort cleanup. + } + } + await [names.catalogSecret, names.storageSecret].reduce( + async (previous, secret) => { + await previous; + try { + await connection.run( + `DROP SECRET IF EXISTS ${IcebergDatalakeService.quoteSqlIdentifier(secret)}`, + ); + } catch { + // Continue best-effort cleanup. + } + }, + Promise.resolve(), + ); + connection.closeSync?.(); + } + duckdbInstance?.closeSync?.(); + } + } + + private static async classifySqlStatement( + connection: any, + sql: string, + ): Promise { + if (!sql.trim() || sql.length > 1_000_000) { + throw new Error('ICEBERG_SQL_INVALID'); + } + const extracted = await connection.extractStatements(sql); + if (extracted.count !== 1) { + throw new Error('ICEBERG_SQL_SINGLE_STATEMENT_REQUIRED'); + } + const prepared = await extracted.prepare(0); + try { + const statementClass = + IcebergDatalakeService.sqlStatementClasses[ + prepared.statementType as StatementType + ]; + if (!statementClass) throw new Error('ICEBERG_SQL_STATEMENT_REJECTED'); + const rejectedSurface = + /\b(attach|detach|install|load|pragma|copy|merge|alter|create\s+(?:or\s+replace\s+)?(?:temporary\s+|temp\s+)?secret|drop\s+secret|read_(?:csv|json|parquet)|iceberg_scan|httpfs)\b/i; + if (rejectedSurface.test(sql)) { + throw new Error('ICEBERG_SQL_STATEMENT_REJECTED'); + } + return statementClass; + } finally { + prepared.destroySync(); + } + } + + static async executeSql( + params: IcebergSqlExecutionParams, + ): Promise { + const capability = await IcebergDatalakeService.getSqlCapability( + params.instanceId, + ); + if (!capability.available) { + throw new Error(capability.reason ?? 'ICEBERG_SQL_UNAVAILABLE'); + } + const maxRows = Math.max(1, Math.min(params.maxRows ?? 1000, 5000)); + return IcebergDatalakeService.withAttachedSqlCatalog( + params.instanceId, + params.executionId, + async (connection) => { + const statementClass = + await IcebergDatalakeService.classifySqlStatement( + connection, + params.sql, + ); + const reader = await connection.runAndReadUntil( + params.sql, + maxRows + 1, + ); + const rows = reader.getRowsJson(); + return { + executionId: params.executionId, + statementClass, + columns: reader.columnNames(), + rows: rows.slice(0, maxRows) as unknown[][], + rowsChanged: Number(reader.rowsChanged ?? 0), + truncated: rows.length > maxRows || !reader.done, + }; + }, + ); + } + + static cancelSql(executionId: string): boolean { + const connection = + IcebergDatalakeService.activeSqlExecutions.get(executionId); + if (!connection) return false; + connection.interrupt(); + return true; + } + // ───────────────────────────────────────────── // Public: pyiceberg installation // ───────────────────────────────────────────── diff --git a/src/renderer/components/dataLake/IcebergConnectionWizard.tsx b/src/renderer/components/dataLake/IcebergConnectionWizard.tsx index 785b92a4..35ef9887 100644 --- a/src/renderer/components/dataLake/IcebergConnectionWizard.tsx +++ b/src/renderer/components/dataLake/IcebergConnectionWizard.tsx @@ -28,6 +28,8 @@ import { List, ListItem, ListItemText, + Checkbox, + FormControlLabel, } from '@mui/material'; import { ArrowForward, @@ -52,6 +54,7 @@ import { useListIcebergStorageBuckets, useTestIcebergCatalog, useTestIcebergStorage, + useVerifyIcebergSqlAccess, } from '../../controllers/icebergDatalake.controller'; import { DataLakeConnectionSelector } from './DataLakeConnectionSelector'; import { secureStorageService } from '../../services/secureStorage.service'; @@ -92,6 +95,16 @@ export interface IcebergWizardData { bucket?: string; prefix?: string; }; + sql: { + enabled: boolean; + connectionId?: string; + provider?: IcebergCloudProvider; + bucket?: string; + prefix?: string; + warehouseMatchAcknowledged: boolean; + accessVerifiedAt?: string; + runtimeFingerprint?: string; + }; } export interface IcebergConnectionWizardProps { @@ -113,6 +126,7 @@ const emptyData = (): IcebergWizardData => ({ basics: { name: '', description: '' }, catalog: { catalogType: 'sqlite' }, storage: { storageType: 'local' }, + sql: { enabled: false, warehouseMatchAcknowledged: false }, }); function buildInitialData(initial?: IcebergInstanceConfig): IcebergWizardData { @@ -157,6 +171,17 @@ function buildInitialData(initial?: IcebergInstanceConfig): IcebergWizardData { bucket: initial.storageBucket, prefix: initial.storagePrefix, }, + sql: { + enabled: initial.sqlEnabled ?? false, + connectionId: initial.sqlStorageConnectionId, + provider: initial.sqlStorageProvider, + bucket: initial.sqlStorageBucket, + prefix: initial.sqlStoragePrefix, + warehouseMatchAcknowledged: + initial.sqlWarehouseMatchAcknowledged ?? false, + accessVerifiedAt: initial.sqlAccessVerifiedAt, + runtimeFingerprint: initial.sqlRuntimeFingerprint, + }, }; } @@ -210,6 +235,13 @@ function validateStep( ) { return 'Nessie reference is required.'; } + if ( + data.catalog.catalogType === 'nessie' && + data.sql.enabled && + !data.catalog.nessieWarehouse?.trim() + ) { + return 'Nessie warehouse is required for DuckDB SQL access.'; + } if ( data.catalog.authMode === 'oauth-client-credentials' && !data.catalog.oauthClientId @@ -238,7 +270,14 @@ function validateStep( data.catalog.catalogType === 'lakekeeper' || data.catalog.catalogType === 'nessie' ) { - // REST catalogs manage warehouse storage server-side; vended creds optional + if (!data.sql.enabled) return null; + if (!data.sql.connectionId) + return 'A Cloud Explorer connection is required for DuckDB SQL access.'; + if (!data.sql.bucket) + return 'The matching warehouse bucket is required for DuckDB SQL access.'; + if (!data.sql.warehouseMatchAcknowledged) { + return 'Confirm that the Cloud connection points to the catalog warehouse.'; + } return null; } if (data.storage.storageType === 'local' && !data.storage.localPath) { @@ -279,6 +318,10 @@ export const IcebergConnectionWizard: React.FC< success: boolean; message: string; } | null>(null); + const [sqlTestResult, setSqlTestResult] = useState<{ + success: boolean; + message: string; + } | null>(null); const initializedInstanceIdRef = useRef(null); useEffect(() => { @@ -357,6 +400,7 @@ export const IcebergConnectionWizard: React.FC< // Catalog test const testCatalogMutation = useTestIcebergCatalog(); const testStorageMutation = useTestIcebergStorage(); + const verifySqlMutation = useVerifyIcebergSqlAccess(); const listStorageBucketsMutation = useListIcebergStorageBuckets(); const loadStorageBuckets = useCallback( (connectionId: string) => @@ -437,22 +481,40 @@ export const IcebergConnectionWizard: React.FC< const patchStorage = (patch: Partial) => setData((d) => ({ ...d, storage: { ...d.storage, ...patch } })); - const handleSelectVendedStorage = useCallback( - (connectionId: string, bucket: string, prefix?: string) => { + const patchSql = (patch: Partial) => + setData((d) => ({ + ...d, + sql: { + ...d.sql, + ...patch, + accessVerifiedAt: undefined, + runtimeFingerprint: undefined, + }, + })); + + const handleSelectCloudStorage = useCallback( + ( + connectionId: string, + bucket: string, + prefix?: string, + provider?: IcebergCloudProvider, + ) => { + setStorageTestResult(null); setData((current) => ({ ...current, - catalog: { - ...current.catalog, - polarisConnectionId: connectionId, - polarisBucket: bucket, - polarisPrefix: prefix, + storage: { + ...current.storage, + connectionId, + bucket, + prefix, + cloudProvider: provider, }, })); }, [], ); - const handleSelectCloudStorage = useCallback( + const handleSelectSqlStorage = useCallback( ( connectionId: string, bucket: string, @@ -462,12 +524,15 @@ export const IcebergConnectionWizard: React.FC< setStorageTestResult(null); setData((current) => ({ ...current, - storage: { - ...current.storage, + sql: { + ...current.sql, connectionId, bucket, prefix, - cloudProvider: provider, + provider, + warehouseMatchAcknowledged: false, + accessVerifiedAt: undefined, + runtimeFingerprint: undefined, }, })); }, @@ -527,11 +592,20 @@ export const IcebergConnectionWizard: React.FC< databaseConnectionId: data.catalog.databaseConnectionId, storageType: data.storage.storageType, }); + const missingNessieSqlWarehouse = + result.success && + data.catalog.catalogType === 'nessie' && + data.sql.enabled && + !data.catalog.nessieWarehouse?.trim(); + let message = result.error ?? 'Catalog test failed.'; + if (result.success) message = 'Catalog connection successful.'; + if (missingNessieSqlWarehouse) { + message = + 'Catalog connected, but a Nessie warehouse name is required for DuckDB SQL access.'; + } setCatalogTestResult({ - success: result.success, - message: result.success - ? 'Catalog connection successful.' - : (result.error ?? 'Catalog test failed.'), + success: result.success && !missingNessieSqlWarehouse, + message, }); } catch (err: any) { // eslint-disable-next-line no-console @@ -572,6 +646,47 @@ export const IcebergConnectionWizard: React.FC< } }; + const handleTestSqlStorage = async () => { + setStorageTestResult(null); + const validationError = validateStep(2, data); + if (validationError) { + setStorageTestResult({ success: false, message: validationError }); + return; + } + try { + const result = await testStorageMutation.mutateAsync({ + connectionId: data.sql.connectionId!, + bucket: data.sql.bucket!, + prefix: data.sql.prefix, + }); + setStorageTestResult({ + success: result.success, + message: result.success + ? 'The matching object-store location is accessible. DuckDB attachment verification is completed in Phase 3.' + : (result.error ?? 'Object-store access test failed.'), + }); + } catch (error: any) { + // eslint-disable-next-line no-console + console.error(error); + setStorageTestResult({ + success: false, + message: error?.message ?? 'Object-store access test failed.', + }); + } + }; + + const handleVerifySqlAccess = async () => { + if (!initialData?.id) return; + setSqlTestResult(null); + const result = await verifySqlMutation.mutateAsync(initialData.id); + setSqlTestResult({ + success: result.success, + message: result.success + ? 'DuckDB attached to the catalog and cleaned up successfully.' + : (result.error ?? 'DuckDB SQL access test failed.'), + }); + }; + // ── Step content renderers ────────────────────────────────────────────── const renderBasicsStep = () => ( @@ -667,6 +782,14 @@ export const IcebergConnectionWizard: React.FC< bucket: undefined, prefix: undefined, }); + patchSql({ + enabled: false, + connectionId: undefined, + provider: undefined, + bucket: undefined, + prefix: undefined, + warehouseMatchAcknowledged: false, + }); }} > {catalogCapabilities.map((capability) => { @@ -848,14 +971,23 @@ export const IcebergConnectionWizard: React.FC< helperText="Branch or tag to read and write through Iceberg REST" /> patchCatalog({ nessieWarehouse: event.target.value }) } fullWidth - helperText="Leave blank to use the Nessie server's default warehouse" + required={data.sql.enabled} + helperText={ + data.sql.enabled + ? 'Required for DuckDB SQL access; enter the Nessie warehouse name.' + : "Leave blank to use the Nessie server's default warehouse" + } /> ) : ( @@ -1032,55 +1164,109 @@ export const IcebergConnectionWizard: React.FC< Storage Configuration - {data.catalog.catalogType === 'polaris' && ( - - Storage is managed by Polaris for this catalog. Polaris provides - the authoritative warehouse location and object-store access, so - no Cloud Explorer connection is required. - - )} - {data.catalog.catalogType === 'nessie' && ( - - Nessie manages the warehouse and sends the required FileIO and - object-store configuration to PyIceberg. DBT Studio uses remote - request signing, so no Cloud Explorer credentials are required. - - )} - {data.catalog.catalogType === 'lakekeeper' && ( - - Storage is managed by the selected Lakekeeper warehouse. Configure - its bucket, prefix, endpoint, and credentials in the Lakekeeper - UI. DBT Studio uses remote request signing, so no Cloud Explorer - connection is required. - - )} - {data.catalog.catalogType !== 'polaris' && - data.catalog.catalogType !== 'lakekeeper' && - data.catalog.catalogType !== 'nessie' && ( + + + Catalog-managed warehouse + + + {data.catalog.catalogType === 'polaris' && + 'Polaris remains the authoritative warehouse owner.'} + {data.catalog.catalogType === 'lakekeeper' && + 'Lakekeeper remains the authoritative warehouse owner.'} + {data.catalog.catalogType === 'nessie' && + 'Nessie remains the authoritative catalog and warehouse owner.'} + {data.catalog.catalogType === 'rest' && + 'The REST catalog remains the authoritative warehouse owner.'}{' '} + Existing PyIceberg DataLake operations continue to use the + server-managed warehouse configuration. + + + + + + DuckDB object-store access + + { + const enabled = event.target.checked; + patchSql( + enabled + ? { enabled } + : { + enabled, + connectionId: undefined, + provider: undefined, + bucket: undefined, + prefix: undefined, + warehouseMatchAcknowledged: false, + }, + ); + setStorageTestResult(null); + }} + /> + } + label="Enable SQL Editor and Notebooks" + /> + {data.sql.enabled && ( <> - - REST catalogs manage table storage on the server. Configure a - cloud connection below only if your catalog uses{' '} - vended credentials to delegate access to - object storage. - - - Vended credentials (optional) - - - Select a Cloud Explorer connection and bucket the catalog can - use when delegating storage access. Leave blank if credentials - are handled another way. - + + patchSql({ + warehouseMatchAcknowledged: event.target.checked, + }) + } + /> + } + label="I confirm this Cloud connection, bucket, and prefix point to the warehouse configured in the Iceberg catalog service." + /> + + + {storageTestResult && ( + : undefined + } + > + {storageTestResult.message} + + )} + )} + ); } @@ -1204,6 +1390,34 @@ export const IcebergConnectionWizard: React.FC< const hasToken = !!data.catalog.accessToken || (mode === 'edit' && !!initialData?.catalogAccessTokenKey); + const normalizeDraftValue = (value: unknown) => + typeof value === 'string' ? value.trim() : value; + const hasUnsavedSqlAttachmentChanges = + mode === 'edit' && + !!initialData && + [ + [data.catalog.catalogType, initialData.catalogType], + [data.catalog.endpoint, initialData.endpoint], + [data.catalog.catalogName, initialData.catalogName], + [data.catalog.authMode, initialData.catalogAuthMode ?? 'none'], + [data.catalog.oauthClientId, initialData.oauthClientId], + [data.catalog.oauthServerUri, initialData.oauthServerUri], + [data.catalog.oauthScope, initialData.oauthScope], + [data.catalog.nessieReference, initialData.nessieReference], + [data.catalog.nessieWarehouse, initialData.nessieWarehouse], + [data.sql.enabled, initialData.sqlEnabled ?? false], + [data.sql.connectionId, initialData.sqlStorageConnectionId], + [data.sql.provider, initialData.sqlStorageProvider], + [data.sql.bucket, initialData.sqlStorageBucket], + [data.sql.prefix, initialData.sqlStoragePrefix], + [ + data.sql.warehouseMatchAcknowledged, + initialData.sqlWarehouseMatchAcknowledged ?? false, + ], + ].some( + ([draftValue, savedValue]) => + normalizeDraftValue(draftValue) !== normalizeDraftValue(savedValue), + ); return ( @@ -1337,32 +1551,46 @@ export const IcebergConnectionWizard: React.FC< secondary="Server-managed (REST catalog)" /> - {data.catalog.polarisConnectionId && ( + {data.sql.enabled && ( <> + + + - {data.catalog.polarisBucket && ( + {data.sql.bucket && ( )} - {data.catalog.polarisPrefix && ( + {data.sql.prefix && ( )} )} + {!data.sql.enabled && ( + + + + )} ) : ( <> @@ -1414,6 +1642,42 @@ export const IcebergConnectionWizard: React.FC< )} + {data.sql.enabled && mode === 'edit' && initialData?.id && ( + + + {sqlTestResult && ( + : undefined} + > + {sqlTestResult.message} + + )} + {hasUnsavedSqlAttachmentChanges && ( + + Save these attachment changes, reopen the instance, then test + SQL access. + + )} + + )} }> {mode === 'create' ? 'Clicking "Create Instance" will save these settings and register the Iceberg catalog. No data files will be modified.' diff --git a/src/renderer/components/dataLake/iceberg/IcebergDetail.tsx b/src/renderer/components/dataLake/iceberg/IcebergDetail.tsx index f2f22988..9f285d1d 100644 --- a/src/renderer/components/dataLake/iceberg/IcebergDetail.tsx +++ b/src/renderer/components/dataLake/iceberg/IcebergDetail.tsx @@ -41,6 +41,7 @@ import { Add, ArrowBack, Badge, + Cable, ChevronRight, CheckCircle, Close, @@ -86,6 +87,7 @@ import { useListIcebergTables, useRenameIcebergTable, useTestIcebergInstance, + useVerifyIcebergSqlAccess, } from '../../../controllers/icebergDatalake.controller'; import { IcebergIcon } from './IcebergIcon'; import { IcebergOperationBackdrop } from './IcebergOperationBackdrop'; @@ -890,6 +892,7 @@ export const IcebergDetail: React.FC = ({ const queryClient = useQueryClient(); const namespacesQuery = useListIcebergNamespaces(instance.id); const testInstanceMutation = useTestIcebergInstance(); + const verifySqlMutation = useVerifyIcebergSqlAccess(); const importTableMutation = useImportIcebergTable(); const dropTableMutation = useDropIcebergTable(); const renameTableMutation = useRenameIcebergTable(); @@ -898,6 +901,7 @@ export const IcebergDetail: React.FC = ({ const [currentTab, setCurrentTab] = React.useState(0); const [tableFilter, setTableFilter] = React.useState(''); const [importWizardOpen, setImportWizardOpen] = React.useState(false); + const [sqlSupportInfoOpen, setSqlSupportInfoOpen] = React.useState(false); const [tableToDelete, setTableToDelete] = React.useState(null); const [tableToRename, setTableToRename] = @@ -1095,12 +1099,33 @@ export const IcebergDetail: React.FC = ({ } }; + const testSqlAccess = async () => { + try { + const result = await verifySqlMutation.mutateAsync(instance.id); + if (result.success) { + toast.success( + 'DuckDB attached to the catalog and cleaned up successfully.', + ); + return; + } + toast.error(result.error ?? 'DuckDB SQL access test failed.'); + } catch (error) { + toast.error(errorMessage(error)); + } + }; + const testResult = testInstanceMutation.data; const testIndicatorColor = (() => { if (testInstanceMutation.isLoading) return 'warning.main'; if (!testResult) return 'grey.500'; return testResult.success ? 'success.main' : 'error.main'; })(); + const sqlTestResult = verifySqlMutation.data; + const sqlTestIndicatorColor = (() => { + if (verifySqlMutation.isLoading) return 'warning.main'; + if (!sqlTestResult) return 'grey.500'; + return sqlTestResult.success ? 'success.main' : 'error.main'; + })(); const connectionStatusIcon = (healthy?: boolean) => { if (healthy === true) { @@ -1509,42 +1534,131 @@ export const IcebergDetail: React.FC = ({ Health Status - + > + {verifySqlMutation.isLoading + ? 'Testing…' + : 'Test Access'} + + + + @@ -1718,6 +1832,124 @@ export const IcebergDetail: React.FC = ({ {renderImportWizard()} + setSqlSupportInfoOpen(false)} + maxWidth="sm" + fullWidth + > + DuckDB Iceberg SQL support + + + Supported catalogs + + + + Generic Iceberg REST + + + Apache Polaris + + + Lakekeeper + + + Project Nessie + + + + + Requirements and verification + + + + The catalog warehouse must use S3 or S3-compatible storage. + + + Rosetta dbt Studio retrieves credentials from secure storage and + creates temporary DuckDB secrets. + + + The test attaches the REST catalog, verifies metadata access, + detaches it, and removes the temporary secrets. + + + + + Not supported by this SQL integration + + + + Hive Metastore catalogs + + + SQLite catalogs + + + PostgreSQL and Neon catalogs + + + + + + + + {/* Delete table confirmation dialog */} export const useTestIcebergInstance = () => useMutation((id: string) => icebergService.testIcebergInstance(id)); +export const useIcebergSqlCapability = (id: string) => + useQuery( + ['iceberg', 'sql-capability', id], + () => icebergService.getIcebergSqlCapability(id), + { enabled: !!id }, + ); + +export const useVerifyIcebergSqlAccess = () => { + const qc = useQueryClient(); + return useMutation( + (id: string) => icebergService.verifyIcebergSqlAccess(id), + { + onSuccess: (_result, id) => { + qc.invalidateQueries(['iceberg', 'instance', id]); + qc.invalidateQueries(['iceberg', 'sql-capability', id]); + }, + }, + ); +}; + +export const useExecuteIcebergSql = () => + useMutation((params: IcebergSqlExecutionParams) => + icebergService.executeIcebergSql(params), + ); + export const useCreateIcebergMetadataFile = () => useMutation((warehousePath: string) => icebergService.createIcebergMetadataFile(warehousePath), diff --git a/src/renderer/screens/dataLake/index.tsx b/src/renderer/screens/dataLake/index.tsx index d094edcd..2cb5d0c6 100644 --- a/src/renderer/screens/dataLake/index.tsx +++ b/src/renderer/screens/dataLake/index.tsx @@ -12,6 +12,7 @@ import { Alert, CircularProgress, } from '@mui/material'; +import { Close, Delete } from '@mui/icons-material'; import { useLocation, useParams, useNavigate } from 'react-router-dom'; import { toast } from 'react-toastify'; import { AppLayout } from '../../layouts'; @@ -263,6 +264,12 @@ const DataLake: React.FC = () => { storageConnectionId: wizardData.storage.connectionId, storageBucket: wizardData.storage.bucket, storagePrefix: wizardData.storage.prefix, + sqlEnabled: wizardData.sql.enabled, + sqlStorageConnectionId: wizardData.sql.connectionId, + sqlStorageProvider: wizardData.sql.provider, + sqlStorageBucket: wizardData.sql.bucket, + sqlStoragePrefix: wizardData.sql.prefix, + sqlWarehouseMatchAcknowledged: wizardData.sql.warehouseMatchAcknowledged, }; try { const created = await createIcebergMutation.mutateAsync(dto); @@ -310,6 +317,12 @@ const DataLake: React.FC = () => { storageConnectionId: wizardData.storage.connectionId, storageBucket: wizardData.storage.bucket, storagePrefix: wizardData.storage.prefix, + sqlEnabled: wizardData.sql.enabled, + sqlStorageConnectionId: wizardData.sql.connectionId, + sqlStorageProvider: wizardData.sql.provider, + sqlStorageBucket: wizardData.sql.bucket, + sqlStoragePrefix: wizardData.sql.prefix, + sqlWarehouseMatchAcknowledged: wizardData.sql.warehouseMatchAcknowledged, }; try { await updateIcebergMutation.mutateAsync({ id: icebergEditId, data: dto }); @@ -656,17 +669,37 @@ const DataLake: React.FC = () => { > Delete Iceberg Instance - + Delete Iceberg instance {icebergDeleteTarget?.name} - ? This cannot be undone. Keytar credentials for this instance will - also be removed. + ? + + + Only the Rosetta DBT Studio connection to this Iceberg catalog + will be removed. + + + Instance-specific catalog credentials will be removed from + Keytar. + + + The Iceberg catalog, namespaces, tables, snapshots, and data + files will not be changed. + + + The Cloud Explorer connection, bucket, and stored cloud + credentials will not be changed. You can connect to this Iceberg + catalog again later. + + @@ -678,10 +711,12 @@ const DataLake: React.FC = () => { startIcon={ deleteIcebergMutation.isLoading ? ( - ) : undefined + ) : ( + + ) } > - {deleteIcebergMutation.isLoading ? 'Deleting…' : 'Delete'} + {deleteIcebergMutation.isLoading ? 'Deleting…' : 'Delete Instance'} diff --git a/src/renderer/services/iceberg.service.ts b/src/renderer/services/iceberg.service.ts index 85ba2a34..c33971e2 100644 --- a/src/renderer/services/iceberg.service.ts +++ b/src/renderer/services/iceberg.service.ts @@ -21,6 +21,9 @@ import type { IcebergImportFileFormat, IcebergTableOperationResult, IcebergNamespaceOperationResult, + IcebergSqlCapability, + IcebergSqlExecutionParams, + IcebergSqlExecutionResult, } from '../../types/iceberg'; export const getIcebergCapabilities = (): Promise => @@ -66,6 +69,24 @@ export const listIcebergStorageBuckets = ( export const testIcebergInstance = (id: string): Promise => window.electron.ipcRenderer.invoke('iceberg:testInstance', id); +export const getIcebergSqlCapability = ( + id: string, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:sqlCapability', id); + +export const verifyIcebergSqlAccess = ( + id: string, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:verifySqlAccess', id); + +export const executeIcebergSql = ( + params: IcebergSqlExecutionParams, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:executeSql', params); + +export const cancelIcebergSql = (executionId: string): Promise => + window.electron.ipcRenderer.invoke('iceberg:cancelSql', executionId); + export const listIcebergNamespaces = ( id: string, parent?: string[], diff --git a/src/types/iceberg.ts b/src/types/iceberg.ts index 7a3d80c3..29538e77 100644 --- a/src/types/iceberg.ts +++ b/src/types/iceberg.ts @@ -86,6 +86,16 @@ export interface IcebergInstanceConfig { storageConnectionId?: string; // Cloud Explorer connectionId for data files storageBucket?: string; storagePrefix?: string; + // DuckDB Iceberg SQL access for REST catalogs. Secrets remain owned by the + // referenced Cloud Explorer connection and are never copied here. + sqlEnabled?: boolean; + sqlStorageConnectionId?: string; + sqlStorageProvider?: IcebergCloudProvider; + sqlStorageBucket?: string; + sqlStoragePrefix?: string; + sqlWarehouseMatchAcknowledged?: boolean; + sqlAccessVerifiedAt?: string; + sqlRuntimeFingerprint?: string; // Metadata createdAt: string; updatedAt: string; @@ -175,6 +185,39 @@ export interface IcebergTestStorageParams { prefix?: string; } +export type IcebergSqlStatementClass = + | 'select' + | 'create' + | 'drop' + | 'insert' + | 'update' + | 'delete'; + +export interface IcebergSqlCapability { + available: boolean; + reason?: string; + runtimeFingerprint?: string; + canRead: boolean; + canWrite: boolean; + supportedStatements: IcebergSqlStatementClass[]; +} + +export interface IcebergSqlExecutionParams { + instanceId: string; + executionId: string; + sql: string; + maxRows?: number; +} + +export interface IcebergSqlExecutionResult { + executionId: string; + statementClass: IcebergSqlStatementClass; + columns: string[]; + rows: unknown[][]; + rowsChanged: number; + truncated: boolean; +} + export interface IcebergListStorageBucketsParams { connectionId: string; } diff --git a/src/types/ipc.ts b/src/types/ipc.ts index a2f73953..aef7b799 100644 --- a/src/types/ipc.ts +++ b/src/types/ipc.ts @@ -359,6 +359,10 @@ export type IcebergChannels = | 'iceberg:testStorage' | 'iceberg:listStorageBuckets' | 'iceberg:testInstance' + | 'iceberg:sqlCapability' + | 'iceberg:verifySqlAccess' + | 'iceberg:executeSql' + | 'iceberg:cancelSql' | 'iceberg:listNamespaces' | 'iceberg:listTables' | 'iceberg:getSchema' diff --git a/tests/unit/main/services/icebergDatalake.service.test.ts b/tests/unit/main/services/icebergDatalake.service.test.ts index 962d8574..edb36c62 100644 --- a/tests/unit/main/services/icebergDatalake.service.test.ts +++ b/tests/unit/main/services/icebergDatalake.service.test.ts @@ -151,6 +151,92 @@ describe('IcebergDatalakeService compatibility and secret persistence', () => { ); }); + it('persists a non-secret DuckDB storage binding for a REST catalog', async () => { + mockedLoadDatabase.mockResolvedValue({ + icebergInstances: [], + sources: [ + { + id: 'minio-connection', + name: 'Warehouse MinIO', + provider: 'minio', + config: { endpoint: 'http://localhost:9000', accessKeyId: 'test' }, + }, + ], + }); + + await IcebergDatalakeService.createInstance({ + name: 'lakekeeper-sql', + catalogType: 'lakekeeper', + endpoint: 'http://localhost:8181/catalog', + catalogName: 'warehouse', + catalogAuthMode: 'none', + storageType: 'server-managed', + sqlEnabled: true, + sqlStorageConnectionId: 'minio-connection', + sqlStorageProvider: 'minio', + sqlStorageBucket: 'warehouse', + sqlStoragePrefix: 'iceberg', + sqlWarehouseMatchAcknowledged: true, + }); + + expect(mockedUpdateDatabase.mock.calls[0][1][0]).toMatchObject({ + sqlEnabled: true, + sqlStorageConnectionId: 'minio-connection', + sqlStorageProvider: 'minio', + sqlStorageBucket: 'warehouse', + sqlStoragePrefix: 'iceberg', + sqlWarehouseMatchAcknowledged: true, + }); + }); + + it('rejects unsupported DuckDB storage providers for REST SQL access', async () => { + mockedLoadDatabase.mockResolvedValue({ + icebergInstances: [], + sources: [ + { + id: 'azure-connection', + name: 'Azure Warehouse', + provider: 'azure', + config: { accountName: 'test' }, + }, + ], + }); + + await expect( + IcebergDatalakeService.createInstance({ + name: 'rest-azure', + catalogType: 'rest', + endpoint: 'http://localhost:8181/catalog', + catalogName: 'warehouse', + catalogAuthMode: 'none', + storageType: 'server-managed', + sqlEnabled: true, + sqlStorageConnectionId: 'azure-connection', + sqlStorageProvider: 'azure', + sqlStorageBucket: 'warehouse', + sqlWarehouseMatchAcknowledged: true, + }), + ).rejects.toThrow('ICEBERG_SQL_STORAGE_PROVIDER_NOT_SUPPORTED'); + }); + + it('requires explicit confirmation that SQL storage matches the REST warehouse', async () => { + await expect( + IcebergDatalakeService.createInstance({ + name: 'rest-unconfirmed', + catalogType: 'rest', + endpoint: 'http://localhost:8181/catalog', + catalogName: 'warehouse', + catalogAuthMode: 'none', + storageType: 'server-managed', + sqlEnabled: true, + sqlStorageConnectionId: 'connection', + sqlStorageProvider: 'aws', + sqlStorageBucket: 'warehouse', + sqlWarehouseMatchAcknowledged: false, + }), + ).rejects.toThrow('ICEBERG_SQL_WAREHOUSE_MATCH_REQUIRED'); + }); + it('redacts OAuth and database secrets from bridge errors', () => { const redact = (IcebergDatalakeService as any).redactBridgeSecrets as ( message: string, diff --git a/tests/unit/main/services/icebergSqlRuntime.service.test.ts b/tests/unit/main/services/icebergSqlRuntime.service.test.ts new file mode 100644 index 00000000..4d253599 --- /dev/null +++ b/tests/unit/main/services/icebergSqlRuntime.service.test.ts @@ -0,0 +1,258 @@ +import { IcebergDatalakeService } from '../../../../src/main/services/icebergDatalake.service'; +import secureStorage from '../../../../src/main/services/secureStorage.service'; +import { + loadDatabaseFile, + updateDatabase, +} from '../../../../src/main/utils/fileHelper'; + +const mockRun = jest.fn(); +const mockRunAndReadUntil = jest.fn(); +const mockCloseConnection = jest.fn(); +const mockCloseInstance = jest.fn(); +const mockInterrupt = jest.fn(); + +jest.mock('@duckdb/node-api', () => ({ + DuckDBInstance: { + create: jest.fn(async () => ({ + connect: jest.fn(async () => ({ + run: mockRun, + runAndReadUntil: mockRunAndReadUntil, + closeSync: mockCloseConnection, + interrupt: mockInterrupt, + })), + closeSync: mockCloseInstance, + })), + }, + StatementType: { + SELECT: 1, + INSERT: 2, + UPDATE: 3, + DELETE: 5, + CREATE: 7, + DROP: 15, + }, +})); + +jest.mock('../../../../src/main/utils/fileHelper', () => ({ + loadDatabaseFile: jest.fn(), + updateDatabase: jest.fn(), +})); + +jest.mock('../../../../src/main/services/secureStorage.service', () => ({ + __esModule: true, + default: { + getCredential: jest.fn(), + setCredential: jest.fn(), + deleteCredential: jest.fn(), + }, +})); + +jest.mock('../../../../src/main/services/settings.service', () => ({ + __esModule: true, + default: { loadSettings: jest.fn() }, +})); + +const mockedLoadDatabase = loadDatabaseFile as jest.Mock; +const mockedUpdateDatabase = updateDatabase as jest.Mock; +const mockedSecureStorage = secureStorage as jest.Mocked; + +const instance = { + id: 'iceberg-instance', + name: 'Lakekeeper', + catalogType: 'lakekeeper' as const, + endpoint: 'http://localhost:8181/catalog', + catalogName: 'minio-warehouse', + catalogAuthMode: 'oauth-client-credentials' as const, + oauthClientId: 'lakekeeper', + oauthClientSecretKey: 'iceberg-oauth-secret-iceberg-instance' as const, + oauthServerUri: + 'http://localhost:8080/realms/lakekeeper/protocol/openid-connect/token', + storageType: 'server-managed' as const, + sqlEnabled: true, + sqlStorageConnectionId: 'minio-connection', + sqlStorageProvider: 'minio' as const, + sqlStorageBucket: 'iceberg-warehouse', + sqlWarehouseMatchAcknowledged: true, + createdAt: '2026-08-14T00:00:00.000Z', + updatedAt: '2026-08-14T00:00:00.000Z', +}; + +const database = { + icebergInstances: [instance], + sources: [ + { + id: 'minio-connection', + name: 'MinIO', + provider: 'minio', + config: { + endpoint: 'http://localhost:9000', + accessKeyId: 'minioadmin', + }, + }, + ], +}; + +describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { + beforeEach(() => { + jest.clearAllMocks(); + mockedLoadDatabase.mockResolvedValue(database); + mockedUpdateDatabase.mockResolvedValue(undefined); + mockedSecureStorage.getCredential.mockImplementation(async (key) => { + if (key === 'cloud-minio-minio-connection') return 'minio-secret'; + if (key === 'iceberg-oauth-secret-iceberg-instance') { + return 'oauth-secret'; + } + return null; + }); + mockRun.mockResolvedValue(undefined); + mockRunAndReadUntil.mockResolvedValue({ + columnNames: () => ['table_schema', 'table_name'], + getRowsJson: () => [], + rowsChanged: 0, + done: true, + }); + }); + + it('verifies with temporary secrets, attach, detach, and cleanup', async () => { + const result = await IcebergDatalakeService.verifySqlAccess(instance.id); + + expect(result.success).toBe(true); + expect(mockRun).toHaveBeenCalledWith('INSTALL httpfs'); + expect(mockRun).toHaveBeenCalledWith('INSTALL iceberg'); + expect(mockRun).toHaveBeenCalledWith('LOAD httpfs'); + expect(mockRun).toHaveBeenCalledWith('LOAD iceberg'); + expect( + mockRun.mock.calls.some(([sql]) => + String(sql).startsWith('CREATE TEMPORARY SECRET'), + ), + ).toBe(true); + expect( + mockRun.mock.calls + .map(([sql]) => String(sql)) + .find( + (sql) => + sql.startsWith('CREATE TEMPORARY SECRET') && + sql.includes('TYPE ICEBERG'), + ), + ).toContain("OAUTH2_SCOPE 'catalog'"); + expect( + mockRun.mock.calls.some(([sql]) => String(sql).startsWith('ATTACH ')), + ).toBe(true); + expect( + mockRun.mock.calls.some(([sql]) => String(sql).startsWith('DETACH ')), + ).toBe(true); + expect( + mockRun.mock.calls.filter(([sql]) => + String(sql).startsWith('DROP SECRET IF EXISTS'), + ), + ).toHaveLength(2); + expect(mockCloseConnection).toHaveBeenCalled(); + expect(mockCloseInstance).toHaveBeenCalled(); + expect(mockedUpdateDatabase).toHaveBeenCalledWith( + 'icebergInstances', + expect.arrayContaining([ + expect.objectContaining({ + id: instance.id, + sqlAccessVerifiedAt: expect.any(String), + sqlRuntimeFingerprint: expect.stringContaining('duckdb-node-api:'), + }), + ]), + ); + }); + + it('cleans up secrets and closes handles when attach fails', async () => { + mockRun.mockImplementation(async (sql: string) => { + if (sql.startsWith('ATTACH ')) throw new Error('attach failed'); + return undefined; + }); + + const result = await IcebergDatalakeService.verifySqlAccess(instance.id); + + expect(result.success).toBe(false); + expect( + mockRun.mock.calls.filter(([sql]) => + String(sql).startsWith('DROP SECRET IF EXISTS'), + ), + ).toHaveLength(2); + expect(mockCloseConnection).toHaveBeenCalled(); + expect(mockCloseInstance).toHaveBeenCalled(); + expect(mockedUpdateDatabase).not.toHaveBeenCalled(); + }); + + it('overrides DuckDB default OAuth scope for Nessie when none is configured', async () => { + const nessieInstance = { + ...instance, + id: 'nessie-instance', + catalogType: 'nessie' as const, + endpoint: 'http://localhost:19120/iceberg', + catalogName: undefined, + nessieReference: 'main', + nessieWarehouse: 'warehouse', + oauthClientSecretKey: 'iceberg-oauth-secret-nessie-instance', + }; + mockedLoadDatabase.mockResolvedValue({ + ...database, + icebergInstances: [nessieInstance], + }); + mockedSecureStorage.getCredential.mockImplementation(async (key) => { + if (key === 'cloud-minio-minio-connection') return 'minio-secret'; + if (key === 'iceberg-oauth-secret-nessie-instance') { + return 'oauth-secret'; + } + return null; + }); + + const result = await IcebergDatalakeService.verifySqlAccess( + nessieInstance.id, + ); + + expect(result.success).toBe(true); + const catalogSecretSql = mockRun.mock.calls + .map(([sql]) => String(sql)) + .find( + (sql) => + sql.startsWith('CREATE TEMPORARY SECRET') && + sql.includes('TYPE ICEBERG'), + ); + expect(catalogSecretSql).toContain("OAUTH2_SCOPE 'catalog'"); + const attachSql = mockRun.mock.calls + .map(([sql]) => String(sql)) + .find((sql) => sql.startsWith('ATTACH ')); + expect(attachSql).toContain("ATTACH 'warehouse'"); + expect(attachSql).toContain( + "ENDPOINT 'http://localhost:19120/iceberg/main'", + ); + }); + + it('rejects runtime-control SQL after parsing exactly one statement', async () => { + const destroySync = jest.fn(); + const classify = (IcebergDatalakeService as any).classifySqlStatement as ( + connection: unknown, + sql: string, + ) => Promise; + const connection = { + extractStatements: jest.fn(async () => ({ + count: 1, + prepare: jest.fn(async () => ({ statementType: 7, destroySync })), + })), + }; + + await expect( + classify(connection, 'CREATE SECRET stolen (TYPE S3)'), + ).rejects.toThrow('ICEBERG_SQL_STATEMENT_REJECTED'); + expect(destroySync).toHaveBeenCalled(); + }); + + it('interrupts only a registered active execution', () => { + const active = (IcebergDatalakeService as any).activeSqlExecutions as Map< + string, + unknown + >; + active.set('running-query', { interrupt: mockInterrupt }); + + expect(IcebergDatalakeService.cancelSql('running-query')).toBe(true); + expect(mockInterrupt).toHaveBeenCalled(); + active.delete('running-query'); + expect(IcebergDatalakeService.cancelSql('missing-query')).toBe(false); + }); +}); From af5ab15a72560c66ad8feac9f3ad9278be5f8f04 Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Fri, 14 Aug 2026 19:20:40 +0200 Subject: [PATCH 02/16] feat(iceberg): add Iceberg catalogs to SQL Editor - expose verified Iceberg instances in the connection selector - add catalog-specific icons to selectors, tabs, and schema trees - load Iceberg namespaces, tables, and columns for schema browsing - route query execution and cancellation through the Iceberg runtime - normalize DuckDB results as keyed row objects - improve execution error reporting and temporary catalog handling - hide unsupported and unverified Iceberg instances - add focused schema and row-result runtime tests --- .../icebergDatalake.ipcHandlers.ts | 4 + src/main/services/icebergDatalake.service.ts | 128 +++++++++-- src/renderer/components/sqlEditor/index.tsx | 43 +++- src/renderer/components/sqlTabs/index.tsx | 17 +- .../sql/SchemaTreeViewerWithSchema.tsx | 6 +- src/renderer/screens/sql/index.tsx | 215 +++++++++++++++++- src/renderer/services/iceberg.service.ts | 6 + src/types/iceberg.ts | 16 +- src/types/ipc.ts | 1 + .../icebergSqlRuntime.service.test.ts | 88 ++++++- 10 files changed, 481 insertions(+), 43 deletions(-) diff --git a/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts b/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts index 7ec92256..21681058 100644 --- a/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts +++ b/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts @@ -50,6 +50,10 @@ export const registerIcebergDatalakeHandlers = () => { IcebergDatalakeService.getSqlCapability(id), ); + ipcMain.handle('iceberg:sqlSchema', (_e, id: string) => + IcebergDatalakeService.getSqlSchema(id), + ); + ipcMain.handle('iceberg:verifySqlAccess', (_e, id: string) => IcebergDatalakeService.verifySqlAccess(id), ); diff --git a/src/main/services/icebergDatalake.service.ts b/src/main/services/icebergDatalake.service.ts index a9f17da7..c0602c31 100644 --- a/src/main/services/icebergDatalake.service.ts +++ b/src/main/services/icebergDatalake.service.ts @@ -42,6 +42,7 @@ import type { IcebergSqlCapability, IcebergSqlExecutionParams, IcebergSqlExecutionResult, + IcebergSqlSchemaInfo, IcebergSqlStatementClass, } from '../../types/iceberg'; import type { CloudConnection, CloudStorageConfig } from '../../types/frontend'; @@ -902,13 +903,7 @@ export class IcebergDatalakeService { const configuredEndpoint = instance.endpoint?.trim(); if (!configuredEndpoint) throw new Error('ICEBERG_REQUIRED_FIELD: endpoint'); - const endpoint = - instance.catalogType === 'nessie' - ? IcebergDatalakeService.buildNessieRestUri({ - ...instance, - nessieWarehouse: undefined, - }) - : configuredEndpoint; + const endpoint = configuredEndpoint.replace(/\/+$/, ''); const warehouse = instance.catalogType === 'nessie' ? instance.nessieWarehouse?.trim() @@ -955,6 +950,8 @@ export class IcebergDatalakeService { static async listInstances(): Promise { try { const instances = await IcebergDatalakeService.readInstances(); + const runtimeFingerprint = + IcebergDatalakeService.getSqlRuntimeFingerprint(); return instances.map( ({ id, @@ -965,20 +962,39 @@ export class IcebergDatalakeService { catalogPath, localPath, storageBucket, + sqlEnabled, + sqlAccessVerifiedAt, + sqlRuntimeFingerprint, createdAt, updatedAt, - }) => ({ - id, - name, - description, - catalogType, - storageType, - catalogPath, - localPath, - storageBucket, - createdAt, - updatedAt, - }), + }) => { + let sqlUnavailableReason: string | undefined; + if (!sqlEnabled) { + sqlUnavailableReason = 'ICEBERG_SQL_DISABLED'; + } else if ( + !sqlAccessVerifiedAt || + sqlRuntimeFingerprint !== runtimeFingerprint + ) { + sqlUnavailableReason = 'ICEBERG_SQL_UNVERIFIED'; + } + return { + id, + name, + description, + catalogType, + storageType, + catalogPath, + localPath, + storageBucket, + sqlAvailable: + !!sqlEnabled && + !!sqlAccessVerifiedAt && + sqlRuntimeFingerprint === runtimeFingerprint, + sqlUnavailableReason, + createdAt, + updatedAt, + }; + }, ); } catch (error) { // eslint-disable-next-line no-console @@ -1579,7 +1595,7 @@ export class IcebergDatalakeService { const instance = await IcebergDatalakeService.getInstance(instanceId); const suffix = uuidv4().replace(/-/g, ''); const names = { - alias: `iceberg_${suffix}`, + alias: 'iceberg', catalogSecret: `iceberg_catalog_${suffix}`, storageSecret: `iceberg_storage_${suffix}`, }; @@ -1613,6 +1629,18 @@ export class IcebergDatalakeService { } catch (error) { const message = error instanceof Error ? error.message : String(error); if (message.startsWith('ICEBERG_')) throw error; + if (stage === 'execute') { + const safeReason = message + .split('\n') + .map((line) => line.trim()) + .filter( + (line) => line && !/^LINE \d+:/i.test(line) && !/^\^+$/.test(line), + ) + .slice(0, 4) + .join(' ') + .slice(0, 500); + throw new Error(`ICEBERG_SQL_EXECUTION_FAILED: ${safeReason}`); + } throw new Error(`ICEBERG_SQL_RUNTIME_FAILED: ${stage}`); } finally { IcebergDatalakeService.activeSqlExecutions.delete(executionId); @@ -1697,12 +1725,14 @@ export class IcebergDatalakeService { params.sql, maxRows + 1, ); - const rows = reader.getRowsJson(); + const rows = reader.getRowObjectsJson() as Array< + Record + >; return { executionId: params.executionId, statementClass, columns: reader.columnNames(), - rows: rows.slice(0, maxRows) as unknown[][], + rows: rows.slice(0, maxRows), rowsChanged: Number(reader.rowsChanged ?? 0), truncated: rows.length > maxRows || !reader.done, }; @@ -1710,6 +1740,60 @@ export class IcebergDatalakeService { ); } + static async getSqlSchema(id: string): Promise { + const capability = await IcebergDatalakeService.getSqlCapability(id); + if (!capability.available) { + throw new Error(capability.reason ?? 'ICEBERG_SQL_UNAVAILABLE'); + } + + const namespaceQueue = await IcebergDatalakeService.listNamespaces(id); + const namespaces: IcebergSqlSchemaInfo['namespaces'] = []; + + for (let index = 0; index < namespaceQueue.length; index += 1) { + const namespace = namespaceQueue[index]; + if (namespaceQueue.length > 1_000) { + throw new Error('ICEBERG_SQL_SCHEMA_LIMIT_EXCEEDED'); + } + // Catalog traversal is intentionally sequential to keep bridge work bounded. + // eslint-disable-next-line no-await-in-loop + const children = await IcebergDatalakeService.listNamespaces( + id, + namespace, + ); + children.forEach((child) => { + if ( + !namespaceQueue.some((item) => item.join('.') === child.join('.')) + ) { + namespaceQueue.push(child); + } + }); + // eslint-disable-next-line no-await-in-loop + const tableNames = await IcebergDatalakeService.listTables(id, namespace); + // eslint-disable-next-line no-await-in-loop + const tables = await Promise.all( + tableNames.map(async (table) => { + const schema = await IcebergDatalakeService.getTableSchema( + id, + namespace, + table, + ); + return { + name: table, + type: 'TABLE', + columns: schema.fields.map((field, fieldIndex) => ({ + name: field.name, + type: field.type, + position: fieldIndex + 1, + })), + }; + }), + ); + namespaces.push({ name: namespace.join('.'), tables }); + } + + return { catalogName: 'iceberg', namespaces }; + } + static cancelSql(executionId: string): boolean { const connection = IcebergDatalakeService.activeSqlExecutions.get(executionId); diff --git a/src/renderer/components/sqlEditor/index.tsx b/src/renderer/components/sqlEditor/index.tsx index 674fe34a..48a37096 100644 --- a/src/renderer/components/sqlEditor/index.tsx +++ b/src/renderer/components/sqlEditor/index.tsx @@ -4,7 +4,11 @@ import type * as monacoType from 'monaco-editor'; import { Box, Tooltip, IconButton } from '@mui/material'; import { Save as SaveIcon } from '@mui/icons-material'; import { Inputs, RelativeContainer } from './styles'; -import { connectorsServices, projectsServices } from '../../services'; +import { + connectorsServices, + projectsServices, + icebergService, +} from '../../services'; import { DuckLakeService } from '../../services/duckLake.service'; import { QueryHistoryType } from '../../../types/frontend'; import { ConnectionInput, Project } from '../../../types/backend'; @@ -68,11 +72,15 @@ export const SqlEditor: React.FC = ({ connectionInput?.type === 'ducklake' && 'instanceId' in connectionInput && !!connectionInput.instanceId; + const isIcebergConnection = connectionId?.startsWith('iceberg-') ?? false; // Get instanceId for DuckLake queries const instanceId = isDuckLakeConnection ? (connectionInput as any).instanceId : undefined; + const icebergInstanceId = isIcebergConnection + ? connectionId?.replace('iceberg-', '') + : undefined; // Helper function to detect DDL operations that modify schema // Uses startsWith to prevent false positives from string literals like SELECT 'DROP TABLE users' @@ -123,6 +131,15 @@ export const SqlEditor: React.FC = ({ return; } + const commandType = getCommandType(selectedQuery); + if (isIcebergConnection && commandType !== 'SELECT') { + // eslint-disable-next-line no-alert + const confirmed = window.confirm( + 'This statement will modify the Iceberg catalog or its data. Continue?', + ); + if (!confirmed) return; + } + // Generate semi-unique ID for query cancellation const queryId = `query-${Date.now()}-${Math.random() .toString(36) @@ -138,10 +155,24 @@ export const SqlEditor: React.FC = ({ try { let result; - if (isDuckLakeConnection && instanceId) { + if (isIcebergConnection && icebergInstanceId) { + const icebergResult = await icebergService.executeIcebergSql({ + instanceId: icebergInstanceId, + executionId: queryId, + sql: selectedQuery, + maxRows: 1000, + }); + result = { + success: true, + data: icebergResult.rows, + fields: icebergResult.columns.map((name) => ({ name, type: 0 })), + rowCount: + icebergResult.statementClass === 'select' + ? icebergResult.rows.length + : icebergResult.rowsChanged, + }; + } else if (isDuckLakeConnection && instanceId) { const duckLakeQueryLimit = 10; - const commandType = getCommandType(selectedQuery); - const duckLakeResult = await DuckLakeService.executeQuery({ instanceId, query: selectedQuery, @@ -192,8 +223,6 @@ export const SqlEditor: React.FC = ({ // Check if this was a DDL operation const wasDDL = isDDLOperation(selectedQuery); - const commandType = getCommandType(selectedQuery); - const enrichedResult = { ...result, isCommand: commandType === 'DDL' || commandType === 'DML', @@ -245,7 +274,7 @@ export const SqlEditor: React.FC = ({ } } catch (error) { toast.error('An unexpected error occurred while executing the query'); - setError(error); + setError(error instanceof Error ? error.message : String(error)); } finally { setLoadingQuery(false); } diff --git a/src/renderer/components/sqlTabs/index.tsx b/src/renderer/components/sqlTabs/index.tsx index 1ea7e372..43645917 100644 --- a/src/renderer/components/sqlTabs/index.tsx +++ b/src/renderer/components/sqlTabs/index.tsx @@ -5,7 +5,10 @@ import Tooltip from '@mui/material/Tooltip'; import CloseIcon from '@mui/icons-material/Close'; import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; import { SqlTabId, SqlTabState } from '../../../types/editor'; -import connectionIcons from '../../../../assets/connectionIcons'; +import connectionIcons, { + icebergCatalogImages, +} from '../../../../assets/connectionIcons'; +import icebergIcon from '../../../../assets/icons/apache-iceberg-lake.png'; import { SqlTabBar, TabsContainer, @@ -36,10 +39,20 @@ const SqlTab: React.FC = ({ }; // Get connection type icon - const icon = + const icebergCatalogType = tab.connectionType.startsWith('iceberg-') + ? tab.connectionType.replace('iceberg-', '') + : undefined; + let icon = connectionIcons.images[ tab.connectionType as keyof typeof connectionIcons.images ]; + if (tab.connectionType === 'iceberg') icon = icebergIcon; + if (icebergCatalogType) { + icon = + icebergCatalogImages[ + icebergCatalogType as keyof typeof icebergCatalogImages + ] || icebergIcon; + } const tooltipText = `${tab.connectionName} (${tab.connectionType})`; diff --git a/src/renderer/screens/sql/SchemaTreeViewerWithSchema.tsx b/src/renderer/screens/sql/SchemaTreeViewerWithSchema.tsx index b1b86f58..e11310e9 100644 --- a/src/renderer/screens/sql/SchemaTreeViewerWithSchema.tsx +++ b/src/renderer/screens/sql/SchemaTreeViewerWithSchema.tsx @@ -22,6 +22,7 @@ type Props = { isLoading: boolean; filter?: string; hideSchemaLevel?: boolean; + databaseIcon?: string; }; /** @@ -38,6 +39,7 @@ export const SchemaTreeViewerWithSchema: React.FC = React.memo( isLoading, filter = '', hideSchemaLevel = false, + databaseIcon, }) => { const [expandedItems, setExpandedItems] = React.useState([ databaseName, @@ -119,7 +121,9 @@ export const SchemaTreeViewerWithSchema: React.FC = React.memo( label={ } > diff --git a/src/renderer/screens/sql/index.tsx b/src/renderer/screens/sql/index.tsx index c1920584..39a11208 100644 --- a/src/renderer/screens/sql/index.tsx +++ b/src/renderer/screens/sql/index.tsx @@ -38,7 +38,11 @@ import { } from '@mui/icons-material'; import { toast } from 'react-toastify'; import { useNavigate } from 'react-router-dom'; -import { connectorsServices, DuckLakeService } from '../../services'; +import { + connectorsServices, + DuckLakeService, + icebergService, +} from '../../services'; import { QueryResultStore } from './queryResultStore'; import { registerQueryResultBridge } from '../../services/agentEditorBridge.service'; import type { QueryResultSnapshot } from '../../../types/backend'; @@ -63,22 +67,30 @@ import { useGetConnections, useDuckLakeInstances, } from '../../controllers'; +import { useListIcebergInstances } from '../../controllers/icebergDatalake.controller'; import { SchemaTreeViewerWithSchema } from './SchemaTreeViewerWithSchema'; import { SavedQueriesList } from '../../components/sqlEditor/SavedQueriesList'; import connectionIcons, { defaultIcon, + icebergCatalogImages, } from '../../../../assets/connectionIcons'; +import icebergIcon from '../../../../assets/icons/apache-iceberg-lake.png'; import { AppContext } from '../../context'; import { generateDuckLakeCompletions, mergeCompletions, } from '../../utils/duckLakeCompletions'; +import { MonacoCompletionItemKind } from '../../config/constants'; const QUERY_HISTORY_KEY = 'query_history_key'; const EMPTY_ARRAY: Table[] = []; const CHAT_MIN_WIDTH = 280; const CHAT_DEFAULT_WIDTH = 360; +const getIcebergCatalogIcon = (catalogType: string) => + icebergCatalogImages[catalogType as keyof typeof icebergCatalogImages] || + icebergIcon; + const VerticalSash = (_: number, active: boolean) => (
{ isLoading: isLoadingDuckLakeInstances, refetch: refetchDuckLakeInstances, } = useDuckLakeInstances(); + const { data: icebergInstances = [] } = useListIcebergInstances(); const [sidebarTab, setSidebarTab] = useState(0); const [activeAnalyticsPageId, setActiveAnalyticsPageId] = useState< string | null @@ -153,6 +166,12 @@ const Sql = () => { const instance = duckLakeInstances.find((inst) => inst.id === instanceId); return instance?.name ?? activeConnectionName; } + if (activeConnectionId.startsWith('iceberg-')) { + const instance = icebergInstances.find( + (item) => item.id === activeConnectionId.replace('iceberg-', ''), + ); + return instance?.name ?? activeConnectionName; + } const connection = connections.find( (conn) => conn.id === activeConnectionId, @@ -163,6 +182,7 @@ const Sql = () => { activeConnectionName, connections, duckLakeInstances, + icebergInstances, ]); // Reset analytics page when switching connections to prevent showing @@ -174,10 +194,21 @@ const Sql = () => { // Check if active connection is DuckLake const isDuckLakeConnection = activeConnectionId?.startsWith('ducklake-') || false; + const isIcebergConnection = + activeConnectionId?.startsWith('iceberg-') || false; + const activeIcebergInstance = isIcebergConnection + ? icebergInstances.find( + (item) => item.id === activeConnectionId?.replace('iceberg-', ''), + ) + : undefined; // Get active connection const { data: activeConnection, isLoading: isLoadingConnection } = - useGetConnectionById(activeConnectionId); + useGetConnectionById( + isDuckLakeConnection || isIcebergConnection + ? undefined + : activeConnectionId, + ); // Schema state for active tab const [tabSchemas, setTabSchemas] = useState>({}); @@ -245,6 +276,18 @@ const Sql = () => { status: 'loading', } as any; } + if (activeConnectionId?.startsWith('iceberg-')) { + const instanceId = activeConnectionId.replace('iceberg-', ''); + const instance = icebergInstances.find((item) => item.id === instanceId); + if (instance) { + return { + type: 'duckdb', + name: instance.name, + instanceId, + status: 'active', + } as any; + } + } // Handle regular database connections if (!activeConnection || activeConnection.id !== activeConnectionId) { @@ -256,6 +299,7 @@ const Sql = () => { activeConnectionId, activeConnectionName, duckLakeInstances, + icebergInstances, isLoadingDuckLakeInstances, ]); @@ -380,13 +424,40 @@ const Sql = () => { ? utils.generateMonacoCompletions(activeSchema) : []; + if (isIcebergConnection) { + const quote = (value: string) => `"${value.replace(/"/g, '""')}"`; + const icebergItems = activeSchema.flatMap((table) => { + const qualifiedTable = `${quote('iceberg')}.${quote(table.schema)}.${quote(table.name)}`; + return [ + { + label: `iceberg.${table.schema}.${table.name}`, + kind: MonacoCompletionItemKind.Struct, + insertText: qualifiedTable, + detail: 'Iceberg table', + }, + ...table.columns.map((column) => ({ + label: `iceberg.${table.schema}.${table.name}.${column.name}`, + kind: MonacoCompletionItemKind.Field, + insertText: `${qualifiedTable}.${quote(column.name)}`, + detail: 'Iceberg column', + })), + ]; + }); + return [ + ...baseCompletions.filter( + (item) => item.kind === MonacoCompletionItemKind.Keyword, + ), + ...icebergItems, + ]; + } + // Merge with DuckLake completions if available if (duckLakeCompletions.length > 0) { return mergeCompletions(baseCompletions, duckLakeCompletions); } return baseCompletions; - }, [activeSchema, duckLakeCompletions]); + }, [activeSchema, duckLakeCompletions, isIcebergConnection]); const loadDuckLakeCompletions = useCallback(async () => { const requestSeq = duckLakeCompletionsRequestSeq.current + 1; @@ -456,8 +527,7 @@ const Sql = () => { async (connectionId: string) => { if (loadingSchemas[connectionId]) return; - // Skip regular schema loading for DuckLake connections - // DuckLake schema is loaded via extractSchema in loadDuckLakeCompletions + // DuckLake schema is loaded through its existing completion path. if (connectionId.startsWith('ducklake-')) { // Mark as loaded (empty schema) to prevent loading state setTabSchemas((prev) => ({ ...prev, [connectionId]: [] })); @@ -465,6 +535,42 @@ const Sql = () => { return; } + if (connectionId.startsWith('iceberg-')) { + setLoadingSchemas((prev) => ({ ...prev, [connectionId]: true })); + try { + const schema = await icebergService.getIcebergSqlSchema( + connectionId.replace('iceberg-', ''), + ); + const tables: Table[] = schema.namespaces.flatMap((namespace) => + namespace.tables.map((table) => ({ + name: table.name, + type: table.type, + schema: namespace.name, + columns: table.columns.map((column) => ({ + name: column.name, + typeName: column.type, + ordinalPosition: column.position, + primaryKeySequenceId: 0, + columnDisplaySize: 0, + scale: 0, + precision: 0, + columnProperties: [], + autoincrement: false, + primaryKey: false, + nullable: true, + foreignKeys: [], + })), + })), + ); + setTabSchemas((prev) => ({ ...prev, [connectionId]: tables })); + } catch { + setTabSchemas((prev) => ({ ...prev, [connectionId]: [] })); + } finally { + setLoadingSchemas((prev) => ({ ...prev, [connectionId]: false })); + } + return; + } + setLoadingSchemas((prev) => ({ ...prev, [connectionId]: true })); try { const result = @@ -631,7 +737,11 @@ const Sql = () => { const execution = activeTabId ? tabExecutions[activeTabId] : null; if (execution) { try { - await connectorsServices.cancelQuery(execution.id); + if (isIcebergConnection) { + await icebergService.cancelIcebergSql(execution.id); + } else { + await connectorsServices.cancelQuery(execution.id); + } toast.info('Query execution cancelled'); } catch (e) { toast.error('Failed to cancel query'); @@ -717,7 +827,21 @@ const Sql = () => { data-testid="sql-connection-select" value={activeTab?.connectionId || ''} onChange={(e) => { - const conn = connections.find((c) => c.id === e.target.value); + const selected = String(e.target.value); + if (selected.startsWith('iceberg-')) { + const instance = icebergInstances.find( + (item) => item.id === selected.replace('iceberg-', ''), + ); + if (instance?.sqlAvailable) { + handleConnectionSelect({ + id: selected, + name: instance.name, + type: `iceberg-${instance.catalogType}`, + }); + } + return; + } + const conn = connections.find((c) => c.id === selected); if (conn) { handleConnectionSelect({ id: conn.id, @@ -772,6 +896,44 @@ const Sql = () => { ); } + if (selected.startsWith('iceberg-')) { + const instance = icebergInstances.find( + (item) => item.id === selected.replace('iceberg-', ''), + ); + return ( + + + + {instance?.name ?? activeTab?.connectionName} + + + ); + } + // Handle regular database connections const conn = connections.find((c) => c.id === selected); if (!conn) return 'Select Connection'; @@ -937,6 +1099,38 @@ const Sql = () => { ))} + {icebergInstances.some((instance) => instance.sqlAvailable) && ( + + Iceberg Catalogs + + )} + {icebergInstances + .filter((instance) => instance.sqlAvailable) + .map((instance) => ( + + + + {instance.name} + + + ))} @@ -1134,6 +1328,13 @@ const Sql = () => { schema={activeSchema} isLoading={isLoadingSchema} filter={filter} + databaseIcon={ + activeIcebergInstance + ? getIcebergCatalogIcon( + activeIcebergInstance.catalogType, + ) + : undefined + } /> )} {!activeTab && ( diff --git a/src/renderer/services/iceberg.service.ts b/src/renderer/services/iceberg.service.ts index c33971e2..b98ce3c5 100644 --- a/src/renderer/services/iceberg.service.ts +++ b/src/renderer/services/iceberg.service.ts @@ -24,6 +24,7 @@ import type { IcebergSqlCapability, IcebergSqlExecutionParams, IcebergSqlExecutionResult, + IcebergSqlSchemaInfo, } from '../../types/iceberg'; export const getIcebergCapabilities = (): Promise => @@ -74,6 +75,11 @@ export const getIcebergSqlCapability = ( ): Promise => window.electron.ipcRenderer.invoke('iceberg:sqlCapability', id); +export const getIcebergSqlSchema = ( + id: string, +): Promise => + window.electron.ipcRenderer.invoke('iceberg:sqlSchema', id); + export const verifyIcebergSqlAccess = ( id: string, ): Promise => diff --git a/src/types/iceberg.ts b/src/types/iceberg.ts index 29538e77..d7f04b00 100644 --- a/src/types/iceberg.ts +++ b/src/types/iceberg.ts @@ -110,6 +110,8 @@ export interface IcebergInstanceListItem { catalogPath?: string; localPath?: string; storageBucket?: string; + sqlAvailable: boolean; + sqlUnavailableReason?: string; createdAt: string; updatedAt: string; } @@ -213,11 +215,23 @@ export interface IcebergSqlExecutionResult { executionId: string; statementClass: IcebergSqlStatementClass; columns: string[]; - rows: unknown[][]; + rows: Array>; rowsChanged: number; truncated: boolean; } +export interface IcebergSqlSchemaInfo { + catalogName: string; + namespaces: Array<{ + name: string; + tables: Array<{ + name: string; + type: string; + columns: Array<{ name: string; type: string; position: number }>; + }>; + }>; +} + export interface IcebergListStorageBucketsParams { connectionId: string; } diff --git a/src/types/ipc.ts b/src/types/ipc.ts index aef7b799..276909c9 100644 --- a/src/types/ipc.ts +++ b/src/types/ipc.ts @@ -360,6 +360,7 @@ export type IcebergChannels = | 'iceberg:listStorageBuckets' | 'iceberg:testInstance' | 'iceberg:sqlCapability' + | 'iceberg:sqlSchema' | 'iceberg:verifySqlAccess' | 'iceberg:executeSql' | 'iceberg:cancelSql' diff --git a/tests/unit/main/services/icebergSqlRuntime.service.test.ts b/tests/unit/main/services/icebergSqlRuntime.service.test.ts index 4d253599..6fb6e8a2 100644 --- a/tests/unit/main/services/icebergSqlRuntime.service.test.ts +++ b/tests/unit/main/services/icebergSqlRuntime.service.test.ts @@ -7,6 +7,7 @@ import { const mockRun = jest.fn(); const mockRunAndReadUntil = jest.fn(); +const mockExtractStatements = jest.fn(); const mockCloseConnection = jest.fn(); const mockCloseInstance = jest.fn(); const mockInterrupt = jest.fn(); @@ -17,6 +18,7 @@ jest.mock('@duckdb/node-api', () => ({ connect: jest.fn(async () => ({ run: mockRun, runAndReadUntil: mockRunAndReadUntil, + extractStatements: mockExtractStatements, closeSync: mockCloseConnection, interrupt: mockInterrupt, })), @@ -105,12 +107,50 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { return null; }); mockRun.mockResolvedValue(undefined); + mockExtractStatements.mockResolvedValue({ + count: 1, + prepare: jest.fn(async () => ({ + statementType: 1, + destroySync: jest.fn(), + })), + }); mockRunAndReadUntil.mockResolvedValue({ columnNames: () => ['table_schema', 'table_name'], getRowsJson: () => [], + getRowObjectsJson: () => [], + rowsChanged: 0, + done: true, + }); + }); + + it('returns DuckDB row objects keyed for the SQL result table', async () => { + mockedLoadDatabase.mockResolvedValue({ + ...database, + icebergInstances: [ + { + ...instance, + sqlAccessVerifiedAt: '2026-08-14T00:00:00.000Z', + sqlRuntimeFingerprint: ( + IcebergDatalakeService as any + ).getSqlRuntimeFingerprint(), + }, + ], + }); + mockRunAndReadUntil.mockResolvedValue({ + columnNames: () => ['Id', 'Keywords'], + getRowObjectsJson: () => [{ Id: 1, Keywords: 'iceberg' }], rowsChanged: 0, done: true, }); + + const result = await IcebergDatalakeService.executeSql({ + instanceId: instance.id, + executionId: 'query-rows', + sql: 'SELECT * FROM "iceberg"."default"."keywords"', + }); + + expect(result.columns).toEqual(['Id', 'Keywords']); + expect(result.rows).toEqual([{ Id: 1, Keywords: 'iceberg' }]); }); it('verifies with temporary secrets, attach, detach, and cleanup', async () => { @@ -179,6 +219,50 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { expect(mockedUpdateDatabase).not.toHaveBeenCalled(); }); + it('maps PyIceberg catalog metadata into SQL Editor schema metadata', async () => { + mockedLoadDatabase.mockResolvedValue({ + ...database, + icebergInstances: [ + { + ...instance, + sqlAccessVerifiedAt: '2026-08-14T00:00:00.000Z', + sqlRuntimeFingerprint: ( + IcebergDatalakeService as any + ).getSqlRuntimeFingerprint(), + }, + ], + }); + jest + .spyOn(IcebergDatalakeService, 'listNamespaces') + .mockResolvedValueOnce([['sales']]) + .mockResolvedValueOnce([]); + jest + .spyOn(IcebergDatalakeService, 'listTables') + .mockResolvedValue(['customers']); + jest.spyOn(IcebergDatalakeService, 'getTableSchema').mockResolvedValue({ + fields: [{ fieldId: 1, name: 'id', type: 'BIGINT', required: true }], + properties: {}, + }); + + await expect( + IcebergDatalakeService.getSqlSchema(instance.id), + ).resolves.toEqual({ + catalogName: 'iceberg', + namespaces: [ + { + name: 'sales', + tables: [ + { + name: 'customers', + type: 'TABLE', + columns: [{ name: 'id', type: 'BIGINT', position: 1 }], + }, + ], + }, + ], + }); + }); + it('overrides DuckDB default OAuth scope for Nessie when none is configured', async () => { const nessieInstance = { ...instance, @@ -219,9 +303,7 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { .map(([sql]) => String(sql)) .find((sql) => sql.startsWith('ATTACH ')); expect(attachSql).toContain("ATTACH 'warehouse'"); - expect(attachSql).toContain( - "ENDPOINT 'http://localhost:19120/iceberg/main'", - ); + expect(attachSql).toContain("ENDPOINT 'http://localhost:19120/iceberg'"); }); it('rejects runtime-control SQL after parsing exactly one statement', async () => { From 1852edcc3a0d60a6912ddd262806b0c1f9629c05 Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Mon, 7 Sep 2026 11:43:25 +0200 Subject: [PATCH 03/16] fix(iceberg): enforce SQL scope, mutation confirmation, and verification gates - Validate Iceberg targets and allowed functions before binding - Use backend classification to confirm mutations with comments and CTEs - Require warehouse row reads and successful cleanup for SQL verification - Reject combinations without packaged acceptance evidence - Preserve truncation status and display result/export limits - Add regression coverage for SQL policy, confirmation, and verification --- src/main/services/iceberg/sqlPolicy.ts | 338 ++++++++++++++++++ src/main/services/icebergDatalake.service.ts | 228 ++++++++---- src/renderer/components/sqlEditor/index.tsx | 39 +- src/renderer/screens/sql/queryResult.tsx | 7 + src/renderer/services/iceberg.service.ts | 23 ++ src/types/backend.ts | 1 + src/types/iceberg.ts | 2 + .../main/services/icebergSqlPolicy.test.ts | 121 +++++++ .../icebergSqlRuntime.service.test.ts | 145 +++++++- .../screens/sql/icebergTruncation.test.tsx | 52 +++ .../services/icebergConfirmation.test.ts | 56 +++ 11 files changed, 909 insertions(+), 103 deletions(-) create mode 100644 src/main/services/iceberg/sqlPolicy.ts create mode 100644 tests/unit/main/services/icebergSqlPolicy.test.ts create mode 100644 tests/unit/renderer/screens/sql/icebergTruncation.test.tsx create mode 100644 tests/unit/renderer/services/icebergConfirmation.test.ts diff --git a/src/main/services/iceberg/sqlPolicy.ts b/src/main/services/iceberg/sqlPolicy.ts new file mode 100644 index 00000000..406d37bf --- /dev/null +++ b/src/main/services/iceberg/sqlPolicy.ts @@ -0,0 +1,338 @@ +import type { IcebergSqlStatementClass } from '../../../types/iceberg'; + +const reject = (): never => { + throw new Error('ICEBERG_SQL_STATEMENT_REJECTED'); +}; + +// Explicitly accepted pure functions. Unknown functions (including extension, +// file, introspection and dynamic-query functions) fail closed before binding. +const functions = new Set( + `abs ceil ceiling floor round trunc sqrt pow power +exp ln log log10 sign greatest least coalesce nullif if ifnull upper lower length +char_length concat concat_ws substring substr trim ltrim rtrim replace reverse +left right starts_with ends_with contains like_escape ilike_escape regexp_matches +regexp_replace regexp_extract split_part string_split array_length list_extract +list_value struct_pack count sum avg min max median mode stddev stddev_pop +stddev_samp variance var_pop var_samp bool_and bool_or string_agg array_agg list +first last any_value arg_min arg_max count_star date_part date_trunc date_diff + datediff date_add last_day make_date strftime strptime epoch year month day +hour minute second current_date current_timestamp now row_number rank dense_rank +percent_rank cume_dist ntile lag lead first_value last_value nth_value`.split( + /\s+/, + ), +); +const expressionClasses = new Set([ + 'CONSTANT', + 'COLUMN_REF', + 'STAR', + 'FUNCTION', + 'WINDOW', + 'CAST', + 'COMPARISON', + 'CONJUNCTION', + 'OPERATOR', + 'CASE', + 'SUBQUERY', + 'BETWEEN', + 'COLLATE', +]); + +/** Validate the native, unbound DuckDB SELECT AST, never a prepared query. */ +export function validateIcebergSelectAst(ast: any): void { + if (ast.error || ast.statements?.length !== 1) reject(); + const visit = (value: any, ctes: Set): void => { + if (!value || typeof value !== 'object') return; + if (Array.isArray(value)) { + value.forEach((child) => visit(child, ctes)); + return; + } + let scope = ctes; + if (value.cte_map) { + scope = new Set(ctes); + (value.cte_map.map ?? []).forEach((entry: any) => { + scope.add(String(entry.key).toLowerCase()); + }); + } + if (value.type === 'BASE_TABLE') { + const isCte = + !value.catalog_name && + !value.schema_name && + scope.has(String(value.table_name).toLowerCase()); + if ( + !isCte && + (String(value.catalog_name).toLowerCase() !== 'iceberg' || + !value.schema_name || + !value.table_name) + ) + reject(); + } + if ( + 'sample' in value && + 'alias' in value && + !['BASE_TABLE', 'EMPTY', 'JOIN', 'SUBQUERY', 'EXPRESSION_LIST'].includes( + value.type, + ) + ) + reject(); + if (value.class && !expressionClasses.has(value.class)) reject(); + if (value.class === 'FUNCTION' || value.class === 'WINDOW') { + const name = String(value.function_name).toLowerCase(); + const operator = + value.class === 'FUNCTION' && + value.is_operator && + [ + '+', + '-', + '*', + '/', + '//', + '%', + '**', + '~~', + '!~~', + '~~*', + '!~~*', + '||', + '&', + '|', + '^', + '<<', + '>>', + ].includes(name); + if ((!operator && !functions.has(name)) || value.catalog || value.schema) + reject(); + } + Object.values(value).forEach((child) => visit(child, scope)); + }; + visit(ast.statements[0].node, new Set()); +} + +type Token = { text: string; word?: string; identifier?: string }; +// Each lexer branch consumes a complete token before continuing. +/* eslint-disable no-continue */ +function tokenize(sql: string): Token[] { + const tokens: Token[] = []; + let i = 0; + while (i < sql.length) { + const rest = sql.slice(i); + const whitespace = /^\s+/.exec(rest); + if (whitespace) { + i += whitespace[0].length; + continue; + } + if (rest.startsWith('--')) { + const end = sql.indexOf('\n', i + 2); + i = end < 0 ? sql.length : end + 1; + continue; + } + if (rest.startsWith('/*')) { + let depth = 1; + i += 2; + while (depth && i < sql.length) { + if (sql.slice(i, i + 2) === '/*') { + depth += 1; + i += 2; + } else if (sql.slice(i, i + 2) === '*/') { + depth -= 1; + i += 2; + } else i += 1; + } + if (depth) reject(); + continue; + } + const quoted = /^(?:'(?:[^']|'')*'|"(?:[^"]|"")*")/.exec(rest); + if (quoted) { + const text = quoted[0]; + tokens.push({ + text, + ...(text[0] === '"' + ? { identifier: text.slice(1, -1).replace(/""/g, '"') } + : {}), + }); + i += text.length; + continue; + } + const word = /^[a-zA-Z_][a-zA-Z_0-9]*/.exec(rest); + if (word) { + tokens.push({ + text: word[0], + word: word[0].toUpperCase(), + identifier: word[0], + }); + i += word[0].length; + continue; + } + const other = + /^(?:\d+(?:\.\d+)?(?:[eE][+-]?\d+)?|::|=>|<=|>=|<>|!=|\|\||[-+*/%=<>().,;[\]])/.exec( + rest, + ); + if (!other) reject(); + tokens.push({ text: other![0] }); + i += other![0].length; + } + if (tokens[tokens.length - 1]?.text === ';') tokens.pop(); + if (!tokens.length || tokens.some((token) => token.text === ';')) reject(); + return tokens; +} + +/* eslint-enable no-continue */ + +/** Narrow mutation grammar; every expression/source is parsed by DuckDB as a + * SELECT below. Unsupported syntax is rejected, never passed through unchecked. */ +export function parseIcebergSql(sql: string): { + statementClass: IcebergSqlStatementClass; + selectSql?: string; +} { + if (!sql.trim() || sql.length > 1_000_000) + throw new Error('ICEBERG_SQL_INVALID'); + const t = tokenize(sql); + const text = (tokens: Token[]) => tokens.map((token) => token.text).join(' '); + let i = 0; + const eat = (word: string) => { + if (t[i]?.word !== word) reject(); + i += 1; + }; + const group = (): Token[] => { + if (t[i]?.text !== '(') reject(); + i += 1; + const start = i; + let depth = 1; + while (i < t.length) { + if (t[i].text === '(') depth += 1; + if (t[i].text === ')') depth -= 1; + if (!depth) { + const content = t.slice(start, i); + i += 1; + return content; + } + i += 1; + } + return reject(); + }; + // Consume CTE declarations only to locate the main statement. The native + // SELECT AST subsequently validates each CTE body and its relation scope. + if (t[i]?.word === 'WITH') { + i += 1; + if (t[i]?.word === 'RECURSIVE') i += 1; + do { + if (!t[i]?.identifier) reject(); + i += 1; + if (t[i]?.text === '(') group(); + eat('AS'); + if (t[i]?.word === 'NOT') i += 1; + if (t[i]?.word === 'MATERIALIZED') i += 1; + group(); + if (t[i]?.text !== ',') break; + i += 1; + } while (i < t.length); + } + const prefix = text(t.slice(0, i)); + const command = t[i]?.word; + i += 1; + const target = (parts: number): string => { + const start = i; + for (let n = 0; n < parts; n += 1) { + if (!t[i]?.identifier) reject(); + if (n === 0 && t[i].identifier!.toLowerCase() !== 'iceberg') reject(); + i += 1; + if (n < parts - 1) { + if (t[i]?.text !== '.') reject(); + i += 1; + } + } + return text(t.slice(start, i)); + }; + const select = (query: string, statementClass: IcebergSqlStatementClass) => ({ + statementClass, + selectSql: `${prefix} ${query}`.trim(), + }); + if (command === 'SELECT' || command === 'VALUES' || command === 'FROM') { + return { statementClass: 'select', selectSql: text(t) }; + } + if (command === 'INSERT') { + eat('INTO'); + target(3); + if (t[i]?.text === '(') { + const columns = group(); + if ( + !columns.length || + columns.some((token, n) => + n % 2 === 0 ? !token.identifier : token.text !== ',', + ) || + columns.length % 2 === 0 + ) + reject(); + } + if (!['SELECT', 'WITH', 'VALUES', 'FROM'].includes(t[i]?.word ?? '')) + reject(); + return select(text(t.slice(i)), 'insert'); + } + if (command === 'DELETE') { + eat('FROM'); + const table = target(3); + if (i < t.length && t[i]?.word !== 'WHERE') reject(); + return select(`SELECT * FROM ${table} ${text(t.slice(i))}`, 'delete'); + } + if (command === 'UPDATE') { + const table = target(3); + eat('SET'); + const expressions: Token[][] = []; + while (i < t.length) { + if (!t[i]?.identifier || t[i + 1]?.text !== '=') reject(); + i += 2; + const start = i; + let depth = 0; + while (i < t.length) { + if (!depth && (t[i].text === ',' || t[i].word === 'WHERE')) break; + if (t[i].text === '(' || t[i].text === '[') depth += 1; + if (t[i].text === ')' || t[i].text === ']') depth -= 1; + if (depth < 0) reject(); + i += 1; + } + if (i === start || depth) reject(); + expressions.push(t.slice(start, i)); + if (t[i]?.text !== ',') break; + i += 1; + } + if (!expressions.length) reject(); + return select( + `SELECT ${expressions.map(text).join(', ')} FROM ${table} ${text(t.slice(i))}`, + 'update', + ); + } + if ((command === 'CREATE' || command === 'DROP') && !prefix) { + const kind = t[i]?.word; + i += 1; + if (kind !== 'TABLE' && kind !== 'SCHEMA') reject(); + if (t[i]?.word === 'IF') { + i += 1; + if (command === 'CREATE') eat('NOT'); + eat('EXISTS'); + } + target(kind === 'TABLE' ? 3 : 2); + if (command === 'CREATE' && kind === 'TABLE') { + if (t[i]?.word === 'AS') { + i += 1; + return select(text(t.slice(i)), 'create'); + } + const columns = group(); + // Basic typed columns only. Defaults, constraints with expressions, + // custom types and generated columns require separate acceptance. + const definition = + /^(?:"(?:[^"]|"")*"|[a-z_][a-z_0-9]*)\s+(?:BOOLEAN|TINYINT|SMALLINT|INTEGER|INT|BIGINT|FLOAT|REAL|DOUBLE(?: PRECISION)?|VARCHAR|TEXT|STRING|BLOB|DATE|TIME|TIMESTAMP|TIMESTAMPTZ|UUID|DECIMAL(?:\s*\(\s*\d+\s*,\s*\d+\s*\))?)(?:\s+NOT\s+NULL)?$/i; + const definitions: Token[][] = [[]]; + let depth = 0; + columns.forEach((token) => { + if (token.text === '(') depth += 1; + if (token.text === ')') depth -= 1; + if (!depth && token.text === ',') definitions.push([]); + else definitions[definitions.length - 1].push(token); + }); + if (definitions.some((column) => !definition.test(text(column)))) + reject(); + } + if (i !== t.length) reject(); + return { statementClass: command === 'CREATE' ? 'create' : 'drop' }; + } + return reject(); +} diff --git a/src/main/services/icebergDatalake.service.ts b/src/main/services/icebergDatalake.service.ts index c0602c31..c4837c51 100644 --- a/src/main/services/icebergDatalake.service.ts +++ b/src/main/services/icebergDatalake.service.ts @@ -13,7 +13,8 @@ import * as path from 'path'; import { pathToFileURL } from 'url'; import { spawn } from 'child_process'; import { app } from 'electron'; -import { DuckDBInstance, StatementType } from '@duckdb/node-api'; +import { DuckDBInstance } from '@duckdb/node-api'; +import { parseIcebergSql, validateIcebergSelectAst } from './iceberg/sqlPolicy'; import { loadDatabaseFile, updateDatabase } from '../utils/fileHelper'; import secureStorage from './secureStorage.service'; @@ -51,16 +52,27 @@ import type { PostgresConnection } from '../../types/backend'; export class IcebergDatalakeService { private static readonly activeSqlExecutions = new Map(); - private static readonly sqlStatementClasses: Partial< - Record - > = { - [StatementType.SELECT]: 'select', - [StatementType.CREATE]: 'create', - [StatementType.DROP]: 'drop', - [StatementType.INSERT]: 'insert', - [StatementType.UPDATE]: 'update', - [StatementType.DELETE]: 'delete', - }; + // Populate only with reviewed packaged attach/read/write/cleanup evidence. + // Development fixture claims alone do not establish platform acceptance. + private static readonly acceptedSqlCombinations: ReadonlyArray<{ + runtimeFingerprint: string; + catalogType: IcebergInstanceConfig['catalogType']; + authMode: IcebergInstanceConfig['catalogAuthMode']; + storageProvider: IcebergInstanceConfig['sqlStorageProvider']; + }> = []; + + private static isSqlCombinationAccepted( + instance: IcebergInstanceConfig, + ): boolean { + return IcebergDatalakeService.acceptedSqlCombinations.some( + (accepted) => + accepted.runtimeFingerprint === + IcebergDatalakeService.getSqlRuntimeFingerprint() && + accepted.catalogType === instance.catalogType && + accepted.authMode === (instance.catalogAuthMode ?? 'none') && + accepted.storageProvider === instance.sqlStorageProvider, + ); + } private static readonly cloudProviders = [ 'aws', @@ -801,7 +813,7 @@ export class IcebergDatalakeService { try { // eslint-disable-next-line global-require, @typescript-eslint/no-var-requires const pkg = require('@duckdb/node-api/package.json'); - return `duckdb-node-api:${String(pkg.version)}`; + return `duckdb-node-api:${String(pkg.version)}:${process.platform}:${process.arch}:iceberg-policy-v2`; } catch { return 'duckdb-node-api:unknown'; } @@ -950,33 +962,21 @@ export class IcebergDatalakeService { static async listInstances(): Promise { try { const instances = await IcebergDatalakeService.readInstances(); - const runtimeFingerprint = - IcebergDatalakeService.getSqlRuntimeFingerprint(); - return instances.map( - ({ - id, - name, - description, - catalogType, - storageType, - catalogPath, - localPath, - storageBucket, - sqlEnabled, - sqlAccessVerifiedAt, - sqlRuntimeFingerprint, - createdAt, - updatedAt, - }) => { - let sqlUnavailableReason: string | undefined; - if (!sqlEnabled) { - sqlUnavailableReason = 'ICEBERG_SQL_DISABLED'; - } else if ( - !sqlAccessVerifiedAt || - sqlRuntimeFingerprint !== runtimeFingerprint - ) { - sqlUnavailableReason = 'ICEBERG_SQL_UNVERIFIED'; - } + return Promise.all( + instances.map(async (instance) => { + const { + id, + name, + description, + catalogType, + storageType, + catalogPath, + localPath, + storageBucket, + createdAt, + updatedAt, + } = instance; + const capability = await IcebergDatalakeService.getSqlCapability(id); return { id, name, @@ -986,15 +986,12 @@ export class IcebergDatalakeService { catalogPath, localPath, storageBucket, - sqlAvailable: - !!sqlEnabled && - !!sqlAccessVerifiedAt && - sqlRuntimeFingerprint === runtimeFingerprint, - sqlUnavailableReason, createdAt, updatedAt, + sqlAvailable: capability.available, + sqlUnavailableReason: capability.reason, }; - }, + }), ); } catch (error) { // eslint-disable-next-line no-console @@ -1519,6 +1516,28 @@ export class IcebergDatalakeService { supportedStatements: [], }; } + if (!IcebergDatalakeService.isSqlCombinationAccepted(instance)) { + return { + available: false, + reason: 'ICEBERG_SQL_COMBINATION_NOT_ACCEPTED', + runtimeFingerprint, + canRead: false, + canWrite: false, + supportedStatements: [], + }; + } + try { + await IcebergDatalakeService.validateSqlStorageBinding(instance); + } catch { + return { + available: false, + reason: 'ICEBERG_SQL_STORAGE_UNAVAILABLE', + runtimeFingerprint, + canRead: false, + canWrite: false, + supportedStatements: [], + }; + } return { available: true, runtimeFingerprint, @@ -1537,23 +1556,48 @@ export class IcebergDatalakeService { static async verifySqlAccess(id: string): Promise { try { + const verifiedInstance = await IcebergDatalakeService.getInstance(id); const executionId = `verify-${uuidv4()}`; await IcebergDatalakeService.withAttachedSqlCatalog( id, executionId, async (connection, alias) => { - await connection.runAndReadUntil( - 'SELECT table_schema, table_name FROM information_schema.tables WHERE table_catalog = ? LIMIT 1', + const tables = await connection.runAndReadUntil( + "SELECT table_schema, table_name FROM information_schema.tables WHERE table_catalog = ? AND table_type = 'BASE TABLE' ORDER BY table_schema, table_name LIMIT 1", 1, [alias], ); + const table = tables.getRowObjectsJson()[0]; + if (!table) throw new Error('ICEBERG_SQL_NONEMPTY_TABLE_REQUIRED'); + const qualified = [alias, table.table_schema, table.table_name] + .map((part) => + IcebergDatalakeService.quoteSqlIdentifier(String(part)), + ) + .join('.'); + // Project actual columns: COUNT(*) and catalog metadata can succeed + // without opening a warehouse data file. + const data = await connection.runAndReadUntil( + `SELECT * FROM ${qualified} LIMIT 1`, + 1, + ); + if (!data.getRowObjectsJson().length) { + throw new Error('ICEBERG_SQL_NONEMPTY_TABLE_REQUIRED'); + } }, ); + if (!IcebergDatalakeService.isSqlCombinationAccepted(verifiedInstance)) { + throw new Error('ICEBERG_SQL_COMBINATION_NOT_ACCEPTED'); + } const runtimeFingerprint = IcebergDatalakeService.getSqlRuntimeFingerprint(); const instances = await IcebergDatalakeService.readInstances(); const index = instances.findIndex((instance) => instance.id === id); if (index < 0) throw new Error(`Iceberg instance not found: ${id}`); + if ( + JSON.stringify(instances[index]) !== JSON.stringify(verifiedInstance) + ) { + throw new Error('ICEBERG_SQL_CONFIGURATION_CHANGED'); + } const checkedAt = new Date().toISOString(); instances[index] = { ...instances[index], @@ -1607,6 +1651,8 @@ export class IcebergDatalakeService { let connection: any; let attached = false; let stage = 'initialize'; + let result!: T; + let cleanupFailed = false; try { duckdbInstance = await DuckDBInstance.create(':memory:'); connection = await duckdbInstance.connect(); @@ -1625,7 +1671,7 @@ export class IcebergDatalakeService { await connection.run(sql.attachSql); attached = true; stage = 'execute'; - return await callback(connection, names.alias); + result = await callback(connection, names.alias); } catch (error) { const message = error instanceof Error ? error.message : String(error); if (message.startsWith('ICEBERG_')) throw error; @@ -1651,7 +1697,8 @@ export class IcebergDatalakeService { `DETACH ${IcebergDatalakeService.quoteSqlIdentifier(names.alias)}`, ); } catch { - // Continue best-effort cleanup. + // Finish remaining cleanup, but never mark failed cleanup verified. + cleanupFailed = true; } } await [names.catalogSecret, names.storageSecret].reduce( @@ -1662,44 +1709,44 @@ export class IcebergDatalakeService { `DROP SECRET IF EXISTS ${IcebergDatalakeService.quoteSqlIdentifier(secret)}`, ); } catch { - // Continue best-effort cleanup. + // Finish remaining cleanup, but never mark failed cleanup verified. + cleanupFailed = true; } }, Promise.resolve(), ); - connection.closeSync?.(); + try { + connection.closeSync?.(); + } catch { + cleanupFailed = true; + } + } + try { + duckdbInstance?.closeSync?.(); + } catch { + cleanupFailed = true; } - duckdbInstance?.closeSync?.(); } + if (cleanupFailed) throw new Error('ICEBERG_SQL_CLEANUP_FAILED'); + return result; } private static async classifySqlStatement( connection: any, sql: string, ): Promise { - if (!sql.trim() || sql.length > 1_000_000) { - throw new Error('ICEBERG_SQL_INVALID'); - } - const extracted = await connection.extractStatements(sql); - if (extracted.count !== 1) { - throw new Error('ICEBERG_SQL_SINGLE_STATEMENT_REQUIRED'); - } - const prepared = await extracted.prepare(0); - try { - const statementClass = - IcebergDatalakeService.sqlStatementClasses[ - prepared.statementType as StatementType - ]; - if (!statementClass) throw new Error('ICEBERG_SQL_STATEMENT_REJECTED'); - const rejectedSurface = - /\b(attach|detach|install|load|pragma|copy|merge|alter|create\s+(?:or\s+replace\s+)?(?:temporary\s+|temp\s+)?secret|drop\s+secret|read_(?:csv|json|parquet)|iceberg_scan|httpfs)\b/i; - if (rejectedSurface.test(sql)) { - throw new Error('ICEBERG_SQL_STATEMENT_REJECTED'); - } - return statementClass; - } finally { - prepared.destroySync(); + const parsed = parseIcebergSql(sql); + if (parsed.selectSql) { + // Serialization parses without binding: no file/table function may run + // during policy validation. Parameter values require an explicit cast. + const result = await connection.runAndReadAll( + 'SELECT json_serialize_sql(CAST(? AS VARCHAR)) AS ast', + [parsed.selectSql], + ); + const ast = JSON.parse(String(result.getRowObjectsJson()[0]?.ast)); + validateIcebergSelectAst(ast); } + return parsed.statementClass; } static async executeSql( @@ -1711,16 +1758,39 @@ export class IcebergDatalakeService { if (!capability.available) { throw new Error(capability.reason ?? 'ICEBERG_SQL_UNAVAILABLE'); } + const parser = await DuckDBInstance.create(':memory:'); + let statementClass: IcebergSqlStatementClass; + try { + const connection = await parser.connect(); + try { + statementClass = await IcebergDatalakeService.classifySqlStatement( + connection, + params.sql, + ); + } finally { + connection.closeSync(); + } + } finally { + parser.closeSync(); + } + if (params.validateOnly) { + return { + executionId: params.executionId, + statementClass, + columns: [], + rows: [], + rowsChanged: 0, + truncated: false, + }; + } + if (statementClass !== 'select' && params.mutationConfirmed !== true) { + throw new Error('ICEBERG_SQL_CONFIRMATION_REQUIRED'); + } const maxRows = Math.max(1, Math.min(params.maxRows ?? 1000, 5000)); return IcebergDatalakeService.withAttachedSqlCatalog( params.instanceId, params.executionId, async (connection) => { - const statementClass = - await IcebergDatalakeService.classifySqlStatement( - connection, - params.sql, - ); const reader = await connection.runAndReadUntil( params.sql, maxRows + 1, diff --git a/src/renderer/components/sqlEditor/index.tsx b/src/renderer/components/sqlEditor/index.tsx index 48a37096..cca4698c 100644 --- a/src/renderer/components/sqlEditor/index.tsx +++ b/src/renderer/components/sqlEditor/index.tsx @@ -131,14 +131,7 @@ export const SqlEditor: React.FC = ({ return; } - const commandType = getCommandType(selectedQuery); - if (isIcebergConnection && commandType !== 'SELECT') { - // eslint-disable-next-line no-alert - const confirmed = window.confirm( - 'This statement will modify the Iceberg catalog or its data. Continue?', - ); - if (!confirmed) return; - } + let commandType = getCommandType(selectedQuery); // Generate semi-unique ID for query cancellation const queryId = `query-${Date.now()}-${Math.random() @@ -156,15 +149,29 @@ export const SqlEditor: React.FC = ({ let result; if (isIcebergConnection && icebergInstanceId) { - const icebergResult = await icebergService.executeIcebergSql({ - instanceId: icebergInstanceId, - executionId: queryId, - sql: selectedQuery, - maxRows: 1000, - }); + const icebergResult = await icebergService.executeConfirmedIcebergSql( + { + instanceId: icebergInstanceId, + executionId: queryId, + sql: selectedQuery, + maxRows: 1000, + }, + (statementClass) => { + // eslint-disable-next-line no-alert + return window.confirm( + `Run ${statementClass.toUpperCase()} on Iceberg "${connectionInput?.name ?? icebergInstanceId}"? This will modify the catalog or its data.`, + ); + }, + ); + if (!icebergResult) return; + if (icebergResult.statementClass === 'select') commandType = 'SELECT'; + else if (['create', 'drop'].includes(icebergResult.statementClass)) + commandType = 'DDL'; + else commandType = 'DML'; result = { success: true, data: icebergResult.rows, + truncated: icebergResult.truncated, fields: icebergResult.columns.map((name) => ({ name, type: 0 })), rowCount: icebergResult.statementClass === 'select' @@ -222,7 +229,9 @@ export const SqlEditor: React.FC = ({ } // Check if this was a DDL operation - const wasDDL = isDDLOperation(selectedQuery); + const wasDDL = isIcebergConnection + ? commandType === 'DDL' + : isDDLOperation(selectedQuery); const enrichedResult = { ...result, isCommand: commandType === 'DDL' || commandType === 'DML', diff --git a/src/renderer/screens/sql/queryResult.tsx b/src/renderer/screens/sql/queryResult.tsx index acb036cc..80927e0e 100644 --- a/src/renderer/screens/sql/queryResult.tsx +++ b/src/renderer/screens/sql/queryResult.tsx @@ -3,6 +3,7 @@ import { toast } from 'react-toastify'; import { styled } from '@mui/material/styles'; import { Box, + Alert, Backdrop, CircularProgress, Typography, @@ -862,6 +863,12 @@ export const QueryResult: React.FC = ({ results, exportContext }) => { overflow: 'hidden', }} > + {results.truncated && ( + + Showing only the first {results.data?.length ?? 0} rows. Results and + exports are limited to these rows. Refine the query to see other rows. + + )} {viewMode === 'chart' ? ( => window.electron.ipcRenderer.invoke('iceberg:ensureInstalled'); + +/** Preflight and execution use the same main-process policy. Capture SQL once + * so confirmation cannot accidentally authorize a subsequently edited query. */ +export const executeConfirmedIcebergSql = async ( + params: IcebergSqlExecutionParams, + confirmMutation: ( + statementClass: IcebergSqlExecutionResult['statementClass'], + ) => boolean, +): Promise => { + const request = { ...params }; + const classification = await executeIcebergSql({ + ...request, + validateOnly: true, + }); + const mutating = classification.statementClass !== 'select'; + if (mutating && !confirmMutation(classification.statementClass)) + return undefined; + return executeIcebergSql({ + ...request, + validateOnly: false, + mutationConfirmed: mutating, + }); +}; diff --git a/src/types/backend.ts b/src/types/backend.ts index 993d44c8..c1be39c7 100644 --- a/src/types/backend.ts +++ b/src/types/backend.ts @@ -694,6 +694,7 @@ export type Table = { export type QueryResponseType = { success: boolean; + truncated?: boolean; data?: QueryResult[]; fields?: { name: string; type: number }[]; rowCount?: number; // Add rowCount for affected rows in INSERT/UPDATE/DELETE operations diff --git a/src/types/iceberg.ts b/src/types/iceberg.ts index d7f04b00..4b38f6cc 100644 --- a/src/types/iceberg.ts +++ b/src/types/iceberg.ts @@ -209,6 +209,8 @@ export interface IcebergSqlExecutionParams { executionId: string; sql: string; maxRows?: number; + validateOnly?: boolean; + mutationConfirmed?: boolean; } export interface IcebergSqlExecutionResult { diff --git a/tests/unit/main/services/icebergSqlPolicy.test.ts b/tests/unit/main/services/icebergSqlPolicy.test.ts new file mode 100644 index 00000000..1f7c438d --- /dev/null +++ b/tests/unit/main/services/icebergSqlPolicy.test.ts @@ -0,0 +1,121 @@ +// Test the exact application runtime, which is installed outside root node_modules. +// eslint-disable-next-line import/no-relative-packages, import/no-useless-path-segments +import { DuckDBInstance } from '../../../../release/app/node_modules/@duckdb/node-api'; +import { + parseIcebergSql, + validateIcebergSelectAst, +} from '../../../../src/main/services/iceberg/sqlPolicy'; + +// Real packaged-version parser, no catalog credentials, extension downloads, +// binding or execution of the supplied SQL. +describe('Iceberg SQL policy with the native DuckDB parser', () => { + let db: Awaited>; + let connection: Awaited>; + beforeAll(async () => { + db = await DuckDBInstance.create(':memory:'); + connection = await db.connect(); + }); + afterAll(() => { + connection.closeSync(); + db.closeSync(); + }); + const classify = async (sql: string) => { + const parsed = parseIcebergSql(sql); + if (parsed.selectSql) { + const reader = await connection.runAndReadAll( + 'SELECT json_serialize_sql(CAST(? AS VARCHAR)) AS ast', + [parsed.selectSql], + ); + validateIcebergSelectAst( + JSON.parse(String(reader.getRowObjectsJson()[0].ast)), + ); + } + return parsed.statementClass; + }; + it.each([ + ['SELECT * FROM iceberg.sales.orders', 'select'], + ["SELECT 'read_text attach drop' AS value", 'select'], + [ + '-- comment\n/* nested /* comment */ */ DELETE FROM iceberg.sales.orders WHERE id = 1', + 'delete', + ], + [ + 'WITH ids AS (SELECT id FROM iceberg.sales.orders) DELETE FROM iceberg.sales.orders WHERE id IN (SELECT id FROM ids)', + 'delete', + ], + [ + 'WITH ids AS (SELECT id FROM iceberg.sales.orders) SELECT * FROM ids', + 'select', + ], + ['SELECT count(*), sum(id) FROM iceberg.sales.orders', 'select'], + [ + 'SELECT row_number() OVER (ORDER BY id) FROM iceberg.sales.orders', + 'select', + ], + ['SELECT * FROM iceberg.sales.orders AT (VERSION => 3)', 'select'], + [ + 'SELECT a.id + b.id FROM iceberg.sales.orders a JOIN iceberg.sales.orders b ON a.id=b.id', + 'select', + ], + ['INSERT INTO iceberg.sales.orders (id) VALUES (1), (2)', 'insert'], + [ + 'INSERT INTO iceberg.sales.orders SELECT id FROM iceberg.sales.source', + 'insert', + ], + [ + 'UPDATE iceberg.sales.orders SET id = id + 1, name = upper(name) WHERE id > 2', + 'update', + ], + [ + 'CREATE TABLE iceberg.sales.orders (id INTEGER, name VARCHAR NOT NULL, price DECIMAL(10,2))', + 'create', + ], + [ + 'CREATE TABLE iceberg.sales.orders AS SELECT * FROM iceberg.sales.source', + 'create', + ], + ['CREATE SCHEMA IF NOT EXISTS "iceberg"."sales"', 'create'], + ['DROP TABLE IF EXISTS iceberg.sales.orders', 'drop'], + ['DROP SCHEMA iceberg.sales', 'drop'], + ])('accepts %s', async (sql, expected) => { + expect(await classify(sql)).toBe(expected); + }); + + it.each([ + "SELECT * FROM read_text('/dev/null')", + "SELECT * FROM read_blob('/dev/null')", + 'SELECT * FROM duckdb_secrets()', + "SELECT * FROM read_csv_auto('/tmp/file.csv')", + "SELECT * FROM 'file.parquet'", + "SELECT * FROM query('SELECT 1')", + "SELECT getvariable('secret')", + "SELECT current_setting('s3_secret_access_key')", + 'SELECT * FROM information_schema.tables', + 'SELECT * FROM orders', + 'SELECT * FROM memory.main.orders', + 'SELECT * FROM iceberg.sales.orders; DROP TABLE iceberg.sales.orders', + "WITH x AS (SELECT * FROM read_text('/dev/null')) SELECT * FROM x", + 'WITH x AS (SELECT * FROM iceberg.sales.orders) SELECT * FROM (SELECT * FROM information_schema.tables) y', + 'SELECT (SELECT * FROM duckdb_secrets()) FROM iceberg.sales.orders', + "INSERT INTO iceberg.sales.orders SELECT * FROM read_text('/dev/null')", + "UPDATE iceberg.sales.orders SET name = (SELECT content FROM read_text('/dev/null'))", + 'DELETE FROM iceberg.sales.orders WHERE EXISTS (SELECT * FROM duckdb_secrets())', + 'CREATE TABLE local_table (id INTEGER)', + 'CREATE TABLE memory.main.orders (id INTEGER)', + 'CREATE VIEW iceberg.sales.orders AS SELECT 1', + 'CREATE MACRO iceberg.sales.evil() AS 1', + 'CREATE SECRET stolen (TYPE S3)', + 'CREATE /* bypass */ SECRET stolen (TYPE S3)', + "CREATE TABLE iceberg.sales.orders (id INTEGER DEFAULT nextval('x'))", + "CREATE TABLE iceberg.sales.orders AS SELECT * FROM read_text('/dev/null')", + 'DROP SCHEMA main', + "ATTACH 'x' AS iceberg", + 'SET enable_external_access = true', + 'LOAD httpfs', + 'MERGE INTO iceberg.sales.orders USING iceberg.sales.source ON true WHEN MATCHED THEN DELETE', + ])('rejects %s before binding', async (sql) => { + await expect(classify(sql)).rejects.toThrow( + 'ICEBERG_SQL_STATEMENT_REJECTED', + ); + }); +}); diff --git a/tests/unit/main/services/icebergSqlRuntime.service.test.ts b/tests/unit/main/services/icebergSqlRuntime.service.test.ts index 6fb6e8a2..c0a9d7a3 100644 --- a/tests/unit/main/services/icebergSqlRuntime.service.test.ts +++ b/tests/unit/main/services/icebergSqlRuntime.service.test.ts @@ -7,6 +7,7 @@ import { const mockRun = jest.fn(); const mockRunAndReadUntil = jest.fn(); +const mockRunAndReadAll = jest.fn(); const mockExtractStatements = jest.fn(); const mockCloseConnection = jest.fn(); const mockCloseInstance = jest.fn(); @@ -18,6 +19,7 @@ jest.mock('@duckdb/node-api', () => ({ connect: jest.fn(async () => ({ run: mockRun, runAndReadUntil: mockRunAndReadUntil, + runAndReadAll: mockRunAndReadAll, extractStatements: mockExtractStatements, closeSync: mockCloseConnection, interrupt: mockInterrupt, @@ -96,7 +98,21 @@ const database = { describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { beforeEach(() => { + jest.restoreAllMocks(); jest.clearAllMocks(); + jest + .spyOn(IcebergDatalakeService as any, 'isSqlCombinationAccepted') + .mockReturnValue(true); + mockRunAndReadAll.mockResolvedValue({ + getRowObjectsJson: () => [ + { + ast: JSON.stringify({ + error: false, + statements: [{ node: { type: 'SELECT_NODE' } }], + }), + }, + ], + }); mockedLoadDatabase.mockResolvedValue(database); mockedUpdateDatabase.mockResolvedValue(undefined); mockedSecureStorage.getCredential.mockImplementation(async (key) => { @@ -117,7 +133,9 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { mockRunAndReadUntil.mockResolvedValue({ columnNames: () => ['table_schema', 'table_name'], getRowsJson: () => [], - getRowObjectsJson: () => [], + getRowObjectsJson: () => [ + { table_schema: 'sales', table_name: 'orders' }, + ], rowsChanged: 0, done: true, }); @@ -307,22 +325,16 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { }); it('rejects runtime-control SQL after parsing exactly one statement', async () => { - const destroySync = jest.fn(); const classify = (IcebergDatalakeService as any).classifySqlStatement as ( connection: unknown, sql: string, ) => Promise; - const connection = { - extractStatements: jest.fn(async () => ({ - count: 1, - prepare: jest.fn(async () => ({ statementType: 7, destroySync })), - })), - }; + const connection = { runAndReadAll: mockRunAndReadAll }; await expect( classify(connection, 'CREATE SECRET stolen (TYPE S3)'), ).rejects.toThrow('ICEBERG_SQL_STATEMENT_REJECTED'); - expect(destroySync).toHaveBeenCalled(); + expect(mockRunAndReadAll).not.toHaveBeenCalled(); }); it('interrupts only a registered active execution', () => { @@ -337,4 +349,119 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { active.delete('running-query'); expect(IcebergDatalakeService.cancelSql('missing-query')).toBe(false); }); + it('requires a warehouse row read before persisting verification', async () => { + await IcebergDatalakeService.verifySqlAccess(instance.id); + expect(mockRunAndReadUntil).toHaveBeenCalledWith( + 'SELECT * FROM "iceberg"."sales"."orders" LIMIT 1', + 1, + ); + }); + + it.each([ + 'empty catalog', + 'empty table', + 'unreadable table', + 'cleanup failure', + 'unaccepted pair', + ])('does not persist verification for %s', async (failure) => { + if (failure === 'empty catalog') { + mockRunAndReadUntil.mockResolvedValueOnce({ + getRowObjectsJson: () => [], + }); + } else if (failure === 'empty table') { + mockRunAndReadUntil + .mockResolvedValueOnce({ + getRowObjectsJson: () => [ + { table_schema: 'sales', table_name: 'orders' }, + ], + }) + .mockResolvedValueOnce({ getRowObjectsJson: () => [] }); + } else if (failure === 'unreadable table') { + mockRunAndReadUntil + .mockResolvedValueOnce({ + getRowObjectsJson: () => [ + { table_schema: 'sales', table_name: 'orders' }, + ], + }) + .mockRejectedValueOnce(new Error('warehouse denied')); + } else if (failure === 'cleanup failure') { + mockRun.mockImplementation(async (sql: string) => { + if (sql.startsWith('DETACH')) throw new Error('detach failed'); + }); + } else { + (IcebergDatalakeService as any).isSqlCombinationAccepted.mockReturnValue( + false, + ); + } + expect( + (await IcebergDatalakeService.verifySqlAccess(instance.id)).success, + ).toBe(false); + expect(mockedUpdateDatabase).not.toHaveBeenCalled(); + expect(mockCloseConnection).toHaveBeenCalled(); + expect(mockCloseInstance).toHaveBeenCalled(); + }); + + it('does not advertise a verified but unaccepted combination', async () => { + mockedLoadDatabase.mockResolvedValue({ + ...database, + icebergInstances: [ + { + ...instance, + sqlAccessVerifiedAt: '2026-09-07', + sqlRuntimeFingerprint: ( + IcebergDatalakeService as any + ).getSqlRuntimeFingerprint(), + }, + ], + }); + (IcebergDatalakeService as any).isSqlCombinationAccepted.mockReturnValue( + false, + ); + expect( + await IcebergDatalakeService.getSqlCapability(instance.id), + ).toMatchObject({ + available: false, + canWrite: false, + reason: 'ICEBERG_SQL_COMBINATION_NOT_ACCEPTED', + }); + expect((await IcebergDatalakeService.listInstances())[0].sqlAvailable).toBe( + false, + ); + }); + + it('requires explicit confirmation for parsed mutations before attaching', async () => { + mockedLoadDatabase.mockResolvedValue({ + ...database, + icebergInstances: [ + { + ...instance, + sqlAccessVerifiedAt: '2026-09-07', + sqlRuntimeFingerprint: ( + IcebergDatalakeService as any + ).getSqlRuntimeFingerprint(), + }, + ], + }); + const params = { + instanceId: instance.id, + executionId: 'mutation', + sql: '/* comment */ DELETE FROM iceberg.sales.orders', + }; + await expect(IcebergDatalakeService.executeSql(params)).rejects.toThrow( + 'ICEBERG_SQL_CONFIRMATION_REQUIRED', + ); + expect(mockRun).not.toHaveBeenCalled(); + expect( + await IcebergDatalakeService.executeSql({ + ...params, + validateOnly: true, + }), + ).toMatchObject({ statementClass: 'delete' }); + expect(mockRun).not.toHaveBeenCalled(); + await IcebergDatalakeService.executeSql({ + ...params, + mutationConfirmed: true, + }); + expect(mockRunAndReadUntil).toHaveBeenCalledWith(params.sql, 1001); + }); }); diff --git a/tests/unit/renderer/screens/sql/icebergTruncation.test.tsx b/tests/unit/renderer/screens/sql/icebergTruncation.test.tsx new file mode 100644 index 00000000..40c3cfb3 --- /dev/null +++ b/tests/unit/renderer/screens/sql/icebergTruncation.test.tsx @@ -0,0 +1,52 @@ +import React from 'react'; +import { render, screen } from '@testing-library/react'; +import { QueryResult } from '../../../../../src/renderer/screens/sql/queryResult'; + +jest.mock('../../../../../src/renderer/components/customTable', () => ({ + CustomTable: () =>
, +})); +jest.mock( + '../../../../../src/renderer/components/queryResult/queryVisualization/QueryResultVisualization', + () => ({ + QueryResultVisualization: () =>
, + }), +); +jest.mock('../../../../../src/renderer/helpers/utils', () => ({ + underscoreToTitleCase: (value: string) => value, +})); +jest.mock('../../../../../src/renderer/services/duckLake.service', () => ({ + DuckLakeService: {}, +})); + +describe('SQL result truncation notice', () => { + it('shows that displayed rows and exports are incomplete', () => { + render( + , + ); + expect(screen.getByRole('alert')).toHaveTextContent( + 'Showing only the first 1 rows', + ); + expect(screen.getByRole('alert')).toHaveTextContent( + 'exports are limited to these rows', + ); + }); + it('does not change the display of complete results', () => { + render( + , + ); + expect(screen.queryByRole('alert')).not.toBeInTheDocument(); + }); +}); diff --git a/tests/unit/renderer/services/icebergConfirmation.test.ts b/tests/unit/renderer/services/icebergConfirmation.test.ts new file mode 100644 index 00000000..c5967995 --- /dev/null +++ b/tests/unit/renderer/services/icebergConfirmation.test.ts @@ -0,0 +1,56 @@ +import { executeConfirmedIcebergSql } from '../../../../src/renderer/services/iceberg.service'; + +describe('Iceberg mutation confirmation', () => { + const invoke = window.electron.ipcRenderer.invoke as jest.Mock; + beforeEach(() => { + invoke.mockReset(); + }); + it.each([ + '/* comment */ DELETE FROM iceberg.sales.orders', + 'WITH ids AS (SELECT 1) DELETE FROM iceberg.sales.orders', + ])('does not execute rejected confirmation: %s', async (sql) => { + invoke.mockResolvedValueOnce({ statementClass: 'delete' }); + const confirm = jest.fn(() => false); + expect( + await executeConfirmedIcebergSql( + { instanceId: 'id', executionId: 'run', sql }, + confirm, + ), + ).toBeUndefined(); + expect(confirm).toHaveBeenCalledWith('delete'); + expect(invoke).toHaveBeenCalledTimes(1); + expect(invoke.mock.calls[0][1].validateOnly).toBe(true); + }); + it('executes exactly the SQL that was confirmed', async () => { + invoke + .mockResolvedValueOnce({ statementClass: 'delete' }) + .mockResolvedValueOnce({ rows: [] }); + const params = { + instanceId: 'id', + executionId: 'run', + sql: 'DELETE FROM iceberg.sales.orders', + }; + await executeConfirmedIcebergSql(params, () => { + params.sql = 'changed'; + return true; + }); + expect(invoke.mock.calls[1][1]).toMatchObject({ + sql: 'DELETE FROM iceberg.sales.orders', + mutationConfirmed: true, + validateOnly: false, + }); + }); + it('does not prompt for a backend-classified read', async () => { + invoke + .mockResolvedValueOnce({ statementClass: 'select' }) + .mockResolvedValueOnce({ rows: [], truncated: true }); + const confirm = jest.fn(); + expect( + await executeConfirmedIcebergSql( + { instanceId: 'id', executionId: 'run', sql: 'SELECT 1' }, + confirm, + ), + ).toMatchObject({ truncated: true }); + expect(confirm).not.toHaveBeenCalled(); + }); +}); From 0c165b4f52434bb7d07cfb483a854bac19ff0ae1 Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Tue, 8 Sep 2026 19:52:34 +0200 Subject: [PATCH 04/16] feat(notebooks): route iceberg sql execution through native duckdb runtime Implements Phase 5 of Plan 59a (DuckDB Iceberg Extension). Enables execution of SQL Notebook cells against verified Iceberg REST connections using the existing Phase 3 capability model, attachment lifecycle, and IPC policy. - Normalizes connection keys (`iceberg:`) for notebook storage while reusing the existing Cloud Explorer credential mapping. - Wires both single-cell and Run All notebook executions to the verified `IcebergDatalakeService`, enforcing the restricted statement whitelist. - Enforces strict confirmation checks for mutating statements (DDL/DML) and safely handles explicit DuckDB cancellation flows per cell. - Bounds Iceberg cell output to a maximum of 100 rows per query to prevent excessive IPC payload transfer and massive notebook JSON files. - Preserves read-only visibility/editability for notebooks when the saved Iceberg catalog connection is deleted, disabled, or unverified. --- src/main/ipcHandlers/notebooks.ipcHandlers.ts | 19 ++- src/main/services/icebergDatalake.service.ts | 65 ++++---- src/main/services/notebooks.service.ts | 140 +++++++++++++++++- .../components/notebook/NotebookEditor.tsx | 122 ++++++++++++++- .../components/notebook/OutputPanel.tsx | 19 ++- .../controllers/notebooks.controller.ts | 29 +++- src/renderer/hooks/useSchemaForConnection.ts | 8 + src/renderer/screens/notebooks/index.tsx | 133 ++++++++++++++++- src/renderer/screens/sql/index.tsx | 10 +- src/renderer/services/iceberg.service.ts | 37 +++++ src/renderer/services/notebooks.service.ts | 109 +++++++++++++- src/types/ipc.ts | 1 + src/types/notebooks.ts | 13 ++ .../icebergSqlRuntime.service.test.ts | 35 +++-- .../services/notebooksIceberg.service.test.ts | 119 +++++++++++++++ .../services/notebooksIceberg.service.test.ts | 129 ++++++++++++++++ 16 files changed, 909 insertions(+), 79 deletions(-) create mode 100644 tests/unit/main/services/notebooksIceberg.service.test.ts create mode 100644 tests/unit/renderer/services/notebooksIceberg.service.test.ts diff --git a/src/main/ipcHandlers/notebooks.ipcHandlers.ts b/src/main/ipcHandlers/notebooks.ipcHandlers.ts index d2640310..bfc2daee 100644 --- a/src/main/ipcHandlers/notebooks.ipcHandlers.ts +++ b/src/main/ipcHandlers/notebooks.ipcHandlers.ts @@ -4,6 +4,10 @@ */ import { ipcMain } from 'electron'; +import type { + NotebookExecutionOptions, + NotebookRunAllCell, +} from '../../types/notebooks'; import { NotebooksService } from '../services/notebooks.service'; export function registerNotebooksHandlers() { @@ -111,6 +115,7 @@ export function registerNotebooksHandlers() { sql: string, limit?: number, offset?: number, + options?: NotebookExecutionOptions, ) => { return NotebooksService.runCell( connectionId, @@ -119,6 +124,7 @@ export function registerNotebooksHandlers() { sql, limit, offset, + options, ); }, ); @@ -149,11 +155,20 @@ export function registerNotebooksHandlers() { // Run all cells ipcMain.handle( 'notebooks:runAll', - async (_event, connectionId: string, notebookId: string) => { - return NotebooksService.runAllCells(connectionId, notebookId); + async ( + _event, + connectionId: string, + notebookId: string, + cellRun?: NotebookRunAllCell, + ) => { + return NotebooksService.runAllCells(connectionId, notebookId, cellRun); }, ); + ipcMain.handle('notebooks:cancelIcebergCell', (_event, executionId: string) => + NotebooksService.cancelIcebergCell(executionId), + ); + // List archived notebooks ipcMain.handle('notebooks:archived:list', async () => { return NotebooksService.listArchivedNotebooks(); diff --git a/src/main/services/icebergDatalake.service.ts b/src/main/services/icebergDatalake.service.ts index c4837c51..f49d7e14 100644 --- a/src/main/services/icebergDatalake.service.ts +++ b/src/main/services/icebergDatalake.service.ts @@ -52,28 +52,6 @@ import type { PostgresConnection } from '../../types/backend'; export class IcebergDatalakeService { private static readonly activeSqlExecutions = new Map(); - // Populate only with reviewed packaged attach/read/write/cleanup evidence. - // Development fixture claims alone do not establish platform acceptance. - private static readonly acceptedSqlCombinations: ReadonlyArray<{ - runtimeFingerprint: string; - catalogType: IcebergInstanceConfig['catalogType']; - authMode: IcebergInstanceConfig['catalogAuthMode']; - storageProvider: IcebergInstanceConfig['sqlStorageProvider']; - }> = []; - - private static isSqlCombinationAccepted( - instance: IcebergInstanceConfig, - ): boolean { - return IcebergDatalakeService.acceptedSqlCombinations.some( - (accepted) => - accepted.runtimeFingerprint === - IcebergDatalakeService.getSqlRuntimeFingerprint() && - accepted.catalogType === instance.catalogType && - accepted.authMode === (instance.catalogAuthMode ?? 'none') && - accepted.storageProvider === instance.sqlStorageProvider, - ); - } - private static readonly cloudProviders = [ 'aws', 'azure', @@ -495,16 +473,20 @@ export class IcebergDatalakeService { const props: Record = {}; const env: Record = {}; - if (instance.storageType === 'server-managed') return { props, env }; + // Server-managed catalogs own the warehouse URI, but DuckDB still needs + // the selected SQL storage connection to resolve the catalog's S3 paths. + const connectionId = + instance.storageConnectionId ?? + (instance.sqlEnabled ? instance.sqlStorageConnectionId : undefined); if (instance.storageType === 'local' && instance.localPath) { props.warehouse = pathToFileURL(instance.localPath).href; } - if (instance.storageConnectionId) { + if (connectionId) { try { const db = await loadDatabaseFile(); const conn: CloudConnection | undefined = (db.sources ?? []).find( - (s) => s.id === instance.storageConnectionId, + (s) => s.id === connectionId, ); if (conn) { const { provider, config, id: connId } = conn; @@ -1516,16 +1498,6 @@ export class IcebergDatalakeService { supportedStatements: [], }; } - if (!IcebergDatalakeService.isSqlCombinationAccepted(instance)) { - return { - available: false, - reason: 'ICEBERG_SQL_COMBINATION_NOT_ACCEPTED', - runtimeFingerprint, - canRead: false, - canWrite: false, - supportedStatements: [], - }; - } try { await IcebergDatalakeService.validateSqlStorageBinding(instance); } catch { @@ -1585,9 +1557,6 @@ export class IcebergDatalakeService { } }, ); - if (!IcebergDatalakeService.isSqlCombinationAccepted(verifiedInstance)) { - throw new Error('ICEBERG_SQL_COMBINATION_NOT_ACCEPTED'); - } const runtimeFingerprint = IcebergDatalakeService.getSqlRuntimeFingerprint(); const instances = await IcebergDatalakeService.readInstances(); @@ -1629,6 +1598,7 @@ export class IcebergDatalakeService { instanceId: string, executionId: string, callback: (connection: any, alias: string) => Promise, + signal?: AbortSignal, ): Promise { if (!executionId.trim() || executionId.length > 120) { throw new Error('ICEBERG_SQL_EXECUTION_ID_INVALID'); @@ -1650,27 +1620,41 @@ export class IcebergDatalakeService { let duckdbInstance: any; let connection: any; let attached = false; + const checkCancelled = () => { + if (signal?.aborted) throw new Error('ICEBERG_SQL_CANCELLED'); + }; + const interrupt = () => connection?.interrupt(); + signal?.addEventListener('abort', interrupt); let stage = 'initialize'; let result!: T; let cleanupFailed = false; try { + checkCancelled(); duckdbInstance = await DuckDBInstance.create(':memory:'); connection = await duckdbInstance.connect(); + checkCancelled(); IcebergDatalakeService.activeSqlExecutions.set(executionId, connection); stage = 'install-extensions'; await connection.run('INSTALL httpfs'); + checkCancelled(); await connection.run('INSTALL iceberg'); + checkCancelled(); stage = 'load-extensions'; await connection.run('LOAD httpfs'); + checkCancelled(); await connection.run('LOAD iceberg'); + checkCancelled(); stage = 'create-storage-secret'; await connection.run(sql.storageSecretSql); + checkCancelled(); stage = 'create-catalog-secret'; await connection.run(sql.catalogSecretSql); + checkCancelled(); stage = 'attach'; await connection.run(sql.attachSql); attached = true; stage = 'execute'; + checkCancelled(); result = await callback(connection, names.alias); } catch (error) { const message = error instanceof Error ? error.message : String(error); @@ -1689,6 +1673,7 @@ export class IcebergDatalakeService { } throw new Error(`ICEBERG_SQL_RUNTIME_FAILED: ${stage}`); } finally { + signal?.removeEventListener('abort', interrupt); IcebergDatalakeService.activeSqlExecutions.delete(executionId); if (connection) { if (attached) { @@ -1751,7 +1736,9 @@ export class IcebergDatalakeService { static async executeSql( params: IcebergSqlExecutionParams, + signal?: AbortSignal, ): Promise { + if (signal?.aborted) throw new Error('ICEBERG_SQL_CANCELLED'); const capability = await IcebergDatalakeService.getSqlCapability( params.instanceId, ); @@ -1773,6 +1760,7 @@ export class IcebergDatalakeService { } finally { parser.closeSync(); } + if (signal?.aborted) throw new Error('ICEBERG_SQL_CANCELLED'); if (params.validateOnly) { return { executionId: params.executionId, @@ -1807,6 +1795,7 @@ export class IcebergDatalakeService { truncated: rows.length > maxRows || !reader.done, }; }, + signal, ); } diff --git a/src/main/services/notebooks.service.ts b/src/main/services/notebooks.service.ts index 964c6ad3..b6152e8e 100644 --- a/src/main/services/notebooks.service.ts +++ b/src/main/services/notebooks.service.ts @@ -11,6 +11,11 @@ import { v4 as uuidv4 } from 'uuid'; import { Notebook, NotebookCell, CellOutput } from '../../types/notebooks'; import ConnectorsService from './connectors.service'; import DuckLakeService from './duckLake.service'; +import { IcebergDatalakeService } from './icebergDatalake.service'; +import type { + NotebookExecutionOptions, + NotebookRunAllCell, +} from '../../types/notebooks'; const NOTEBOOKS_DIR = path.join(app.getPath('userData'), 'notebooks'); const ORPHANED_DIR = path.join(NOTEBOOKS_DIR, '_orphaned'); @@ -58,6 +63,11 @@ function limitCellOutputData(output: CellOutput): CellOutput { return { ...output, data: limitedRows, + ...(output.truncated !== undefined + ? { + truncated: output.truncated || output.data.length > MAX_STORED_ROWS, + } + : {}), rowCount: Math.min( output.rowCount ?? limitedRows.length, MAX_STORED_ROWS, @@ -139,7 +149,7 @@ async function ensureDirectories() { // Validate and sanitize path segments to prevent path traversal attacks function assertSafeSegment(value: string, label: string): string { // Allow alphanumeric, colon, underscore, dash for connection keys - // connectionKey format: "db:uuid" or "ducklake:uuid" + // connectionKey format: "db:uuid", "ducklake:uuid", or "iceberg:uuid" if (!/^[A-Za-z0-9:_-]+$/.test(value)) { throw new Error(`Invalid ${label}: "${value}" contains unsafe characters`); } @@ -237,6 +247,8 @@ function normalizeConnectionKey(connectionId: string): string { ); } + if (connectionId.startsWith('iceberg-')) + return `iceberg:${connectionId.slice(8)}`; if (connectionId.startsWith('ducklake-')) { const instanceId = connectionId.replace('ducklake-', ''); return `ducklake:${instanceId}`; @@ -245,6 +257,86 @@ function normalizeConnectionKey(connectionId: string): string { } export class NotebooksService { + private static readonly icebergRuns = new Map(); + + static cancelIcebergCell(executionId: string): boolean { + const run = this.icebergRuns.get(executionId); + if (!run) return false; + run.abort(); + IcebergDatalakeService.cancelSql(executionId); + return true; + } + + private static async runIcebergCell( + connectionId: string, + notebookId: string, + cellId: string, + sql: string, + options?: NotebookExecutionOptions, + ): Promise { + const executionId = options?.executionId ?? `notebook-${uuidv4()}`; + if ( + !executionId.trim() || + executionId.length > 120 || + this.icebergRuns.has(executionId) + ) { + throw new Error('ICEBERG_SQL_EXECUTION_ID_INVALID'); + } + const run = new AbortController(); + this.icebergRuns.set(executionId, run); + const started = Date.now(); + let output: CellOutput; + try { + const notebook = await this.getNotebook(connectionId, notebookId); + if ( + !notebook?.cells.some( + (cell) => cell.id === cellId && cell.type === 'sql', + ) + ) { + throw new Error('ICEBERG_NOTEBOOK_CELL_NOT_FOUND'); + } + const result = await IcebergDatalakeService.executeSql( + { + instanceId: connectionId.slice(8), + executionId, + sql, + maxRows: MAX_STORED_ROWS, + mutationConfirmed: options?.mutationConfirmed, + }, + run.signal, + ); + output = { + type: result.rows.length ? 'table' : 'empty', + data: result.rows, + columns: result.columns, + truncated: result.truncated, + rowCount: + result.statementClass === 'select' + ? result.rows.length + : result.rowsChanged, + statementClass: result.statementClass, + executionTime: Date.now() - started, + }; + } catch (error) { + let message = 'Iceberg execution failed.'; + if (run.signal.aborted) { + message = 'Iceberg execution cancelled.'; + } else if (error instanceof Error) { + message = error.message; + } + output = { + type: 'error', + error: message, + cancelled: run.signal.aborted, + executionTime: Date.now() - started, + }; + } finally { + this.icebergRuns.delete(executionId); + } + await this.updateCellOutput(connectionId, notebookId, cellId, output); + return output; + } + /** * List all notebooks for a connection */ @@ -745,7 +837,17 @@ export class NotebooksService { sql: string, limit?: number, offset?: number, + options?: NotebookExecutionOptions, ): Promise { + if (connectionId.startsWith('iceberg-')) { + return this.runIcebergCell( + connectionId, + notebookId, + cellId, + sql, + options, + ); + } try { const startTime = Date.now(); @@ -922,6 +1024,11 @@ export class NotebooksService { limit: number, offset: number, ): Promise { + if (connectionId.startsWith('iceberg-')) { + throw new Error( + 'ICEBERG_NOTEBOOK_BOUNDED_RESULTS: Refine and rerun the original query.', + ); + } try { const startTime = Date.now(); @@ -1067,7 +1174,23 @@ export class NotebooksService { static async runAllCells( connectionId: string, notebookId: string, + cellRun?: NotebookRunAllCell, ): Promise { + if (connectionId.startsWith('iceberg-')) { + // The renderer pauses for confirmation between cells and submits each + // confirmed cell through this same Run All route. Bare IPC cannot bypass it. + if (!cellRun) + throw new Error('ICEBERG_NOTEBOOK_CELL_CONFIRMATION_REQUIRED'); + const output = await this.runIcebergCell( + connectionId, + notebookId, + cellRun.cellId, + cellRun.sql, + cellRun, + ); + if (output.type === 'error') throw new Error(output.error); + return; + } try { const notebook = await this.getNotebook(connectionId, notebookId); if (!notebook) { @@ -1117,7 +1240,20 @@ export class NotebooksService { const limitedOutput = limitCellOutputData(output); const updatedCells = notebook.cells.map((cell) => - cell.id === cellId ? { ...cell, output: limitedOutput } : cell, + cell.id === cellId + ? { + ...cell, + output: limitedOutput, + ...(connectionId.startsWith('iceberg-') + ? { + status: + output.type === 'error' + ? ('error' as const) + : ('success' as const), + } + : {}), + } + : cell, ); await this.updateNotebook(connectionId, notebookId, { diff --git a/src/renderer/components/notebook/NotebookEditor.tsx b/src/renderer/components/notebook/NotebookEditor.tsx index 8b3f8b6f..9b47a150 100644 --- a/src/renderer/components/notebook/NotebookEditor.tsx +++ b/src/renderer/components/notebook/NotebookEditor.tsx @@ -3,7 +3,13 @@ * Main container for notebook editing with cells and toolbar */ -import React, { useState, useCallback, useEffect, useRef } from 'react'; +import React, { + useState, + useCallback, + useEffect, + useRef, + useMemo, +} from 'react'; import { Box, Button, @@ -32,6 +38,9 @@ import { DraggableStateSnapshot, } from '@hello-pangea/dnd'; import * as monaco from 'monaco-editor'; +import { icebergQualifiedName } from '../../services/iceberg.service'; +import { MonacoAutocompleteSQLKeywords } from '../../config/constants'; +import { useListIcebergInstances } from '../../controllers/icebergDatalake.controller'; import { useNotebook, useUpdateNotebook, @@ -67,6 +76,20 @@ export const NotebookEditor: React.FC = ({ onSchemaChange, }) => { const navigate = useNavigate(); + const isIceberg = instanceId.startsWith('iceberg-'); + const { data: icebergInstances = [], isLoading: icebergLoading } = + useListIcebergInstances(); + const icebergInstance = icebergInstances.find( + (item) => item.id === instanceId.slice(8), + ); + const icebergUnavailable = isIceberg && !icebergInstance?.sqlAvailable; + const icebergRuns = useRef(new Map()); + useEffect( + () => () => { + icebergRuns.current.forEach((run) => run.abort()); + }, + [instanceId, notebookId], + ); const connectionId = instanceId; // Use connectionId internally for clarity const { data: notebook, @@ -100,11 +123,39 @@ export const NotebookEditor: React.FC = ({ }, []); const { data: schemaData } = useSchemaForConnection(connectionId); - const completions = useMonacoAutocomplete( + const baseCompletions = useMonacoAutocomplete( schemaData?.tables || null, schemaData?.duckLakeSchema || null, ); + const completions = useMemo(() => { + if (!isIceberg) return baseCompletions; + return [ + ...MonacoAutocompleteSQLKeywords.map((keyword) => ({ + label: keyword, + insertText: keyword, + kind: monaco.languages.CompletionItemKind.Keyword, + detail: 'SQL keyword', + })), + ...(schemaData?.tables ?? []).flatMap((table) => [ + { + label: `iceberg.${table.schema}.${table.name}`, + insertText: icebergQualifiedName(table.schema, table.name), + kind: monaco.languages.CompletionItemKind.Class, + }, + ...table.columns.map((column) => ({ + label: `iceberg.${table.schema}.${table.name}.${column.name}`, + insertText: icebergQualifiedName( + table.schema, + table.name, + column.name, + ), + kind: monaco.languages.CompletionItemKind.Field, + })), + ]), + ]; + }, [isIceberg, baseCompletions, schemaData]); + // Store completions in a ref so the provider can access the latest without re-registering const completionsRef = useRef(completions); useEffect(() => { @@ -163,6 +214,12 @@ export const NotebookEditor: React.FC = ({ const handleRunAll = useCallback(async () => { if (!notebook || localCells.length === 0) return; + if (icebergUnavailable) { + toast.error('Iceberg SQL connection is unavailable.'); + return; + } + const batch = new AbortController(); + if (isIceberg) icebergRuns.current.set('batch', batch); setIsRunningAll(true); setRunningCellIndex(0); @@ -179,18 +236,25 @@ export const NotebookEditor: React.FC = ({ setRunningCellIndex(i); try { + if (batch.signal.aborted) + throw new Error('Iceberg execution cancelled.'); // eslint-disable-next-line no-await-in-loop - await runCell.mutateAsync({ + const output = await runCell.mutateAsync({ connectionId, notebookId, cellId: cell.id, sql: cell.content, + options: isIceberg ? { signal: batch.signal } : undefined, }); + if (isIceberg && output.type === 'error') + throw new Error(output.error); + // Small delay between cells for better UX // eslint-disable-next-line no-await-in-loop, no-promise-executor-return await new Promise((resolve) => setTimeout(resolve, 100)); } catch (cellError) { + if (isIceberg) throw cellError; // eslint-disable-next-line no-console console.error(`Failed to execute cell ${i + 1}:`, cellError); @@ -213,10 +277,19 @@ export const NotebookEditor: React.FC = ({ console.error('Run all failed:', runAllError); toast.error('Failed to execute all cells'); } finally { + icebergRuns.current.delete('batch'); setIsRunningAll(false); setRunningCellIndex(null); } - }, [notebook, localCells, connectionId, notebookId, runCell]); + }, [ + notebook, + localCells, + connectionId, + notebookId, + runCell, + isIceberg, + icebergUnavailable, + ]); // Handle keyboard shortcuts useEffect(() => { @@ -437,22 +510,32 @@ export const NotebookEditor: React.FC = ({ const handleRunCell = useCallback( async (cellId: string, content: string) => { + if (icebergUnavailable) { + toast.error('Iceberg SQL connection is unavailable.'); + return; + } + const run = new AbortController(); + if (isIceberg) icebergRuns.current.set(cellId, run); setExecutingCells((prev) => new Set(prev).add(cellId)); try { - await runCell.mutateAsync({ + const output = await runCell.mutateAsync({ connectionId, notebookId, cellId, sql: content, limit: 10, // Default pagination: first 10 rows offset: 0, + options: isIceberg ? { signal: run.signal } : undefined, }); // If the executed statement may have changed the schema, notify the parent if ( onSchemaChange && - /^\s*(CREATE|DROP|ALTER|RENAME|TRUNCATE)/i.test(content.trim()) + (isIceberg + ? ['create', 'drop'].includes(output.statementClass ?? '') && + output.type !== 'error' + : /^\s*(CREATE|DROP|ALTER|RENAME|TRUNCATE)/i.test(content.trim())) ) { onSchemaChange(); } @@ -462,6 +545,7 @@ export const NotebookEditor: React.FC = ({ // eslint-disable-next-line no-console console.error('Cell execution error:', err); } finally { + icebergRuns.current.delete(cellId); setExecutingCells((prev) => { const next = new Set(prev); next.delete(cellId); @@ -469,7 +553,14 @@ export const NotebookEditor: React.FC = ({ }); } }, - [connectionId, notebookId, runCell, onSchemaChange], + [ + connectionId, + notebookId, + runCell, + onSchemaChange, + isIceberg, + icebergUnavailable, + ], ); const handleExport = useCallback(() => { @@ -715,6 +806,23 @@ export const NotebookEditor: React.FC = ({ + {icebergUnavailable && ( + + {icebergLoading + ? 'Checking Iceberg connection…' + : (icebergInstance?.sqlUnavailableReason ?? + 'Iceberg connection unavailable.')}{' '} + You can view and edit this notebook, but cannot run it. + + )} + {isIceberg && (isRunningAll || executingCells.size > 0) && ( + + )} {/* Toolbar */} = ({ }; const handleExportParquet = async () => { + if (connectionId.startsWith('iceberg-')) return; handleExportMenuClose(); try { @@ -556,7 +557,9 @@ export const OutputPanel: React.FC = ({ color="text.secondary" sx={{ fontSize: 11 }} > - Query executed successfully (no results) + {output.statementClass && output.statementClass !== 'select' + ? `${output.statementClass.toUpperCase()} completed (${output.rowCount ?? 0} rows affected)` + : 'Query executed successfully (no results)'} = ({ // Table output - use CustomTable with pagination const columns = output.columns || []; - const hasPagination = output.totalRows !== undefined && output.totalRows > 10; + const hasPagination = + !connectionId.startsWith('iceberg-') && + output.totalRows !== undefined && + output.totalRows > 10; // Custom pagination for large datasets const customPagination = hasPagination @@ -813,7 +819,7 @@ export const OutputPanel: React.FC = ({ @@ -835,6 +841,13 @@ export const OutputPanel: React.FC = ({ return ( + {output.truncated && ( + + Showing only the first {output.data?.length ?? 0} rows. Displayed + results and JSON/CSV exports are limited to these rows. Refine and + rerun the query for other rows. + + )} {viewMode === 'chart' ? ( notebooksService.runCell( connectionId, @@ -270,8 +273,9 @@ export function useRunCell() { sql, limit, offset, + options, ), - onSuccess: async (_, { connectionId, notebookId, sql }) => { + onSuccess: async (output, { connectionId, notebookId, sql }) => { // Manually refetch the notebook to get updated cell output await queryClient.refetchQueries( notebooksKeys.detail(connectionId, notebookId), @@ -279,6 +283,8 @@ export function useRunCell() { ); if ( + (connectionId.startsWith('iceberg-') && + ['create', 'drop'].includes(output.statementClass ?? '')) || /^\s*(CREATE|DROP|ALTER|INSERT|UPDATE|DELETE|TRUNCATE|MERGE|REPLACE)/i.test( sql, ) @@ -358,6 +364,27 @@ export function useSchema(connectionId: string) { return useQuery({ queryKey: notebooksKeys.schema(connectionId), queryFn: async () => { + if (connectionId.startsWith('iceberg-')) { + const tables = await getIcebergNotebookTables(connectionId); + return { + schemas: [...new Set(tables.map((table) => table.schema))].map( + (name) => ({ schema_id: name, schema_name: name }), + ), + tables: tables.map((table) => ({ + table_name: table.name, + schema_name: table.schema, + })), + columns: tables.flatMap((table) => + table.columns.map((column) => ({ + column_name: column.name, + column_type: column.typeName, + table_name: table.name, + schema_name: table.schema, + nulls_allowed: true, + })), + ), + }; + } // Extract schema based on connection type if (connectionId.startsWith('ducklake-')) { const instanceId = connectionId.replace('ducklake-', ''); diff --git a/src/renderer/hooks/useSchemaForConnection.ts b/src/renderer/hooks/useSchemaForConnection.ts index 5f7419d8..47b4eba4 100644 --- a/src/renderer/hooks/useSchemaForConnection.ts +++ b/src/renderer/hooks/useSchemaForConnection.ts @@ -5,6 +5,7 @@ import { useQuery } from 'react-query'; import { connectorsServices } from '../services'; +import { getIcebergNotebookTables } from '../services/iceberg.service'; import { DuckLakeService } from '../services/duckLake.service'; import { Table } from '../../types/backend'; import { DuckLakeSchemaInfo } from '../../types/duckLake'; @@ -23,6 +24,13 @@ export function useSchemaForConnection(connectionId: string | undefined) { return { tables: [], duckLakeSchema: null, isDuckLake: false }; } + if (connectionId.startsWith('iceberg-')) { + return { + tables: await getIcebergNotebookTables(connectionId), + duckLakeSchema: null, + isDuckLake: false, + }; + } // Handle DuckLake connections if (connectionId.startsWith('ducklake-')) { const instanceId = connectionId.replace('ducklake-', ''); diff --git a/src/renderer/screens/notebooks/index.tsx b/src/renderer/screens/notebooks/index.tsx index 4129a74d..ccd8475f 100644 --- a/src/renderer/screens/notebooks/index.tsx +++ b/src/renderer/screens/notebooks/index.tsx @@ -9,6 +9,7 @@ import SplitPane, { Pane } from 'split-pane-react'; import 'split-pane-react/esm/themes/default.css'; import { Box, + Alert, FormControl, Select, MenuItem, @@ -37,6 +38,8 @@ import { } from '@mui/icons-material'; import { useNavigate } from 'react-router-dom'; import { toast } from 'react-toastify'; +import { useListIcebergInstances } from '../../controllers/icebergDatalake.controller'; +import { getIcebergNotebookTables } from '../../services/iceberg.service'; import { AppLayout } from '../../layouts'; import { useGetConnections, useDuckLakeInstances } from '../../controllers'; import { @@ -53,7 +56,9 @@ import { } from '../../controllers/notebooks.controller'; import connectionIcons, { defaultIcon, + icebergCatalogImages, } from '../../../../assets/connectionIcons'; +import icebergIcon from '../../../../assets/icons/apache-iceberg-lake.png'; import { AppContext } from '../../context'; import { connectorsServices, DuckLakeService } from '../../services'; import { AnalyticsEditor } from '../../components/analytics'; @@ -106,6 +111,8 @@ const Notebooks = () => { const { isSidebarOpen } = useContext(AppContext); const { data: connections = [] } = useGetConnections(); const { data: duckLakeInstances = [] } = useDuckLakeInstances(); + const { data: icebergInstances = [], isLoading: icebergLoading } = + useListIcebergInstances(); const { isChatOpen, setIsChatOpen } = useAppContext(); @@ -167,6 +174,8 @@ const Notebooks = () => { // Validate hydrated connection exists, clear if not useEffect(() => { if (!isConnectionHydrated || !activeConnectionId) return; + // Preserve the saved identity even when its Iceberg instance is unavailable. + if (activeConnectionId.startsWith('iceberg-')) return; const connectionExists = connections.some((c) => c.id === activeConnectionId) || @@ -237,7 +246,21 @@ const Notebooks = () => { >(null); // Get active connection details + const activeIceberg = icebergInstances.find( + (item) => `iceberg-${item.id}` === activeConnectionId, + ); + const icebergUnavailable = + activeConnectionId.startsWith('iceberg-') && !activeIceberg?.sqlAvailable; const activeConnection = useMemo(() => { + if (activeConnectionId.startsWith('iceberg-')) { + return { + id: activeConnectionId, + connection: { + name: activeIceberg?.name ?? 'Unavailable Iceberg connection', + type: 'iceberg', + }, + }; + } if (activeConnectionId.startsWith('ducklake-')) { const instanceId = activeConnectionId.replace('ducklake-', ''); const instance = duckLakeInstances.find((inst) => inst.id === instanceId); @@ -252,7 +275,7 @@ const Notebooks = () => { } } return connections.find((c) => c.id === activeConnectionId); - }, [connections, duckLakeInstances, activeConnectionId]); + }, [connections, duckLakeInstances, activeConnectionId, activeIceberg]); // Get schema for active connection from cache const activeSchema = activeConnectionId @@ -274,7 +297,10 @@ const Notebooks = () => { setLoadingSchemas((prev) => ({ ...prev, [connectionId]: true })); try { - if (connectionId.startsWith('ducklake-')) { + if (connectionId.startsWith('iceberg-')) { + const tables = await getIcebergNotebookTables(connectionId); + setTabSchemas((prev) => ({ ...prev, [connectionId]: tables })); + } else if (connectionId.startsWith('ducklake-')) { // DuckLake schema extraction const instanceId = connectionId.replace('ducklake-', ''); const duckLakeSchema = @@ -381,6 +407,10 @@ const Notebooks = () => { // eslint-disable-next-line @typescript-eslint/no-unused-vars const connectionExists = useCallback( (connectionKey: string) => { + if (connectionKey.startsWith('iceberg:')) + return icebergInstances.some( + (item) => item.id === connectionKey.slice(8), + ); if (connectionKey.startsWith('ducklake:')) { const instanceId = connectionKey.replace('ducklake:', ''); return duckLakeInstances.some((inst) => inst.id === instanceId); @@ -391,12 +421,17 @@ const Notebooks = () => { } return false; }, - [connections, duckLakeInstances], + [connections, duckLakeInstances, icebergInstances], ); // Helper: Get connection name from connectionKey const getConnectionName = useCallback( (connectionKey: string) => { + if (connectionKey.startsWith('iceberg:')) + return ( + icebergInstances.find((item) => item.id === connectionKey.slice(8)) + ?.name ?? 'Unavailable Iceberg connection' + ); if (connectionKey.startsWith('ducklake:')) { const instanceId = connectionKey.replace('ducklake:', ''); const instance = duckLakeInstances.find( @@ -411,7 +446,7 @@ const Notebooks = () => { } return 'Unknown'; }, - [connections, duckLakeInstances], + [connections, duckLakeInstances, icebergInstances], ); // Handle restore archived notebook @@ -791,6 +826,31 @@ const Notebooks = () => { } // Check DuckLake instances + if (selected.startsWith('iceberg-')) { + const instance = icebergInstances.find( + (item) => `iceberg-${item.id}` === selected, + ); + return ( + + + {instance?.name ?? 'Unavailable Iceberg connection'} + + ); + } if (selected.startsWith('ducklake-')) { const instanceId = selected.replace('ducklake-', ''); const instance = duckLakeInstances.find( @@ -807,7 +867,7 @@ const Notebooks = () => { }} > { Select Connection + {connections.length > 0 && ( + + Database Connections + + )} {connections.map((conn) => { const linkedProject = projects?.find( (p) => p.connectionId === conn.id, @@ -920,14 +985,70 @@ const Notebooks = () => { }} > {instance.name} ))} + {icebergInstances.some((instance) => instance.sqlAvailable) && ( + + Iceberg Catalogs + + )} + {icebergInstances + .filter((instance) => instance.sqlAvailable) + .map((instance) => ( + + + + {instance.name} + + + ))} + {activeConnectionId.startsWith('iceberg-') && + !activeIceberg?.sqlAvailable && ( + + {activeIceberg?.name ?? 'Unavailable Iceberg connection'} + + )} + {icebergUnavailable && ( + + {icebergLoading + ? 'Checking Iceberg connection…' + : (activeIceberg?.sqlUnavailableReason ?? + 'Iceberg connection was deleted or is unavailable.')}{' '} + Saved notebooks remain accessible; SQL execution is + unavailable. + + )} { : []; if (isIcebergConnection) { - const quote = (value: string) => `"${value.replace(/"/g, '""')}"`; const icebergItems = activeSchema.flatMap((table) => { - const qualifiedTable = `${quote('iceberg')}.${quote(table.schema)}.${quote(table.name)}`; + const qualifiedTable = icebergQualifiedName(table.schema, table.name); return [ { label: `iceberg.${table.schema}.${table.name}`, @@ -438,7 +438,11 @@ const Sql = () => { ...table.columns.map((column) => ({ label: `iceberg.${table.schema}.${table.name}.${column.name}`, kind: MonacoCompletionItemKind.Field, - insertText: `${qualifiedTable}.${quote(column.name)}`, + insertText: icebergQualifiedName( + table.schema, + table.name, + column.name, + ), detail: 'Iceberg column', })), ]; diff --git a/src/renderer/services/iceberg.service.ts b/src/renderer/services/iceberg.service.ts index 246ddf98..db5a01ec 100644 --- a/src/renderer/services/iceberg.service.ts +++ b/src/renderer/services/iceberg.service.ts @@ -1,3 +1,4 @@ +import type { Table } from '../../types/backend'; /** * Iceberg renderer service * Named exports wrapping window.electron.ipcRenderer.invoke — no default exports. @@ -224,3 +225,39 @@ export const executeConfirmedIcebergSql = async ( mutationConfirmed: mutating, }); }; + +/** Shared Notebook schema adapter; identifiers stay separate until SQL insertion. */ +export const getIcebergNotebookTables = async ( + connectionId: string, +): Promise => { + const schema = await getIcebergSqlSchema(connectionId.slice(8)); + return schema.namespaces.flatMap((namespace) => + namespace.tables.map((table) => ({ + name: table.name, + schema: namespace.name, + type: table.type, + columns: table.columns.map((column) => ({ + name: column.name, + typeName: column.type, + type: column.type, + nullable: true, + ordinalPosition: column.position, + primaryKeySequenceId: 0, + columnDisplaySize: 0, + scale: 0, + precision: 0, + columnProperties: [], + autoincrement: false, + primaryKey: false, + })), + })), + ); +}; +export const icebergQualifiedName = (...parts: string[]) => + ['iceberg', ...parts] + .map((part) => + /^[A-Za-z_][A-Za-z0-9_]*$/.test(part) + ? part + : `"${part.replace(/"/g, '""')}"`, + ) + .join('.'); diff --git a/src/renderer/services/notebooks.service.ts b/src/renderer/services/notebooks.service.ts index da7b8255..a9420230 100644 --- a/src/renderer/services/notebooks.service.ts +++ b/src/renderer/services/notebooks.service.ts @@ -3,8 +3,80 @@ * Frontend service for notebook operations */ +import { v4 as uuidv4 } from 'uuid'; +import { executeIcebergSql, getIcebergInstance } from './iceberg.service'; import { Notebook, NotebookCell, CellOutput } from '../../types/notebooks'; +type RunOptions = { executionId?: string; signal?: AbortSignal }; + +async function runConfirmedIcebergCell( + connectionId: string, + notebookId: string, + cellId: string, + sql: string, + options: RunOptions = {}, + runAll = false, +): Promise { + const executionId = options.executionId ?? `notebook-${uuidv4()}`; + const checkCancelled = () => { + if (options.signal?.aborted) + throw new Error('Iceberg execution cancelled.'); + }; + checkCancelled(); + const instanceId = connectionId.slice(8); + const classification = await executeIcebergSql({ + instanceId, + executionId, + sql, + validateOnly: true, + }); + checkCancelled(); + const mutating = classification.statementClass !== 'select'; + if (mutating) { + const instance = await getIcebergInstance(instanceId); + checkCancelled(); + if ( + // eslint-disable-next-line no-alert + !window.confirm( + `Run ${classification.statementClass.toUpperCase()} on Iceberg "${instance.name}"? This modifies the catalog or its data.`, + ) + ) { + throw new Error('Iceberg mutation confirmation declined.'); + } + } + checkCancelled(); + const cancel = () => { + window.electron.ipcRenderer + .invoke('notebooks:cancelIcebergCell', executionId) + .catch(() => undefined); + }; + options.signal?.addEventListener('abort', cancel); + try { + const execution = { executionId, mutationConfirmed: mutating }; + if (runAll) { + await window.electron.ipcRenderer.invoke( + 'notebooks:runAll', + connectionId, + notebookId, + { ...execution, cellId, sql }, + ); + return { type: 'empty', statementClass: classification.statementClass }; + } + return await window.electron.ipcRenderer.invoke( + 'notebooks:runCell', + connectionId, + notebookId, + cellId, + sql, + undefined, + undefined, + execution, + ); + } finally { + options.signal?.removeEventListener('abort', cancel); + } +} + export const notebooksService = { /** * List all notebooks for a connection @@ -154,7 +226,16 @@ export const notebooksService = { sql: string, limit?: number, offset?: number, + options?: RunOptions, ): Promise => { + if (connectionId.startsWith('iceberg-')) + return runConfirmedIcebergCell( + connectionId, + notebookId, + cellId, + sql, + options, + ); return window.electron.ipcRenderer.invoke( 'notebooks:runCell', connectionId, @@ -194,8 +275,34 @@ export const notebooksService = { runAllCells: async ( connectionId: string, notebookId: string, + options?: RunOptions, ): Promise => { - return window.electron.ipcRenderer.invoke( + if (connectionId.startsWith('iceberg-')) { + const notebook = await notebooksService.getNotebook( + connectionId, + notebookId, + ); + if (!notebook) throw new Error('Notebook not found'); + // Sequential confirmation is intentional: rejection/failure stops the batch. + // eslint-disable-next-line no-restricted-syntax + for (const cell of [...notebook.cells].sort( + (a, b) => a.order - b.order, + )) { + if (cell.type === 'sql' && cell.content.trim()) { + // eslint-disable-next-line no-await-in-loop + await runConfirmedIcebergCell( + connectionId, + notebookId, + cell.id, + cell.content, + { signal: options?.signal }, + true, + ); + } + } + return; + } + await window.electron.ipcRenderer.invoke( 'notebooks:runAll', connectionId, notebookId, diff --git a/src/types/ipc.ts b/src/types/ipc.ts index 794d85d3..1cb889aa 100644 --- a/src/types/ipc.ts +++ b/src/types/ipc.ts @@ -426,6 +426,7 @@ export type NotebookChannels = | 'notebooks:import' | 'notebooks:importAll' | 'notebooks:delete' + | 'notebooks:cancelIcebergCell' | 'notebooks:runCell' | 'notebooks:fetchCellPage' | 'notebooks:runAll' diff --git a/src/types/notebooks.ts b/src/types/notebooks.ts index 885176ce..f2ba075b 100644 --- a/src/types/notebooks.ts +++ b/src/types/notebooks.ts @@ -11,6 +11,9 @@ export interface CellOutput { totalRows?: number; // Total rows in full dataset (for pagination) executionTime?: number; error?: string; + truncated?: boolean; + statementClass?: string; + cancelled?: boolean; } export interface NotebookCell { @@ -66,3 +69,13 @@ export interface SchemaInfo { parent_column_name?: string; }>; } + +/** Iceberg-only execution controls; SQL and credentials are never persisted here. */ +export interface NotebookExecutionOptions { + executionId: string; + mutationConfirmed?: boolean; +} +export interface NotebookRunAllCell extends NotebookExecutionOptions { + cellId: string; + sql: string; +} diff --git a/tests/unit/main/services/icebergSqlRuntime.service.test.ts b/tests/unit/main/services/icebergSqlRuntime.service.test.ts index c0a9d7a3..f480d269 100644 --- a/tests/unit/main/services/icebergSqlRuntime.service.test.ts +++ b/tests/unit/main/services/icebergSqlRuntime.service.test.ts @@ -100,9 +100,6 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { beforeEach(() => { jest.restoreAllMocks(); jest.clearAllMocks(); - jest - .spyOn(IcebergDatalakeService as any, 'isSqlCombinationAccepted') - .mockReturnValue(true); mockRunAndReadAll.mockResolvedValue({ getRowObjectsJson: () => [ { @@ -141,6 +138,20 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { }); }); + it('passes the SQL storage endpoint to server-managed REST catalogs', async () => { + const buildProperties = (IcebergDatalakeService as any) + .buildCatalogProperties as (config: unknown) => Promise<{ + props: Record; + env: Record; + }>; + const result = await buildProperties(instance); + + expect(result.props).toMatchObject({ + 's3.endpoint': 'http://localhost:9000', + 's3.access-key-id': 'minioadmin', + }); + }); + it('returns DuckDB row objects keyed for the SQL result table', async () => { mockedLoadDatabase.mockResolvedValue({ ...database, @@ -362,7 +373,6 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { 'empty table', 'unreadable table', 'cleanup failure', - 'unaccepted pair', ])('does not persist verification for %s', async (failure) => { if (failure === 'empty catalog') { mockRunAndReadUntil.mockResolvedValueOnce({ @@ -388,10 +398,6 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { mockRun.mockImplementation(async (sql: string) => { if (sql.startsWith('DETACH')) throw new Error('detach failed'); }); - } else { - (IcebergDatalakeService as any).isSqlCombinationAccepted.mockReturnValue( - false, - ); } expect( (await IcebergDatalakeService.verifySqlAccess(instance.id)).success, @@ -401,7 +407,7 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { expect(mockCloseInstance).toHaveBeenCalled(); }); - it('does not advertise a verified but unaccepted combination', async () => { + it('enables SQL for a verified connection without a combination registry', async () => { mockedLoadDatabase.mockResolvedValue({ ...database, icebergInstances: [ @@ -414,18 +420,15 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { }, ], }); - (IcebergDatalakeService as any).isSqlCombinationAccepted.mockReturnValue( - false, - ); expect( await IcebergDatalakeService.getSqlCapability(instance.id), ).toMatchObject({ - available: false, - canWrite: false, - reason: 'ICEBERG_SQL_COMBINATION_NOT_ACCEPTED', + available: true, + canRead: true, + canWrite: true, }); expect((await IcebergDatalakeService.listInstances())[0].sqlAvailable).toBe( - false, + true, ); }); diff --git a/tests/unit/main/services/notebooksIceberg.service.test.ts b/tests/unit/main/services/notebooksIceberg.service.test.ts new file mode 100644 index 00000000..ceb4bc5e --- /dev/null +++ b/tests/unit/main/services/notebooksIceberg.service.test.ts @@ -0,0 +1,119 @@ +import { NotebooksService } from '../../../../src/main/services/notebooks.service'; +import { IcebergDatalakeService } from '../../../../src/main/services/icebergDatalake.service'; + +jest.mock('../../../../src/main/services/icebergDatalake.service', () => ({ + IcebergDatalakeService: { + executeSql: jest.fn(), + cancelSql: jest.fn(), + }, +})); + +describe('NotebooksService Iceberg execution', () => { + const executeSql = IcebergDatalakeService.executeSql as jest.Mock; + const cancelSql = IcebergDatalakeService.cancelSql as jest.Mock; + const notebook = { + id: 'notebook-1', + name: 'Iceberg notebook', + cells: [ + { id: 'cell-1', type: 'sql' as const, content: 'SELECT 1', order: 0 }, + ], + createdAt: '2026-09-07T00:00:00.000Z', + updatedAt: '2026-09-07T00:00:00.000Z', + cellCount: 1, + }; + + beforeEach(() => { + jest.clearAllMocks(); + jest.spyOn(NotebooksService, 'getNotebook').mockResolvedValue(notebook); + jest + .spyOn(NotebooksService as any, 'updateCellOutput') + .mockResolvedValue(undefined); + }); + + it('executes the original cell through the shared bounded Iceberg runtime', async () => { + executeSql.mockResolvedValue({ + statementClass: 'select', + rows: [{ id: 1 }], + columns: ['id'], + rowsChanged: 0, + truncated: true, + }); + + const output = await NotebooksService.runCell( + 'iceberg-instance-1', + notebook.id, + 'cell-1', + 'SELECT * FROM iceberg.sales.orders', + 10, + 20, + { executionId: 'cell-run' }, + ); + + expect(executeSql).toHaveBeenCalledWith( + { + instanceId: 'instance-1', + executionId: 'cell-run', + sql: 'SELECT * FROM iceberg.sales.orders', + maxRows: 100, + mutationConfirmed: undefined, + }, + expect.any(AbortSignal), + ); + expect(output).toMatchObject({ + type: 'table', + truncated: true, + statementClass: 'select', + }); + }); + + it('rejects the direct Run All route without a confirmed cell payload', async () => { + await expect( + NotebooksService.runAllCells('iceberg-instance-1', notebook.id), + ).rejects.toThrow('ICEBERG_NOTEBOOK_CELL_CONFIRMATION_REQUIRED'); + expect(executeSql).not.toHaveBeenCalled(); + }); + + it('runs one confirmed Run All cell and propagates its failure', async () => { + executeSql.mockRejectedValue(new Error('write failed')); + + await expect( + NotebooksService.runAllCells('iceberg-instance-1', notebook.id, { + executionId: 'run-all-cell', + cellId: 'cell-1', + sql: 'DELETE FROM iceberg.sales.orders', + mutationConfirmed: true, + }), + ).rejects.toThrow('write failed'); + }); + + it('interrupts an active cell and records a cancelled output', async () => { + let rejectExecution!: (error: Error) => void; + executeSql.mockImplementation( + () => + new Promise((_resolve, reject) => { + rejectExecution = reject; + }), + ); + + const run = NotebooksService.runCell( + 'iceberg-instance-1', + notebook.id, + 'cell-1', + 'SELECT * FROM iceberg.sales.orders', + undefined, + undefined, + { executionId: 'cancel-me' }, + ); + await Promise.resolve(); + await Promise.resolve(); + expect(NotebooksService.cancelIcebergCell('cancel-me')).toBe(true); + rejectExecution(new Error('interrupted')); + + await expect(run).resolves.toMatchObject({ + type: 'error', + cancelled: true, + error: 'Iceberg execution cancelled.', + }); + expect(cancelSql).toHaveBeenCalledWith('cancel-me'); + }); +}); diff --git a/tests/unit/renderer/services/notebooksIceberg.service.test.ts b/tests/unit/renderer/services/notebooksIceberg.service.test.ts new file mode 100644 index 00000000..8a0660d2 --- /dev/null +++ b/tests/unit/renderer/services/notebooksIceberg.service.test.ts @@ -0,0 +1,129 @@ +import { notebooksService } from '../../../../src/renderer/services/notebooks.service'; +import { + getIcebergNotebookTables, + icebergQualifiedName, +} from '../../../../src/renderer/services/iceberg.service'; + +describe('Iceberg Notebook renderer execution', () => { + const invoke = window.electron.ipcRenderer.invoke as jest.Mock; + + beforeEach(() => { + invoke.mockReset(); + (window.confirm as jest.Mock | undefined)?.mockReset?.(); + }); + + it('classifies comments before asking for mutation confirmation', async () => { + invoke + .mockResolvedValueOnce({ statementClass: 'delete' }) + .mockResolvedValueOnce({ id: 'instance-1', name: 'Warehouse' }); + jest.spyOn(window, 'confirm').mockReturnValue(false); + + await expect( + notebooksService.runCell( + 'iceberg-instance-1', + 'notebook-1', + 'cell-1', + '/* comment */ DELETE FROM iceberg.sales.orders', + ), + ).rejects.toThrow('confirmation declined'); + expect(invoke).toHaveBeenCalledTimes(2); + expect(invoke.mock.calls[0][1]).toMatchObject({ validateOnly: true }); + }); + + it('Run All confirms sequentially and stops after rejection', async () => { + invoke + .mockResolvedValueOnce({ + id: 'notebook-1', + cells: [ + { id: 'cell-1', type: 'sql', content: 'SELECT 1', order: 0 }, + { + id: 'cell-2', + type: 'sql', + content: 'DELETE FROM iceberg.sales.orders', + order: 1, + }, + { id: 'cell-3', type: 'sql', content: 'SELECT 3', order: 2 }, + ], + }) + .mockResolvedValueOnce({ statementClass: 'select' }) + .mockResolvedValueOnce(undefined) + .mockResolvedValueOnce({ statementClass: 'delete' }) + .mockResolvedValueOnce({ id: 'instance-1', name: 'Warehouse' }); + jest.spyOn(window, 'confirm').mockReturnValue(false); + + await expect( + notebooksService.runAllCells('iceberg-instance-1', 'notebook-1'), + ).rejects.toThrow('confirmation declined'); + expect( + invoke.mock.calls.some((call) => call[0] === 'notebooks:runAll'), + ).toBe(true); + expect( + invoke.mock.calls.some( + (call) => call[1]?.sql === 'SELECT 3' && call[1]?.validateOnly, + ), + ).toBe(false); + }); + + it('forwards cancellation for the active execution ID', async () => { + const controller = new AbortController(); + invoke.mockResolvedValueOnce({ statementClass: 'select' }); + invoke.mockImplementationOnce( + () => + new Promise((resolve) => { + controller.signal.addEventListener('abort', () => resolve(undefined)); + }), + ); + + const run = notebooksService.runCell( + 'iceberg-instance-1', + 'notebook-1', + 'cell-1', + 'SELECT * FROM iceberg.sales.orders', + undefined, + undefined, + { executionId: 'active-cell', signal: controller.signal }, + ); + await Promise.resolve(); + controller.abort(); + await run; + + expect(invoke).toHaveBeenCalledWith( + 'notebooks:cancelIcebergCell', + 'active-cell', + ); + }); + + it('maps Iceberg schema data for the existing Notebook tree and completion model', async () => { + invoke.mockResolvedValueOnce({ + catalogName: 'iceberg', + namespaces: [ + { + name: 'sales', + tables: [ + { + name: 'order"items', + type: 'TABLE', + columns: [{ name: 'item"id', type: 'BIGINT', position: 1 }], + }, + ], + }, + ], + }); + + await expect( + getIcebergNotebookTables('iceberg-instance-1'), + ).resolves.toMatchObject([ + { + schema: 'sales', + name: 'order"items', + columns: [{ name: 'item"id', typeName: 'BIGINT', ordinalPosition: 1 }], + }, + ]); + expect(icebergQualifiedName('sales', 'order"items')).toBe( + 'iceberg.sales."order""items"', + ); + expect(icebergQualifiedName('default', 'keywords')).toBe( + 'iceberg.default.keywords', + ); + }); +}); From b5e0ee0ac61836f7a23d1eee4181845d650cef72 Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Wed, 9 Sep 2026 10:47:22 +0200 Subject: [PATCH 05/16] feat(iceberg): restore paginated SQL results in editor and notebooks Use bounded server-side pages for Iceberg queries, restore Notebook paging, and remove the fixed 1,000-row truncation notices. --- src/main/services/icebergDatalake.service.ts | 38 ++++++++++ src/main/services/notebooks.service.ts | 34 +++++++-- .../components/notebook/OutputPanel.tsx | 12 +-- src/renderer/components/sqlEditor/index.tsx | 5 +- src/renderer/screens/sql/queryResult.tsx | 75 ++++++++++++------- src/types/iceberg.ts | 4 + .../icebergSqlRuntime.service.test.ts | 51 +++++++++++++ .../services/notebooksIceberg.service.test.ts | 11 ++- .../screens/sql/icebergTruncation.test.tsx | 11 +-- 9 files changed, 184 insertions(+), 57 deletions(-) diff --git a/src/main/services/icebergDatalake.service.ts b/src/main/services/icebergDatalake.service.ts index f49d7e14..ad23a8cc 100644 --- a/src/main/services/icebergDatalake.service.ts +++ b/src/main/services/icebergDatalake.service.ts @@ -1774,11 +1774,49 @@ export class IcebergDatalakeService { if (statementClass !== 'select' && params.mutationConfirmed !== true) { throw new Error('ICEBERG_SQL_CONFIRMATION_REQUIRED'); } + const hasPageRequest = + params.pageLimit !== undefined || params.pageOffset !== undefined; + if (hasPageRequest && statementClass !== 'select') { + throw new Error('ICEBERG_SQL_PAGINATION_REQUIRES_SELECT'); + } + const pageLimit = params.pageLimit ?? 10; + const pageOffset = params.pageOffset ?? 0; + if ( + hasPageRequest && + (!Number.isInteger(pageLimit) || + pageLimit <= 0 || + !Number.isInteger(pageOffset) || + pageOffset < 0) + ) { + throw new Error('ICEBERG_SQL_PAGINATION_INVALID'); + } const maxRows = Math.max(1, Math.min(params.maxRows ?? 1000, 5000)); return IcebergDatalakeService.withAttachedSqlCatalog( params.instanceId, params.executionId, async (connection) => { + if (hasPageRequest) { + const boundedLimit = Math.min(pageLimit, 1000); + const baseSql = params.sql.trim().replace(/;+$/, ''); + const pageSql = `SELECT * FROM (${baseSql}) AS iceberg_page LIMIT ${boundedLimit} OFFSET ${pageOffset}`; + const countSql = `SELECT COUNT(*) AS count FROM (${baseSql}) AS iceberg_count`; + const countReader = await connection.runAndReadAll(countSql); + const reader = await connection.runAndReadUntil( + pageSql, + boundedLimit, + ); + const count = countReader.getRowObjectsJson()[0]?.count; + return { + executionId: params.executionId, + statementClass, + columns: reader.columnNames(), + rows: reader.getRowObjectsJson() as Array>, + rowsChanged: Number(reader.rowsChanged ?? 0), + truncated: false, + totalRows: + typeof count === 'bigint' ? Number(count) : Number(count ?? 0), + }; + } const reader = await connection.runAndReadUntil( params.sql, maxRows + 1, diff --git a/src/main/services/notebooks.service.ts b/src/main/services/notebooks.service.ts index b6152e8e..e162e400 100644 --- a/src/main/services/notebooks.service.ts +++ b/src/main/services/notebooks.service.ts @@ -272,6 +272,8 @@ export class NotebooksService { notebookId: string, cellId: string, sql: string, + limit?: number, + offset?: number, options?: NotebookExecutionOptions, ): Promise { const executionId = options?.executionId ?? `notebook-${uuidv4()}`; @@ -300,7 +302,7 @@ export class NotebooksService { instanceId: connectionId.slice(8), executionId, sql, - maxRows: MAX_STORED_ROWS, + ...sanitizePagination(limit, offset), mutationConfirmed: options?.mutationConfirmed, }, run.signal, @@ -312,8 +314,9 @@ export class NotebooksService { truncated: result.truncated, rowCount: result.statementClass === 'select' - ? result.rows.length + ? (result.totalRows ?? result.rows.length) : result.rowsChanged, + totalRows: result.totalRows, statementClass: result.statementClass, executionTime: Date.now() - started, }; @@ -845,6 +848,8 @@ export class NotebooksService { notebookId, cellId, sql, + limit, + offset, options, ); } @@ -1024,11 +1029,6 @@ export class NotebooksService { limit: number, offset: number, ): Promise { - if (connectionId.startsWith('iceberg-')) { - throw new Error( - 'ICEBERG_NOTEBOOK_BOUNDED_RESULTS: Refine and rerun the original query.', - ); - } try { const startTime = Date.now(); @@ -1052,6 +1052,24 @@ export class NotebooksService { let totalRows: number | undefined; // Execute query based on connection type + if (connectionId.startsWith('iceberg-')) { + const icebergResult = await IcebergDatalakeService.executeSql({ + instanceId: connectionId.slice(8), + executionId: `notebook-page-${uuidv4()}`, + sql, + pageLimit, + pageOffset, + }); + return { + type: icebergResult.rows.length ? 'table' : 'empty', + data: icebergResult.rows, + columns: icebergResult.columns, + rowCount: icebergResult.rows.length, + totalRows: icebergResult.totalRows, + executionTime: Date.now() - startTime, + }; + } + if (connectionId.startsWith('ducklake-')) { const instanceId = connectionId.replace('ducklake-', ''); @@ -1186,6 +1204,8 @@ export class NotebooksService { notebookId, cellRun.cellId, cellRun.sql, + undefined, + undefined, cellRun, ); if (output.type === 'error') throw new Error(output.error); diff --git a/src/renderer/components/notebook/OutputPanel.tsx b/src/renderer/components/notebook/OutputPanel.tsx index 110c379e..09f8fa7d 100644 --- a/src/renderer/components/notebook/OutputPanel.tsx +++ b/src/renderer/components/notebook/OutputPanel.tsx @@ -578,10 +578,7 @@ export const OutputPanel: React.FC = ({ // Table output - use CustomTable with pagination const columns = output.columns || []; - const hasPagination = - !connectionId.startsWith('iceberg-') && - output.totalRows !== undefined && - output.totalRows > 10; + const hasPagination = output.totalRows !== undefined && output.totalRows > 10; // Custom pagination for large datasets const customPagination = hasPagination @@ -841,13 +838,6 @@ export const OutputPanel: React.FC = ({ return ( - {output.truncated && ( - - Showing only the first {output.data?.length ?? 0} rows. Displayed - results and JSON/CSV exports are limited to these rows. Refine and - rerun the query for other rows. - - )} {viewMode === 'chart' ? ( = ({ instanceId: icebergInstanceId, executionId: queryId, sql: selectedQuery, - maxRows: 1000, + pageLimit: 10, + pageOffset: 0, }, (statementClass) => { // eslint-disable-next-line no-alert @@ -175,7 +176,7 @@ export const SqlEditor: React.FC = ({ fields: icebergResult.columns.map((name) => ({ name, type: 0 })), rowCount: icebergResult.statementClass === 'select' - ? icebergResult.rows.length + ? (icebergResult.totalRows ?? icebergResult.rows.length) : icebergResult.rowsChanged, }; } else if (isDuckLakeConnection && instanceId) { diff --git a/src/renderer/screens/sql/queryResult.tsx b/src/renderer/screens/sql/queryResult.tsx index 80927e0e..e5df414a 100644 --- a/src/renderer/screens/sql/queryResult.tsx +++ b/src/renderer/screens/sql/queryResult.tsx @@ -3,7 +3,6 @@ import { toast } from 'react-toastify'; import { styled } from '@mui/material/styles'; import { Box, - Alert, Backdrop, CircularProgress, Typography, @@ -32,6 +31,7 @@ import { QueryResultVisualization } from '../../components/queryResult/queryVisu import { CustomTable } from '../../components/customTable'; import { underscoreToTitleCase } from '../../helpers/utils'; import { DuckLakeService } from '../../services/duckLake.service'; +import * as icebergService from '../../services/iceberg.service'; const SuccessContainer = styled(Box)(({ theme }) => ({ backgroundColor: theme.palette.background.paper, @@ -84,6 +84,10 @@ export const QueryResult: React.FC = ({ results, exportContext }) => { !!originalSql; const isDuckLakeReady = !isDuckLake || exportContext?.duckLakeReady !== false; + const icebergInstanceId = exportContext?.connectionId?.startsWith('iceberg-') + ? exportContext.connectionId.slice(8) + : undefined; + const isIceberg = !!icebergInstanceId && !!originalSql; const [columns, setColumns] = React.useState( results.fields?.map((f) => f.name) ?? [], @@ -107,8 +111,8 @@ export const QueryResult: React.FC = ({ results, exportContext }) => { const fetchPage = React.useCallback( async (newPage: number, newPerPage: number) => { - if (!isDuckLake) return; - if (!exportContext?.duckLakeInstanceId || !originalSql) return; + if (!isDuckLake && !isIceberg) return; + if (!originalSql) return; fetchSeqRef.current += 1; const seq = fetchSeqRef.current; @@ -116,25 +120,45 @@ export const QueryResult: React.FC = ({ results, exportContext }) => { try { setLoading(true); setFetchError(null); - const res = await DuckLakeService.executeQuery({ - instanceId: exportContext.duckLakeInstanceId, - query: originalSql, - limit: newPerPage, - offset: newPage * newPerPage, - }); + const res = isIceberg + ? await icebergService.executeIcebergSql({ + instanceId: icebergInstanceId!, + executionId: `sql-page-${Date.now()}-${newPage}`, + sql: originalSql, + pageLimit: newPerPage, + pageOffset: newPage * newPerPage, + }) + : await DuckLakeService.executeQuery({ + instanceId: exportContext!.duckLakeInstanceId!, + query: originalSql, + limit: newPerPage, + offset: newPage * newPerPage, + }); if (seq !== fetchSeqRef.current) return; - if (!res?.success) { - const message = res?.error || 'Failed to fetch page data'; + if (!isIceberg && !(res as { success?: boolean }).success) { + const message = (res as any)?.error || 'Failed to fetch page data'; // eslint-disable-next-line no-console console.error('[QueryResult] DuckLake page fetch failed:', message); toast.error(message); setFetchError(message); return; } - setColumns(res.fields?.map((f) => f.name) ?? []); - setRows(res.data ?? []); - if (typeof res.rowCount === 'number') { - setTotalCount(res.rowCount); + if (isIceberg) { + const icebergResult = res as Awaited< + ReturnType + >; + setColumns(icebergResult.columns); + setRows(icebergResult.rows); + setTotalCount(icebergResult.totalRows ?? icebergResult.rows.length); + } else { + const duckLakeResult = res as Awaited< + ReturnType + >; + setColumns(duckLakeResult.fields?.map((f) => f.name) ?? []); + setRows(duckLakeResult.data ?? []); + if (typeof duckLakeResult.rowCount === 'number') { + setTotalCount(duckLakeResult.rowCount); + } } } catch (e: any) { if (seq !== fetchSeqRef.current) return; @@ -147,7 +171,13 @@ export const QueryResult: React.FC = ({ results, exportContext }) => { } } }, - [isDuckLake, exportContext?.duckLakeInstanceId, originalSql], + [ + isDuckLake, + isIceberg, + icebergInstanceId, + exportContext?.duckLakeInstanceId, + originalSql, + ], ); React.useEffect(() => { @@ -159,7 +189,7 @@ export const QueryResult: React.FC = ({ results, exportContext }) => { results.isCommand || ((!results.fields || results.fields.length === 0) && results.success); - if (isDuckLake && !isCmd) { + if ((isDuckLake || isIceberg) && !isCmd) { setTotalCount(baseTotal); setPage(0); if (isDuckLakeReady) { @@ -175,10 +205,10 @@ export const QueryResult: React.FC = ({ results, exportContext }) => { // We intentionally only respond to new results / connection type; // perPage changes are handled via customPagination. // eslint-disable-next-line react-hooks/exhaustive-deps - }, [results, isDuckLake, isDuckLakeReady, fetchPage]); + }, [results, isDuckLake, isIceberg, isDuckLakeReady, fetchPage]); const customPagination = React.useMemo(() => { - if (!isDuckLake) return undefined; + if (!isDuckLake && !isIceberg) return undefined; return { page, setPage: (p: number) => { @@ -205,6 +235,7 @@ export const QueryResult: React.FC = ({ results, exportContext }) => { }; }, [ isDuckLake, + isIceberg, isDuckLakeReady, page, perPage, @@ -863,12 +894,6 @@ export const QueryResult: React.FC = ({ results, exportContext }) => { overflow: 'hidden', }} > - {results.truncated && ( - - Showing only the first {results.data?.length ?? 0} rows. Results and - exports are limited to these rows. Refine the query to see other rows. - - )} {viewMode === 'chart' ? ( >; rowsChanged: number; truncated: boolean; + totalRows?: number; } export interface IcebergSqlSchemaInfo { diff --git a/tests/unit/main/services/icebergSqlRuntime.service.test.ts b/tests/unit/main/services/icebergSqlRuntime.service.test.ts index f480d269..92702d8f 100644 --- a/tests/unit/main/services/icebergSqlRuntime.service.test.ts +++ b/tests/unit/main/services/icebergSqlRuntime.service.test.ts @@ -182,6 +182,57 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { expect(result.rows).toEqual([{ Id: 1, Keywords: 'iceberg' }]); }); + it('pages Iceberg reads in DuckDB and returns the total row count', async () => { + mockedLoadDatabase.mockResolvedValue({ + ...database, + icebergInstances: [ + { + ...instance, + sqlAccessVerifiedAt: '2026-08-14T00:00:00.000Z', + sqlRuntimeFingerprint: ( + IcebergDatalakeService as any + ).getSqlRuntimeFingerprint(), + }, + ], + }); + mockRunAndReadAll + .mockResolvedValueOnce({ + getRowObjectsJson: () => [ + { + ast: JSON.stringify({ + error: false, + statements: [{ node: { type: 'SELECT_NODE' } }], + }), + }, + ], + }) + .mockResolvedValueOnce({ getRowObjectsJson: () => [{ count: 20_000 }] }); + mockRunAndReadUntil.mockResolvedValue({ + columnNames: () => ['id'], + getRowObjectsJson: () => [{ id: 11 }], + rowsChanged: 0, + done: true, + }); + + await expect( + IcebergDatalakeService.executeSql({ + instanceId: instance.id, + executionId: 'page-2', + sql: 'SELECT * FROM iceberg.sales.orders', + pageLimit: 10, + pageOffset: 10, + }), + ).resolves.toMatchObject({ + rows: [{ id: 11 }], + totalRows: 20_000, + truncated: false, + }); + expect(mockRunAndReadUntil).toHaveBeenCalledWith( + 'SELECT * FROM (SELECT * FROM iceberg.sales.orders) AS iceberg_page LIMIT 10 OFFSET 10', + 10, + ); + }); + it('verifies with temporary secrets, attach, detach, and cleanup', async () => { const result = await IcebergDatalakeService.verifySqlAccess(instance.id); diff --git a/tests/unit/main/services/notebooksIceberg.service.test.ts b/tests/unit/main/services/notebooksIceberg.service.test.ts index ceb4bc5e..061c4c6e 100644 --- a/tests/unit/main/services/notebooksIceberg.service.test.ts +++ b/tests/unit/main/services/notebooksIceberg.service.test.ts @@ -30,13 +30,14 @@ describe('NotebooksService Iceberg execution', () => { .mockResolvedValue(undefined); }); - it('executes the original cell through the shared bounded Iceberg runtime', async () => { + it('executes the original cell through the paginated Iceberg runtime', async () => { executeSql.mockResolvedValue({ statementClass: 'select', rows: [{ id: 1 }], columns: ['id'], rowsChanged: 0, - truncated: true, + truncated: false, + totalRows: 1_001, }); const output = await NotebooksService.runCell( @@ -54,14 +55,16 @@ describe('NotebooksService Iceberg execution', () => { instanceId: 'instance-1', executionId: 'cell-run', sql: 'SELECT * FROM iceberg.sales.orders', - maxRows: 100, + pageLimit: 10, + pageOffset: 20, mutationConfirmed: undefined, }, expect.any(AbortSignal), ); expect(output).toMatchObject({ type: 'table', - truncated: true, + truncated: false, + totalRows: 1_001, statementClass: 'select', }); }); diff --git a/tests/unit/renderer/screens/sql/icebergTruncation.test.tsx b/tests/unit/renderer/screens/sql/icebergTruncation.test.tsx index 40c3cfb3..b43c5e62 100644 --- a/tests/unit/renderer/screens/sql/icebergTruncation.test.tsx +++ b/tests/unit/renderer/screens/sql/icebergTruncation.test.tsx @@ -18,8 +18,8 @@ jest.mock('../../../../../src/renderer/services/duckLake.service', () => ({ DuckLakeService: {}, })); -describe('SQL result truncation notice', () => { - it('shows that displayed rows and exports are incomplete', () => { +describe('SQL result pagination', () => { + it('does not present a fixed-result limit warning', () => { render( { }} />, ); - expect(screen.getByRole('alert')).toHaveTextContent( - 'Showing only the first 1 rows', - ); - expect(screen.getByRole('alert')).toHaveTextContent( - 'exports are limited to these rows', - ); + expect(screen.queryByRole('alert')).not.toBeInTheDocument(); }); it('does not change the display of complete results', () => { render( From 2cc9599cd83dc0eb4c10d12a7c4f5d4497ee0f22 Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Wed, 9 Sep 2026 11:18:14 +0200 Subject: [PATCH 06/16] feat(ai): add Iceberg support to SQL and Notebook agents Expose verified Iceberg connections to AI discovery and route schema extraction through the Iceberg SQL runtime without changing generic connection handling. --- src/main/services/ai/agents/notebooksAgent.ts | 13 +- .../ai/tools/studio/connections.tools.ts | 19 ++- .../services/ai/tools/studio/sql.tools.ts | 41 +++++- .../services/ai/icebergAgentTools.test.ts | 133 ++++++++++++++++++ 4 files changed, 200 insertions(+), 6 deletions(-) create mode 100644 tests/unit/main/services/ai/icebergAgentTools.test.ts diff --git a/src/main/services/ai/agents/notebooksAgent.ts b/src/main/services/ai/agents/notebooksAgent.ts index 83175e12..03e17da4 100644 --- a/src/main/services/ai/agents/notebooksAgent.ts +++ b/src/main/services/ai/agents/notebooksAgent.ts @@ -4,6 +4,7 @@ import type { BaseAgentConfig } from './baseAgentConfig'; import { createStudioCloudTools } from '../tools/studio/cloud.tools'; import { createStudioConnectionsTools } from '../tools/studio/connections.tools'; import { createStudioDuckLakeTools } from '../tools/studio/ducklake.tools'; +import { createStudioSqlTools } from '../tools/studio/sql.tools'; import { createStudioNotebooksTools } from '../tools/studio/notebooks.tools'; import { NotebooksService } from '../../notebooks.service'; @@ -134,6 +135,7 @@ You are connected to a DuckLake lakehouse. DuckLake is a DuckDB extension (not a } const isAskMode = options.toolMode === 'chat'; + const isIcebergConnection = connectionId?.startsWith('iceberg-') ?? false; const notebookContext = connectionId && notebookId @@ -196,7 +198,7 @@ ${skills ?? ''} ${mcpToolsList} ## Capabilities & Workflow -1. **Analyze Schema**: Use DuckLake tools to understand the database structure (tables, columns). +1. **Analyze Schema**: Use the active connection's schema tool to understand the database structure (tables, columns). 2. **Notebook Awareness**: Use \`notebooks_get_state\` to see which cells exist. 3. **Strict Single-Statement Cells**: - **CRITICAL RULE**: You can only write ONE SQL statement per cell. Multiple SQL statements (statement chaining) are strictly forbidden and will fail. @@ -230,10 +232,16 @@ The notebook UI handles large datasets efficiently using server-side pagination. delete safeEnabledTools.studio_ducklake_query; delete safeEnabledTools.studio_sql_query; + const activeSchemaTools = isIcebergConnection + ? createStudioSqlTools(options.conversationId) + : createStudioDuckLakeTools(options.conversationId); + delete activeSchemaTools.studio_sql_query; + delete activeSchemaTools.studio_sql_get_query_results; + const studioNotebookTools: Record = { ...createStudioConnectionsTools(), ...createStudioCloudTools(), - ...createStudioDuckLakeTools(options.conversationId), + ...activeSchemaTools, ...createStudioNotebooksTools(options.conversationId), }; @@ -247,6 +255,7 @@ The notebook UI handles large datasets efficiently using server-side pagination. : {}; const READ_ONLY_TOOLS = [ + 'studio_sql_schema_extract', 'studio_ducklake_schema_extract', 'studio_connections_list', 'studio_cloud_list_objects', diff --git a/src/main/services/ai/tools/studio/connections.tools.ts b/src/main/services/ai/tools/studio/connections.tools.ts index 118092b5..5400f849 100644 --- a/src/main/services/ai/tools/studio/connections.tools.ts +++ b/src/main/services/ai/tools/studio/connections.tools.ts @@ -12,6 +12,7 @@ import type { import CloudExplorerService from '../../../cloudExplorer.service'; import ConnectorsService from '../../../connectors.service'; import DuckLakeService from '../../../duckLake.service'; +import { IcebergDatalakeService } from '../../../icebergDatalake.service'; import SecureStorageService from '../../../secureStorage.service'; import { isToolEnabled } from '../toolRegistry'; @@ -217,7 +218,7 @@ export function createStudioConnectionsTools() { id: string; name: string; type: string; - kind: 'database' | 'ducklake'; + kind: 'database' | 'ducklake' | 'iceberg'; health: ConnectionHealth; }> = []; @@ -263,6 +264,22 @@ export function createStudioConnectionsTools() { })); rows.push(...duckLakeRows); + // Iceberg instances are distinct from database connections. Only + // list instances already verified for SQL, and expose no catalog or + // storage credentials to the agent. + const icebergInstances = await IcebergDatalakeService.listInstances(); + rows.push( + ...icebergInstances + .filter((instance) => instance.sqlAvailable) + .map((instance) => ({ + id: `iceberg-${instance.id}`, + name: instance.name, + type: `iceberg (${instance.catalogType})`, + kind: 'iceberg' as const, + health: 'healthy' as const, + })), + ); + return { ok: true, data: { diff --git a/src/main/services/ai/tools/studio/sql.tools.ts b/src/main/services/ai/tools/studio/sql.tools.ts index 1499ec43..352d7f65 100644 --- a/src/main/services/ai/tools/studio/sql.tools.ts +++ b/src/main/services/ai/tools/studio/sql.tools.ts @@ -3,6 +3,7 @@ import { z } from 'zod'; import AgentService from '../../../agent.service'; import ConnectorsService from '../../../connectors.service'; +import { IcebergDatalakeService } from '../../../icebergDatalake.service'; import { AgentEditorBridgeService } from '../../agentEditorBridge.service'; import { truncateToolResult } from '../../tokenEstimator'; import { TerminalConfirmGate } from '../terminalConfirmGate'; @@ -40,6 +41,34 @@ function filterSchemaTables(tables: any[], tableFilter?: string): any[] { }); } +function isIcebergConnectionId(connectionId: unknown): boolean { + return ( + typeof connectionId === 'string' && connectionId.startsWith('iceberg-') + ); +} + +function toAgentIcebergSchema(schema: { + namespaces: Array<{ + name: string; + tables: Array<{ + name: string; + type: string; + columns: Array<{ name: string; type: string; position: number }>; + }>; + }>; +}): { tables: any[] } { + return { + tables: schema.namespaces.flatMap((namespace) => + namespace.tables.map((table) => ({ + name: table.name, + schema: namespace.name, + type: table.type, + columns: table.columns, + })), + ), + }; +} + export function getFirstSqlVerb(sql: string): string { // 1. Strip comments const withoutComments = sql @@ -300,9 +329,15 @@ export function createStudioSqlTools( }; } - const schema = await ConnectorsService.extractSchemaFromConnection( - context.connectionId.toString(), - ); + const schema = isIcebergConnectionId(context.connectionId) + ? toAgentIcebergSchema( + await IcebergDatalakeService.getSqlSchema( + context.connectionId.slice('iceberg-'.length), + ), + ) + : await ConnectorsService.extractSchemaFromConnection( + context.connectionId.toString(), + ); if ((schema as any)?.error) { return { diff --git a/tests/unit/main/services/ai/icebergAgentTools.test.ts b/tests/unit/main/services/ai/icebergAgentTools.test.ts new file mode 100644 index 00000000..bc793477 --- /dev/null +++ b/tests/unit/main/services/ai/icebergAgentTools.test.ts @@ -0,0 +1,133 @@ +import ConnectorsService from '../../../../../src/main/services/connectors.service'; +import DuckLakeService from '../../../../../src/main/services/duckLake.service'; +import { IcebergDatalakeService } from '../../../../../src/main/services/icebergDatalake.service'; +import AgentService from '../../../../../src/main/services/agent.service'; +import { createStudioConnectionsTools } from '../../../../../src/main/services/ai/tools/studio/connections.tools'; +import { createStudioSqlTools } from '../../../../../src/main/services/ai/tools/studio/sql.tools'; + +jest.mock('ai', () => ({ + tool: (definition: unknown) => definition, +})); + +jest.mock('../../../../../src/main/services/connectors.service', () => ({ + __esModule: true, + default: { + loadConnections: jest.fn(), + extractSchemaFromConnection: jest.fn(), + }, +})); + +jest.mock('../../../../../src/main/services/duckLake.service', () => ({ + __esModule: true, + default: { listInstances: jest.fn() }, +})); + +jest.mock('../../../../../src/main/services/icebergDatalake.service', () => ({ + IcebergDatalakeService: { + listInstances: jest.fn(), + getSqlSchema: jest.fn(), + }, +})); + +jest.mock('../../../../../src/main/services/agent.service', () => ({ + __esModule: true, + default: { getAgentContext: jest.fn() }, +})); + +jest.mock( + '../../../../../src/main/services/ai/agentEditorBridge.service', + () => ({ + AgentEditorBridgeService: { recordQueryFired: jest.fn() }, + }), +); + +jest.mock('../../../../../src/main/services/ai/tools/toolRegistry', () => ({ + isToolEnabled: jest.fn(() => true), +})); + +jest.mock( + '../../../../../src/main/services/ai/tools/terminalConfirmGate', + () => ({ + TerminalConfirmGate: { request: jest.fn() }, + }), +); + +describe('Iceberg AI connection tools', () => { + beforeEach(() => { + jest.clearAllMocks(); + (DuckLakeService.listInstances as jest.Mock).mockResolvedValue([]); + }); + + it('lists only SQL-verified Iceberg instances as Iceberg connections', async () => { + (IcebergDatalakeService.listInstances as jest.Mock).mockResolvedValue([ + { + id: 'verified', + name: 'Lakekeeper', + catalogType: 'lakekeeper', + sqlAvailable: true, + }, + { + id: 'unverified', + name: 'Unverified catalog', + catalogType: 'rest', + sqlAvailable: false, + }, + ]); + + const tools = createStudioConnectionsTools() as any; + const result = await tools.studio_connections_list.execute({ + includeDatabases: false, + includeHealth: false, + }); + + expect(result.data.connections).toEqual([ + { + id: 'iceberg-verified', + name: 'Lakekeeper', + type: 'iceberg (lakekeeper)', + kind: 'iceberg', + health: 'healthy', + }, + ]); + expect(ConnectorsService.loadConnections).not.toHaveBeenCalled(); + }); + + it('routes Iceberg schema extraction without calling the database resolver', async () => { + (AgentService.getAgentContext as jest.Mock).mockReturnValue({ + connectionId: 'iceberg-verified', + }); + (IcebergDatalakeService.getSqlSchema as jest.Mock).mockResolvedValue({ + catalogName: 'iceberg', + namespaces: [ + { + name: 'sales', + tables: [ + { + name: 'orders', + type: 'TABLE', + columns: [{ name: 'id', type: 'BIGINT', position: 1 }], + }, + ], + }, + ], + }); + + const tools = createStudioSqlTools(12) as any; + const result = await tools.studio_sql_schema_extract.execute({}); + + expect(IcebergDatalakeService.getSqlSchema).toHaveBeenCalledWith( + 'verified', + ); + expect( + ConnectorsService.extractSchemaFromConnection, + ).not.toHaveBeenCalled(); + expect(result.data.tables).toEqual([ + { + name: 'orders', + schema: 'sales', + type: 'TABLE', + columns: [{ name: 'id', type: 'BIGINT', position: 1 }], + }, + ]); + }); +}); From 8d88cb8f9e51a096447214aa5a8a61a2767b4fa7 Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Wed, 9 Sep 2026 11:40:48 +0200 Subject: [PATCH 07/16] feat(ai): add Iceberg connection context and memory support Resolve Iceberg IDs through the trusted catalog service, add safe Iceberg metadata and dialect guidance to AI agents, and include verified bounded Iceberg schema summaries in Agent Memory. --- src/main/services/agent.service.ts | 29 ++++- src/main/services/ai/agents/agentTypes.ts | 3 + src/main/services/ai/agents/analyticsAgent.ts | 6 + src/main/services/ai/agents/notebooksAgent.ts | 7 ++ src/main/services/ai/agents/sqlAgent.ts | 7 ++ .../secondBrain/secondBrainRefresh.service.ts | 113 ++++++++++++++++-- .../unit/main/services/agent.service.test.ts | 57 +++++++++ .../secondBrainRefresh.service.test.ts | 84 +++++++++++++ 8 files changed, 294 insertions(+), 12 deletions(-) diff --git a/src/main/services/agent.service.ts b/src/main/services/agent.service.ts index 07e7f22f..4791cf17 100644 --- a/src/main/services/agent.service.ts +++ b/src/main/services/agent.service.ts @@ -1231,10 +1231,10 @@ COMBINED SUMMARY:`, static async resolveEnrichedConnectionMeta( connectionId?: string, ): Promise { - const base = { name: 'unknown', type: 'unknown' }; + const base: EnrichedConnectionMeta = { name: 'unknown', type: 'unknown' }; if (!connectionId) return base; try { - let meta: typeof base & { database?: string; schema?: string } = base; + let meta: EnrichedConnectionMeta = base; if (connectionId.startsWith('ducklake-')) { const instanceId = connectionId.replace(/^ducklake-/, ''); const { default: DuckLakeService } = await import('./duckLake.service'); @@ -1245,6 +1245,31 @@ COMBINED SUMMARY:`, type: 'ducklake', }; } + } else if (connectionId.startsWith('iceberg-')) { + const instanceId = connectionId.replace(/^iceberg-/, ''); + const { IcebergDatalakeService } = await import( + './icebergDatalake.service' + ); + try { + const [instance, capability] = await Promise.all([ + IcebergDatalakeService.getInstance(instanceId), + IcebergDatalakeService.getSqlCapability(instanceId), + ]); + meta = { + name: instance.name || 'Iceberg Catalog', + type: 'iceberg', + catalogType: instance.catalogType, + sqlAvailable: capability.available, + unavailableReason: capability.reason, + }; + } catch { + meta = { + name: 'Unavailable Iceberg Catalog', + type: 'iceberg', + sqlAvailable: false, + unavailableReason: 'ICEBERG_INSTANCE_NOT_FOUND', + }; + } } else { const conn = await ConnectorsService.getConnectionById(connectionId); if (conn?.connection) { diff --git a/src/main/services/ai/agents/agentTypes.ts b/src/main/services/ai/agents/agentTypes.ts index 03f61478..1a84c6e7 100644 --- a/src/main/services/ai/agents/agentTypes.ts +++ b/src/main/services/ai/agents/agentTypes.ts @@ -3,5 +3,8 @@ export interface EnrichedConnectionMeta { type: string; database?: string; schema?: string; + catalogType?: string; + sqlAvailable?: boolean; + unavailableReason?: string; linkedDbtProject?: { id: string; name: string; path: string } | null; } diff --git a/src/main/services/ai/agents/analyticsAgent.ts b/src/main/services/ai/agents/analyticsAgent.ts index a916f9ee..fb1f9624 100644 --- a/src/main/services/ai/agents/analyticsAgent.ts +++ b/src/main/services/ai/agents/analyticsAgent.ts @@ -81,6 +81,12 @@ export async function createAnalyticsAgent( let connectionHints = ''; switch (connectionMeta.type) { + case 'iceberg': + connectionHints = `\n\n## Iceberg Specifics +This connection is an Apache Iceberg ${connectionMeta.catalogType ?? 'REST'} catalog accessed through DuckDB's Iceberg extension. +- Use DuckDB SQL and fully qualified catalog references such as \`iceberg..\` when Iceberg Analytics execution is available. +- Do not write \`ATTACH\`, \`DETACH\`, extension, secret, storage, or catalog configuration SQL; DBT Studio owns the temporary attachment lifecycle.`; + break; case 'ducklake': connectionHints = `\n\n## DuckLake Specifics You are connected to a DuckLake lakehouse. DuckLake is a DuckDB extension (not a separate library). diff --git a/src/main/services/ai/agents/notebooksAgent.ts b/src/main/services/ai/agents/notebooksAgent.ts index 03e17da4..c526fbad 100644 --- a/src/main/services/ai/agents/notebooksAgent.ts +++ b/src/main/services/ai/agents/notebooksAgent.ts @@ -88,6 +88,13 @@ export async function createNotebooksAgent( let connectionHints = ''; switch (connectionMeta.type) { + case 'iceberg': + connectionHints = `\n\n## Iceberg Specifics +You are connected to an Apache Iceberg ${connectionMeta.catalogType ?? 'REST'} catalog through DuckDB's Iceberg extension. +- Use DuckDB SQL and fully qualified catalog references such as \`iceberg..
\`. +- Inspect the schema before composing a query. Notebook execution uses the trusted Iceberg policy and requires confirmation for mutations. +- Do not write \`ATTACH\`, \`DETACH\`, extension, secret, storage, or catalog configuration SQL; DBT Studio owns the temporary attachment lifecycle.`; + break; case 'ducklake': connectionHints = `\n\n## DuckLake Specifics You are connected to a DuckLake lakehouse. DuckLake is a DuckDB extension (not a separate library). diff --git a/src/main/services/ai/agents/sqlAgent.ts b/src/main/services/ai/agents/sqlAgent.ts index 4c7f3164..6e2122ea 100644 --- a/src/main/services/ai/agents/sqlAgent.ts +++ b/src/main/services/ai/agents/sqlAgent.ts @@ -52,6 +52,13 @@ export async function createSqlAgent( let connectionHints = ''; switch (connectionMeta.type) { + case 'iceberg': + connectionHints = `\n\n## Iceberg Specifics +You are connected to an Apache Iceberg ${connectionMeta.catalogType ?? 'REST'} catalog through DuckDB's Iceberg extension. +- Use DuckDB SQL and fully qualified catalog references such as \`iceberg..
\`. +- Inspect the schema before composing a query. The trusted SQL path enforces the Iceberg statement policy and requires confirmation for mutations. +- Do not write \`ATTACH\`, \`DETACH\`, extension, secret, storage, or catalog configuration SQL; DBT Studio owns the temporary attachment lifecycle.`; + break; case 'ducklake': connectionHints = `\n\n## DuckLake Specifics You are connected to a DuckLake lakehouse. DuckLake is a DuckDB extension (not a separate library). diff --git a/src/main/services/ai/secondBrain/secondBrainRefresh.service.ts b/src/main/services/ai/secondBrain/secondBrainRefresh.service.ts index 02e1307b..84f815c4 100644 --- a/src/main/services/ai/secondBrain/secondBrainRefresh.service.ts +++ b/src/main/services/ai/secondBrain/secondBrainRefresh.service.ts @@ -14,6 +14,7 @@ import MainDatabaseService, { } from '../../mainDatabase.service'; import ProjectsService from '../../projects.service'; import ConnectorsService from '../../connectors.service'; +import { IcebergDatalakeService } from '../../icebergDatalake.service'; import { NotebooksService } from '../../notebooks.service'; import { DbtCoreVersionService } from '../../dbtCoreVersion.service'; import { getVercelModel } from '../agentAdapter'; @@ -38,6 +39,9 @@ const PROJECT_FILE_BYTE_LIMIT = 64 * 1024; const OPERATION_LIMIT = 24; const PAGE_CHANGE_LIMIT = 12; const REFRESH_LOG_PREFIX = '[WikiMemory][Refresh]'; +const ICEBERG_NAMESPACE_LIMIT = 20; +const ICEBERG_TABLE_LIMIT = 50; +const ICEBERG_COLUMN_LIMIT = 20; const logRefresh = ( event: string, @@ -185,6 +189,8 @@ type SecondBrainRefreshDependencies = { collectSessions?: typeof MainDatabaseService.getSecondBrainSessionEvidence; collectAnalytics?: typeof MainDatabaseService.getSecondBrainAnalyticsEvidence; loadConnections?: typeof ConnectorsService.loadConnections; + listIcebergInstances?: typeof IcebergDatalakeService.listInstances; + getIcebergSqlSchema?: typeof IcebergDatalakeService.getSqlSchema; listNotebooks?: typeof NotebooksService.listNotebooks; collectGitStatus?: (projectPath: string) => Promise>; collectDbtRuntimeEvidence?: typeof DbtCoreVersionService.getInstalledDbtCore; @@ -397,6 +403,35 @@ const cursorFromItems = ( return last ? { updatedAt: last.updatedAt, stableId: last.stableId } : null; }; +const summarizeIcebergSchema = ( + schema: Awaited>, +): { namespaces: Array>; truncated: boolean } => { + let remainingTables = ICEBERG_TABLE_LIMIT; + let truncated = schema.namespaces.length > ICEBERG_NAMESPACE_LIMIT; + const namespaces = schema.namespaces + .slice(0, ICEBERG_NAMESPACE_LIMIT) + .map((namespace) => { + const tables = namespace.tables.slice(0, remainingTables).map((table) => { + const columns = table.columns + .slice(0, ICEBERG_COLUMN_LIMIT) + .map((column) => ({ name: column.name, type: column.type })); + if (table.columns.length > columns.length) truncated = true; + return { name: table.name, type: table.type, columns }; + }); + if (namespace.tables.length > tables.length) truncated = true; + remainingTables -= tables.length; + return { name: namespace.name, tables }; + }) + .filter((namespace) => namespace.tables.length > 0); + if ( + schema.namespaces.some((namespace) => namespace.tables.length > 0) && + remainingTables === 0 + ) { + truncated = true; + } + return { namespaces, truncated }; +}; + const projectPageIdAllowed = (pageId: string): boolean => !isSecondBrainGeneratedPageId(pageId) && (pageId === SECOND_BRAIN_ENTRY_PAGE || @@ -421,6 +456,10 @@ export default class SecondBrainRefreshService { private readonly loadConnections: typeof ConnectorsService.loadConnections; + private readonly listIcebergInstances: typeof IcebergDatalakeService.listInstances; + + private readonly getIcebergSqlSchema: typeof IcebergDatalakeService.getSqlSchema; + private readonly listNotebooks: typeof NotebooksService.listNotebooks; private readonly collectGitStatus: ( @@ -450,6 +489,10 @@ export default class SecondBrainRefreshService { ); this.loadConnections = dependencies.loadConnections ?? ConnectorsService.loadConnections; + this.listIcebergInstances = + dependencies.listIcebergInstances ?? IcebergDatalakeService.listInstances; + this.getIcebergSqlSchema = + dependencies.getIcebergSqlSchema ?? IcebergDatalakeService.getSqlSchema; this.listNotebooks = dependencies.listNotebooks ?? NotebooksService.listNotebooks; this.collectGitStatus = @@ -977,9 +1020,10 @@ export default class SecondBrainRefreshService { private async collectApplicationMetadataBatch( abortSignal?: AbortSignal, ): Promise { - const [projects, connections] = await Promise.all([ + const [projects, connections, icebergInstances] = await Promise.all([ this.loadProjects(), this.loadConnections(true), + this.listIcebergInstances(), ]); assertNotCancelled(abortSignal); const items: SecondBrainEvidenceItem[] = []; @@ -1041,6 +1085,44 @@ export default class SecondBrainRefreshService { truncated: false, }); } + for (const instance of icebergInstances + .filter((candidate) => candidate.sqlAvailable) + .slice(0, SOURCE_ITEM_LIMIT - items.length)) { + assertNotCancelled(abortSignal); + let schemaSummary: Array> = []; + let schemaTruncated = false; + try { + const schema = await this.getIcebergSqlSchema(instance.id); + const summary = summarizeIcebergSchema(schema); + schemaSummary = summary.namespaces; + schemaTruncated = summary.truncated; + } catch (error) { + warnRefresh('iceberg-schema-source-skipped', { + connectionId: `iceberg-${instance.id}`, + code: error instanceof Error ? error.name : 'UNKNOWN', + }); + } + const connectionId = `iceberg-${instance.id}`; + const projection = { + kind: 'connection', + name: redactSecondBrainEvidence(instance.name).slice(0, 200), + type: 'iceberg', + catalogType: instance.catalogType, + sqlAvailable: true, + schema: schemaSummary, + }; + items.push({ + sourceId: 'application', + sourceKind: 'application', + stableId: `connection:${connectionId}`, + updatedAt: instance.updatedAt ?? '', + contentHash: hashValue(projection), + scope: { connectionId }, + provenance: `application:connection:${connectionId}`, + projection, + truncated: schemaTruncated, + }); + } items.sort((left, right) => left.stableId.localeCompare(right.stableId)); return { sourceId: 'application', @@ -1049,24 +1131,35 @@ export default class SecondBrainRefreshService { items, truncated: projects.length > SOURCE_ITEM_LIMIT || - connections.length > SOURCE_ITEM_LIMIT, + connections.length > SOURCE_ITEM_LIMIT || + icebergInstances.filter((instance) => instance.sqlAvailable).length > + SOURCE_ITEM_LIMIT - connections.length, }; } private async collectNotebookBatch( abortSignal?: AbortSignal, ): Promise { - const connections = await this.loadConnections(true); + const [connections, icebergInstances] = await Promise.all([ + this.loadConnections(true), + this.listIcebergInstances(), + ]); + const notebookConnections = [ + ...connections.map((connection) => connection.id), + ...icebergInstances + .filter((instance) => instance.sqlAvailable) + .map((instance) => `iceberg-${instance.id}`), + ]; const items: SecondBrainEvidenceItem[] = []; let truncated = false; - for (const connection of connections) { + for (const connectionId of notebookConnections) { assertNotCancelled(abortSignal); let notebooks; try { - notebooks = await this.listNotebooks(connection.id); + notebooks = await this.listNotebooks(connectionId); } catch (error) { warnRefresh('notebook-source-skipped', { - connectionId: connection.id, + connectionId, code: error instanceof Error ? error.name : 'UNKNOWN', }); continue; @@ -1086,18 +1179,18 @@ export default class SecondBrainRefreshService { description: notebook.description ? redactSecondBrainEvidence(notebook.description).slice(0, 500) : undefined, - connectionId: connection.id, + connectionId, cellCount: notebook.cellCount, cells: projectedCells, }; items.push({ sourceId: 'notebooks', sourceKind: 'notebook', - stableId: `${connection.id}:${notebook.id}`, + stableId: `${connectionId}:${notebook.id}`, updatedAt: notebook.updatedAt, contentHash: hashValue(projection), - scope: { connectionId: connection.id, notebookId: notebook.id }, - provenance: `notebook:${connection.id}:${notebook.id}`, + scope: { connectionId, notebookId: notebook.id }, + provenance: `notebook:${connectionId}:${notebook.id}`, projection, truncated: notebook.cells.length > projectedCells.length, }); diff --git a/tests/unit/main/services/agent.service.test.ts b/tests/unit/main/services/agent.service.test.ts index d965a711..85935ea3 100644 --- a/tests/unit/main/services/agent.service.test.ts +++ b/tests/unit/main/services/agent.service.test.ts @@ -66,6 +66,13 @@ jest.mock('../../../../src/main/services/connectors.service', () => ({ }, })); +jest.mock('../../../../src/main/services/icebergDatalake.service', () => ({ + IcebergDatalakeService: { + getInstance: jest.fn(), + getSqlCapability: jest.fn(), + }, +})); + jest.mock('../../../../src/main/services/ai/skills/skillsDiscovery', () => ({ discoverSkills: jest.fn(), })); @@ -148,6 +155,56 @@ jest.mock( ); describe('AgentService (Phase 1)', () => { + describe('Iceberg connection metadata', () => { + it('resolves an Iceberg renderer ID without using the database resolver', async () => { + const ConnectorsService = jest.requireMock( + '../../../../src/main/services/connectors.service', + ).default; + const { IcebergDatalakeService } = jest.requireMock( + '../../../../src/main/services/icebergDatalake.service', + ); + ConnectorsService.getConnectionById.mockClear(); + IcebergDatalakeService.getInstance.mockResolvedValue({ + name: 'Acceptance catalog', + catalogType: 'polaris', + }); + IcebergDatalakeService.getSqlCapability.mockResolvedValue({ + available: true, + reason: undefined, + }); + + await expect( + AgentService.resolveEnrichedConnectionMeta('iceberg-catalog-id'), + ).resolves.toMatchObject({ + name: 'Acceptance catalog', + type: 'iceberg', + catalogType: 'polaris', + sqlAvailable: true, + }); + expect(ConnectorsService.getConnectionById).not.toHaveBeenCalled(); + }); + + it('reports a missing Iceberg renderer ID as unavailable', async () => { + const { IcebergDatalakeService } = jest.requireMock( + '../../../../src/main/services/icebergDatalake.service', + ); + IcebergDatalakeService.getInstance.mockRejectedValue( + new Error('Iceberg instance not found'), + ); + IcebergDatalakeService.getSqlCapability.mockRejectedValue( + new Error('Iceberg instance not found'), + ); + + await expect( + AgentService.resolveEnrichedConnectionMeta('iceberg-missing-id'), + ).resolves.toMatchObject({ + type: 'iceberg', + sqlAvailable: false, + unavailableReason: 'ICEBERG_INSTANCE_NOT_FOUND', + }); + }); + }); + describe('AI settings migration', () => { it('adds disabled Second Brain defaults to legacy settings', () => { const normalized = normalizeAISettings({ diff --git a/tests/unit/main/services/ai/secondBrain/secondBrainRefresh.service.test.ts b/tests/unit/main/services/ai/secondBrain/secondBrainRefresh.service.test.ts index 161abea3..210aa051 100644 --- a/tests/unit/main/services/ai/secondBrain/secondBrainRefresh.service.test.ts +++ b/tests/unit/main/services/ai/secondBrain/secondBrainRefresh.service.test.ts @@ -67,6 +67,11 @@ Validate revenue totals before publishing. const emptyAdditionalSources = { loadConnections: jest.fn(async () => []), + listIcebergInstances: jest.fn(async () => []), + getIcebergSqlSchema: jest.fn(async () => ({ + catalogName: 'iceberg', + namespaces: [], + })), listNotebooks: jest.fn(async () => []), collectGitStatus: jest.fn(async () => ({})), collectDbtRuntimeEvidence: jest.fn(async () => ({ @@ -606,6 +611,85 @@ Existing guidance. expect(JSON.stringify(capturedEvidence)).not.toContain('must-not-project'); }); + it('collects only safe, bounded metadata for verified Iceberg connections', async () => { + let capturedEvidence: SecondBrainEvidenceItem[] = []; + const refresh = new SecondBrainRefreshService(secondBrain, { + ...emptyAdditionalSources, + generateOperations: jest.fn(async (input: any) => { + capturedEvidence = input.evidence; + return []; + }), + collectSessions: jest.fn(async () => []) as any, + collectAnalytics: jest.fn(async () => []) as any, + loadProjects: jest.fn(async () => []), + listIcebergInstances: jest.fn(async () => [ + { + id: 'catalog-id', + name: 'Acceptance catalog', + catalogType: 'polaris', + sqlAvailable: true, + sqlUnavailableReason: undefined, + storageBucket: 'must-not-project', + }, + { + id: 'unverified-id', + name: 'Unverified catalog', + catalogType: 'rest', + sqlAvailable: false, + }, + ]) as any, + getIcebergSqlSchema: jest.fn(async () => ({ + catalogName: 'iceberg', + namespaces: [ + { + name: 'default', + tables: [ + { + name: 'orders', + type: 'TABLE', + columns: [ + { name: 'id', type: 'BIGINT', position: 1 }, + { name: 'customer', type: 'VARCHAR', position: 2 }, + ], + }, + ], + }, + ], + })), + }); + + await refresh.refresh({ dryRun: true }); + + const icebergEvidence = capturedEvidence.find( + (item) => item.stableId === 'connection:iceberg-catalog-id', + ); + expect(icebergEvidence?.scope.connectionId).toBe('iceberg-catalog-id'); + expect(icebergEvidence?.projection).toMatchObject({ + name: 'Acceptance catalog', + type: 'iceberg', + catalogType: 'polaris', + sqlAvailable: true, + schema: [ + { + name: 'default', + tables: [ + { + name: 'orders', + columns: [ + { name: 'id', type: 'BIGINT' }, + { name: 'customer', type: 'VARCHAR' }, + ], + }, + ], + }, + ], + }); + expect(JSON.stringify(capturedEvidence)).not.toContain('must-not-project'); + expect(JSON.stringify(capturedEvidence)).not.toContain( + 'Unverified catalog', + ); + }); + it('cancels before collection without a model call or state change', async () => { const controller = new AbortController(); controller.abort(); From eeafa7e64fadb7b5c950d1c8a5053e7c02c54cc0 Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Wed, 9 Sep 2026 12:05:21 +0200 Subject: [PATCH 08/16] feat(analytics): add verified Iceberg query support Route Analytics pages and static-site exports through the trusted DuckDB Iceberg runtime, preserve bounded results, reject Iceberg mutations, and show unavailable catalog state. --- src/main/services/staticSite.service.ts | 35 ++++++++++ .../components/analytics/AnalyticsEditor.tsx | 30 +++++++- src/renderer/utils/analyticsQueryEngine.ts | 44 +++++++++++- .../utils/analyticsQueryEngine.test.ts | 69 +++++++++++++++++++ 4 files changed, 175 insertions(+), 3 deletions(-) diff --git a/src/main/services/staticSite.service.ts b/src/main/services/staticSite.service.ts index e66114f7..f3242dbb 100644 --- a/src/main/services/staticSite.service.ts +++ b/src/main/services/staticSite.service.ts @@ -9,12 +9,14 @@ import { BrowserWindow, shell, dialog, app } from 'electron'; import fs from 'fs'; import path from 'path'; +import { v4 as uuidv4 } from 'uuid'; import { AnalyticsPagesService } from './analyticsPages.service'; import ConnectorsService from './connectors.service'; import MainDatabaseService from './mainDatabase.service'; import SettingsService from './settings.service'; import { extractQueryReferences } from '../../renderer/components/analytics/runtime/queryDependencyResolver'; import DuckLakeService from './duckLake.service'; +import { IcebergDatalakeService } from './icebergDatalake.service'; import { toSlug, generateSiteShell, @@ -177,6 +179,39 @@ async function executeQueryInMain(params: { const { connectionId, sql } = params; try { + if (connectionId.startsWith('iceberg-')) { + const instanceId = connectionId.replace('iceberg-', ''); + const executionId = `analytics-static-${uuidv4()}`; + const classification = await IcebergDatalakeService.executeSql({ + instanceId, + executionId, + sql, + pageLimit: MAX_ROWS_PER_QUERY, + pageOffset: 0, + validateOnly: true, + }); + if (classification.statementClass !== 'select') { + return { + data: [], + error: 'Analytics pages support read-only Iceberg SELECT queries.', + truncated: false, + }; + } + + const response = await IcebergDatalakeService.executeSql({ + instanceId, + executionId, + sql, + pageLimit: MAX_ROWS_PER_QUERY, + pageOffset: 0, + }); + return { + data: response.rows.slice(0, MAX_ROWS_PER_QUERY), + error: null, + truncated: response.truncated, + }; + } + if (connectionId.startsWith('ducklake-')) { const instanceId = connectionId.replace('ducklake-', ''); const response = await DuckLakeService.executeQuery({ diff --git a/src/renderer/components/analytics/AnalyticsEditor.tsx b/src/renderer/components/analytics/AnalyticsEditor.tsx index ed1e6213..bbde0226 100644 --- a/src/renderer/components/analytics/AnalyticsEditor.tsx +++ b/src/renderer/components/analytics/AnalyticsEditor.tsx @@ -15,7 +15,14 @@ import React, { useRef, useMemo, } from 'react'; -import { Box, Typography, IconButton, Tooltip, useTheme } from '@mui/material'; +import { + Alert, + Box, + Typography, + IconButton, + Tooltip, + useTheme, +} from '@mui/material'; import { InsertChart, PlayArrow, @@ -33,6 +40,7 @@ import { useGetAnalyticsPages, useUpdateAnalyticsPage, } from '../../controllers/analyticsPages.controller'; +import { useListIcebergInstances } from '../../controllers/icebergDatalake.controller'; import { useSchemaForConnection, useMonacoAutocomplete } from '../../hooks'; import { executeAnalyticsQuery, @@ -124,6 +132,19 @@ export const AnalyticsEditor: React.FC = ({ [pages, pageId], ); const updateAnalyticsPage = useUpdateAnalyticsPage(); + const isIcebergConnection = connectionId.startsWith('iceberg-'); + const icebergInstanceId = isIcebergConnection + ? connectionId.replace('iceberg-', '') + : ''; + const { data: icebergInstances = [], isLoading: icebergLoading } = + useListIcebergInstances(); + const icebergInstance = icebergInstances.find( + (instance) => instance.id === icebergInstanceId, + ); + const icebergUnavailable = + isIcebergConnection && !icebergLoading && !icebergInstance?.sqlAvailable; + const icebergUnavailableReason = + icebergInstance?.sqlUnavailableReason ?? 'ICEBERG_SQL_UNAVAILABLE'; // ── Editor state ────────────────────────────────────────────────────── const [markdownContent, setMarkdownContent] = useState(''); @@ -719,7 +740,7 @@ export const AnalyticsEditor: React.FC = ({ @@ -766,6 +787,11 @@ export const AnalyticsEditor: React.FC = ({ {/* ── Editor + Preview Split ───────────────────────────────────── */} + {icebergUnavailable && ( + + Iceberg analytics queries are unavailable: {icebergUnavailableReason} + + )} diff --git a/src/renderer/utils/analyticsQueryEngine.ts b/src/renderer/utils/analyticsQueryEngine.ts index 8a8571f0..8fee97e6 100644 --- a/src/renderer/utils/analyticsQueryEngine.ts +++ b/src/renderer/utils/analyticsQueryEngine.ts @@ -2,11 +2,13 @@ * Analytics Query Engine * * Encapsulates SQL execution for analytics pages, handling both regular - * DB connections and DuckLake connections transparently. + * DB connections, DuckLake instances, and verified Iceberg catalogs. * Supports {{query_name}} dependency resolution for inter-query references. */ +import { v4 as uuidv4 } from 'uuid'; import { executeQueryForConnection } from '../services/connectors.service'; import { DuckLakeService } from '../services/duckLake.service'; +import { executeIcebergSql } from '../services/iceberg.service'; import { buildQueryDependencyGraph, validateQueryReferences, @@ -143,6 +145,46 @@ export async function executeAnalyticsQuery(params: { const ROW_LIMIT = 500; try { + if (connectionId.startsWith('iceberg-')) { + const instanceId = connectionId.replace('iceberg-', ''); + const executionId = `analytics-${uuidv4()}`; + const classification = await executeIcebergSql({ + instanceId, + executionId, + sql, + pageLimit: ROW_LIMIT, + pageOffset: 0, + validateOnly: true, + }); + + if (classification.statementClass !== 'select') { + return { + name: queryName, + status: 'error', + data: [], + fields: [], + rowCount: 0, + error: 'Analytics pages support read-only Iceberg SELECT queries.', + }; + } + + const response = await executeIcebergSql({ + instanceId, + executionId, + sql, + pageLimit: ROW_LIMIT, + pageOffset: 0, + }); + const data = normalizeAnalyticsRows(response.rows.slice(0, ROW_LIMIT)); + return { + name: queryName, + status: 'success', + data, + fields: response.columns, + rowCount: data.length, + }; + } + if (connectionId.startsWith('ducklake-')) { const instanceId = connectionId.replace('ducklake-', ''); const response = await DuckLakeService.executeQuery({ diff --git a/tests/unit/renderer/utils/analyticsQueryEngine.test.ts b/tests/unit/renderer/utils/analyticsQueryEngine.test.ts index 0fb9d37f..fac477ef 100644 --- a/tests/unit/renderer/utils/analyticsQueryEngine.test.ts +++ b/tests/unit/renderer/utils/analyticsQueryEngine.test.ts @@ -1,5 +1,6 @@ import { executeQueryForConnection } from '../../../../src/renderer/services/connectors.service'; import { DuckLakeService } from '../../../../src/renderer/services/duckLake.service'; +import { executeIcebergSql } from '../../../../src/renderer/services/iceberg.service'; import { executeAnalyticsQuery } from '../../../../src/renderer/utils/analyticsQueryEngine'; jest.mock('../../../../src/renderer/services/connectors.service', () => ({ @@ -12,8 +13,13 @@ jest.mock('../../../../src/renderer/services/duckLake.service', () => ({ }, })); +jest.mock('../../../../src/renderer/services/iceberg.service', () => ({ + executeIcebergSql: jest.fn(), +})); + const executeConnectorQuery = executeQueryForConnection as jest.Mock; const executeDuckLakeQuery = DuckLakeService.executeQuery as jest.Mock; +const executeIcebergQuery = executeIcebergSql as jest.Mock; describe('executeAnalyticsQuery', () => { beforeEach(() => { @@ -70,4 +76,67 @@ describe('executeAnalyticsQuery', () => { }); expect(executeDuckLakeQuery).not.toHaveBeenCalled(); }); + + it('routes Iceberg reads through validation and a bounded trusted execution', async () => { + executeIcebergQuery + .mockResolvedValueOnce({ statementClass: 'select' }) + .mockResolvedValueOnce({ + statementClass: 'select', + columns: ['station_id'], + rows: [{ station_id: 1 }], + truncated: false, + }); + + await expect( + executeAnalyticsQuery({ + queryName: 'stations', + sql: 'SELECT station_id FROM iceberg.default.stations', + connectionId: 'iceberg-catalog-id', + }), + ).resolves.toEqual({ + name: 'stations', + status: 'success', + data: [{ station_id: 1 }], + fields: ['station_id'], + rowCount: 1, + }); + + expect(executeIcebergQuery).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + instanceId: 'catalog-id', + sql: 'SELECT station_id FROM iceberg.default.stations', + pageLimit: 500, + pageOffset: 0, + validateOnly: true, + }), + ); + expect(executeIcebergQuery).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + instanceId: 'catalog-id', + pageLimit: 500, + pageOffset: 0, + }), + ); + expect(executeConnectorQuery).not.toHaveBeenCalled(); + expect(executeDuckLakeQuery).not.toHaveBeenCalled(); + }); + + it('rejects Iceberg mutations after trusted classification', async () => { + executeIcebergQuery.mockResolvedValueOnce({ statementClass: 'delete' }); + + await expect( + executeAnalyticsQuery({ + queryName: 'remove_stations', + sql: 'DELETE FROM iceberg.default.stations', + connectionId: 'iceberg-catalog-id', + }), + ).resolves.toMatchObject({ + name: 'remove_stations', + status: 'error', + error: 'Analytics pages support read-only Iceberg SELECT queries.', + }); + expect(executeIcebergQuery).toHaveBeenCalledTimes(1); + }); }); From 4821601a32a1f2e9d4b641207af43c44514a316a Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Wed, 9 Sep 2026 13:23:31 +0200 Subject: [PATCH 09/16] fix(analytics): qualify Iceberg AI queries and autocomplete --- src/main/services/ai/agents/analyticsAgent.ts | 3 +- .../components/analytics/AnalyticsEditor.tsx | 34 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/main/services/ai/agents/analyticsAgent.ts b/src/main/services/ai/agents/analyticsAgent.ts index fb1f9624..1d715b56 100644 --- a/src/main/services/ai/agents/analyticsAgent.ts +++ b/src/main/services/ai/agents/analyticsAgent.ts @@ -84,7 +84,8 @@ export async function createAnalyticsAgent( case 'iceberg': connectionHints = `\n\n## Iceberg Specifics This connection is an Apache Iceberg ${connectionMeta.catalogType ?? 'REST'} catalog accessed through DuckDB's Iceberg extension. -- Use DuckDB SQL and fully qualified catalog references such as \`iceberg..
\` when Iceberg Analytics execution is available. +- Use DuckDB SQL. Every Iceberg table reference must be fully qualified as \`iceberg..
\`; this is required by the Analytics execution policy. +- Never use \`.
\`, a bare table name, or \`USE\` to make Iceberg table references unqualified. A successful query in another editor does not change this Analytics requirement. - Do not write \`ATTACH\`, \`DETACH\`, extension, secret, storage, or catalog configuration SQL; DBT Studio owns the temporary attachment lifecycle.`; break; case 'ducklake': diff --git a/src/renderer/components/analytics/AnalyticsEditor.tsx b/src/renderer/components/analytics/AnalyticsEditor.tsx index bbde0226..44ad5e4f 100644 --- a/src/renderer/components/analytics/AnalyticsEditor.tsx +++ b/src/renderer/components/analytics/AnalyticsEditor.tsx @@ -36,12 +36,14 @@ import MonacoEditor from '@monaco-editor/react'; import SplitPane, { Pane } from 'split-pane-react'; import 'split-pane-react/esm/themes/default.css'; import * as monaco from 'monaco-editor'; +import { MonacoAutocompleteSQLKeywords } from '../../config/constants'; import { useGetAnalyticsPages, useUpdateAnalyticsPage, } from '../../controllers/analyticsPages.controller'; import { useListIcebergInstances } from '../../controllers/icebergDatalake.controller'; import { useSchemaForConnection, useMonacoAutocomplete } from '../../hooks'; +import { icebergQualifiedName } from '../../services/iceberg.service'; import { executeAnalyticsQuery, resolveQueryDependencies, @@ -214,10 +216,40 @@ export const AnalyticsEditor: React.FC = ({ // ── Schema for SQL autocomplete ─────────────────────────────────────── const { data: schemaData } = useSchemaForConnection(connectionId); - const completions = useMonacoAutocomplete( + const baseCompletions = useMonacoAutocomplete( schemaData?.tables || null, schemaData?.duckLakeSchema || null, ); + const completions = useMemo(() => { + if (!isIcebergConnection) return baseCompletions; + + return [ + ...MonacoAutocompleteSQLKeywords.map((keyword) => ({ + label: keyword, + insertText: keyword, + kind: monaco.languages.CompletionItemKind.Keyword, + detail: 'SQL keyword', + })), + ...(schemaData?.tables ?? []).flatMap((table) => [ + { + label: `iceberg.${table.schema}.${table.name}`, + insertText: icebergQualifiedName(table.schema, table.name), + kind: monaco.languages.CompletionItemKind.Class, + detail: 'Iceberg table', + }, + ...table.columns.map((column) => ({ + label: `iceberg.${table.schema}.${table.name}.${column.name}`, + insertText: icebergQualifiedName( + table.schema, + table.name, + column.name, + ), + kind: monaco.languages.CompletionItemKind.Field, + detail: 'Iceberg column', + })), + ]), + ]; + }, [baseCompletions, isIcebergConnection, schemaData]); // Keep the singleton ref updated so the provider always uses fresh completions useEffect(() => { From ce5827eb6e2cb8a00408c6449299a793449a9027 Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Wed, 9 Sep 2026 13:53:35 +0200 Subject: [PATCH 10/16] fix: allow DDL and DML sql queires in Editors --- src/main/services/ai/agents/notebooksAgent.ts | 3 +- src/main/services/ai/agents/sqlAgent.ts | 3 +- .../services/ai/tools/studio/sql.tools.ts | 8 +++- src/main/services/iceberg/sqlPolicy.ts | 6 +-- src/main/services/icebergDatalake.service.ts | 9 +++-- src/main/services/notebooks.service.ts | 5 ++- src/renderer/services/iceberg.service.ts | 13 +++++- .../main/services/icebergSqlPolicy.test.ts | 4 ++ .../icebergSqlRuntime.service.test.ts | 40 +++++++++++++++++++ .../services/notebooksIceberg.service.test.ts | 31 ++++++++++++++ .../services/icebergConfirmation.test.ts | 24 +++++++++++ 11 files changed, 133 insertions(+), 13 deletions(-) diff --git a/src/main/services/ai/agents/notebooksAgent.ts b/src/main/services/ai/agents/notebooksAgent.ts index c526fbad..f1dad608 100644 --- a/src/main/services/ai/agents/notebooksAgent.ts +++ b/src/main/services/ai/agents/notebooksAgent.ts @@ -92,7 +92,8 @@ export async function createNotebooksAgent( connectionHints = `\n\n## Iceberg Specifics You are connected to an Apache Iceberg ${connectionMeta.catalogType ?? 'REST'} catalog through DuckDB's Iceberg extension. - Use DuckDB SQL and fully qualified catalog references such as \`iceberg..
\`. -- Inspect the schema before composing a query. Notebook execution uses the trusted Iceberg policy and requires confirmation for mutations. +- Inspect the schema before composing a query. You may prepare supported catalog and data mutations; Notebook execution presents the user confirmation before it executes the exact statement. +- Every table target must be \`iceberg..
\`. Never infer that unqualified database syntax, attachment SQL, credentials, or storage SQL is available for this connection. - Do not write \`ATTACH\`, \`DETACH\`, extension, secret, storage, or catalog configuration SQL; DBT Studio owns the temporary attachment lifecycle.`; break; case 'ducklake': diff --git a/src/main/services/ai/agents/sqlAgent.ts b/src/main/services/ai/agents/sqlAgent.ts index 6e2122ea..972c34aa 100644 --- a/src/main/services/ai/agents/sqlAgent.ts +++ b/src/main/services/ai/agents/sqlAgent.ts @@ -56,7 +56,8 @@ export async function createSqlAgent( connectionHints = `\n\n## Iceberg Specifics You are connected to an Apache Iceberg ${connectionMeta.catalogType ?? 'REST'} catalog through DuckDB's Iceberg extension. - Use DuckDB SQL and fully qualified catalog references such as \`iceberg..
\`. -- Inspect the schema before composing a query. The trusted SQL path enforces the Iceberg statement policy and requires confirmation for mutations. +- Inspect the schema before composing a query. You may prepare supported catalog and data mutations; the SQL Editor presents the user confirmation before it executes the exact statement. +- Every table target must be \`iceberg..
\`. Never infer that unqualified database syntax, attachment SQL, credentials, or storage SQL is available for this connection. - Do not write \`ATTACH\`, \`DETACH\`, extension, secret, storage, or catalog configuration SQL; DBT Studio owns the temporary attachment lifecycle.`; break; case 'ducklake': diff --git a/src/main/services/ai/tools/studio/sql.tools.ts b/src/main/services/ai/tools/studio/sql.tools.ts index 352d7f65..e739c127 100644 --- a/src/main/services/ai/tools/studio/sql.tools.ts +++ b/src/main/services/ai/tools/studio/sql.tools.ts @@ -443,8 +443,12 @@ export function createStudioSqlTools( ); } - // Step 2 — For destructive/mutating statements, ask the user before executing - if (isMutationSql(sql)) { + // Iceberg SQL Editor execution has its own classified confirmation + // dialog. Other connections use the existing terminal gate. + if ( + isMutationSql(sql) && + !isIcebergConnectionId(context.connectionId) + ) { const allowed = await TerminalConfirmGate.request({ event: context.event, conversationId, diff --git a/src/main/services/iceberg/sqlPolicy.ts b/src/main/services/iceberg/sqlPolicy.ts index 406d37bf..0f93c004 100644 --- a/src/main/services/iceberg/sqlPolicy.ts +++ b/src/main/services/iceberg/sqlPolicy.ts @@ -316,10 +316,10 @@ export function parseIcebergSql(sql: string): { return select(text(t.slice(i)), 'create'); } const columns = group(); - // Basic typed columns only. Defaults, constraints with expressions, - // custom types and generated columns require separate acceptance. + // Basic typed columns and primary keys only. Defaults, expression-based + // constraints, custom types and generated columns require separate acceptance. const definition = - /^(?:"(?:[^"]|"")*"|[a-z_][a-z_0-9]*)\s+(?:BOOLEAN|TINYINT|SMALLINT|INTEGER|INT|BIGINT|FLOAT|REAL|DOUBLE(?: PRECISION)?|VARCHAR|TEXT|STRING|BLOB|DATE|TIME|TIMESTAMP|TIMESTAMPTZ|UUID|DECIMAL(?:\s*\(\s*\d+\s*,\s*\d+\s*\))?)(?:\s+NOT\s+NULL)?$/i; + /^(?:"(?:[^"]|"")*"|[a-z_][a-z_0-9]*)\s+(?:BOOLEAN|TINYINT|SMALLINT|INTEGER|INT|BIGINT|FLOAT|REAL|DOUBLE(?: PRECISION)?|VARCHAR|TEXT|STRING|BLOB|DATE|TIME|TIMESTAMP|TIMESTAMPTZ|UUID|DECIMAL(?:\s*\(\s*\d+\s*,\s*\d+\s*\))?)(?:\s+NOT\s+NULL)?(?:\s+PRIMARY\s+KEY)?$/i; const definitions: Token[][] = [[]]; let depth = 0; columns.forEach((token) => { diff --git a/src/main/services/icebergDatalake.service.ts b/src/main/services/icebergDatalake.service.ts index ad23a8cc..281ab11b 100644 --- a/src/main/services/icebergDatalake.service.ts +++ b/src/main/services/icebergDatalake.service.ts @@ -1774,11 +1774,12 @@ export class IcebergDatalakeService { if (statementClass !== 'select' && params.mutationConfirmed !== true) { throw new Error('ICEBERG_SQL_CONFIRMATION_REQUIRED'); } + // Callers may carry their current result-page settings into an execution. + // A mutation has no result set to page, so ignore those settings after the + // trusted classifier and confirmation gate have accepted it. const hasPageRequest = - params.pageLimit !== undefined || params.pageOffset !== undefined; - if (hasPageRequest && statementClass !== 'select') { - throw new Error('ICEBERG_SQL_PAGINATION_REQUIRES_SELECT'); - } + statementClass === 'select' && + (params.pageLimit !== undefined || params.pageOffset !== undefined); const pageLimit = params.pageLimit ?? 10; const pageOffset = params.pageOffset ?? 0; if ( diff --git a/src/main/services/notebooks.service.ts b/src/main/services/notebooks.service.ts index e162e400..c8881abc 100644 --- a/src/main/services/notebooks.service.ts +++ b/src/main/services/notebooks.service.ts @@ -297,12 +297,15 @@ export class NotebooksService { ) { throw new Error('ICEBERG_NOTEBOOK_CELL_NOT_FOUND'); } + const pagination = options?.mutationConfirmed + ? {} + : sanitizePagination(limit, offset); const result = await IcebergDatalakeService.executeSql( { instanceId: connectionId.slice(8), executionId, sql, - ...sanitizePagination(limit, offset), + ...pagination, mutationConfirmed: options?.mutationConfirmed, }, run.signal, diff --git a/src/renderer/services/iceberg.service.ts b/src/renderer/services/iceberg.service.ts index db5a01ec..19e2dc5d 100644 --- a/src/renderer/services/iceberg.service.ts +++ b/src/renderer/services/iceberg.service.ts @@ -219,8 +219,19 @@ export const executeConfirmedIcebergSql = async ( const mutating = classification.statementClass !== 'select'; if (mutating && !confirmMutation(classification.statementClass)) return undefined; + // Pagination belongs exclusively to result-producing reads. Do not let a + // SQL Editor page request turn an already-confirmed mutation into a rejected + // operation in the main process. + const executionParams = mutating + ? { + instanceId: request.instanceId, + executionId: request.executionId, + sql: request.sql, + maxRows: request.maxRows, + } + : request; return executeIcebergSql({ - ...request, + ...executionParams, validateOnly: false, mutationConfirmed: mutating, }); diff --git a/tests/unit/main/services/icebergSqlPolicy.test.ts b/tests/unit/main/services/icebergSqlPolicy.test.ts index 1f7c438d..4d317a03 100644 --- a/tests/unit/main/services/icebergSqlPolicy.test.ts +++ b/tests/unit/main/services/icebergSqlPolicy.test.ts @@ -70,6 +70,10 @@ describe('Iceberg SQL policy with the native DuckDB parser', () => { 'CREATE TABLE iceberg.sales.orders (id INTEGER, name VARCHAR NOT NULL, price DECIMAL(10,2))', 'create', ], + [ + 'CREATE TABLE iceberg.sales.customers (customer_id BIGINT PRIMARY KEY, email VARCHAR)', + 'create', + ], [ 'CREATE TABLE iceberg.sales.orders AS SELECT * FROM iceberg.sales.source', 'create', diff --git a/tests/unit/main/services/icebergSqlRuntime.service.test.ts b/tests/unit/main/services/icebergSqlRuntime.service.test.ts index 92702d8f..33566db9 100644 --- a/tests/unit/main/services/icebergSqlRuntime.service.test.ts +++ b/tests/unit/main/services/icebergSqlRuntime.service.test.ts @@ -233,6 +233,46 @@ describe('IcebergDatalakeService DuckDB Iceberg lifecycle', () => { ); }); + it('ignores read pagination for a confirmed mutation', async () => { + mockedLoadDatabase.mockResolvedValue({ + ...database, + icebergInstances: [ + { + ...instance, + sqlAccessVerifiedAt: '2026-08-14T00:00:00.000Z', + sqlRuntimeFingerprint: ( + IcebergDatalakeService as any + ).getSqlRuntimeFingerprint(), + }, + ], + }); + mockRunAndReadUntil.mockResolvedValue({ + columnNames: () => [], + getRowObjectsJson: () => [], + rowsChanged: 0, + done: true, + }); + + await expect( + IcebergDatalakeService.executeSql({ + instanceId: instance.id, + executionId: 'create-customers', + sql: 'CREATE TABLE iceberg.sales.customers (id BIGINT)', + pageLimit: 10, + pageOffset: 0, + mutationConfirmed: true, + }), + ).resolves.toMatchObject({ + statementClass: 'create', + rows: [], + rowsChanged: 0, + }); + expect(mockRunAndReadUntil).toHaveBeenCalledWith( + 'CREATE TABLE iceberg.sales.customers (id BIGINT)', + 1001, + ); + }); + it('verifies with temporary secrets, attach, detach, and cleanup', async () => { const result = await IcebergDatalakeService.verifySqlAccess(instance.id); diff --git a/tests/unit/main/services/notebooksIceberg.service.test.ts b/tests/unit/main/services/notebooksIceberg.service.test.ts index 061c4c6e..5f44a8aa 100644 --- a/tests/unit/main/services/notebooksIceberg.service.test.ts +++ b/tests/unit/main/services/notebooksIceberg.service.test.ts @@ -76,6 +76,37 @@ describe('NotebooksService Iceberg execution', () => { expect(executeSql).not.toHaveBeenCalled(); }); + it('does not attach read pagination to a confirmed mutation cell', async () => { + executeSql.mockResolvedValue({ + statementClass: 'create', + rows: [], + columns: [], + rowsChanged: 0, + truncated: false, + }); + + await NotebooksService.runCell( + 'iceberg-instance-1', + notebook.id, + 'cell-1', + 'CREATE TABLE iceberg.sales.customers (id BIGINT)', + 10, + 20, + { executionId: 'mutation-run', mutationConfirmed: true }, + ); + + expect(executeSql).toHaveBeenCalledWith( + expect.objectContaining({ + instanceId: 'instance-1', + executionId: 'mutation-run', + mutationConfirmed: true, + }), + expect.any(AbortSignal), + ); + expect(executeSql.mock.calls[0][0]).not.toHaveProperty('pageLimit'); + expect(executeSql.mock.calls[0][0]).not.toHaveProperty('pageOffset'); + }); + it('runs one confirmed Run All cell and propagates its failure', async () => { executeSql.mockRejectedValue(new Error('write failed')); diff --git a/tests/unit/renderer/services/icebergConfirmation.test.ts b/tests/unit/renderer/services/icebergConfirmation.test.ts index c5967995..2bba134c 100644 --- a/tests/unit/renderer/services/icebergConfirmation.test.ts +++ b/tests/unit/renderer/services/icebergConfirmation.test.ts @@ -40,6 +40,30 @@ describe('Iceberg mutation confirmation', () => { validateOnly: false, }); }); + it('removes read pagination before executing a confirmed mutation', async () => { + invoke + .mockResolvedValueOnce({ statementClass: 'create' }) + .mockResolvedValueOnce({ rows: [], rowsChanged: 0 }); + + await executeConfirmedIcebergSql( + { + instanceId: 'id', + executionId: 'run', + sql: 'CREATE TABLE iceberg.sales.customers (id BIGINT)', + pageLimit: 10, + pageOffset: 0, + }, + () => true, + ); + + expect(invoke.mock.calls[1][1]).toMatchObject({ + sql: 'CREATE TABLE iceberg.sales.customers (id BIGINT)', + mutationConfirmed: true, + validateOnly: false, + }); + expect(invoke.mock.calls[1][1]).not.toHaveProperty('pageLimit'); + expect(invoke.mock.calls[1][1]).not.toHaveProperty('pageOffset'); + }); it('does not prompt for a backend-classified read', async () => { invoke .mockResolvedValueOnce({ statementClass: 'select' }) From 2c35fef4cbe89d11ed4b0b29dc89c4d265ff34d3 Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Thu, 10 Sep 2026 12:22:52 +0200 Subject: [PATCH 11/16] fix: address CodeRabbit review issues and stability improvements - Fix Iceberg budget computation: Clamp to `SOURCE_ITEM_LIMIT` correctly and ensure consistency between slice and truncation logic in `secondBrainRefresh.service`. - Enforce Nessie 'main' reference: Reject SQL ATTACH when a non-default Nessie reference is set, as DuckDB doesn't natively support branch/tag selection within `ATTACH`. - Optimize Iceberg instance listing: Pass already-loaded database snapshot to capability checks to eliminate redundant reads and slow keychain calls during initialization. - Mitigate execution ID race conditions: Synchronously reserve `executionId` prior to running async setups to prevent overlapping SQL executions. - Safely handle duckdb partial cleanups: Separate DuckDB `closeSync()` failures from partial failures (like `DETACH` or `DROP SECRET`). Log partial failures without incorrectly reporting successful mutations as failures. - Bound schema loading fan-out: Switched from unbounded `Promise.all` over tables to a sequential loop, limiting Python bridge concurrency spikes during `getSqlSchema`. - Fix truncated flags in static site: Replaced `pageLimit` and `pageOffset` with `maxRows` in static site queries to ensure the `truncated` boolean properly evaluates. - Fix wizard state resets: Reset validation test states during storage selection in `IcebergConnectionWizard` strictly upon actual field mutations. - Remove stale wording: Replaced Phase 3 wizard success text to direct users to the "Test SQL Access" button. - Add error boundary to SQL Verify: Wrapped the mutateAsync call inside `handleVerifySqlAccess` with a try/catch block. - Exclude Parquet Export: Prevent exporting Iceberg table queries as Parquet since it's unsupported under connector constraints. - Fix Linter/TS Errors: Refactored DuckDB cleanup state outside the `finally` block to fix ESLint `no-unsafe-finally`, and asserted Zod Schema types to bypass TypeScript `TS2589` infinite recursion. --- .../secondBrain/secondBrainRefresh.service.ts | 10 +- src/main/services/icebergDatalake.service.ts | 99 +++++++++++++++---- src/main/services/staticSite.service.ts | 6 +- .../dataLake/IcebergConnectionWizard.tsx | 62 +++++++----- src/renderer/screens/sql/queryResult.tsx | 3 +- 5 files changed, 128 insertions(+), 52 deletions(-) diff --git a/src/main/services/ai/secondBrain/secondBrainRefresh.service.ts b/src/main/services/ai/secondBrain/secondBrainRefresh.service.ts index 84f815c4..d284afb2 100644 --- a/src/main/services/ai/secondBrain/secondBrainRefresh.service.ts +++ b/src/main/services/ai/secondBrain/secondBrainRefresh.service.ts @@ -215,8 +215,9 @@ const operationZodSchema: z.ZodType = z.object({ // Present a shallow AI SDK Schema to generateObject. Passing the nested Zod // type directly makes TypeScript recursively infer the entire provider/schema // result graph and can trigger TS2589 in the editor language service. -const operationSchema: Schema = - zodSchema(operationZodSchema); +const operationSchema: Schema = zodSchema( + operationZodSchema as any, +) as any as Schema; const stableJson = (value: unknown): string => { if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`; @@ -1085,9 +1086,10 @@ export default class SecondBrainRefreshService { truncated: false, }); } + const icebergBudget = Math.max(0, SOURCE_ITEM_LIMIT - items.length); for (const instance of icebergInstances .filter((candidate) => candidate.sqlAvailable) - .slice(0, SOURCE_ITEM_LIMIT - items.length)) { + .slice(0, icebergBudget)) { assertNotCancelled(abortSignal); let schemaSummary: Array> = []; let schemaTruncated = false; @@ -1133,7 +1135,7 @@ export default class SecondBrainRefreshService { projects.length > SOURCE_ITEM_LIMIT || connections.length > SOURCE_ITEM_LIMIT || icebergInstances.filter((instance) => instance.sqlAvailable).length > - SOURCE_ITEM_LIMIT - connections.length, + icebergBudget, }; } diff --git a/src/main/services/icebergDatalake.service.ts b/src/main/services/icebergDatalake.service.ts index 30847260..d7701750 100644 --- a/src/main/services/icebergDatalake.service.ts +++ b/src/main/services/icebergDatalake.service.ts @@ -909,6 +909,17 @@ export class IcebergDatalakeService { : instance.catalogName; if (!warehouse) throw new Error('ICEBERG_REQUIRED_FIELD: catalogName'); + // DuckDB's Iceberg ATTACH has no branch/tag selector: honour the configured + // reference by rejecting SQL access when a non-default reference is set. + if (instance.catalogType === 'nessie') { + const ref = instance.nessieReference?.trim(); + if (ref && ref !== 'main') { + throw new Error( + 'ICEBERG_SQL_NESSIE_REFERENCE_UNSUPPORTED: DuckDB ATTACH does not support Nessie branch or tag selection; configure the main branch or use the catalog API.', + ); + } + } + return { storageSecretSql: `CREATE TEMPORARY SECRET ${IcebergDatalakeService.quoteSqlIdentifier( names.storageSecret, @@ -949,6 +960,8 @@ export class IcebergDatalakeService { static async listInstances(): Promise { try { const instances = await IcebergDatalakeService.readInstances(); + // Reuse the already-loaded instances and avoid redundant DB reads + + // keychain lookups that getSqlCapability(id) would incur per instance. return Promise.all( instances.map(async (instance) => { const { @@ -963,7 +976,8 @@ export class IcebergDatalakeService { createdAt, updatedAt, } = instance; - const capability = await IcebergDatalakeService.getSqlCapability(id); + const capability = + await IcebergDatalakeService.getSqlCapabilityFromInstance(instance); return { id, name, @@ -1477,8 +1491,20 @@ export class IcebergDatalakeService { } } + /** Public: look up by id (for callers that don't already have the instance). */ static async getSqlCapability(id: string): Promise { const instance = await IcebergDatalakeService.getInstance(id); + return IcebergDatalakeService.getSqlCapabilityFromInstance(instance); + } + + /** + * Private: compute capability from an already-loaded instance config. + * Used by listInstances to avoid redundant readInstances / loadDatabaseFile + * and secureStorage.getCredential calls for every item in the list. + */ + private static async getSqlCapabilityFromInstance( + instance: IcebergInstanceConfig, + ): Promise { const runtimeFingerprint = IcebergDatalakeService.getSqlRuntimeFingerprint(); if (!instance.sqlEnabled) { @@ -1599,11 +1625,18 @@ export class IcebergDatalakeService { } } + /** + * @param strictCleanup When true (used by verifySqlAccess), any cleanup + * failure throws. When false (default), only instance-closure failure + * throws; DETACH / DROP SECRET failures are logged and the completed + * result is returned so committed mutations are not silently discarded. + */ private static async withAttachedSqlCatalog( instanceId: string, executionId: string, callback: (connection: any, alias: string) => Promise, signal?: AbortSignal, + strictCleanup = false, ): Promise { if (!executionId.trim() || executionId.length > 120) { throw new Error('ICEBERG_SQL_EXECUTION_ID_INVALID'); @@ -1611,6 +1644,9 @@ export class IcebergDatalakeService { if (IcebergDatalakeService.activeSqlExecutions.has(executionId)) { throw new Error('ICEBERG_SQL_EXECUTION_ID_DUPLICATE'); } + // Reserve the id synchronously so a concurrent call cannot pass the + // duplicate check while the async setup is still in progress. + IcebergDatalakeService.activeSqlExecutions.set(executionId, null); const instance = await IcebergDatalakeService.getInstance(instanceId); const suffix = uuidv4().replace(/-/g, ''); const names = { @@ -1633,6 +1669,7 @@ export class IcebergDatalakeService { let stage = 'initialize'; let result!: T; let cleanupFailed = false; + let instanceCloseFailed = false; try { checkCancelled(); duckdbInstance = await DuckDBInstance.create(':memory:'); @@ -1715,9 +1752,27 @@ export class IcebergDatalakeService { duckdbInstance?.closeSync?.(); } catch { cleanupFailed = true; + instanceCloseFailed = true; + } + } + // If the instance could not be closed, session + credentials may still + // be open: always throw so the caller does not treat this as success. + if (instanceCloseFailed) { + throw new Error('ICEBERG_SQL_CLEANUP_FAILED'); + } + if (cleanupFailed) { + if (strictCleanup) { + // verifySqlAccess must not persist verification after any cleanup failure. + throw new Error('ICEBERG_SQL_CLEANUP_FAILED'); } + // DETACH or DROP SECRET failed, but the DuckDB instance is closed. + // Session state cannot leak. Log a warning and return the completed + // result so committed mutations are not falsely reported as failures. + // eslint-disable-next-line no-console + console.error( + '[IcebergDatalakeService] SQL session partial cleanup failed (DETACH/DROP SECRET)', + ); } - if (cleanupFailed) throw new Error('ICEBERG_SQL_CLEANUP_FAILED'); return result; } @@ -1872,25 +1927,27 @@ export class IcebergDatalakeService { }); // eslint-disable-next-line no-await-in-loop const tableNames = await IcebergDatalakeService.listTables(id, namespace); - // eslint-disable-next-line no-await-in-loop - const tables = await Promise.all( - tableNames.map(async (table) => { - const schema = await IcebergDatalakeService.getTableSchema( - id, - namespace, - table, - ); - return { - name: table, - type: 'TABLE', - columns: schema.fields.map((field, fieldIndex) => ({ - name: field.name, - type: field.type, - position: fieldIndex + 1, - })), - }; - }), - ); + // Each bridge call spawns a Python process; keep the fan-out bounded by + // loading table schemas sequentially (mirrors the namespace traversal above). + const tables: IcebergSqlSchemaInfo['namespaces'][number]['tables'] = []; + // eslint-disable-next-line no-restricted-syntax + for (const table of tableNames) { + // eslint-disable-next-line no-await-in-loop + const schema = await IcebergDatalakeService.getTableSchema( + id, + namespace, + table, + ); + tables.push({ + name: table, + type: 'TABLE', + columns: schema.fields.map((field, fieldIndex) => ({ + name: field.name, + type: field.type, + position: fieldIndex + 1, + })), + }); + } namespaces.push({ name: namespace.join('.'), tables }); } diff --git a/src/main/services/staticSite.service.ts b/src/main/services/staticSite.service.ts index f3242dbb..3c3f4c9d 100644 --- a/src/main/services/staticSite.service.ts +++ b/src/main/services/staticSite.service.ts @@ -186,8 +186,7 @@ async function executeQueryInMain(params: { instanceId, executionId, sql, - pageLimit: MAX_ROWS_PER_QUERY, - pageOffset: 0, + maxRows: MAX_ROWS_PER_QUERY, validateOnly: true, }); if (classification.statementClass !== 'select') { @@ -202,8 +201,7 @@ async function executeQueryInMain(params: { instanceId, executionId, sql, - pageLimit: MAX_ROWS_PER_QUERY, - pageOffset: 0, + maxRows: MAX_ROWS_PER_QUERY, }); return { data: response.rows.slice(0, MAX_ROWS_PER_QUERY), diff --git a/src/renderer/components/dataLake/IcebergConnectionWizard.tsx b/src/renderer/components/dataLake/IcebergConnectionWizard.tsx index 35ef9887..01ad3ad5 100644 --- a/src/renderer/components/dataLake/IcebergConnectionWizard.tsx +++ b/src/renderer/components/dataLake/IcebergConnectionWizard.tsx @@ -521,20 +521,31 @@ export const IcebergConnectionWizard: React.FC< prefix?: string, provider?: IcebergCloudProvider, ) => { - setStorageTestResult(null); - setData((current) => ({ - ...current, - sql: { - ...current.sql, - connectionId, - bucket, - prefix, - provider, - warehouseMatchAcknowledged: false, - accessVerifiedAt: undefined, - runtimeFingerprint: undefined, - }, - })); + setData((current) => { + const isSame = + current.sql.connectionId === connectionId && + current.sql.bucket === bucket && + current.sql.prefix === prefix && + current.sql.provider === provider; + + if (isSame) return current; + + setTimeout(() => setStorageTestResult(null), 0); + + return { + ...current, + sql: { + ...current.sql, + connectionId, + bucket, + prefix, + provider, + warehouseMatchAcknowledged: false, + accessVerifiedAt: undefined, + runtimeFingerprint: undefined, + }, + }; + }); }, [], ); @@ -662,7 +673,7 @@ export const IcebergConnectionWizard: React.FC< setStorageTestResult({ success: result.success, message: result.success - ? 'The matching object-store location is accessible. DuckDB attachment verification is completed in Phase 3.' + ? 'The matching object-store location is accessible. Use "Test SQL Access" on the Review step to verify DuckDB attachment.' : (result.error ?? 'Object-store access test failed.'), }); } catch (error: any) { @@ -678,13 +689,20 @@ export const IcebergConnectionWizard: React.FC< const handleVerifySqlAccess = async () => { if (!initialData?.id) return; setSqlTestResult(null); - const result = await verifySqlMutation.mutateAsync(initialData.id); - setSqlTestResult({ - success: result.success, - message: result.success - ? 'DuckDB attached to the catalog and cleaned up successfully.' - : (result.error ?? 'DuckDB SQL access test failed.'), - }); + try { + const result = await verifySqlMutation.mutateAsync(initialData.id); + setSqlTestResult({ + success: result.success, + message: result.success + ? 'DuckDB attached to the catalog and cleaned up successfully.' + : (result.error ?? 'DuckDB SQL access test failed.'), + }); + } catch (error: any) { + setSqlTestResult({ + success: false, + message: error?.message ?? 'DuckDB SQL access test failed.', + }); + } }; // ── Step content renderers ────────────────────────────────────────────── diff --git a/src/renderer/screens/sql/queryResult.tsx b/src/renderer/screens/sql/queryResult.tsx index e5df414a..00591ff3 100644 --- a/src/renderer/screens/sql/queryResult.tsx +++ b/src/renderer/screens/sql/queryResult.tsx @@ -408,7 +408,8 @@ export const QueryResult: React.FC = ({ results, exportContext }) => { const canExportParquet = !!exportContext && !!resolvedOriginalSql && - (exportContext.connectionType === 'duckdb' || + ((exportContext.connectionType === 'duckdb' && + !exportContext.connectionId?.startsWith('iceberg-')) || exportContext.connectionType === 'ducklake'); const handleExportParquet = async () => { From 230d579eae59771335385f01c735ea251690935b Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Thu, 10 Sep 2026 16:36:05 +0200 Subject: [PATCH 12/16] refactor: fix indentation in notebooks service cell update logic --- src/main/services/notebooks.service.ts | 32 +++++++++++++------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/main/services/notebooks.service.ts b/src/main/services/notebooks.service.ts index 7d29178f..e1c4d248 100644 --- a/src/main/services/notebooks.service.ts +++ b/src/main/services/notebooks.service.ts @@ -1357,22 +1357,22 @@ export class NotebooksService { // Limit output data size to prevent massive files const limitedOutput = limitCellOutputData(output); - const updatedCells = notebook.cells.map((cell) => - cell.id === cellId - ? { - ...cell, - output: limitedOutput, - ...(connectionId.startsWith('iceberg-') - ? { - status: - output.type === 'error' - ? ('error' as const) - : ('success' as const), - } - : {}), - } - : cell, - ); + const updatedCells = notebook.cells.map((cell) => + cell.id === cellId + ? { + ...cell, + output: limitedOutput, + ...(connectionId.startsWith('iceberg-') + ? { + status: + output.type === 'error' + ? ('error' as const) + : ('success' as const), + } + : {}), + } + : cell, + ); const updatedNotebook: Notebook = { ...notebook, From 24d2774771890df7151631ed64456cba50d01fbf Mon Sep 17 00:00:00 2001 From: Nuri Lacka Date: Wed, 16 Sep 2026 16:03:33 +0200 Subject: [PATCH 13/16] feat(iceberg): improve wizard SQL access testing and table controls - Use the configured projects directory as the default catalog path - Add disabled Create Table and Register Table actions - Simplify namespace selection with default selected - Move Test SQL Access into Storage configuration - Test SQL access using the current wizard state - Add DuckDB icons to SQL access controls - Improve long error message wrapping - Align the Description field with the DuckLake wizard --- .../icebergDatalake.ipcHandlers.ts | 4 +- src/main/services/icebergDatalake.service.ts | 28 +- .../dataLake/IcebergConnectionWizard.tsx | 261 ++++++++++++------ .../dataLake/iceberg/IcebergDetail.tsx | 71 +++-- .../iceberg/IcebergTableImportWizard.tsx | 72 +---- .../controllers/icebergDatalake.controller.ts | 13 +- src/renderer/services/iceberg.service.ts | 3 +- src/types/iceberg.ts | 2 +- 8 files changed, 285 insertions(+), 169 deletions(-) diff --git a/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts b/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts index 21681058..aad518d4 100644 --- a/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts +++ b/src/main/ipcHandlers/icebergDatalake.ipcHandlers.ts @@ -54,8 +54,8 @@ export const registerIcebergDatalakeHandlers = () => { IcebergDatalakeService.getSqlSchema(id), ); - ipcMain.handle('iceberg:verifySqlAccess', (_e, id: string) => - IcebergDatalakeService.verifySqlAccess(id), + ipcMain.handle('iceberg:verifySqlAccess', (_e, id: string, draft) => + IcebergDatalakeService.verifySqlAccess(id, draft), ); ipcMain.handle('iceberg:executeSql', (_e, params) => diff --git a/src/main/services/icebergDatalake.service.ts b/src/main/services/icebergDatalake.service.ts index d7701750..de1320ac 100644 --- a/src/main/services/icebergDatalake.service.ts +++ b/src/main/services/icebergDatalake.service.ts @@ -84,7 +84,7 @@ export class IcebergDatalakeService { }, { type: 'sql', - label: 'PostgreSQL / Neon', + label: 'PostgreSQL', pyicebergType: 'sql', enabled: true, requiredFields: ['databaseConnectionId', 'catalogName'], @@ -1557,9 +1557,15 @@ export class IcebergDatalakeService { }; } - static async verifySqlAccess(id: string): Promise { + static async verifySqlAccess( + id: string, + draft?: Partial, + ): Promise { try { - const verifiedInstance = await IcebergDatalakeService.getInstance(id); + const savedInstance = await IcebergDatalakeService.getInstance(id); + const verifiedInstance = draft + ? ({ ...savedInstance, ...draft } as IcebergInstanceConfig) + : savedInstance; const executionId = `verify-${uuidv4()}`; await IcebergDatalakeService.withAttachedSqlCatalog( id, @@ -1587,7 +1593,18 @@ export class IcebergDatalakeService { throw new Error('ICEBERG_SQL_NONEMPTY_TABLE_REQUIRED'); } }, + undefined, + verifiedInstance, + true, ); + if (draft) { + return { + success: true, + catalogConnected: true, + warehouseConnected: true, + checkedAt: new Date().toISOString(), + }; + } const runtimeFingerprint = IcebergDatalakeService.getSqlRuntimeFingerprint(); const instances = await IcebergDatalakeService.readInstances(); @@ -1636,6 +1653,7 @@ export class IcebergDatalakeService { executionId: string, callback: (connection: any, alias: string) => Promise, signal?: AbortSignal, + instanceOverride?: IcebergInstanceConfig, strictCleanup = false, ): Promise { if (!executionId.trim() || executionId.length > 120) { @@ -1647,7 +1665,9 @@ export class IcebergDatalakeService { // Reserve the id synchronously so a concurrent call cannot pass the // duplicate check while the async setup is still in progress. IcebergDatalakeService.activeSqlExecutions.set(executionId, null); - const instance = await IcebergDatalakeService.getInstance(instanceId); + const instance = + instanceOverride ?? + (await IcebergDatalakeService.getInstance(instanceId)); const suffix = uuidv4().replace(/-/g, ''); const names = { alias: 'iceberg', diff --git a/src/renderer/components/dataLake/IcebergConnectionWizard.tsx b/src/renderer/components/dataLake/IcebergConnectionWizard.tsx index 01ad3ad5..81e1afdd 100644 --- a/src/renderer/components/dataLake/IcebergConnectionWizard.tsx +++ b/src/renderer/components/dataLake/IcebergConnectionWizard.tsx @@ -47,7 +47,11 @@ import type { IcebergInstanceConfig, IcebergStorageType, } from '../../../types/iceberg'; -import { useFilePicker, useGetConnections } from '../../controllers'; +import { + useFilePicker, + useGetConnections, + useGetSettings, +} from '../../controllers'; import { useCreateIcebergMetadataFile, useIcebergCapabilities, @@ -58,7 +62,10 @@ import { } from '../../controllers/icebergDatalake.controller'; import { DataLakeConnectionSelector } from './DataLakeConnectionSelector'; import { secureStorageService } from '../../services/secureStorage.service'; -import { icebergCatalogImages } from '../../../../assets/connectionIcons'; +import { + databaseIcons, + icebergCatalogImages, +} from '../../../../assets/connectionIcons'; // ─── Wizard Data ───────────────────────────────────────────────────────────── @@ -205,7 +212,7 @@ function validateStep( data.catalog.catalogType === 'sql' && !data.catalog.databaseConnectionId ) { - return 'A PostgreSQL or Neon connection is required.'; + return 'A PostgreSQL connection is required.'; } if (data.catalog.catalogType === 'sql' && !data.catalog.catalogName) { return 'SQL catalog name is required.'; @@ -323,6 +330,8 @@ export const IcebergConnectionWizard: React.FC< message: string; } | null>(null); const initializedInstanceIdRef = useRef(null); + const { data: settings } = useGetSettings(); + const defaultProjectPath = settings?.projectsDirectory?.trim() || ''; useEffect(() => { if (!initialData || initializedInstanceIdRef.current === initialData.id) { @@ -422,7 +431,7 @@ export const IcebergConnectionWizard: React.FC< const pickFolder = (setter: (path: string) => void) => { getFiles( - { properties: ['openDirectory'] }, + { properties: ['openDirectory'], defaultPath: defaultProjectPath }, { onSuccess: (filePaths) => { if (filePaths && filePaths.length > 0) { @@ -478,10 +487,29 @@ export const IcebergConnectionWizard: React.FC< const patchCatalog = (patch: Partial) => setData((d) => ({ ...d, catalog: { ...d.catalog, ...patch } })); + useEffect(() => { + if ( + mode === 'create' && + activeStep === 1 && + data.catalog.catalogType === 'sqlite' && + !data.catalog.catalogPath && + defaultProjectPath + ) { + patchCatalog({ catalogPath: defaultProjectPath }); + } + }, [ + activeStep, + data.catalog.catalogPath, + data.catalog.catalogType, + defaultProjectPath, + mode, + ]); + const patchStorage = (patch: Partial) => setData((d) => ({ ...d, storage: { ...d.storage, ...patch } })); - const patchSql = (patch: Partial) => + const patchSql = (patch: Partial) => { + setSqlTestResult(null); setData((d) => ({ ...d, sql: { @@ -491,6 +519,7 @@ export const IcebergConnectionWizard: React.FC< runtimeFingerprint: undefined, }, })); + }; const handleSelectCloudStorage = useCallback( ( @@ -673,7 +702,7 @@ export const IcebergConnectionWizard: React.FC< setStorageTestResult({ success: result.success, message: result.success - ? 'The matching object-store location is accessible. Use "Test SQL Access" on the Review step to verify DuckDB attachment.' + ? 'The matching object-store location is accessible. Use "Test SQL Access" below to verify DuckDB attachment.' : (result.error ?? 'Object-store access test failed.'), }); } catch (error: any) { @@ -690,7 +719,29 @@ export const IcebergConnectionWizard: React.FC< if (!initialData?.id) return; setSqlTestResult(null); try { - const result = await verifySqlMutation.mutateAsync(initialData.id); + const result = await verifySqlMutation.mutateAsync({ + id: initialData.id, + draft: { + catalogType: data.catalog.catalogType, + catalogPath: data.catalog.catalogPath, + endpoint: data.catalog.endpoint, + catalogName: data.catalog.catalogName, + catalogAuthMode: data.catalog.authMode, + databaseConnectionId: data.catalog.databaseConnectionId, + storageType: data.storage.storageType, + localPath: data.storage.localPath, + cloudProvider: data.storage.cloudProvider, + storageConnectionId: data.storage.connectionId, + storageBucket: data.storage.bucket, + storagePrefix: data.storage.prefix, + sqlEnabled: data.sql.enabled, + sqlStorageConnectionId: data.sql.connectionId, + sqlStorageProvider: data.sql.provider, + sqlStorageBucket: data.sql.bucket, + sqlStoragePrefix: data.sql.prefix, + sqlWarehouseMatchAcknowledged: data.sql.warehouseMatchAcknowledged, + }, + }); setSqlTestResult({ success: result.success, message: result.success @@ -723,14 +774,11 @@ export const IcebergConnectionWizard: React.FC< helperText="A unique name to identify this Iceberg instance (max 80 chars)" /> patchBasics({ description: e.target.value })} fullWidth - multiline - rows={2} - helperText="Optional human-readable description" /> ); @@ -855,8 +903,8 @@ export const IcebergConnectionWizard: React.FC< onChange={(e) => patchCatalog({ catalogPath: e.target.value })} fullWidth required - placeholder="/data/my-catalog/pyiceberg_catalog.db" - helperText="Choose a folder to initialize a SQLite catalog and local warehouse" + placeholder="Local project folder path" + helperText="Defaults to the current project folder" slotProps={{ input: { endAdornment: ( @@ -884,10 +932,10 @@ export const IcebergConnectionWizard: React.FC< {data.catalog.catalogType === 'sql' && ( <> - PostgreSQL / Neon Connection + PostgreSQL Connection