diff --git a/.gitignore b/.gitignore index cc154914b..1ed8c6916 100644 --- a/.gitignore +++ b/.gitignore @@ -37,9 +37,12 @@ yarn-error.log [Rr]elease/ [Rr]eleases/ api/lib/ -# we need to include the installer image +# we need to include the installer image build/ bld/ + +# DuckDB extension build output +src/main/build/duckdb-extensions/ [Oo]bj/ lib/cpp/loot/loot/ lib/cpp/loot/loot_api/docs/ diff --git a/InstallAssets.json b/InstallAssets.json index a4d058c93..a723b34c6 100644 --- a/InstallAssets.json +++ b/InstallAssets.json @@ -1,6 +1,12 @@ { "spawn": [], "copy": [ + { + "srcPath": "src/main/build/duckdb-extensions/**/*.duckdb_extension", + "outPath": "duckdb-extensions", + "skipPaths": 4, + "target": ["out", "dist"] + }, { "srcPath": "./LICENSE.md", "outPath": "", diff --git a/package.json b/package.json index c608db6a9..599e072d2 100644 --- a/package.json +++ b/package.json @@ -18,11 +18,11 @@ "build": "pnpm run typecheck && pnpm --filter \"@vortex/*\" -r run build", "build:all": "pnpm run build && pnpm run build:extensions && pnpm run build:assets", "build:extensions": "pnpm run api && pnpm run typecheck:extensions && pnpm --filter \"./extensions/**\" run build", - "build:assets": "node ./scripts/dependency-report.mjs && node ./InstallAssets.mjs ./src/main/out && pnpm sass --style compressed --silence-deprecation=import ./src/stylesheets/loadingScreen.scss ./src/main/out/assets/css/loadingScreen.css && pnpm tailwindcss -i ./src/stylesheets/tailwind-v4.css -o ./src/main/out/assets/css/tailwind-v4.css -m", + "build:assets": "npx tsx scripts/download-duckdb-extensions.ts && node ./scripts/dependency-report.mjs && node ./InstallAssets.mjs ./src/main/out && pnpm sass --style compressed --silence-deprecation=import ./src/stylesheets/loadingScreen.scss ./src/main/out/assets/css/loadingScreen.css && pnpm tailwindcss -i ./src/stylesheets/tailwind-v4.css -o ./src/main/out/assets/css/tailwind-v4.css -m", "dist": "pnpm run typecheck && pnpm --filter \"@vortex/*\" -r run dist", "dist:all": "pnpm run dist && pnpm run dist:extensions && pnpm run dist:assets", "dist:extensions": "pnpm run api && pnpm --filter \"./extensions/**\" run dist", - "dist:assets": "node ./scripts/dependency-report.mjs && node ./InstallAssets.mjs ./src/main/dist && pnpm sass --style compressed --silence-deprecation=import ./src/stylesheets/loadingScreen.scss ./src/main/dist/assets/css/loadingScreen.css && pnpm tailwindcss -i ./src/stylesheets/tailwind-v4.css -o ./src/main/dist/assets/css/tailwind-v4.css -m", + "dist:assets": "npx tsx scripts/download-duckdb-extensions.ts && node ./scripts/dependency-report.mjs && node ./InstallAssets.mjs ./src/main/dist && pnpm sass --style compressed --silence-deprecation=import ./src/stylesheets/loadingScreen.scss ./src/main/dist/assets/css/loadingScreen.css && pnpm tailwindcss -i ./src/stylesheets/tailwind-v4.css -o ./src/main/dist/assets/css/tailwind-v4.css -m", "package": "pnpm run dist:all && pnpm -F @vortex/main run package", "package:nosign": "pnpm run dist:all && pnpm -F @vortex/main run package:nosign", "typecheck": "pnpm -F @vortex/shared run build && pnpm -F @vortex/paths run build && pnpm --filter \"@vortex/*\" -r run typecheck", diff --git a/scripts/download-duckdb-extensions.test.ts b/scripts/download-duckdb-extensions.test.ts new file mode 100644 index 000000000..109903006 --- /dev/null +++ b/scripts/download-duckdb-extensions.test.ts @@ -0,0 +1,56 @@ +import { describe, it, expect } from "vitest"; +import { parseDuckDBVersion, buildExtensionUrl } from "./download-duckdb-extensions"; + +describe("parseDuckDBVersion", () => { + it("strips the -r.X suffix and prepends v", () => { + expect(parseDuckDBVersion("1.5.1-r.1")).toBe("v1.5.1"); + }); + + it("works with higher revision numbers", () => { + expect(parseDuckDBVersion("1.10.0-r.42")).toBe("v1.10.0"); + }); + + it("throws on unexpected version format", () => { + expect(() => parseDuckDBVersion("1.5.1")).toThrow(/unexpected/i); + expect(() => parseDuckDBVersion("1.5.1-rc.1")).toThrow(/unexpected/i); + expect(() => parseDuckDBVersion("not-a-version")).toThrow(/unexpected/i); + }); +}); + +describe("buildExtensionUrl", () => { + it("builds a correct http extension URL", () => { + const url = buildExtensionUrl({ + type: "http", + name: "level_pivot", + repository: "https://halgari.github.io/duckdb-level-pivot/current_release", + version: "v1.5.1", + platform: "windows_amd64", + }); + expect(url).toBe( + "https://halgari.github.io/duckdb-level-pivot/current_release/v1.5.1/windows_amd64/level_pivot.duckdb_extension.gz" + ); + }); + + it("builds a correct community extension URL", () => { + const url = buildExtensionUrl({ + type: "community", + name: "delta", + version: "v1.5.1", + platform: "linux_amd64", + }); + expect(url).toBe( + "https://community-extensions.duckdb.org/v1/v1.5.1/linux_amd64/delta.duckdb_extension.gz" + ); + }); + + it("throws when http extension is missing repository", () => { + expect(() => + buildExtensionUrl({ + type: "http", + name: "my_ext", + version: "v1.5.1", + platform: "windows_amd64", + }) + ).toThrow(/repository/i); + }); +}); diff --git a/scripts/download-duckdb-extensions.ts b/scripts/download-duckdb-extensions.ts new file mode 100644 index 000000000..d971777cd --- /dev/null +++ b/scripts/download-duckdb-extensions.ts @@ -0,0 +1,185 @@ +import * as fs from "node:fs"; +import * as path from "node:path"; +import * as https from "node:https"; +import * as zlib from "node:zlib"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface ExtensionConfig { + platforms: string[]; + outputDir: string; + extensions: ExtensionEntry[]; +} + +interface ExtensionEntry { + name: string; + type: "http" | "community"; + repository?: string; +} + +interface BuildUrlOptions { + type: "http" | "community"; + name: string; + version: string; + platform: string; + repository?: string; +} + +// --------------------------------------------------------------------------- +// Pure functions (exported for testing) +// --------------------------------------------------------------------------- + +/** + * Parses a @duckdb/node-api version string (e.g. "1.5.1-r.1") into the + * DuckDB core version string (e.g. "v1.5.1") used in extension download URLs. + */ +export function parseDuckDBVersion(rawVersion: string): string { + const match = rawVersion.match(/^(\d+\.\d+\.\d+)-r\.\d+$/); + if (match === null) { + throw new Error( + `Unexpected @duckdb/node-api version format: "${rawVersion}". ` + + `Expected pattern: "..-r."` + ); + } + return `v${match[1]}`; +} + +/** + * Constructs the download URL for a single extension/platform combination. + */ +export function buildExtensionUrl(opts: BuildUrlOptions): string { + const { type, name, version, platform, repository } = opts; + + if (type === "community") { + return `https://community-extensions.duckdb.org/v1/${version}/${platform}/${name}.duckdb_extension.gz`; + } + + if (type === "http") { + if (!repository) { + throw new Error( + `Extension "${name}" has type "http" but is missing a "repository" field.` + ); + } + return `${repository}/${version}/${platform}/${name}.duckdb_extension.gz`; + } + + throw new Error(`Unknown extension type: "${type as string}"`); +} + +// --------------------------------------------------------------------------- +// I/O helpers +// --------------------------------------------------------------------------- + +function downloadFile(url: string, destPath: string): Promise { + return new Promise((resolve, reject) => { + const dir = path.dirname(destPath); + fs.mkdirSync(dir, { recursive: true }); + + const file = fs.createWriteStream(destPath); + + const request = (url: string) => { + https.get(url, (res) => { + if (res.statusCode === 301 || res.statusCode === 302) { + // Follow redirect + file.destroy(); + fs.unlinkSync(destPath); + request(res.headers.location!); + return; + } + if (res.statusCode !== 200) { + file.destroy(); + fs.unlinkSync(destPath); + reject(new Error(`HTTP ${res.statusCode} downloading ${url}`)); + return; + } + const gunzip = zlib.createGunzip(); + res.pipe(gunzip).pipe(file); + file.on("finish", () => { + file.close(); + resolve(); + }); + gunzip.on("error", (err) => { + file.destroy(); + fs.unlinkSync(destPath); + reject(err); + }); + file.on("error", (err) => { + fs.unlinkSync(destPath); + reject(err); + }); + }).on("error", reject); + }; + + request(url); + }); +} + +// --------------------------------------------------------------------------- +// Main +// --------------------------------------------------------------------------- + +async function main(): Promise { + const configPath = path.resolve(__dirname, "duckdb-extensions.json"); + const config: ExtensionConfig = JSON.parse(fs.readFileSync(configPath, "utf8")); + + // Detect DuckDB version from the installed @duckdb/node-api package + const nodeApiPkgPath = path.resolve( + __dirname, + "../src/main/node_modules/@duckdb/node-api/package.json" + ); + const nodeApiPkg = JSON.parse(fs.readFileSync(nodeApiPkgPath, "utf8")); + const duckdbVersion = parseDuckDBVersion(nodeApiPkg.version as string); + + console.log(`DuckDB version: ${duckdbVersion}`); + + const outputDir = path.resolve(__dirname, "..", config.outputDir); + + for (const ext of config.extensions) { + for (const platform of config.platforms) { + const url = buildExtensionUrl({ + type: ext.type, + name: ext.name, + version: duckdbVersion, + platform, + repository: ext.repository, + }); + + const destPath = path.join( + outputDir, + duckdbVersion, + platform, + `${ext.name}.duckdb_extension` + ); + + if (fs.existsSync(destPath)) { + console.log(` skip ${ext.name} [${platform}] — already exists`); + continue; + } + + console.log(` download ${ext.name} [${platform}]`); + console.log(` from: ${url}`); + console.log(` to: ${destPath}`); + await downloadFile(url, destPath); + console.log(` ✓ ${ext.name} [${platform}]`); + } + } + + console.log("Done."); +} + +// Only run when executed directly (not when imported for testing) +const isMain = + typeof process.argv[1] === "string" && + import.meta.url === `file:///${process.argv[1].replace(/\\/g, "/")}`; + +if (isMain) { + main().catch((err: unknown) => { + console.error(err); + process.exit(1); + }); +} diff --git a/scripts/duckdb-extensions.json b/scripts/duckdb-extensions.json new file mode 100644 index 000000000..24b0779a5 --- /dev/null +++ b/scripts/duckdb-extensions.json @@ -0,0 +1,11 @@ +{ + "platforms": ["windows_amd64", "linux_amd64"], + "outputDir": "src/main/build/duckdb-extensions", + "extensions": [ + { + "name": "level_pivot", + "type": "http", + "repository": "https://halgari.github.io/duckdb-level-pivot/current_release" + } + ] +} diff --git a/scripts/vitest.config.ts b/scripts/vitest.config.ts new file mode 100644 index 000000000..c944129c8 --- /dev/null +++ b/scripts/vitest.config.ts @@ -0,0 +1,9 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + name: "scripts", + environment: "node", + include: ["*.test.ts"], + }, +}); diff --git a/src/main/electron-builder.config.json b/src/main/electron-builder.config.json index 959276248..f215fa01d 100644 --- a/src/main/electron-builder.config.json +++ b/src/main/electron-builder.config.json @@ -65,7 +65,8 @@ "node_modules/@nexusmods/fomod-installer-ipc/dist/*.exe", "assets/*.exe", "assets/css/**", - "**/*.node" + "**/*.node", + "duckdb-extensions" ], "buildDependenciesFromSource": false, "npmRebuild": false diff --git a/src/main/src/getVortexPath.ts b/src/main/src/getVortexPath.ts index 78ccaeb0c..f60eb6608 100644 --- a/src/main/src/getVortexPath.ts +++ b/src/main/src/getVortexPath.ts @@ -22,6 +22,7 @@ const electronAppInfoEnv: { [key: string]: string | undefined } = bundledPlugins: process.env.ELECTRON_BUNDLEDPLUGINS, locales: process.env.ELECTRON_LOCALES, base: process.env.ELECTRON_BASE, + base_unpacked: process.env.ELECTRON_BASE_UNPACKED, application: process.env.ELECTRON_APPLICATION, package: process.env.ELECTRON_PACKAGE, package_unpacked: process.env.ELECTRON_PACKAGE_UNPACKED, @@ -165,6 +166,8 @@ export function getVortexPath(id: keyof VortexPaths): string { return cachedAppPath("desktop"); case "base": return basePath; + case "base_unpacked": + return isAsar ? basePath + ".unpacked" : basePath; case "application": return applicationPath; case "package": diff --git a/src/main/src/ipcHandlers.ts b/src/main/src/ipcHandlers.ts index c63b24947..5ecb4dca3 100644 --- a/src/main/src/ipcHandlers.ts +++ b/src/main/src/ipcHandlers.ts @@ -46,6 +46,7 @@ export function init() { function resolveVortexPaths(): VortexPaths { const paths: VortexPaths = { base: getVortexPath("base"), + base_unpacked: getVortexPath("base_unpacked"), assets: getVortexPath("assets"), assets_unpacked: getVortexPath("assets_unpacked"), modules: getVortexPath("modules"), diff --git a/src/main/src/main.ts b/src/main/src/main.ts index 3a0bc8166..eb3fedef3 100644 --- a/src/main/src/main.ts +++ b/src/main/src/main.ts @@ -252,6 +252,7 @@ async function main(): Promise { ), ELECTRON_LOCALES: path.resolve(app.getAppPath(), "..", "locales"), ELECTRON_BASE: app.getAppPath(), + ELECTRON_BASE_UNPACKED: app.getAppPath() + ".unpacked", ELECTRON_APPLICATION: path.resolve(app.getAppPath(), ".."), ELECTRON_PACKAGE: app.getAppPath(), ELECTRON_PACKAGE_UNPACKED: path.join( diff --git a/src/main/src/store/DuckDBSingleton.ts b/src/main/src/store/DuckDBSingleton.ts index 842b76cda..b2eeb94e2 100644 --- a/src/main/src/store/DuckDBSingleton.ts +++ b/src/main/src/store/DuckDBSingleton.ts @@ -26,10 +26,15 @@ class DuckDBSingleton { } /** - * Initialize the shared DuckDB instance, installing and loading level_pivot. + * Initialize the shared DuckDB instance, loading level_pivot from the + * pre-downloaded extension cache directory. * Safe to call multiple times -- only initializes once. + * + * @param extensionDir - Path to the duckdb-extensions folder produced by the + * download script (e.g. `/duckdb-extensions`). DuckDB looks for + * extensions under `{extensionDir}/{version}/{platform}/`. */ - public initialize(): Promise { + public initialize(extensionDir: string): Promise { if (this.#mInitialized) { return Promise.resolve(); } @@ -40,15 +45,14 @@ class DuckDBSingleton { } this.#mInitPromise = (async () => { - log("debug", "duckdb-singleton: creating shared instance"); + log("debug", "duckdb-singleton: creating shared instance", { extensionDir }); this.#mDuckDB = await DuckDBInstance.create(":memory:", { allow_unsigned_extensions: "true", + extension_directory: extensionDir, }); const connection = await this.#mDuckDB.connect(); try { - log("debug", "duckdb-singleton: installing level_pivot"); - await connection.run("FORCE INSTALL level_pivot FROM 'https://halgari.github.io/duckdb-level-pivot/current_release'"); log("debug", "duckdb-singleton: loading level_pivot"); await connection.run("LOAD level_pivot"); } finally { diff --git a/src/main/src/store/LevelPersist.ts b/src/main/src/store/LevelPersist.ts index 934b25191..9f948e017 100644 --- a/src/main/src/store/LevelPersist.ts +++ b/src/main/src/store/LevelPersist.ts @@ -5,7 +5,10 @@ import type { DuckDBConnection } from "@duckdb/node-api"; import { unknownToError } from "@vortex/shared"; import { DataInvalid } from "@vortex/shared/errors"; +import * as path from "node:path"; + import { log } from "../logging"; +import { getVortexPath } from "../getVortexPath"; import DuckDBSingleton from "./DuckDBSingleton"; const SEPARATOR: string = "###"; @@ -35,7 +38,8 @@ class LevelPersist implements IPersistor { } try { const singleton = DuckDBSingleton.getInstance(); - await singleton.initialize(); + const extensionDir = path.join(getVortexPath("base_unpacked"), "duckdb-extensions"); + await singleton.initialize(extensionDir); const alias = singleton.nextAlias(); const connection = await singleton.attachDatabase(persistPath, alias); diff --git a/src/shared/src/types/ipc.ts b/src/shared/src/types/ipc.ts index d1511384c..31adc0471 100644 --- a/src/shared/src/types/ipc.ts +++ b/src/shared/src/types/ipc.ts @@ -67,6 +67,7 @@ export interface UpdateStatus { /** Vortex application paths */ export type VortexPaths = { base: string; + base_unpacked: string; assets: string; assets_unpacked: string; modules: string; diff --git a/vitest.config.ts b/vitest.config.ts index 5b82dc3ff..e721f8559 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,6 +9,7 @@ export default defineConfig({ test: { projects: [ "./src/main", + "./scripts", "./src/main/vitest.integration.config.ts", "./src/renderer", "./src/shared",