feat(APP-204): pre-cache DuckDB extensions, remove runtime download

.gitignore

@@ -37,9 +37,12 @@ yarn-error.log
 [Rr]elease/
 [Rr]eleases/
 api/lib/
-# we need to include the installer image 
+# we need to include the installer image
 build/
 bld/
+
+# DuckDB extension build output
+src/main/build/duckdb-extensions/
 [Oo]bj/
 lib/cpp/loot/loot/
 lib/cpp/loot/loot_api/docs/

InstallAssets.json

@@ -1,6 +1,12 @@
 {
   "spawn": [],
   "copy": [
+    {
+      "srcPath": "src/main/build/duckdb-extensions/**/*.duckdb_extension",
+      "outPath": "duckdb-extensions",
+      "skipPaths": 4,
+      "target": ["out", "dist"]
+    },
     {
       "srcPath": "./LICENSE.md",
       "outPath": "",

package.json

@@ -18,11 +18,11 @@
     "build": "pnpm run typecheck && pnpm --filter \"@vortex/*\" -r run build",
     "build:all": "pnpm run build && pnpm run build:extensions && pnpm run build:assets",
     "build:extensions": "pnpm run api && pnpm run typecheck:extensions && pnpm --filter \"./extensions/**\" run build",
-    "build:assets": "node ./scripts/dependency-report.mjs && node ./InstallAssets.mjs ./src/main/out && pnpm sass --style compressed --silence-deprecation=import ./src/stylesheets/loadingScreen.scss ./src/main/out/assets/css/loadingScreen.css && pnpm tailwindcss -i ./src/stylesheets/tailwind-v4.css -o ./src/main/out/assets/css/tailwind-v4.css -m",
+    "build:assets": "npx tsx scripts/download-duckdb-extensions.ts && node ./scripts/dependency-report.mjs && node ./InstallAssets.mjs ./src/main/out && pnpm sass --style compressed --silence-deprecation=import ./src/stylesheets/loadingScreen.scss ./src/main/out/assets/css/loadingScreen.css && pnpm tailwindcss -i ./src/stylesheets/tailwind-v4.css -o ./src/main/out/assets/css/tailwind-v4.css -m",
     "dist": "pnpm run typecheck && pnpm --filter \"@vortex/*\" -r run dist",
     "dist:all": "pnpm run dist && pnpm run dist:extensions && pnpm run dist:assets",
     "dist:extensions": "pnpm run api && pnpm --filter \"./extensions/**\" run dist",
-    "dist:assets": "node ./scripts/dependency-report.mjs && node ./InstallAssets.mjs ./src/main/dist && pnpm sass --style compressed --silence-deprecation=import ./src/stylesheets/loadingScreen.scss ./src/main/dist/assets/css/loadingScreen.css && pnpm tailwindcss -i ./src/stylesheets/tailwind-v4.css -o ./src/main/dist/assets/css/tailwind-v4.css -m",
+    "dist:assets": "npx tsx scripts/download-duckdb-extensions.ts && node ./scripts/dependency-report.mjs && node ./InstallAssets.mjs ./src/main/dist && pnpm sass --style compressed --silence-deprecation=import ./src/stylesheets/loadingScreen.scss ./src/main/dist/assets/css/loadingScreen.css && pnpm tailwindcss -i ./src/stylesheets/tailwind-v4.css -o ./src/main/dist/assets/css/tailwind-v4.css -m",
     "package": "pnpm run dist:all && pnpm -F @vortex/main run package",
     "package:nosign": "pnpm run dist:all && pnpm -F @vortex/main run package:nosign",
     "typecheck": "pnpm -F @vortex/shared run build && pnpm -F @vortex/paths run build && pnpm --filter \"@vortex/*\" -r run typecheck",

scripts/download-duckdb-extensions.test.ts

@@ -0,0 +1,56 @@
+import { describe, it, expect } from "vitest";
+import { parseDuckDBVersion, buildExtensionUrl } from "./download-duckdb-extensions";
+
+describe("parseDuckDBVersion", () => {
+  it("strips the -r.X suffix and prepends v", () => {
+    expect(parseDuckDBVersion("1.5.1-r.1")).toBe("v1.5.1");
+  });
+
+  it("works with higher revision numbers", () => {
+    expect(parseDuckDBVersion("1.10.0-r.42")).toBe("v1.10.0");
+  });
+
+  it("throws on unexpected version format", () => {
+    expect(() => parseDuckDBVersion("1.5.1")).toThrow(/unexpected/i);
+    expect(() => parseDuckDBVersion("1.5.1-rc.1")).toThrow(/unexpected/i);
+    expect(() => parseDuckDBVersion("not-a-version")).toThrow(/unexpected/i);
+  });
+});
+
+describe("buildExtensionUrl", () => {
+  it("builds a correct http extension URL", () => {
+    const url = buildExtensionUrl({
+      type: "http",
+      name: "level_pivot",
+      repository: "https://halgari.github.io/duckdb-level-pivot/current_release",
+      version: "v1.5.1",
+      platform: "windows_amd64",
+    });
+    expect(url).toBe(
+      "https://halgari.github.io/duckdb-level-pivot/current_release/v1.5.1/windows_amd64/level_pivot.duckdb_extension.gz"
+    );
+  });
+
+  it("builds a correct community extension URL", () => {
+    const url = buildExtensionUrl({
+      type: "community",
+      name: "delta",
+      version: "v1.5.1",
+      platform: "linux_amd64",
+    });
+    expect(url).toBe(
+      "https://community-extensions.duckdb.org/v1/v1.5.1/linux_amd64/delta.duckdb_extension.gz"
+    );
+  });
+
+  it("throws when http extension is missing repository", () => {
+    expect(() =>
+      buildExtensionUrl({
+        type: "http",
+        name: "my_ext",
+        version: "v1.5.1",
+        platform: "windows_amd64",
+      })
+    ).toThrow(/repository/i);
+  });
+});

scripts/download-duckdb-extensions.ts

@@ -0,0 +1,185 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as https from "node:https";
+import * as zlib from "node:zlib";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+// ---------------------------------------------------------------------------
+// Types
+// ---------------------------------------------------------------------------
+
+interface ExtensionConfig {
+  platforms: string[];
+  outputDir: string;
+  extensions: ExtensionEntry[];
+}
+
+interface ExtensionEntry {
+  name: string;
+  type: "http" | "community";
+  repository?: string;
+}
+
+interface BuildUrlOptions {
+  type: "http" | "community";
+  name: string;
+  version: string;
+  platform: string;
+  repository?: string;
+}
+
+// ---------------------------------------------------------------------------
+// Pure functions (exported for testing)
+// ---------------------------------------------------------------------------
+
+/**
+ * Parses a @duckdb/node-api version string (e.g. "1.5.1-r.1") into the
+ * DuckDB core version string (e.g. "v1.5.1") used in extension download URLs.
+ */
+export function parseDuckDBVersion(rawVersion: string): string {
+  const match = rawVersion.match(/^(\d+\.\d+\.\d+)-r\.\d+$/);
+  if (match === null) {
+    throw new Error(
+      `Unexpected @duckdb/node-api version format: "${rawVersion}". ` +
+        `Expected pattern: "<major>.<minor>.<patch>-r.<n>"`
+    );
+  }
+  return `v${match[1]}`;
+}
+
+/**
+ * Constructs the download URL for a single extension/platform combination.
+ */
+export function buildExtensionUrl(opts: BuildUrlOptions): string {
+  const { type, name, version, platform, repository } = opts;
+
+  if (type === "community") {
+    return `https://community-extensions.duckdb.org/v1/${version}/${platform}/${name}.duckdb_extension.gz`;
+  }
+
+  if (type === "http") {
+    if (!repository) {
+      throw new Error(
+        `Extension "${name}" has type "http" but is missing a "repository" field.`
+      );
+    }
+    return `${repository}/${version}/${platform}/${name}.duckdb_extension.gz`;
+  }
+
+  throw new Error(`Unknown extension type: "${type as string}"`);
+}
+
+// ---------------------------------------------------------------------------
+// I/O helpers
+// ---------------------------------------------------------------------------
+
+function downloadFile(url: string, destPath: string): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const dir = path.dirname(destPath);
+    fs.mkdirSync(dir, { recursive: true });
+
+    const file = fs.createWriteStream(destPath);
+
+    const request = (url: string) => {
+      https.get(url, (res) => {
+        if (res.statusCode === 301 || res.statusCode === 302) {
+          // Follow redirect
+          file.destroy();
+          fs.unlinkSync(destPath);
+          request(res.headers.location!);
+          return;
+        }
+        if (res.statusCode !== 200) {
+          file.destroy();
+          fs.unlinkSync(destPath);
+          reject(new Error(`HTTP ${res.statusCode} downloading ${url}`));
+          return;
+        }
+        const gunzip = zlib.createGunzip();
+        res.pipe(gunzip).pipe(file);
+        file.on("finish", () => {
+          file.close();
+          resolve();
+        });
+        gunzip.on("error", (err) => {
+          file.destroy();
+          fs.unlinkSync(destPath);
+          reject(err);
+        });
+        file.on("error", (err) => {
+          fs.unlinkSync(destPath);
+          reject(err);
+        });
+      }).on("error", reject);
+    };
+
+    request(url);
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+async function main(): Promise<void> {
+  const configPath = path.resolve(__dirname, "duckdb-extensions.json");
+  const config: ExtensionConfig = JSON.parse(fs.readFileSync(configPath, "utf8"));
+
+  // Detect DuckDB version from the installed @duckdb/node-api package
+  const nodeApiPkgPath = path.resolve(
+    __dirname,
+    "../src/main/node_modules/@duckdb/node-api/package.json"
+  );
+  const nodeApiPkg = JSON.parse(fs.readFileSync(nodeApiPkgPath, "utf8"));
+  const duckdbVersion = parseDuckDBVersion(nodeApiPkg.version as string);
+
+  console.log(`DuckDB version: ${duckdbVersion}`);
+
+  const outputDir = path.resolve(__dirname, "..", config.outputDir);
+
+  for (const ext of config.extensions) {
+    for (const platform of config.platforms) {
+      const url = buildExtensionUrl({
+        type: ext.type,
+        name: ext.name,
+        version: duckdbVersion,
+        platform,
+        repository: ext.repository,
+      });
+
+      const destPath = path.join(
+        outputDir,
+        duckdbVersion,
+        platform,
+        `${ext.name}.duckdb_extension`
+      );
+
+      if (fs.existsSync(destPath)) {
+        console.log(`  skip  ${ext.name} [${platform}] — already exists`);
+        continue;
+      }
+
+      console.log(`  download  ${ext.name} [${platform}]`);
+      console.log(`    from: ${url}`);
+      console.log(`    to:   ${destPath}`);
+      await downloadFile(url, destPath);
+      console.log(`  ✓ ${ext.name} [${platform}]`);
+    }
+  }
+
+  console.log("Done.");
+}
+
+// Only run when executed directly (not when imported for testing)
+const isMain =
+  typeof process.argv[1] === "string" &&
+  import.meta.url === `file:///${process.argv[1].replace(/\\/g, "/")}`;
+
+if (isMain) {
+  main().catch((err: unknown) => {
+    console.error(err);
+    process.exit(1);
+  });
+}

scripts/duckdb-extensions.json

@@ -0,0 +1,11 @@
+{
+  "platforms": ["windows_amd64", "linux_amd64"],
+  "outputDir": "src/main/build/duckdb-extensions",
+  "extensions": [
+    {
+      "name": "level_pivot",
+      "type": "http",
+      "repository": "https://halgari.github.io/duckdb-level-pivot/current_release"
+    }
+  ]
+}

scripts/vitest.config.ts

@@ -0,0 +1,9 @@
+import { defineConfig } from "vitest/config";
+
+export default defineConfig({
+  test: {
+    name: "scripts",
+    environment: "node",
+    include: ["*.test.ts"],
+  },
+});

src/main/electron-builder.config.json

@@ -65,7 +65,8 @@
     "node_modules/@nexusmods/fomod-installer-ipc/dist/*.exe",
     "assets/*.exe",
     "assets/css/**",
-    "**/*.node"
+    "**/*.node",
+    "duckdb-extensions"
   ],
   "buildDependenciesFromSource": false,
   "npmRebuild": false

src/main/src/getVortexPath.ts

@@ -22,6 +22,7 @@ const electronAppInfoEnv: { [key: string]: string | undefined } =
         bundledPlugins: process.env.ELECTRON_BUNDLEDPLUGINS,
         locales: process.env.ELECTRON_LOCALES,
         base: process.env.ELECTRON_BASE,
+        base_unpacked: process.env.ELECTRON_BASE_UNPACKED,
         application: process.env.ELECTRON_APPLICATION,
         package: process.env.ELECTRON_PACKAGE,
         package_unpacked: process.env.ELECTRON_PACKAGE_UNPACKED,
@@ -165,6 +166,8 @@ export function getVortexPath(id: keyof VortexPaths): string {
       return cachedAppPath("desktop");
     case "base":
       return basePath;
+    case "base_unpacked":
+      return isAsar ? basePath + ".unpacked" : basePath;
     case "application":
       return applicationPath;
     case "package":

src/main/src/ipcHandlers.ts

@@ -46,6 +46,7 @@ export function init() {
   function resolveVortexPaths(): VortexPaths {
     const paths: VortexPaths = {
       base: getVortexPath("base"),
+      base_unpacked: getVortexPath("base_unpacked"),
       assets: getVortexPath("assets"),
       assets_unpacked: getVortexPath("assets_unpacked"),
       modules: getVortexPath("modules"),

src/main/src/main.ts

@@ -252,6 +252,7 @@ async function main(): Promise<void> {
           ),
           ELECTRON_LOCALES: path.resolve(app.getAppPath(), "..", "locales"),
           ELECTRON_BASE: app.getAppPath(),
+          ELECTRON_BASE_UNPACKED: app.getAppPath() + ".unpacked",
           ELECTRON_APPLICATION: path.resolve(app.getAppPath(), ".."),
           ELECTRON_PACKAGE: app.getAppPath(),
           ELECTRON_PACKAGE_UNPACKED: path.join(

src/main/src/store/DuckDBSingleton.ts

@@ -26,10 +26,15 @@ class DuckDBSingleton {
   }
 
   /**
-   * Initialize the shared DuckDB instance, installing and loading level_pivot.
+   * Initialize the shared DuckDB instance, loading level_pivot from the
+   * pre-downloaded extension cache directory.
    * Safe to call multiple times -- only initializes once.
+   *
+   * @param extensionDir - Path to the duckdb-extensions folder produced by the
+   *   download script (e.g. `<appBase>/duckdb-extensions`). DuckDB looks for
+   *   extensions under `{extensionDir}/{version}/{platform}/`.
    */
-  public initialize(): Promise<void> {
+  public initialize(extensionDir: string): Promise<void> {
     if (this.#mInitialized) {
       return Promise.resolve();
     }
@@ -40,15 +45,14 @@ class DuckDBSingleton {
     }
 
     this.#mInitPromise = (async () => {
-      log("debug", "duckdb-singleton: creating shared instance");
+      log("debug", "duckdb-singleton: creating shared instance", { extensionDir });
       this.#mDuckDB = await DuckDBInstance.create(":memory:", {
         allow_unsigned_extensions: "true",
+        extension_directory: extensionDir,
       });
 
       const connection = await this.#mDuckDB.connect();
       try {
-        log("debug", "duckdb-singleton: installing level_pivot");
-        await connection.run("FORCE INSTALL level_pivot FROM 'https://halgari.github.io/duckdb-level-pivot/current_release'");
         log("debug", "duckdb-singleton: loading level_pivot");
         await connection.run("LOAD level_pivot");
       } finally {