refactor: studio database store

e2e/fixtures/electron-seeded.fixture.ts

@@ -3,6 +3,7 @@ import { test as base, ElectronApplication, Page } from '@playwright/test';
 import * as path from 'path';
 import * as fs from 'fs';
 import * as os from 'os';
+import { CURRENT_SCHEMA_VERSION } from '../../src/main/database/migrations';
 
 export type TestFixtures = {
   electronApp: ElectronApplication;
@@ -47,6 +48,7 @@ export const test = base.extend<TestFixtures>({
 
     // Seed database.json with test project and connection
     const databaseJson = {
+      schemaVersion: CURRENT_SCHEMA_VERSION,
       settings: {
         isSetup: 'true',
         pythonPath:

e2e/fixtures/electron.fixture.ts

@@ -13,13 +13,23 @@ import { test as base, ElectronApplication, Page } from '@playwright/test';
 import * as path from 'path';
 import * as fs from 'fs';
 import * as os from 'os';
+import { CURRENT_SCHEMA_VERSION } from '../../src/main/database/migrations';
 
 // Type definitions for our fixtures
 export type ElectronFixtures = {
   /** Isolated user data directory for this test */
   userData: string;
   /** Whether to automatically skip the setup wizard by seeding database.json */
   autoSkipSetup: boolean;
+  /**
+   * Project names to seed into database.json before the app launches. Each
+   * gets a minimal project directory + dbt_project.yml created automatically.
+   * Use this instead of writing to database.json after the app is already
+   * running — the app only ever re-reads the file on its own operations, so
+   * an external write made while it's live has no defined way to be picked
+   * up short of restarting it.
+   */
+  extraProjects: string[];
   /** The Electron application instance */
   electronApp: ElectronApplication;
   /** The main browser window */
@@ -32,6 +42,7 @@ export type ElectronFixtures = {
 export const test = base.extend<ElectronFixtures>({
   // Default to skipping setup for convenience in most tests
   autoSkipSetup: [true, { option: true }],
+  extraProjects: [[], { option: true }],
 
   // Create isolated userData directory for each test
   // biome-ignore lint/complexity/noEmptyPattern: Playwright requires object destructuring
@@ -58,7 +69,11 @@ export const test = base.extend<ElectronFixtures>({
   },
 
   // Launch Electron app
-  electronApp: async ({ userData, autoSkipSetup }, use, testInfo) => {
+  electronApp: async (
+    { userData, autoSkipSetup, extraProjects },
+    use,
+    testInfo,
+  ) => {
     // Helper to seed database if skipping setup
     if (autoSkipSetup) {
       // Create projects directory
@@ -80,7 +95,24 @@ export const test = base.extend<ElectronFixtures>({
 
       const dbPath = path.join(userData, 'database.json');
 
+      const seededProjects = extraProjects.map((name) => {
+        const projectPath = path.join(userData, 'projects', name);
+        fs.mkdirSync(projectPath, { recursive: true });
+        fs.writeFileSync(
+          path.join(projectPath, 'dbt_project.yml'),
+          `name: ${name}\nversion: 1.0.0\nconfig-version: 2\n`,
+        );
+        return {
+          id: `${name}-id`,
+          name,
+          path: projectPath,
+          createdAt: new Date().toISOString(),
+          isExtracted: false,
+        };
+      });
+
       const settings = {
+        schemaVersion: CURRENT_SCHEMA_VERSION,
         settings: {
           isSetup: 'true',
           pythonPath:
@@ -94,7 +126,7 @@ export const test = base.extend<ElectronFixtures>({
           dbtSampleDirectory: path.join(userData, 'dbt_sample'),
           sampleRosettaMainConf: path.join(userData, 'main.conf'),
         },
-        projects: [],
+        projects: seededProjects,
         connections: [],
       };
       fs.writeFileSync(dbPath, JSON.stringify(settings, null, 2));

e2e/tests/projects/project-lifecycle.spec.ts

@@ -1,6 +1,4 @@
 import { Page, ElectronApplication } from '@playwright/test';
-import * as fs from 'fs';
-import * as path from 'path';
 import { test, expect } from '../../fixtures/electron.fixture';
 import { ProjectSelectionPage } from '../../page-objects/screens/ProjectSelection';
 import { AppHelper } from '../../helpers/app.helper';
@@ -68,119 +66,53 @@ test.describe('Project Lifecycle', () => {
     await expect(sidebar).toBeVisible();
   });
 
-  test('should open existing project', async ({ electronApp, userData }) => {
-    // Seed project manually since tests run in isolation
-    const projectPath = path.join(userData, 'projects', 'Test_Project');
-    const dbPath = path.join(userData, 'database.json');
-
-    // Create project directory and minimal dbt_project.yml
-    if (!fs.existsSync(projectPath)) {
-      fs.mkdirSync(projectPath, { recursive: true });
-      fs.writeFileSync(
-        path.join(projectPath, 'dbt_project.yml'),
-        'name: Test_Project\nversion: 1.0.0\nconfig-version: 2\n',
-      );
-    }
-
-    // Update database.json
-    const db = JSON.parse(fs.readFileSync(dbPath, 'utf8'));
-    // Avoid duplicate seeding
-    if (!db.projects.some((p: any) => p.name === 'Test_Project')) {
-      db.projects.push({
-        id: 'test-project-id',
-        name: 'Test_Project',
-        path: projectPath,
-        createdAt: new Date().toISOString(),
-        isExtracted: false,
-      });
-      fs.writeFileSync(dbPath, JSON.stringify(db, null, 2));
-    }
+  test.describe('with an existing project seeded', () => {
+    // Seeded into database.json before the app launches (see
+    // electron.fixture.ts) rather than written to the file mid-test — the
+    // app only re-reads database.json on its own operations, so a write
+    // made to it while the app is already running has no defined way to be
+    // observed short of restarting the app.
+    test.use({ extraProjects: ['Test_Project'] });
 
-    const stableWindow = await findStableWindow(electronApp);
-    // Reload window to ensure renderer picks up the DB changes
-    await stableWindow.reload();
-    await stableWindow.waitForLoadState('domcontentloaded');
+    test('should open existing project', async ({ electronApp }) => {
+      const stableWindow = await findStableWindow(electronApp);
+      const projectSelection = new ProjectSelectionPage(stableWindow);
 
-    // Wait for project selection screen again
-    await stableWindow.waitForSelector('[data-testid="project-selection"]', {
-      timeout: 10000,
-    });
+      await projectSelection.selectProject('Test_Project');
 
-    // Re-attach console listener after reload if needed (Playwright usually keeps it on Page, but handle might change?)
-    // Actually finding stableWindow again might return same page object.
+      // Verify project details screen is shown (or main app)
+      const sidebar = stableWindow.locator('[data-testid="sidebar"]');
+      await expect(sidebar).toBeVisible();
+    });
 
-    const projectSelection = new ProjectSelectionPage(stableWindow);
+    test('should delete a project', async ({ electronApp }) => {
+      const stableWindow = await findStableWindow(electronApp);
 
-    // Verify and Select
-    await projectSelection.selectProject('Test_Project');
+      // Get project card for verification
+      const projectCard = stableWindow.locator(
+        '[data-testid="project-card-Test_Project"]',
+      );
 
-    // Verify project details screen is shown (or main app)
-    const sidebar = stableWindow.locator('[data-testid="sidebar"]');
-    await expect(sidebar).toBeVisible();
-  });
+      // Click options button
+      const optionsBtn = stableWindow.locator(
+        '[data-testid="project-options-Test_Project"]',
+      );
+      await optionsBtn.click();
 
-  test('should delete a project', async ({ electronApp, userData }) => {
-    // Seed project manually since tests run in isolation
-    const projectPath = path.join(userData, 'projects', 'Test_Project');
-    const dbPath = path.join(userData, 'database.json');
-
-    // Create project directory and minimal dbt_project.yml
-    if (!fs.existsSync(projectPath)) {
-      fs.mkdirSync(projectPath, { recursive: true });
-      fs.writeFileSync(
-        path.join(projectPath, 'dbt_project.yml'),
-        'name: Test_Project\nversion: 1.0.0\nconfig-version: 2\n',
+      // Click delete option
+      const deleteOption = stableWindow.locator(
+        '[data-testid="context-menu-delete"]',
       );
-    }
-
-    // Update database.json
-    const db = JSON.parse(fs.readFileSync(dbPath, 'utf8'));
-    // Avoid duplicate seeding
-    if (!db.projects.some((p: any) => p.name === 'Test_Project')) {
-      db.projects.push({
-        id: 'test-project-id',
-        name: 'Test_Project',
-        path: projectPath,
-        createdAt: new Date().toISOString(),
-        isExtracted: false,
-      });
-      fs.writeFileSync(dbPath, JSON.stringify(db, null, 2));
-    }
+      await deleteOption.click();
 
-    const stableWindow = await findStableWindow(electronApp);
-    // Reload window to ensure renderer picks up the DB changes
-    await stableWindow.reload();
-    await stableWindow.waitForLoadState('domcontentloaded');
+      // Confirm deletion
+      const confirmBtn = stableWindow.locator(
+        '[data-testid="confirm-delete-btn"]',
+      );
+      await confirmBtn.click();
 
-    // Wait for project selection screen again
-    await stableWindow.waitForSelector('[data-testid="project-selection"]', {
-      timeout: 10000,
+      // Verify project is removed
+      await expect(projectCard).not.toBeVisible();
     });
-
-    // Get project card for verification
-    const projectCard = stableWindow.locator(
-      '[data-testid="project-card-Test_Project"]',
-    );
-
-    // Click options button
-    const optionsBtn = stableWindow.locator(
-      '[data-testid="project-options-Test_Project"]',
-    );
-    await optionsBtn.click();
-
-    // Click delete option
-    const deleteOption = stableWindow.locator(
-      '[data-testid="context-menu-delete"]',
-    );
-    await deleteOption.click();
-
-    // Confirm deletion
-    const confirmBtn = stableWindow.locator(
-      '[data-testid="confirm-delete-btn"]',
-    );
-    await confirmBtn.click();
-
-    // Verify project is removed
-    await expect(projectCard).not.toBeVisible();
   });
 });

src/main/database/index.ts

@@ -0,0 +1,12 @@
+import { DB_FILE } from '../utils/setupHelpers';
+import { DatabaseStore } from './store';
+
+// Single shared instance for the whole main process — every service should
+// import this rather than constructing its own DatabaseStore, otherwise the
+// serialized queue that makes concurrent access safe would be per-instance
+// instead of per-file.
+const databaseStore = new DatabaseStore(DB_FILE);
+
+export default databaseStore;
+export { DatabaseStore } from './store';
+export { CURRENT_SCHEMA_VERSION } from './migrations';

src/main/database/migrations.ts

@@ -0,0 +1,102 @@
+import { DataBase } from '../../types/backend';
+
+// Bump this whenever a migration is added below. Every database.json ever
+// written before this system existed has no `schemaVersion` field at all,
+// so it is treated as version 0.
+export const CURRENT_SCHEMA_VERSION = 1;
+
+type RawShape = Record<string, unknown> & { schemaVersion?: number };
+
+interface Migration {
+  // The version this migration produces, i.e. it runs for any file whose
+  // recorded version is lower than this number.
+  version: number;
+  migrate: (raw: RawShape) => RawShape;
+}
+
+// Ordered by version, ascending. Each entry transforms the *previous*
+// shape into this version's shape. Keep old migrations forever — they are
+// what let an install from years ago open successfully today.
+const migrations: Migration[] = [
+  {
+    // No structural change: this just stamps every pre-versioning file
+    // (implicit version 0) so it — and every future file — carries an
+    // explicit, auditable schema version from here on.
+    version: 1,
+    migrate: (raw) => ({ ...raw, schemaVersion: 1 }),
+  },
+];
+
+export function pendingMigrations(fromVersion: number): Migration[] {
+  return migrations
+    .filter((m) => m.version > fromVersion)
+    .sort((a, b) => a.version - b.version);
+}
+
+function toShape(rawInput: unknown): RawShape {
+  return typeof rawInput === 'object' && rawInput !== null
+    ? (rawInput as RawShape)
+    : {};
+}
+
+// Fills in safe defaults for every field this build reads/writes.
+// `preserveUnknownKeys` controls whether fields this build doesn't
+// recognize are kept (spread in) or dropped — migrate() drops them
+// because it's reconstructing a known-old shape into the current one;
+// passthroughNewerVersion() keeps them because it has no business
+// deleting fields a *newer* build understands and this one doesn't.
+function withSafeDefaults(
+  shape: RawShape,
+  schemaVersion: number,
+  preserveUnknownKeys: boolean,
+): DataBase {
+  return {
+    ...(preserveUnknownKeys ? shape : {}),
+    schemaVersion,
+    projects: (shape.projects as DataBase['projects']) ?? [],
+    settings:
+      (shape.settings as DataBase['settings']) ?? ({} as DataBase['settings']),
+    selectedProject: shape.selectedProject as DataBase['selectedProject'],
+    queries: (shape.queries as DataBase['queries']) ?? {},
+    savedQueries: shape.savedQueries as DataBase['savedQueries'],
+    connections: (shape.connections as DataBase['connections']) ?? [],
+    sources: (shape.sources as DataBase['sources']) ?? [],
+    recentItems: (shape.recentItems as DataBase['recentItems']) ?? [],
+    icebergInstances: shape.icebergInstances as DataBase['icebergInstances'],
+  } as DataBase;
+}
+
+/**
+ * Runs every pending migration in order and defensively fills in any field
+ * still missing afterwards (covers installs older than the oldest
+ * migration, and callers that hand in a bare `{}`). Never throws on a
+ * merely-incomplete shape — only a genuinely corrupt/unparseable file
+ * should reach the store's recovery path instead of this function.
+ */
+export function migrate(rawInput: unknown): DataBase {
+  let shape = toShape(rawInput);
+
+  const startVersion = shape.schemaVersion ?? 0;
+  pendingMigrations(startVersion).forEach((m) => {
+    shape = m.migrate(shape);
+  });
+
+  return withSafeDefaults(shape, CURRENT_SCHEMA_VERSION, false);
+}
+
+/**
+ * For a file written by a newer build than this one (e.g. after a
+ * downgrade): unlike migrate(), never reconstructs the object from a fixed
+ * key whitelist — any top-level field this build doesn't recognize
+ * survives untouched, and the recorded schemaVersion is left as whatever
+ * the newer build wrote. This build only acts on the fields it knows;
+ * anything it doesn't stays intact so an eventual re-upgrade loses nothing.
+ */
+export function passthroughNewerVersion(rawInput: unknown): DataBase {
+  const shape = toShape(rawInput);
+  return withSafeDefaults(
+    shape,
+    shape.schemaVersion ?? CURRENT_SCHEMA_VERSION,
+    true,
+  );
+}