Speed up status snapshot loading

native/macos/CuedNative/Sources/CuedNative/RootWindowController.swift

@@ -203,11 +203,6 @@ final class RootWindowController: NSWindowController {
         return
       }
 
-      let globalSkill = Self.decodeJSON(
-        daemonSupervisor: daemonSupervisor,
-        SkillStatus.self,
-        arguments: ["skill", "status"]
-      ) ?? snapshot.globalSkill
       let setupIntegrations = snapshot.setupIntegrations.filter {
         $0.capability.setupVisible
       }
@@ -222,7 +217,7 @@ final class RootWindowController: NSWindowController {
         self.isRefreshing = false
         self.viewModel.apply(
           permissions: permissions,
-          globalSkill: globalSkill,
+          globalSkill: snapshot.globalSkill,
           relationshipSummary: snapshot.relationshipSummary ?? RelationshipSummary(),
           allIntegrations: self.cachedAllIntegrations,
           integrations: setupIntegrations

src/cli.ts

@@ -58,6 +58,7 @@ import {
 import { readMenuBarStatusCache } from "./runtime/menu-bar-status-cache.js";
 import { buildOnboardingSnapshot } from "./runtime/onboarding.js";
 import { runProjectionWorkerFromEnv } from "./runtime/projection/worker.js";
+import { hydrateIntegrationProjectionCounts } from "./runtime/status.js";
 import {
   checkForUpdates,
   clearUpdateHelperPendingState,
@@ -584,7 +585,10 @@ async function main(): Promise<void> {
           lastUpdatedAt: Date.now(),
           source: "menu_bar_cache",
         },
-        integrations: listMenuBarIntegrationStates(db),
+        integrations: hydrateIntegrationProjectionCounts(
+          listMenuBarIntegrationStates(db),
+          overview.integrationBreakdown,
+        ),
         update: getUpdateStatus(db),
         socketRunning: existsSync(CUED_SOCKET_PATH),
         dbPath: db.dbPath,

src/db/database.test.ts

@@ -70,6 +70,114 @@ describe("CuedDatabase", () => {
     db.close();
   });
 
+  it("uses the normalized schema expression index for raw event diagnostics", () => {
+    const db = createDb();
+    const sqlite = (
+      db as unknown as {
+        sqlite: {
+          prepare: (sql: string) => {
+            all: (...params: unknown[]) => Array<{ detail: string }>;
+          };
+        };
+      }
+    ).sqlite;
+    const phonePlan = sqlite
+      .prepare(
+        `
+        EXPLAIN QUERY PLAN
+        SELECT COUNT(*)
+        FROM raw_events
+        WHERE platform = ?
+          AND account_key = ?
+          AND COALESCE(normalized_schema, entity_kind || '.' || event_kind || '@1') = ?
+      `,
+      )
+      .all("imessage", "local", "call.observed@1");
+    const schemaPlan = sqlite
+      .prepare(
+        `
+        EXPLAIN QUERY PLAN
+        SELECT
+          COALESCE(normalized_schema, entity_kind || '.' || event_kind || '@1') AS schema_key,
+          COUNT(*)
+        FROM raw_events
+        WHERE platform = ? AND account_key = ?
+        GROUP BY schema_key
+      `,
+      )
+      .all("imessage", "local");
+
+    expect(phonePlan.map((row) => row.detail).join("\n")).toContain(
+      "idx_raw_events_platform_account_normalized_schema",
+    );
+    expect(schemaPlan.map((row) => row.detail).join("\n")).toContain(
+      "idx_raw_events_platform_account_normalized_schema",
+    );
+    expect(schemaPlan.map((row) => row.detail).join("\n")).not.toContain(
+      "USE TEMP B-TREE FOR GROUP BY",
+    );
+
+    db.close();
+  });
+
+  it("groups menu bar projection counts by integration", () => {
+    const db = createDb();
+    const sqlite = (
+      db as unknown as {
+        sqlite: { exec: (sql: string) => void };
+      }
+    ).sqlite;
+    sqlite.exec(`
+      INSERT INTO contacts (id, kind, name, archived, created_at, updated_at)
+      VALUES ('contact-1', 'person', 'Ava', 0, 1, 1);
+      INSERT INTO contact_sources (
+        id, contact_id, platform, account_key, source_entity_key, first_seen_at, last_seen_at
+      ) VALUES ('source-1', 'contact-1', 'imessage', 'local', 'ava', 1, 1);
+      INSERT INTO conversations (
+        id, platform, account_key, source_conversation_key, type, is_active,
+        unread_count, created_at, updated_at
+      ) VALUES
+        ('conversation-1', 'imessage', 'local', 'chat-1', 'dm', 1, 0, 1, 1),
+        ('conversation-2', 'slack', 'workspace', 'channel-1', 'channel', 1, 0, 1, 1);
+      INSERT INTO messages (
+        id, platform, account_key, platform_message_id, conversation_id, sent_at,
+        is_from_me, is_deleted, is_edited, attachment_count, reaction_count, created_at, updated_at
+      ) VALUES
+        ('message-1', 'imessage', 'local', 'message-1', 'conversation-1', 1, 0, 0, 0, 0, 0, 1, 1),
+        ('message-2', 'slack', 'workspace', 'message-2', 'conversation-2', 1, 0, 0, 0, 0, 0, 1, 1);
+    `);
+
+    expect(db.listProjectionCountsByIntegration()).toEqual(
+      expect.arrayContaining([
+        { platform: "imessage", accountKey: "local", contacts: 1, messages: 1 },
+        { platform: "slack", accountKey: "workspace", contacts: 0, messages: 1 },
+      ]),
+    );
+    sqlite.exec(`
+      INSERT INTO messages (
+        id, platform, account_key, platform_message_id, conversation_id, sent_at,
+        is_from_me, is_deleted, is_edited, attachment_count, reaction_count, created_at, updated_at
+      ) VALUES (
+        'message-3', 'imessage', 'local', 'message-3', 'conversation-1', 2,
+        0, 0, 0, 0, 0, 2, 2
+      );
+    `);
+    expect(
+      db
+        .listProjectionCountsByIntegration()
+        .find((row) => row.platform === "imessage" && row.accountKey === "local")?.messages,
+    ).toBe(1);
+
+    db.upsertProjectionState({ projectionWatermark: 1 });
+    expect(
+      db
+        .listProjectionCountsByIntegration()
+        .find((row) => row.platform === "imessage" && row.accountKey === "local")?.messages,
+    ).toBe(2);
+
+    db.close();
+  });
+
   function insertContact(
     db: CuedDatabase,
     input: {

src/db/database.ts

@@ -446,6 +446,13 @@ type WhatsAppSendCandidate = {
   resolution: Extract<WhatsAppSendResolution["resolution"], "whatsapp_jid" | "phone">;
 };
 
+export interface IntegrationProjectionCount {
+  platform: Platform;
+  accountKey: string;
+  contacts: number;
+  messages: number;
+}
+
 const SIGNAL_UUID_PATTERN =
   /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
 
@@ -581,6 +588,11 @@ function buildRawEventValues(event: RawEventInput) {
 export class CuedDatabase {
   private readonly sqlite: Database.Database;
   private readonly db: LocalDrizzleDatabase;
+  private menuBarProjectionCountsCache: {
+    projectionWatermark: number;
+    lastRebuildAt: number | null;
+    rows: IntegrationProjectionCount[];
+  } | null = null;
 
   constructor(
     public readonly dbPath: string = CUED_DB_PATH,
@@ -1416,8 +1428,22 @@ export class CuedDatabase {
       platform: Platform;
       messages: number;
     }>;
+    integrationBreakdown: IntegrationProjectionCount[];
   } {
-    const messageBreakdown = this.listMessageCountsByPlatform();
+    const integrationBreakdown = this.listProjectionCountsByIntegration();
+    const messagesByPlatform = new Map<Platform, number>();
+    for (const row of integrationBreakdown) {
+      messagesByPlatform.set(
+        row.platform,
+        (messagesByPlatform.get(row.platform) ?? 0) + row.messages,
+      );
+    }
+    const messageBreakdown = [...messagesByPlatform]
+      .map(([platform, messages]) => ({ platform, messages }))
+      .sort(
+        (left, right) =>
+          right.messages - left.messages || left.platform.localeCompare(right.platform),
+      );
     return {
       contacts: this.countRows(contacts),
       conversations: this.countRows(conversations),
@@ -1427,9 +1453,68 @@ export class CuedDatabase {
       integrations: this.countRows(integrationStates),
       authSessions: this.countRows(authSessions),
       messageBreakdown,
+      integrationBreakdown,
     };
   }
 
+  listProjectionCountsByIntegration(): IntegrationProjectionCount[] {
+    const projectionState = this.getProjectionState({ initialize: false });
+    if (
+      this.menuBarProjectionCountsCache?.projectionWatermark ===
+        projectionState.projection_watermark &&
+      this.menuBarProjectionCountsCache.lastRebuildAt === projectionState.last_rebuild_at
+    ) {
+      return this.menuBarProjectionCountsCache.rows;
+    }
+
+    const byIntegration = new Map<string, IntegrationProjectionCount>();
+    const getOrCreate = (platform: Platform, accountKey: string) => {
+      const key = `${platform}\u0000${accountKey}`;
+      const existing = byIntegration.get(key);
+      if (existing) {
+        return existing;
+      }
+      const created = { platform, accountKey, contacts: 0, messages: 0 };
+      byIntegration.set(key, created);
+      return created;
+    };
+    const contactCounts = this.sqlite
+      .prepare(
+        `
+        SELECT platform, account_key, COUNT(*) AS count
+        FROM contact_sources
+        GROUP BY platform, account_key
+      `,
+      )
+      .all() as Array<{ platform: Platform; account_key: string; count: number | null }>;
+    for (const row of contactCounts) {
+      getOrCreate(row.platform, row.account_key).contacts = Number(row.count ?? 0);
+    }
+    const messageCounts = this.sqlite
+      .prepare(
+        `
+        SELECT platform, account_key, COUNT(*) AS count
+        FROM messages
+        GROUP BY platform, account_key
+      `,
+      )
+      .all() as Array<{ platform: Platform; account_key: string; count: number | null }>;
+    for (const row of messageCounts) {
+      getOrCreate(row.platform, row.account_key).messages = Number(row.count ?? 0);
+    }
+    const rows = [...byIntegration.values()];
+    this.menuBarProjectionCountsCache = {
+      projectionWatermark: projectionState.projection_watermark,
+      lastRebuildAt: projectionState.last_rebuild_at,
+      rows,
+    };
+    return rows;
+  }
+
+  invalidateMenuBarProjectionCounts(): void {
+    this.menuBarProjectionCountsCache = null;
+  }
+
   listMessageCountsByPlatform(): Array<{
     platform: Platform;
     messages: number;
@@ -1602,7 +1687,7 @@ export class CuedDatabase {
       .prepare(
         `
         SELECT
-          COALESCE(normalized_schema, entity_kind || '.' || event_kind) AS schema_key,
+          COALESCE(normalized_schema, entity_kind || '.' || event_kind || '@1') AS schema_key,
           COUNT(*) AS count
         FROM raw_events
         WHERE platform = ? AND account_key = ?
@@ -4908,6 +4993,7 @@ export class CuedDatabase {
       tx.delete(contactSources).run();
       tx.delete(contacts).run();
     });
+    this.invalidateMenuBarProjectionCounts();
   }
 
   private countRows(table: SQLiteTable): number {

src/db/migrations.ts

@@ -2229,4 +2229,20 @@ export const MIGRATIONS: Migration[] = [
       `);
     },
   },
+  {
+    id: "0021_raw_event_normalized_schema_index",
+    apply: (db) => {
+      if (!tableExists(db, "raw_events")) {
+        return;
+      }
+      db.exec(`
+        CREATE INDEX IF NOT EXISTS idx_raw_events_platform_account_normalized_schema
+        ON raw_events(
+          platform,
+          account_key,
+          COALESCE(normalized_schema, entity_kind || '.' || event_kind || '@1')
+        );
+      `);
+    },
+  },
 ];

src/platforms/core/state/status.ts

@@ -131,6 +131,11 @@ const AUTH_SESSION_STALE_AFTER_MS = 15 * 60 * 1000;
 const AUTH_SESSION_ABANDONED_ERROR = "Auth session ended before Cued received a completion event";
 const AUTH_SESSION_EXPIRED_ERROR = "Auth session expired before completion";
 
+interface IntegrationStatusOptions {
+  includeLiveLocalIntegrations?: boolean;
+  includeDiagnostics?: boolean;
+}
+
 export function now(): number {
   return Date.now();
 }
@@ -687,7 +692,8 @@ function compareSetupIntegrations(
 
 function buildSetupIntegrations(
   db: CuedDatabase,
-  options: { includeLiveLocalIntegrations?: boolean; includeDiagnostics?: boolean } = {},
+  options: IntegrationStatusOptions = {},
+  existingIntegrations?: IntegrationStateSummary[],
 ): IntegrationStateSummary[] {
   const onboardingOrder: Platform[] = [
     "contacts",
@@ -700,7 +706,7 @@ function buildSetupIntegrations(
     "signal",
   ];
   const byPlatform = new Map<Platform, IntegrationStateSummary>();
-  for (const integration of listIntegrationStates(db, options)) {
+  for (const integration of existingIntegrations ?? listIntegrationStates(db, options)) {
     if (!isOnboardingVisiblePlatform(integration.platform)) {
       continue;
     }
@@ -895,15 +901,16 @@ export function getPlatformRuntimeDefaults(platform: Platform): {
 
 export function buildIntegrationStatus(
   db: CuedDatabase,
-  options: { includeLiveLocalIntegrations?: boolean; includeDiagnostics?: boolean } = {},
+  options: IntegrationStatusOptions = {},
 ): {
   hostOs: ReturnType<typeof resolveHostOS>;
   integrations: IntegrationStateSummary[];
   setupIntegrations: IntegrationStateSummary[];
 } {
+  const integrations = listIntegrationStates(db, options);
   return {
     hostOs: resolveHostOS(),
-    integrations: listIntegrationStates(db, options),
-    setupIntegrations: buildSetupIntegrations(db, options),
+    integrations,
+    setupIntegrations: buildSetupIntegrations(db, options, integrations),
   };
 }