feat(chat): dictate into the composer with local speech-to-text

apps/desktop/resources/entitlements.mac.inherit.plist

@@ -6,5 +6,7 @@
   <true/>
   <key>com.apple.security.cs.disable-library-validation</key>
   <true/>
+  <key>com.apple.security.device.audio-input</key>
+  <true/>
 </dict>
 </plist>

apps/desktop/resources/entitlements.mac.plist

@@ -6,5 +6,7 @@
   <true/>
   <key>com.apple.security.cs.disable-library-validation</key>
   <true/>
+  <key>com.apple.security.device.audio-input</key>
+  <true/>
 </dict>
 </plist>

apps/desktop/src/settings/DesktopClientSettings.test.ts

@@ -19,6 +19,8 @@ const clientSettings: ClientSettings = {
   confirmThreadDelete: false,
   wrapUpThreadsOnPullRequestSettled: true,
   dismissedProviderUpdateNotificationKeys: [],
+  dictationHoldToRecord: true,
+  dictationMicrophoneDeviceId: null,
   diffChangesOnly: false,
   diffIgnoreWhitespace: true,
   diffRenderMode: "stacked",

apps/desktop/src/window/DesktopWindow.test.ts

@@ -62,6 +62,9 @@ function makeFakeBrowserWindow(input?: {
     openDevTools: vi.fn(),
     replaceMisspelling: vi.fn(),
     send: vi.fn(),
+    session: {
+      setPermissionRequestHandler: vi.fn(),
+    },
     setWindowOpenHandler: vi.fn(),
   };
 

apps/desktop/src/window/DesktopWindow.ts

@@ -13,7 +13,7 @@ import * as PlatformError from "effect/PlatformError";
 import * as Ref from "effect/Ref";
 import * as Schema from "effect/Schema";
 
-import type * as Electron from "electron";
+import * as Electron from "electron";
 
 import * as DesktopAssets from "../app/DesktopAssets.ts";
 import * as DesktopEnvironment from "../app/DesktopEnvironment.ts";
@@ -484,6 +484,27 @@ const make = Effect.gen(function* () {
       }
     });
 
+    // Composer dictation is the only thing here that asks for a device. Grant
+    // audio-only capture (behind the OS prompt on macOS) and leave every other
+    // permission on the answer Electron gives today.
+    window.webContents.session.setPermissionRequestHandler(
+      (_contents, permission, callback, details) => {
+        const mediaTypes = "mediaTypes" in details ? (details.mediaTypes ?? []) : [];
+        const audioOnly =
+          permission === "media" &&
+          mediaTypes.length > 0 &&
+          mediaTypes.every((mediaType) => mediaType === "audio");
+        if (!audioOnly || environment.platform !== "darwin") {
+          callback(true);
+          return;
+        }
+        void Electron.systemPreferences
+          .askForMediaAccess("microphone")
+          .then((granted) => callback(granted))
+          .catch(() => callback(false));
+      },
+    );
+
     if (
       !environment.marketingCaptureMode &&
       Option.isSome(persistedWindowState) &&

apps/server/src/bin.ts

@@ -7,6 +7,7 @@ import { Command } from "effect/unstable/cli";
 import * as NetService from "@threadlines/shared/Net";
 import packageJson from "../package.json" with { type: "json" };
 import { authCommand } from "./cli/auth.ts";
+import { dictationWorkerCommand } from "./cli/dictationWorker.ts";
 import { sharedServerCommandFlags } from "./cli/config.ts";
 import { assertSingleEffectRuntime } from "./cli/effectRuntimeCheck.ts";
 import { projectCommand } from "./cli/project.ts";
@@ -17,7 +18,13 @@ const CliRuntimeLayer = Layer.mergeAll(NodeServices.layer, NetService.layer);
 export const cli = Command.make("threadlines", { ...sharedServerCommandFlags }).pipe(
   Command.withDescription("Run the Threadlines server."),
   Command.withHandler((flags) => runServerCommand(flags)),
-  Command.withSubcommands([startCommand, serveCommand, authCommand, projectCommand]),
+  Command.withSubcommands([
+    startCommand,
+    serveCommand,
+    authCommand,
+    projectCommand,
+    dictationWorkerCommand,
+  ]),
 );
 
 if (import.meta.main) {

apps/server/src/cli/dictationWorker.ts

@@ -0,0 +1,15 @@
+import * as Effect from "effect/Effect";
+import { Command } from "effect/unstable/cli";
+
+import { runDictationWorker } from "../dictation/worker.ts";
+
+/**
+ * Internal entry point the server forks for speech-to-text. Routing it through
+ * the CLI means the same path (`process.argv[1]`) works in dev, npm and
+ * desktop builds. The worker exits itself when the parent disconnects, so the
+ * handler simply never completes.
+ */
+export const dictationWorkerCommand = Command.make("dictation-worker").pipe(
+  Command.withDescription("Internal: run the local speech-to-text worker process."),
+  Command.withHandler(() => Effect.sync(runDictationWorker).pipe(Effect.andThen(Effect.never))),
+);

apps/server/src/cli/marketingStudioSeed.ts

@@ -174,6 +174,7 @@ const makeServerConfig = (input: MarketingStudioSeedInput): ServerConfigShape =>
     environmentIdPath: NodePath.join(stateDir, "environment-id"),
     serverRuntimeStatePath: NodePath.join(stateDir, "server-runtime.json"),
     secretsDir: NodePath.join(stateDir, "secrets"),
+    speechModelsDir: NodePath.join(input.baseDir, "models", "speech"),
   };
 };
 

apps/server/src/config.ts

@@ -43,6 +43,8 @@ export interface ServerDerivedPaths {
   readonly environmentIdPath: string;
   readonly serverRuntimeStatePath: string;
   readonly secretsDir: string;
+  /** Shared by dev and userdata: speech models are large and platform-wide. */
+  readonly speechModelsDir: string;
 }
 
 /**
@@ -135,6 +137,7 @@ export const deriveServerPaths = Effect.fn(function* (
     environmentIdPath: join(stateDir, "environment-id"),
     serverRuntimeStatePath: join(stateDir, "server-runtime.json"),
     secretsDir: join(stateDir, "secrets"),
+    speechModelsDir: join(baseDir, "models", "speech"),
   };
 });
 
@@ -155,6 +158,7 @@ export const ensureServerDirectories = Effect.fn(function* (derivedPaths: Server
       fs.makeDirectory(derivedPaths.providerStatusCacheDir, { recursive: true }),
       fs.makeDirectory(path.dirname(derivedPaths.anonymousIdPath), { recursive: true }),
       fs.makeDirectory(path.dirname(derivedPaths.serverRuntimeStatePath), { recursive: true }),
+      fs.makeDirectory(derivedPaths.speechModelsDir, { recursive: true }),
     ],
     { concurrency: "unbounded" },
   );