different-ai · benjaminshafii · May 22, 2026 · May 22, 2026 · May 22, 2026 · May 22, 2026
diff --git a/apps/app/src/app/constants.ts b/apps/app/src/app/constants.ts
@@ -166,6 +166,17 @@ export const MCP_QUICK_CONNECT: McpDirectoryInfo[] = [
     iconSrc: "/ext-openai.svg",
     composerPrompt: "Use the OpenAI Image Gen extension to ",
   },
+  {
+    id: "openwork-voice",
+    name: "Voice Mode",
+    serverName: "openwork-voice",
+    description: "Talk to OpenWork through a Realtime voice panel that drives the same semantic UI controls as OpenWork UI MCP.",
+    oauth: false,
+    kind: "extension",
+    iconSrc: "/openwork-mark.svg",
+    composerPrompt: "Use Voice Mode to ",
+    defaultEnabled: true,
+  },
   {
     id: "ollama",
     name: "Ollama",

diff --git a/apps/app/src/app/lib/openwork-server.ts b/apps/app/src/app/lib/openwork-server.ts
@@ -1388,6 +1388,22 @@ export function createOpenworkServerClient(options: { baseUrl: string; token?: s
         method: "DELETE",
         timeoutMs: timeouts.config,
       }),
+
+    createVoiceRealtimeSession: (payload?: { model?: string }) =>
+      requestJson<{
+        ok: true;
+        clientSecret: string;
+        expiresAt: number | null;
+        model: string;
+        transcriptionModel: string;
+        tools: string[];
+      }>(baseUrl, "/voice/realtime/session", {
+        token,
+        hostToken,
+        method: "POST",
+        body: payload ?? {},
+        timeoutMs: timeouts.config,
+      }),
   };
 }
 

diff --git a/apps/app/src/react-app/domains/session/chat/session-page.tsx b/apps/app/src/react-app/domains/session/chat/session-page.tsx
@@ -2,9 +2,10 @@
 import type { CSSProperties } from "react";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { usePanelRef } from "react-resizable-panels";
-import { FileText, Globe, Settings2, Zap } from "lucide-react";
+import { FileText, Globe, Mic2, Settings2, Zap } from "lucide-react";
 
 import { t } from "../../../../i18n";
+import { OPENWORK_EXTENSION_CATALOG } from "../../../../app/constants";
 import { type OpenworkServerClient, type OpenworkServerStatus } from "../../../../app/lib/openwork-server";
 import { getDisplaySessionTitle } from "../../../../app/lib/session-title";
 import type { BootPhase } from "../../../../app/lib/startup-boot";
@@ -39,13 +40,16 @@ import { StatusBar, type StatusBarProps } from "./status-bar";
 import { OwDotTicker } from "../../../shell/dot-ticker";
 import { useReactRenderWatchdog } from "../../../shell/react-render-watchdog";
 import { useShellConfig } from "../../../shell/shell-config";
-import { type SidePanelItem, useUiStateStore } from "../../../shell/ui-state-store";
+import { getSidePanelState, type SidePanelItem, useUiStateStore } from "../../../shell/ui-state-store";
 
 import { isElectronRuntime } from "../../../../app/utils";
 import { BrowserPanel } from "../browser/browser-panel";
 import { ArtifactPanel } from "../artifacts/artifact-panel";
 import { isCollectibleArtifactTarget, isLocalhostBrowserTarget, type OpenTarget } from "../artifacts/open-target";
+import { VoicePanel } from "../voice/voice-panel";
 import { useWorkspaceShellLayout } from "../../../shell/workspace-shell-layout";
+import { useControlAction, type OpenworkControlAction } from "../../../shell/control/control-provider";
+import { getExtensionId, isOpenWorkExtensionEnabled, OPENWORK_EXTENSION_STATE_CHANGED } from "../../settings/extension-state";
 import { cn } from "@/lib/utils";
 
 const STARTUP_SKELETON_ROWS = [
@@ -123,6 +127,7 @@ export type SessionPageProps = {
   clientConnected: boolean;
   openworkServerStatus: OpenworkServerStatus;
   openworkServerClient: OpenworkServerClient | null;
+  hostOpenworkServerClient?: OpenworkServerClient | null;
   openworkServerToken?: string | null;
   developerMode: boolean;
   headerStatus: string;
@@ -217,14 +222,13 @@ export function SessionPage(props: SessionPageProps) {
   const { config: shellConfig } = useShellConfig();
   const sidebarOpen = useUiStateStore((state) => state.sidebarOpen);
   const setSidebarOpen = useUiStateStore((state) => state.setSidebarOpen);
-  const activeSidePanel = useUiStateStore((state) => (
-    props.selectedSessionId ? state.sidePanelState[props.selectedSessionId] ?? null : null
-  ));
+  const activeSidePanel = useUiStateStore((state) => getSidePanelState(state, props.selectedSessionId));
   const setSidePanelState = useUiStateStore((state) => state.setSidePanelState);
   const toggleSidePanelState = useUiStateStore((state) => state.toggleSidePanelState);
   const [artifactTarget, setArtifactTarget] = useState<OpenTarget | null>(null);
   const [openTargets, setOpenTargets] = useState<OpenTarget[]>([]);
   const [hiddenAccessibleTargetIds, setHiddenAccessibleTargetIds] = useState<Set<string>>(() => new Set());
+  const [, setExtensionStateVersion] = useState(0);
   const loadedHiddenTargetsKeyRef = useRef<string | null>(null);
   const accessibleTargets = useMemo(
     () => openTargets.filter((target) => isTrackableAccessibleTarget(target) && !hiddenAccessibleTargetIds.has(target.id)),
@@ -238,6 +242,12 @@ export function SessionPage(props: SessionPageProps) {
   const browserRailActive = activeSidePanel === "browser";
   const artifactRailActive = activeSidePanel === "artifacts";
   const extensionsRailActive = activeSidePanel === "extensions";
+  const voiceRailActive = activeSidePanel === "voice";
+  const voiceExtension = useMemo(
+    () => OPENWORK_EXTENSION_CATALOG.find((entry) => getExtensionId(entry) === "openwork-voice") ?? null,
+    [],
+  );
+  const voiceExtensionEnabled = voiceExtension ? isOpenWorkExtensionEnabled(voiceExtension) : false;
 
   useReactRenderWatchdog("SessionPage", {
     selectedSessionId: props.selectedSessionId,
@@ -364,6 +374,9 @@ export function SessionPage(props: SessionPageProps) {
   const openExtensionsRailPane = useCallback(() => {
     toggleCurrentSidePanel("extensions");
   }, [toggleCurrentSidePanel]);
+  const openVoiceRailPane = useCallback(() => {
+    toggleCurrentSidePanel("voice");
+  }, [toggleCurrentSidePanel]);
   const removeAccessibleTarget = useCallback((target: OpenTarget) => {
     setHiddenAccessibleTargetIds((current) => new Set(current).add(target.id));
     setArtifactTarget((current) => current?.id === target.id ? null : current);
@@ -393,8 +406,50 @@ export function SessionPage(props: SessionPageProps) {
     window.addEventListener("openwork-close-right-pane", handler);
     return () => window.removeEventListener("openwork-close-right-pane", handler);
   }, [setCurrentSidePanel]);
+  useEffect(() => {
+    const refresh = () => setExtensionStateVersion((value) => value + 1);
+    window.addEventListener(OPENWORK_EXTENSION_STATE_CHANGED, refresh);
+    window.addEventListener("storage", refresh);
+    return () => {
+      window.removeEventListener(OPENWORK_EXTENSION_STATE_CHANGED, refresh);
+      window.removeEventListener("storage", refresh);
+    };
+  }, []);
+  useEffect(() => {
+    if (activeSidePanel === "voice" && !voiceExtensionEnabled) {
+      setCurrentSidePanel(null);
+    }
+  }, [activeSidePanel, setCurrentSidePanel, voiceExtensionEnabled]);
   const [showDelayedSessionLoadingState, setShowDelayedSessionLoadingState] = useState(false);
 
+  const openVoicePanelControlAction = useMemo<OpenworkControlAction | null>(() => (
+    voiceExtensionEnabled && props.selectedSessionId ? {
+      id: "voice.panel.open",
+      label: "Open Voice Mode",
+      description: "Open the Voice Mode right-side panel for the active session.",
+      sideEffect: "none",
+      execute: () => {
+        setCurrentSidePanel("voice");
+        return { open: true };
+      },
+    } : null
+  ), [props.selectedSessionId, setCurrentSidePanel, voiceExtensionEnabled]);
+  useControlAction(openVoicePanelControlAction);
+
+  const closeVoicePanelControlAction = useMemo<OpenworkControlAction | null>(() => (
+    voiceExtensionEnabled && activeSidePanel === "voice" ? {
+      id: "voice.panel.close",
+      label: "Close Voice Mode",
+      description: "Close the Voice Mode right-side panel.",
+      sideEffect: "none",
+      execute: () => {
+        setCurrentSidePanel(null);
+        return { open: false };
+      },
+    } : null
+  ), [activeSidePanel, setCurrentSidePanel, voiceExtensionEnabled]);
+  useControlAction(closeVoicePanelControlAction);
+
   const selectedSessionTitle = useMemo(
     () => sessionTitleForId(props.sidebar.workspaceSessionGroups, props.selectedSessionId),
     [props.selectedSessionId, props.sidebar.workspaceSessionGroups],
@@ -441,6 +496,12 @@ export function SessionPage(props: SessionPageProps) {
     selectedWorkspaceGroupError ||
     "";
   const showSelectedWorkspaceError = Boolean(selectedWorkspaceErrorMessage);
+  const rightPanelDefaultSize = activeSidePanel === "extensions"
+    ? `${Math.max(browserPanelDefaultWidth, 480)}px`
+    : activeSidePanel === "voice"
+      ? `${Math.max(browserPanelDefaultWidth, 380)}px`
+      : `${browserPanelDefaultWidth}px`;
+  const rightPanelMinSize = activeSidePanel === "extensions" ? "420px" : activeSidePanel === "voice" ? "360px" : "320px";
 
   const reactSessionBaseUrl = props.opencodeBaseUrl?.trim() ?? "";
   const reactSessionToken =
@@ -822,15 +883,21 @@ export function SessionPage(props: SessionPageProps) {
                 <ResizableHandle withHandle className="hidden lg:flex" />
                 <ResizablePanel
                   panelRef={browserPanelRef}
-                  defaultSize={`${activeSidePanel === "extensions" ? Math.max(browserPanelDefaultWidth, 480) : browserPanelDefaultWidth}px`}
-                  minSize={activeSidePanel === "extensions" ? "420px" : "320px"}
+                  defaultSize={rightPanelDefaultSize}
+                  minSize={rightPanelMinSize}
                   maxSize="70%"
                   className="min-h-0 overflow-hidden lg:flex lg:flex-col"
                 >
                   {activeSidePanel === "extensions" && props.settingsSlot ? (
                     <div className="flex h-full min-h-0 flex-col overflow-y-auto bg-background">
                       {props.settingsSlot}
                     </div>
+                  ) : activeSidePanel === "voice" ? (
+                    <VoicePanel
+                      client={props.hostOpenworkServerClient ?? props.openworkServerClient}
+                      sessionId={props.selectedSessionId}
+                      onClose={closeRightPane}
+                    />
                   ) : activeSidePanel === "artifacts" && visibleArtifactTarget && props.openworkServerClient && props.runtimeWorkspaceId ? (
                     <ArtifactPanel
                       client={props.openworkServerClient}
@@ -866,6 +933,23 @@ export function SessionPage(props: SessionPageProps) {
                 <Globe size={17} />
               </Button>
             ) : null}
+            {voiceExtensionEnabled ? (
+              <Button
+                variant="ghost"
+                size="icon-sm"
+                className={cn(
+                  "rounded-xl transition-colors hover:bg-muted hover:text-foreground",
+                  voiceRailActive && "bg-primary/10 text-primary hover:bg-primary/15 hover:text-primary",
+                )}
+                onClick={openVoiceRailPane}
+                title={props.selectedSessionId ? "Voice Mode" : "Open a session to use Voice Mode"}
+                aria-label={props.selectedSessionId ? "Voice Mode" : "Open a session to use Voice Mode"}
+                aria-pressed={voiceRailActive}
+                disabled={!props.selectedSessionId}
+              >
+                <Mic2 size={17} />
+              </Button>
+            ) : null}
             <Button
               variant="ghost"
               size="icon-sm"

diff --git a/apps/app/src/react-app/domains/session/settings/extensions-pane-slot.tsx b/apps/app/src/react-app/domains/session/settings/extensions-pane-slot.tsx
@@ -25,6 +25,7 @@ import type { ProviderListItem } from "../../../../app/types";
 import "../../settings/openai-image-gen-config";
 import "../../settings/ollama-config";
 import "../../settings/browser-extension-config";
+import "../../settings/openwork-voice-config";
 
 export type ExtensionsPaneSlotProps = {
   openworkClient: OpenworkServerClient | null;
@@ -49,6 +50,10 @@ export function ExtensionsPaneSlot(props: ExtensionsPaneSlotProps) {
   const [lpBusy, setLpBusy] = useState(false);
   const [lpStatus, setLpStatus] = useState<string | null>(null);
   const [lpError, setLpError] = useState<string | null>(null);
+  const [voiceBusy, setVoiceBusy] = useState(false);
+  const [voiceStatus, setVoiceStatus] = useState<string | null>(null);
+  const [voiceError, setVoiceError] = useState<string | null>(null);
+  const [userEnvKeys, setUserEnvKeys] = useState<string[]>([]);
   const [, setExtensionStateVersion] = useState(0);
 
   useEffect(() => {
@@ -72,6 +77,16 @@ export function ExtensionsPaneSlot(props: ExtensionsPaneSlotProps) {
     return () => { cancelled = true; };
   }, [props.workspaceClient, props.workspaceId]);
 
+  useEffect(() => {
+    const client = props.openworkClient;
+    if (!client) { setUserEnvKeys([]); return; }
+    let cancelled = false;
+    void client.listUserEnvKeys()
+      .then((response) => { if (!cancelled) setUserEnvKeys(response.keys); })
+      .catch(() => { if (!cancelled) setUserEnvKeys([]); });
+    return () => { cancelled = true; };
+  }, [props.openworkClient]);
+
   const installImage = useCallback(async (apiKey: string) => {
     const client = props.workspaceClient;
     const wid = props.workspaceId?.trim();
@@ -120,6 +135,28 @@ export function ExtensionsPaneSlot(props: ExtensionsPaneSlotProps) {
     } catch (e) { setLpError(e instanceof Error ? e.message : String(e)); } finally { setLpBusy(false); }
   }, [props]);
 
+  const saveVoiceApiKey = useCallback(async (apiKey: string) => {
+    const client = props.openworkClient;
+    const value = apiKey.trim();
+    if (!client || !value) { setVoiceError("API key required."); return; }
+    setVoiceBusy(true); setVoiceStatus(null); setVoiceError(null);
+    try {
+      await client.upsertUserEnv([{ key: "OPENAI_API_KEY", value }]);
+      setUserEnvKeys((current) => Array.from(new Set([...current, "OPENAI_API_KEY"])));
+      setVoiceStatus("Saved OPENAI_API_KEY for Voice Mode and other OpenAI extensions.");
+    } catch (e) { setVoiceError(e instanceof Error ? e.message : String(e)); } finally { setVoiceBusy(false); }
+  }, [props.openworkClient]);
+
+  const testVoiceSession = useCallback(async () => {
+    const client = props.openworkClient;
+    if (!client) { setVoiceError("OpenWork host connection required."); return; }
+    setVoiceBusy(true); setVoiceStatus(null); setVoiceError(null);
+    try {
+      const session = await client.createVoiceRealtimeSession();
+      setVoiceStatus(`Realtime ready with ${session.model} (${session.tools.length} OpenWork tools).`);
+    } catch (e) { setVoiceError(e instanceof Error ? e.message : String(e)); } finally { setVoiceBusy(false); }
+  }, [props.openworkClient]);
+
   const configCtx: ExtensionConfigContext = {
     imageExtension: {
       busy: imageBusy || genBusy,
@@ -129,6 +166,18 @@ export function ExtensionsPaneSlot(props: ExtensionsPaneSlotProps) {
       onInstall: installImage,
       onTestGenerate: testGen,
     },
+    voiceExtension: {
+      busy: voiceBusy,
+      status: voiceStatus,
+      error: voiceError,
+      envKeyDetected:
+        userEnvKeys.includes("OPENAI_REALTIME_API_KEY") ||
+        userEnvKeys.includes("OPENAI_API_KEY") ||
+        props.providers.some((p) => p.id === "openai" && p.source === "env") ||
+        props.providerConnectedIds.includes("openai"),
+      onSaveApiKey: saveVoiceApiKey,
+      onTestSession: testVoiceSession,
+    },
     localProvider: { busy: lpBusy, status: lpStatus, error: lpError, onInstall: installLocal },
   };
 

diff --git a/apps/app/src/react-app/domains/session/surface/session-surface.tsx b/apps/app/src/react-app/domains/session/surface/session-surface.tsx
@@ -873,6 +873,24 @@ export function SessionSurface(props: SessionSurfaceProps) {
     await waitForControl(40);
   }, [props.sessionId, setComposerDraft]);
 
+  useEffect(() => {
+    const handleVoiceTranscript = (event: Event) => {
+      if (!(event instanceof CustomEvent)) return;
+      const detail: unknown = event.detail;
+      if (!detail || typeof detail !== "object" || Array.isArray(detail) || !("text" in detail) || typeof detail.text !== "string") return;
+      const text = detail.text;
+      void typeComposerText(text);
+      props.onDraftChange(buildDraft(text, attachments));
+      recordInspectorEvent("voice.transcript.applied", {
+        workspaceId: props.workspaceId,
+        sessionId: props.sessionId,
+        length: text.length,
+      });
+    };
+    window.addEventListener("openwork:voice-transcript", handleVoiceTranscript);
+    return () => window.removeEventListener("openwork:voice-transcript", handleVoiceTranscript);
+  }, [attachments, buildDraft, props.onDraftChange, props.sessionId, props.workspaceId, typeComposerText]);
+
   const composerSetTextControlAction = useMemo<OpenworkControlAction>(() => ({
     id: "composer.set_text",
     label: "Type into the composer",