feat(app): add HandsFree computer use extension

apps/app/src/app/constants.ts

@@ -156,6 +156,18 @@ export const MCP_QUICK_CONNECT: McpDirectoryInfo[] = [
     composerPrompt: "Use the Chrome extension to ",
     defaultEnabled: true,
   },
+  {
+    id: "handsfree-computer-use",
+    name: "HandsFree Computer Use",
+    serverName: "handsfree-computer-use",
+    description: "Control macOS apps through semantic accessibility refs, background-safe clicks, screenshots, keyboard input, and strict mode.",
+    type: "local",
+    command: ["npx", "-y", "@openwork/handsfree", "mcp"],
+    oauth: false,
+    kind: "extension",
+    iconSrc: "/openwork-mark.svg",
+    composerPrompt: "Use HandsFree Computer Use to ",
+  },
   {
     id: "openai-image-gen",
     name: "OpenAI Image Gen",

apps/app/src/app/lib/desktop.ts

@@ -157,6 +157,7 @@ declare global {
         initialDeepLinks?: string[];
         platform?: "darwin" | "linux" | "windows";
         version?: string;
+        browserCdpPort?: string;
       };
     };
   }

apps/app/src/react-app/design-system/extension-card.tsx

@@ -108,15 +108,11 @@ export function ExtensionCard(props: ExtensionCardProps) {
         <div className="min-w-0 flex-1">
           <div className="flex items-center gap-2">
             <h4 className="text-sm font-semibold text-dls-text">{name}</h4>
-            {connected ? (
-              <span className="rounded-md bg-green-3 px-1.5 py-0.5 text-[10px] font-medium text-green-11">
-                Connected
-              </span>
-            ) : (
+            {!connected ? (
               <span className={`rounded-md px-1.5 py-0.5 text-[10px] font-medium ${kindStyle[kind]}`}>
                 {kindLabel[kind]}
               </span>
-            )}
+            ) : null}
           </div>
           <p className="mt-0.5 line-clamp-2 text-xs text-dls-secondary">{description}</p>
           {!connected && !connecting && actionLabel ? (

apps/app/src/react-app/design-system/extension-detail-modal.tsx

@@ -255,10 +255,10 @@ export function ExtensionDetailModal(props: ExtensionDetailModalProps) {
                     </div>
                   ) : null}
 
-                  {kind === "ui-control" ? (
+                  {launchCommand ? (
                     <div className="flex items-center justify-between text-sm">
                       <span className="text-muted-foreground">Launch</span>
-                      <span className="max-w-[300px] truncate font-mono text-xs text-card-foreground">{(launchCommand ?? fallbackUiControlCommand).join(" ")}</span>
+                      <span className="max-w-[300px] truncate font-mono text-xs text-card-foreground">{launchCommand.join(" ")}</span>
                     </div>
                   ) : null}
 

apps/app/src/react-app/domains/connections/store.ts

@@ -273,6 +273,16 @@ export function createConnectionsStore(options: {
   };
 
   const resolveLocalMcpCommand = async (entry: McpDirectoryInfo) => {
+    if (entry.serverName === "handsfree-computer-use") {
+      try {
+        const command = await (window as any).__OPENWORK_ELECTRON__?.invokeDesktop?.("getHandsFreeMcpCommand");
+        if (Array.isArray(command) && command.every((part) => typeof part === "string") && command.length > 0) {
+          return command;
+        }
+      } catch {
+        // Fall through to the published package command.
+      }
+    }
     if (entry.serverName !== "openwork-ui") {
       return entry.command;
     }

apps/app/src/react-app/domains/session/surface/composer/composer.tsx

@@ -678,7 +678,16 @@ export function ReactSessionComposer(props: ComposerProps) {
   };
 
   const applyExtensionSelection = (entry: McpDirectoryInfo) => {
-    props.onDraftChange(entry.composerPrompt ?? `Use ${entry.name} to `);
+    if (entry.id === "openwork-browser") {
+      const port = window.__OPENWORK_ELECTRON__?.meta?.browserCdpPort?.trim();
+      props.onDraftChange(
+        port
+          ? `Use the OpenWork Browser extension with browser_url "http://127.0.0.1:${port}". Do not use any other browser_url. `
+          : entry.composerPrompt ?? `Use ${entry.name} to `,
+      );
+    } else {
+      props.onDraftChange(entry.composerPrompt ?? `Use ${entry.name} to `);
+    }
     setToolMenuOpen(false);
   };
 

apps/app/src/react-app/domains/settings/pages/mcp-view.tsx

@@ -196,6 +196,7 @@ export function McpView(props: McpViewProps) {
   const [detailSkillContent, setDetailSkillContent] = useState<string | null>(null);
   const [openworkUiMcpCommand, setOpenworkUiMcpCommand] = useState<string[] | null>(null);
   const [openworkUiMcpEnvironment, setOpenworkUiMcpEnvironment] = useState<Record<string, string> | null>(null);
+  const [handsFreeMcpCommand, setHandsFreeMcpCommand] = useState<string[] | null>(null);
   const [search, setSearch] = useState("");
   const [filter, setFilter] = useState<ExtensionFilter>("all");
   const [, setExtensionStateVersion] = useState(0);
@@ -266,9 +267,14 @@ export function McpView(props: McpViewProps) {
             ),
           ));
         }
+        const handsFreeCommand = await (window as any).__OPENWORK_ELECTRON__?.invokeDesktop?.("getHandsFreeMcpCommand");
+        if (Array.isArray(handsFreeCommand) && handsFreeCommand.every((part) => typeof part === "string")) {
+          setHandsFreeMcpCommand(handsFreeCommand);
+        }
       } catch {
         setOpenworkUiMcpCommand(null);
         setOpenworkUiMcpEnvironment(null);
+        setHandsFreeMcpCommand(null);
       }
     })();
   }, []);
@@ -341,6 +347,15 @@ export function McpView(props: McpViewProps) {
   const isQuickConnectConfigured = (entry: McpDirectoryInfo) =>
     props.mcpServers.some((server) => server.name === getMcpIdentityKey(entry));
 
+  const isMcpBackedExtension = (entry: McpDirectoryInfo) =>
+    entry.kind === "extension" && Boolean(entry.type || entry.command?.length || entry.url);
+
+  const launchCommandForEntry = (entry: McpDirectoryInfo) => {
+    if (entry.serverName === "openwork-ui") return openworkUiMcpCommand ?? undefined;
+    if (entry.serverName === "handsfree-computer-use") return handsFreeMcpCommand ?? entry.command;
+    return entry.command;
+  };
+
   const supportsOauth = (entry: McpServerEntry) =>
     entry.config.type === "remote" && entry.config.oauth !== false;
 
@@ -468,7 +483,7 @@ export function McpView(props: McpViewProps) {
         busy={props.busy}
         connectingName={props.mcpConnectingName}
         isConfigured={(entry) =>
-          entry.kind === "extension"
+          entry.kind === "extension" && !isMcpBackedExtension(entry)
             ? (entry.defaultEnabled ? isOpenWorkExtensionEnabled(entry) : props.isExtensionConnected?.(entry) ?? false)
             : isQuickConnectConfigured(entry)
         }
@@ -575,7 +590,7 @@ export function McpView(props: McpViewProps) {
       {detailEntry ? (() => {
         const extensionConfigSlot = props.configSlotForEntry?.(detailEntry) ?? null;
         const hasConfigSlot = extensionConfigSlot !== null;
-        const isConnected = detailEntry.kind === "extension"
+        const isConnected = detailEntry.kind === "extension" && !isMcpBackedExtension(detailEntry)
           ? (detailEntry.defaultEnabled ? isOpenWorkExtensionEnabled(detailEntry) : props.isExtensionConnected?.(detailEntry) ?? false)
           : isQuickConnectConfigured(detailEntry);
         return (
@@ -590,19 +605,19 @@ export function McpView(props: McpViewProps) {
             kind={detailEntry.kind ?? "mcp"}
             connected={isConnected}
             connecting={props.mcpConnectingName === detailEntry.name}
-            launchCommand={detailEntry.serverName === "openwork-ui" ? openworkUiMcpCommand ?? undefined : undefined}
+            launchCommand={launchCommandForEntry(detailEntry)}
             environment={detailEntry.serverName === "openwork-ui" ? openworkUiMcpEnvironment ?? undefined : undefined}
             url={typeof detailEntry.url === "string" ? detailEntry.url : undefined}
             oauth={detailEntry.oauth}
             configSlot={extensionConfigSlot}
-            onConnect={detailEntry.defaultEnabled ? () => {
+            onConnect={detailEntry.defaultEnabled && !isMcpBackedExtension(detailEntry) ? () => {
               setOpenWorkExtensionEnabled(detailEntry, true);
               setDetailEntry(null);
             } : hasConfigSlot ? undefined : () => {
               props.connectMcp(detailEntry);
               setDetailEntry(null);
             }}
-            onUninstall={detailEntry.defaultEnabled && isConnected ? () => {
+            onUninstall={detailEntry.defaultEnabled && !isMcpBackedExtension(detailEntry) && isConnected ? () => {
               setOpenWorkExtensionEnabled(detailEntry, false);
             } : isQuickConnectConfigured(detailEntry) ? () => {
               const slug = getMcpIdentityKey(detailEntry);

apps/app/src/react-app/domains/settings/shell/settings-page.tsx

@@ -3,7 +3,6 @@ import type * as React from "react";
 import {
   ArrowLeft,
   Bug,
-  ChevronDown,
   CloudCog,
   Cog,
   Container,
@@ -33,12 +32,6 @@ import {
   SidebarMenuButton,
   SidebarMenuItem,
 } from "@/components/ui/sidebar";
-import {
-  DropdownMenu,
-  DropdownMenuContent,
-  DropdownMenuItem,
-  DropdownMenuTrigger,
-} from "@/components/ui/dropdown-menu";
 import { t } from "../../../../i18n";
 import type { SettingsTab } from "../../../../app/types";
 import {
@@ -53,7 +46,6 @@ import {
   SettingsPanelToolbarMessage,
   SettingsPanelToolbarStatus,
 } from "./panel";
-import { WorkspaceIcon } from "../../../design-system/workspace-icon";
 
 export function getSettingsTabIcon(tab: SettingsTab) {
   switch (tab) {
@@ -234,31 +226,6 @@ export function SettingsSidebar(props: SettingsSidebarProps) {
               <span>{t("dashboard.back_to_app")}</span>
             </SidebarMenuButton>
           </SidebarMenuItem>
-          <SidebarMenuItem>
-            <DropdownMenu>
-              <DropdownMenuTrigger
-                render={
-                  <SidebarMenuButton type="button">
-                    <WorkspaceIcon seed={props.selectedWorkspaceName} sizeClass="size-4" />
-                    <span className="truncate">{props.selectedWorkspaceName}</span>
-                    <ChevronDown className="ml-auto" />
-                  </SidebarMenuButton>
-                }
-              />
-              <DropdownMenuContent className="w-(--anchor-width)">
-                {props.workspaces.map((workspace) => (
-                  <DropdownMenuItem
-                    key={workspace.id}
-                    onClick={() => props.onSelectWorkspace(workspace.id)}
-                    disabled={workspace.id === props.selectedWorkspaceId}
-                  >
-                    <WorkspaceIcon seed={workspace.name} sizeClass="size-4" />
-                    <span className="truncate">{workspace.name}</span>
-                  </DropdownMenuItem>
-                ))}
-              </DropdownMenuContent>
-            </DropdownMenu>
-          </SidebarMenuItem>
         </SidebarMenu>
       </SidebarHeader>
       <SidebarContent>

apps/desktop/electron/main.mjs

@@ -226,15 +226,15 @@ if (process.platform === "darwin" && APP_ICON_IMAGE && !APP_ICON_IMAGE.isEmpty()
 
 // Expose Chrome DevTools Protocol so the opencode-chrome-devtools plugin can
 // drive the built-in browser panel.  Use OPENWORK_ELECTRON_REMOTE_DEBUG_PORT to
-// pin a specific port; otherwise pick a default (9223) that stays out of the
-// way of common dev-tools ports (9222 = Chrome, 9229 = Node inspector).
+// pin a specific port. Prod defaults to 9223; dev defaults to 9823 so both
+// apps can run side by side without the dev browser tools attaching to prod.
 const explicitCdpPort = Number.parseInt(
   process.env.OPENWORK_ELECTRON_REMOTE_DEBUG_PORT?.trim() ?? "",
   10,
 );
 const remoteDebugPort = Number.isFinite(explicitCdpPort) && explicitCdpPort > 0
   ? explicitCdpPort
-  : 9223;
+  : isDevMode ? 9823 : 9223;
 app.commandLine.appendSwitch("remote-debugging-port", String(remoteDebugPort));
 app.commandLine.appendSwitch("remote-debugging-address", "127.0.0.1");
 // Make the port available to the embedded server so it can pass it to OpenCode.
@@ -2362,6 +2362,12 @@ async function handleDesktopInvoke(event, command, ...args) {
       }
       return ["npx", "-y", "openwork-ui-mcp"];
     }
+    case "getHandsFreeMcpCommand": {
+      if (process.env.OPENWORK_DEV_MODE === "1") {
+        return ["node", path.resolve(__dirname, "../../..", "packages/handsfree/bin/openwork-handsfree-computer-use.mjs"), "mcp"];
+      }
+      return ["npx", "-y", "@openwork/handsfree", "mcp"];
+    }
     case "getOpenworkUiMcpEnvironment": {
       return {
         OPENWORK_UI_CONTROL_DISCOVERY: path.join(app.getPath("userData"), "openwork-ui-control.json"),

apps/desktop/electron/preload.mjs

@@ -133,6 +133,7 @@ contextBridge.exposeInMainWorld("__OPENWORK_ELECTRON__", {
     initialDeepLinks: [],
     platform: normalizePlatform(process.platform),
     version: process.versions.electron,
+    browserCdpPort: process.env.OPENWORK_ELECTRON_REMOTE_DEBUG_PORT || undefined,
   },
 });
 

apps/server/src/workspace-init.ts

@@ -47,6 +47,7 @@ Browser tools (\`browser_navigate\`, \`browser_snapshot\`, \`browser_click\`, \`
 - \`browser_url\`: always use \`"http://127.0.0.1:{{BROWSER_CDP_PORT}}"\`.
 - Use for general browsing tasks. The user sees what you do in real time.
 - Always call \`browser_list\` first to discover available targets, then use the appropriate \`target_id\`.
+- Do not scan common CDP ports or fall back to another port. If this endpoint is unavailable, report that the built-in browser is unavailable.
 
 **Chrome (external browser)**:
 - Use when the user needs their real cookies, sign-ins, or extensions.

packages/handsfree/README.md

@@ -0,0 +1,26 @@
+# OpenWork HandsFree Computer Use
+
+Native macOS computer-use runtime imported from the HandsFree prototype.
+
+This package focuses on the reusable control layer:
+
+- Semantic AX snapshots with compact refs like `{e1}`.
+- Strict background mode that avoids foreground cursor/HID fallbacks.
+- Target-window screenshots via `CGWindowListCreateImage(.optionIncludingWindow)`.
+- Background input through `CGEvent.postToPid` with window-addressing fields.
+- Background activation using per-process event taps plus AppKit and center-click primers.
+- Non-UI orchestration modules from the original Electron prototype: realtime tool schemas/instructions and the GPT computer-use loop.
+
+Build the native stdio server:
+
+```bash
+pnpm --filter @openwork/handsfree check:native
+```
+
+Run it as an MCP-compatible adapter:
+
+```bash
+pnpm --filter @openwork/handsfree exec openwork-handsfree-computer-use mcp
+```
+
+The core runtime is intentionally MCP-independent. `ComputerUseRuntime` exposes a small direct surface (`snapshot`, `click`, `typeText`, `pressKey`, `scroll`, `wait`, `setValue`, `performAction`); `MCPServer` is only a thin stdio wrapper.

packages/handsfree/bin/openwork-handsfree-computer-use.mjs

@@ -0,0 +1,41 @@
+#!/usr/bin/env node
+
+import { spawn } from "node:child_process";
+import { existsSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const packageRoot = path.resolve(__dirname, "..");
+const swiftPackagePath = path.join(packageRoot, "native", "HandsFree");
+
+const explicitBinary = process.env.HANDSFREE_COMPUTER_USE_BINARY?.trim();
+const candidates = [
+  explicitBinary,
+  path.join(swiftPackagePath, ".build", "release", "HandsFreeComputerUse"),
+  path.join(swiftPackagePath, ".build", "arm64-apple-macosx", "release", "HandsFreeComputerUse"),
+  path.join(swiftPackagePath, ".build", "debug", "HandsFreeComputerUse"),
+  path.join(swiftPackagePath, ".build", "arm64-apple-macosx", "debug", "HandsFreeComputerUse"),
+].filter(Boolean);
+
+const args = process.argv.slice(2);
+const binary = candidates.find((candidate) => existsSync(candidate));
+const command = binary ?? "swift";
+const commandArgs = binary
+  ? args
+  : ["run", "--package-path", swiftPackagePath, "HandsFreeComputerUse", ...args];
+
+const child = spawn(command, commandArgs, {
+  stdio: "inherit",
+  env: process.env,
+});
+
+child.on("exit", (code, signal) => {
+  if (signal) process.kill(process.pid, signal);
+  process.exit(code ?? 0);
+});
+
+child.on("error", (error) => {
+  console.error(`Failed to start HandsFreeComputerUse: ${error.message}`);
+  process.exit(1);
+});

packages/handsfree/native/HandsFree/.gitignore

@@ -0,0 +1,2 @@
+.build/
+.swiftpm/

packages/handsfree/native/HandsFree/Package.swift

@@ -0,0 +1,13 @@
+// swift-tools-version: 5.9
+import PackageDescription
+
+let package = Package(
+    name: "HandsFree",
+    platforms: [.macOS(.v14)],
+    targets: [
+        .executableTarget(
+            name: "HandsFreeComputerUse",
+            path: "Sources/ComputerUse"
+        ),
+    ]
+)