From 0e12c3492e99e72cde7c7b392f367b779701fb8c Mon Sep 17 00:00:00 2001 From: Benjamin Shafii Date: Mon, 4 May 2026 12:21:39 -0700 Subject: [PATCH 1/4] feat: expose UI control MCP bridge --- AGENTS.md | 1 + ARCHITECTURE.md | 36 ++ .../domains/session/chat/status-bar.tsx | 39 +- .../control/session-control-actions.ts | 171 +++++++ .../session/surface/session-surface.tsx | 186 +++++++- apps/app/src/react-app/shell/app-root.tsx | 100 +++-- .../shell/control/control-provider.tsx | 420 ++++++++++++++++++ .../app/src/react-app/shell/session-route.tsx | 27 ++ apps/desktop/electron/main.mjs | 146 +++++- docs/mcp-ui-control-profile.md | 142 ++++++ packages/openwork-ui-mcp/index.mjs | 180 ++++++++ packages/openwork-ui-mcp/package.json | 16 + pnpm-lock.yaml | 49 +- 13 files changed, 1421 insertions(+), 92 deletions(-) create mode 100644 apps/app/src/react-app/domains/session/control/session-control-actions.ts create mode 100644 apps/app/src/react-app/shell/control/control-provider.tsx create mode 100644 docs/mcp-ui-control-profile.md create mode 100644 packages/openwork-ui-mcp/index.mjs create mode 100644 packages/openwork-ui-mcp/package.json diff --git a/AGENTS.md b/AGENTS.md index c80b2be216..9958c15e2b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,6 +57,7 @@ Read `ARCHITECTURE.md` for runtime flow, server-vs-shell ownership, and architec * **Open source**: keep the repo portable; no secrets committed. * **Slick and fluid**: 60fps animations, micro-interactions, premium feel. * **Mobile-native**: touch targets, gestures, and layouts optimized for small screens. +* **Provider-neutral control**: expose app actions through OpenWork-owned control surfaces first; provider-specific controllers should drive those surfaces rather than hardwiring provider logic into the app UI. ## Task Intake (Required) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 81d6dd1d61..ab75b712c0 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -200,6 +200,42 @@ These are all opencode primitives you can read the docs to find out exactly how OpenWork is a client experience that consumes OpenWork server surfaces. +### Provider-neutral app control surface + +OpenWork app control mode is owned by the UI runtime. The app exposes a +provider-neutral action registry through `window.__openworkControl` so external +controllers can inspect the current route, discover visible/safe actions, and +request an action by ID without depending on DOM scraping or a specific model +provider. + +Guidelines: + +- The app owns visible, screen-local state: which actions are available, which + element should be spotlighted, and how actions are choreographed so users can + see control happen. +- Controllers such as MCP bridges, test harnesses, or optional external drivers should + call the app control surface instead of reaching into app internals. +- Provider/API secrets and privileged filesystem or server mutations remain + server-owned; the app control surface should route those through OpenWork + server APIs rather than adding provider-specific behavior to the UI. +- Raw screenshot or coordinate-based control is a fallback for uninstrumented + surfaces, not the default architecture. + +### MCP UI Control profile + +OpenWork should standardize external app control through MCP where possible. The +app-local `window.__openworkControl` registry remains the source of current UI +affordances, but public integrations should expose those affordances as MCP +tools that follow `docs/mcp-ui-control-profile.md`: + +- `ui.snapshot` for current semantic app state +- `ui.list_actions` for currently available action metadata and input schemas +- `ui.execute_action` for running one semantic action by ID + +Standalone control clients such as HandsFree should be MCP clients first: they +can connect to any configured MCP server and call generic MCP tools. OpenWork's +local UI bridge is an implementation detail behind the OpenWork MCP surface. + OpenWork supports two product runtime modes for users: - desktop diff --git a/apps/app/src/react-app/domains/session/chat/status-bar.tsx b/apps/app/src/react-app/domains/session/chat/status-bar.tsx index ce97a3ceb2..f4521e36a9 100644 --- a/apps/app/src/react-app/domains/session/chat/status-bar.tsx +++ b/apps/app/src/react-app/domains/session/chat/status-bar.tsx @@ -1,9 +1,10 @@ /** @jsxImportSource react */ -import { useEffect, useState } from "react"; +import { useEffect, useMemo, useRef, useState } from "react"; import { BookOpen, MessageCircle, Settings } from "lucide-react"; import { t } from "../../../../i18n"; import { usePlatform } from "../../../kernel/platform"; +import { useControlAction, type OpenworkControlAction } from "../../../shell/control/control-provider"; import type { OpenworkServerStatus } from "../../../../app/lib/openwork-server"; const DOCS_URL = "https://openworklabs.com/docs"; @@ -103,6 +104,9 @@ function deriveStatusCopy(props: StatusBarProps): StatusCopy { export function StatusBar(props: StatusBarProps) { const platform = usePlatform(); + const docsButtonRef = useRef(null); + const feedbackButtonRef = useRef(null); + const settingsButtonRef = useRef(null); const [initializing, setInitializing] = useState( () => Date.now() - STATUS_BAR_BOOT_STARTED_AT < STATUS_BAR_INITIALIZING_MS, ); @@ -118,6 +122,36 @@ export function StatusBar(props: StatusBarProps) { }, [initializing]); const statusCopy = deriveStatusCopy({ ...props, initializing }); + const docsControlAction = useMemo(() => ({ + id: "status.docs.open", + label: "Open OpenWork docs", + description: "Open the documentation from the status bar.", + sideEffect: "external", + targetRef: docsButtonRef, + execute: () => platform.openLink(DOCS_URL), + }), [platform]); + useControlAction(docsControlAction); + + const feedbackControlAction = useMemo(() => ({ + id: "status.feedback.open", + label: "Send feedback", + description: "Open the OpenWork feedback surface from the status bar.", + sideEffect: "external", + targetRef: feedbackButtonRef, + execute: props.onSendFeedback, + }), [props.onSendFeedback]); + useControlAction(feedbackControlAction); + + const settingsControlAction = useMemo(() => ({ + id: "status.settings.open", + label: props.settingsOpen ? "Go back from settings" : "Open settings from the status bar", + description: "Use the visible settings button in the status bar.", + sideEffect: "navigation", + disabled: props.showSettingsButton === false, + targetRef: settingsButtonRef, + execute: props.onOpenSettings, + }), [props.onOpenSettings, props.settingsOpen, props.showSettingsButton]); + useControlAction(settingsControlAction); return (
@@ -143,6 +177,7 @@ export function StatusBar(props: StatusBarProps) {