feat(computer-use): add native desktop control over MCP

.env.example

@@ -13,3 +13,7 @@ TUNNEL_TOKEN=replace-with-cloudflare-tunnel-token
 # 可选：浏览器自动化镜像。
 # AGENTDOCK_IMAGE=ghcr.io/uvwt/agentdock:browser-latest
 # AGENTDOCK_BROWSER_ENABLED=true
+
+# 可选：允许 MCP 客户端查看并控制本机交互式桌面。高权限能力，默认关闭。
+# AGENTDOCK_COMPUTER_USE_ENABLED=true
+# AGENTDOCK_COMPUTER_USE_ALLOW_SYSTEM_KEYS=false

.gitignore

@@ -15,6 +15,8 @@
 /bin/
 /dist/
 /coverage.out
+/.cache/
+/.t/
 /*.test
 /agentdock.killed*
 

README.md

@@ -146,7 +146,13 @@ AgentDock can optionally act as a native ACP client and host a local coding-agen
 - Navigate, click, type, select, and wait
 - Inspect page text, interactive elements, errors, and network responses
 - Persist login state, use dedicated browser profiles, and capture screenshots
-- Use system Chrome and macOS desktop automation
+- Opt in to native Computer Use so authenticated MCP clients can inspect the interactive desktop and run bounded mouse/keyboard action batches
+- Bind every desktop action to the latest screenshot id to prevent stale-coordinate input; system-level shortcuts remain separately disabled by default
+- Use native dependency-free Windows capture/input, macOS system automation, or Linux desktop command adapters
+
+Computer Use is disabled by default. Enable `AGENTDOCK_COMPUTER_USE_ENABLED=true` (or `--computer-use-enabled`) and restart AgentDock; enable `AGENTDOCK_COMPUTER_USE_ALLOW_SYSTEM_KEYS=true` only when the connected client also needs app switching, quitting, locking, or similar shortcuts. Reconnect the MCP client or start a new web chat after restarting so it refreshes the tool list. The client then uses `computer_apps`, `computer_snapshot`, and `computer_act`; screenshots are returned as MCP image content, so a web client does not need local filesystem access. Keep Token or OAuth authentication enabled before exposing `/mcp` beyond localhost.
+
+Windows uses the interactive user session and has no extra runtime dependency. On macOS, grant the AgentDock process Screen Recording and Accessibility/Automation access when prompted. Linux input requires `xdotool`; window inventory uses `wmctrl` or falls back to `xdotool`; screenshots use one of `grim`, `gnome-screenshot`, `scrot`, ImageMagick `import`, or the dependency-light X11 `xwd` fallback. `xdotool` input is intended for X11, while Wayland support depends on the compositor.
 
 ### Recoverable tasks
 

README.zh-CN.md

@@ -150,7 +150,13 @@ AgentDock 可以选择作为 ACP Client 原生托管本地 Coding Agent adapter
 - 页面跳转、点击、输入、选择和等待
 - 页面文本、可交互元素、错误和网络响应检查
 - 登录状态、持久化浏览器 Profile 和截图
-- macOS 系统 Chrome 与桌面自动化支持
+- 可显式启用原生 Computer Use，让已认证的 MCP 客户端查看交互式桌面并批量执行鼠标、键盘动作
+- 每次桌面动作都绑定最新截图 ID，拒绝陈旧坐标；切换应用、退出、锁屏等系统级快捷键默认独立禁用
+- Windows 使用无外部依赖的原生截图与输入，macOS 使用系统自动化，Linux 使用本机桌面命令适配
+
+Computer Use 默认关闭。设置 `AGENTDOCK_COMPUTER_USE_ENABLED=true`（或启动参数 `--computer-use-enabled`）并重启 AgentDock 后启用；只有连接端确实需要切换应用、退出、锁屏等快捷键时，才设置 `AGENTDOCK_COMPUTER_USE_ALLOW_SYSTEM_KEYS=true`。重启后请重新连接 MCP，或在网页端新建对话，让客户端刷新工具列表。客户端随后通过 `computer_apps`、`computer_snapshot` 和 `computer_act` 操作；截图会直接作为 MCP 图片内容返回，因此网页客户端不需要访问本机文件系统。对 localhost 以外开放 `/mcp` 前必须保持 Token 或 OAuth 鉴权。
+
+Windows 运行在交互式用户会话内，无额外运行时依赖。macOS 首次使用时需按系统提示向 AgentDock 进程授予“屏幕录制”和“辅助功能/自动化”权限。Linux 输入依赖 `xdotool`；窗口清单优先使用 `wmctrl`，缺失时回退到 `xdotool`；截图可使用 `grim`、`gnome-screenshot`、`scrot`、ImageMagick `import`，或依赖较少的 X11 `xwd` 回退。`xdotool` 输入主要适用于 X11，Wayland 能力取决于桌面合成器。
 
 ### 可恢复任务
 

cmd/agentdock/server.go

@@ -58,6 +58,8 @@ func runServer(ctx context.Context, args []string, stderr io.Writer) error {
 	flags.StringVar(&cfg.BrowserExecutablePath, "browser-executable-path", cfg.BrowserExecutablePath, "optional absolute Chrome, Chromium, or Edge executable path")
 	flags.StringVar(&cfg.BrowserCDPURL, "browser-cdp-url", cfg.BrowserCDPURL, "optional existing Chromium CDP endpoint to attach")
 	flags.BoolVar(&cfg.BrowserReuseExistingCDP, "browser-reuse-existing-cdp", cfg.BrowserReuseExistingCDP, "discover and reuse a unique local existing CDP browser before launching one")
+	flags.BoolVar(&cfg.ComputerUseEnabled, "computer-use-enabled", cfg.ComputerUseEnabled, "expose native local computer-control tools")
+	flags.BoolVar(&cfg.ComputerUseAllowSystemKeys, "computer-use-allow-system-keys", cfg.ComputerUseAllowSystemKeys, "allow system-level key combinations such as app switching and locking")
 	flags.BoolVar(&cfg.Stdio, "stdio", cfg.Stdio, "serve JSON-RPC over stdio")
 	if err := flags.Parse(args); err != nil {
 		if errors.Is(err, flag.ErrHelp) {
@@ -89,7 +91,7 @@ func runServer(ctx context.Context, args []string, stderr io.Writer) error {
 		// 失败不应阻断 MCP 服务启动；保留明确日志并在下次启动继续重试。
 		slog.Warn("desktop runtime repair skipped", "error", err)
 	}
-	slog.Info("server starting", "agentdock_home", cfg.AgentDockHome, "agentdock_default_dir", cfg.AgentDockDefaultDir, "path_model", config.PathModel, "host", cfg.Host, "port", cfg.Port, "stdio", cfg.Stdio, "log_level", cfg.LogLevel, "recall_enabled", cfg.NexusEndpoint != "", "nexus_enabled", cfg.NexusEndpoint != "", "mcp_apps_enabled", cfg.MCPAppsEnabled, "browser_enabled", cfg.BrowserEnabled)
+	slog.Info("server starting", "agentdock_home", cfg.AgentDockHome, "agentdock_default_dir", cfg.AgentDockDefaultDir, "path_model", config.PathModel, "host", cfg.Host, "port", cfg.Port, "stdio", cfg.Stdio, "log_level", cfg.LogLevel, "recall_enabled", cfg.NexusEndpoint != "", "nexus_enabled", cfg.NexusEndpoint != "", "mcp_apps_enabled", cfg.MCPAppsEnabled, "browser_enabled", cfg.BrowserEnabled, "computer_use_enabled", cfg.ComputerUseEnabled)
 	runtime, err := app.NewRuntime(cfg)
 	if err != nil {
 		return err

core-skills/agentdock-user-guide/references/configuration.md

@@ -15,6 +15,8 @@
 | `AGENTDOCK_BROWSER_EXECUTABLE_PATH` | 显式浏览器可执行文件 | Docker/服务器/高级运行环境 |
 | `AGENTDOCK_BROWSER_CDP_URL` | 复用已有 Chromium 的 CDP 地址 | Desktop 设置或启动环境 |
 | `AGENTDOCK_BROWSER_REUSE_EXISTING_CDP` | 自动复用唯一已发现 CDP | Desktop 设置或启动环境 |
+| `AGENTDOCK_COMPUTER_USE_ENABLED` | 是否向 MCP 客户端开放本机桌面截图、鼠标和键盘控制；默认关闭 | Desktop 设置或启动环境 |
+| `AGENTDOCK_COMPUTER_USE_ALLOW_SYSTEM_KEYS` | 是否额外允许切换应用、退出、锁屏等系统级快捷键；默认关闭 | Desktop 设置或启动环境 |
 | `AGENTDOCK_COMMAND_ENV_FROM_ENV_JSON` | 显式允许 `exec_command` 从 Core 宿主环境复制的变量映射 | Linux/Docker/直接启动的高级配置 |
 | `AGENTDOCK_ACP_ENABLED` | 是否启用 ACP Client | Desktop 设置或启动环境 |
 | `AGENTDOCK_ACP_PROFILES_JSON` | 多 ACP Profile JSON 数组；每项包含 `id/kind/command/args/env_from_env/enabled` | Desktop 设置或高级启动环境 |
@@ -43,6 +45,7 @@ Coding Agent 的发现、Codex / Claude Adapter 安装、Grok stdio 模式、平
 - Linux 官方安装器默认把环境文件按 root:root、0600 写入，并通过 systemd/OpenRC 注入服务进程；不要为了方便把权限放宽。
 - Docker 的环境变量属于容器创建配置。Compose 文件或 env file 修改后，如果容器没有被重新创建，新进程可能仍使用旧的容器配置。
 - `AGENTDOCK_COMMAND_ENV_FROM_ENV_JSON` 只允许显式映射；它不会自动把登录 Shell 的全部环境传给 `exec_command`。
+- Computer Use 是整机级授权边界。启用后，已通过 AgentDock 认证的 MCP 客户端可看到交互式桌面并控制所有应用；公网连接必须保持 Token 或 OAuth 鉴权。坐标动作只接受最新截图返回的 `snapshot_id`，动作失败后必须重新截图。
 
 ## 判断“配置没生效”时
 

desktop/macos/AgentDockApp/Resources/en.lproj/Localizable.strings

@@ -61,6 +61,12 @@
 "Use as default" = "Use as default";
 "Enable MCP Apps UI" = "Enable MCP Apps UI";
 "Enable browser CDP control" = "Enable browser CDP control";
+"Allow connected AI clients to control this computer" = "Allow connected AI clients to control this computer";
+"Also allow system-level shortcuts" = "Also allow system-level shortcuts (app switching, quitting, locking)";
+"Computer Use grants screenshot, mouse, and keyboard access to every desktop app. Keep authentication enabled." = "Computer Use grants screenshot, mouse, and keyboard access to every desktop app. Keep authentication enabled.";
+"Enable Computer Use?" = "Enable Computer Use?";
+"Connected AI clients will be able to see the desktop and control the mouse and keyboard in every app. Only continue if AgentDock authentication is configured and you trust all connected clients." = "Connected AI clients will be able to see the desktop and control the mouse and keyboard in every app. Only continue if AgentDock authentication is configured and you trust all connected clients.";
+"Enable" = "Enable";
 "Enter the CDP address to connect to." = "Enter the CDP address to connect to.";
 "Enter the NexusDock address and one-time pairing code." = "Enter the NexusDock address and one-time pairing code.";
 "Enter the absolute path to an executable ACP Adapter" = "Enter the absolute path to an executable ACP Adapter";

desktop/macos/AgentDockApp/Resources/zh-Hans.lproj/Localizable.strings

@@ -61,6 +61,12 @@
 "Use as default" = "设为默认";
 "Enable MCP Apps UI" = "启用 MCP Apps UI";
 "Enable browser CDP control" = "启用浏览器 CDP 控制";
+"Allow connected AI clients to control this computer" = "允许已连接的 AI 客户端控制这台电脑";
+"Also allow system-level shortcuts" = "同时允许系统级快捷键（切换应用、退出、锁屏）";
+"Computer Use grants screenshot, mouse, and keyboard access to every desktop app. Keep authentication enabled." = "Computer Use 会授予所有桌面应用的截图、鼠标和键盘权限。请始终启用身份验证。";
+"Enable Computer Use?" = "确定启用 Computer Use 吗？";
+"Connected AI clients will be able to see the desktop and control the mouse and keyboard in every app. Only continue if AgentDock authentication is configured and you trust all connected clients." = "已连接的 AI 客户端将能够查看桌面，并在所有应用中控制鼠标和键盘。仅当 AgentDock 已配置身份验证且你信任所有连接客户端时继续。";
+"Enable" = "启用";
 "Enter the CDP address to connect to." = "请输入要连接的 CDP 地址。";
 "Enter the NexusDock address and one-time pairing code." = "请填写 NexusDock 地址和一次性配对码。";
 "Enter the absolute path to an executable ACP Adapter" = "请填写可执行的 ACP Adapter 绝对路径";