From 315c76c9997c965b2725092f365d55b424ba1a58 Mon Sep 17 00:00:00 2001 From: zcai7675-bot <233973883+.zcai7675-bot@users.noreply.github.com> Date: Sun, 13 Sep 2026 10:48:23 +0800 Subject: [PATCH 1/2] feat(tools): add macOS computer-use built-in tools Give the model eyes and hands on macOS via five built-in tools, gated to darwin like the speech tools: - computer_screenshot: capture the full screen or a region via screencapture and return it as image content (createToolCallResponse) plus a text note; the temp capture file is always cleaned up. - computer_click: left/right/double clicks at coordinates through CoreGraphics events posted from osascript's JXA ObjC bridge (a move settles hover state first; double clicks set ClickState 2). - computer_scroll: scroll-wheel events in any direction, vertical and horizontal axes. - computer_type: keystroke for ASCII text (with escaping); non-ASCII text goes through a clipboard paste and says so in the result. - computer_key: named special keys mapped to macOS key codes, with cmd/ctrl/alt/shift/fn modifier combinations. The OS bridge (run/readFile/removeFile/temporaryPath) is injected, so tests drive every tool and error path without touching the real screen or clipboard. macOS prompts for Screen Recording and Accessibility permissions on first use; the tool descriptions say so. Icons are registered in the playground's built-in tool icon map. --- .../tools/built-in/built-in-tools-module.ts | 5 + .../runtime/src/tools/built-in/computer.ts | 528 ++++++++++++++++++ .../built-in/built-in-tools-module.test.ts | 11 +- .../tests/tools/built-in/computer.test.ts | 363 ++++++++++++ .../tool/built-in-tool-icon.ts | 13 + 5 files changed, 919 insertions(+), 1 deletion(-) create mode 100644 packages/runtime/src/tools/built-in/computer.ts create mode 100644 packages/runtime/tests/tools/built-in/computer.test.ts diff --git a/packages/runtime/src/tools/built-in/built-in-tools-module.ts b/packages/runtime/src/tools/built-in/built-in-tools-module.ts index f36f6c88..2440c50c 100644 --- a/packages/runtime/src/tools/built-in/built-in-tools-module.ts +++ b/packages/runtime/src/tools/built-in/built-in-tools-module.ts @@ -1,6 +1,7 @@ import type { RuntimeModule } from "../runtime-module"; import { calculatorBuiltInTools } from "./calculator"; +import { createComputerBuiltInTools } from "./computer"; import { dateDifferenceBuiltInTools } from "./date-difference"; import { createExecCodeBuiltInTools, @@ -47,6 +48,10 @@ export function createBuiltInToolsModule( id: "llm-space.built-in-tools.speech", entries: createSpeechBuiltInTools(speech), }); + tools.register({ + id: "llm-space.built-in-tools.computer", + entries: createComputerBuiltInTools(), + }); } tools.register({ id: "llm-space.built-in-tools.misc", diff --git a/packages/runtime/src/tools/built-in/computer.ts b/packages/runtime/src/tools/built-in/computer.ts new file mode 100644 index 00000000..e3baa2ea --- /dev/null +++ b/packages/runtime/src/tools/built-in/computer.ts @@ -0,0 +1,528 @@ +import { execFile } from "node:child_process"; +import { readFile, unlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +import type { BuiltinTool } from "@llm-space/core"; + +import { createToolCallResponse } from "../tool-registry"; +import type { ToolEntry } from "../tool-registry"; + +const SCREENCAPURE = "/usr/sbin/screencapture"; +const OSASCRIPT = "/usr/bin/osascript"; +const COMMAND_TIMEOUT_MS = 15_000; + +const execFileAsync = promisify(execFile); + +/** + * Injectable OS bridge. The defaults shell out to macOS's built-in + * `screencapture` and `osascript`; tests substitute fakes so nothing touches + * the real screen or clipboard. + */ +export interface ComputerDependencies { + /** Run a helper binary and resolve with its stdout. */ + run(command: string, args: string[]): Promise; + readFile(filePath: string): Promise; + removeFile(filePath: string): Promise; + temporaryPath(extension: string): string; +} + +let screenshotCounter = 0; + +export const defaultComputerDependencies: ComputerDependencies = { + async run(command, args) { + const { stdout } = await execFileAsync(command, args, { + timeout: COMMAND_TIMEOUT_MS, + maxBuffer: 1024 * 1024, + }); + return stdout; + }, + readFile: (filePath) => readFile(filePath), + removeFile: (filePath) => unlink(filePath), + temporaryPath(extension) { + screenshotCounter += 1; + return path.join( + os.tmpdir(), + `llm-space-computer-${Date.now()}-${screenshotCounter}${extension}` + ); + }, +}; + +// -- AppleScript / JXA helpers ------------------------------------------------- + +/** Escape a string for a double-quoted AppleScript string literal. */ +function _escapeAppleScriptString(text: string): string { + return text + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"') + .replace(/\r/g, "\\r") + .replace(/\n/g, "\\n") + .replace(/\t/g, "\\t"); +} + +const MOUSE_BUTTONS = { + left: { + down: "$.kCGEventLeftMouseDown", + up: "$.kCGEventLeftMouseUp", + cg: "$.kCGMouseButtonLeft", + }, + right: { + down: "$.kCGEventRightMouseDown", + up: "$.kCGEventRightMouseUp", + cg: "$.kCGMouseButtonRight", + }, +} as const; + +/** + * Post mouse events through CoreGraphics via the JXA ObjC bridge. Posting a + * move first lets hover states settle before the click lands, mirroring what + * a physical pointer does. + */ +function _clickScript( + x: number, + y: number, + button: keyof typeof MOUSE_BUTTONS, + clickCount: number +): string { + const types = MOUSE_BUTTONS[button]; + const head = ` +ObjC.import('CoreGraphics'); +const point = $.CGPointMake(${x}, ${y}); +const tap = $.kCGHIDEventTap; +const moved = $.CGEventCreateMouseEvent($(), $.kCGEventMouseMoved, point, ${types.cg}); +$.CGEventPost(tap, moved);`; + if (clickCount === 1) { + return `${head} +const down = $.CGEventCreateMouseEvent($(), ${types.down}, point, ${types.cg}); +const up = $.CGEventCreateMouseEvent($(), ${types.up}, point, ${types.cg}); +$.CGEventPost(tap, down); +$.CGEventPost(tap, up);`; + } + // A real double-click posts a second down/up pair with ClickState 2, so the + // target app recognizes it as a double rather than two singles. + return `${head} +const down1 = $.CGEventCreateMouseEvent($(), ${types.down}, point, ${types.cg}); +const up1 = $.CGEventCreateMouseEvent($(), ${types.up}, point, ${types.cg}); +$.CGEventPost(tap, down1); +$.CGEventPost(tap, up1); +const down2 = $.CGEventCreateMouseEvent($(), ${types.down}, point, ${types.cg}); +const up2 = $.CGEventCreateMouseEvent($(), ${types.up}, point, ${types.cg}); +$.CGEventSetIntegerValueField(down2, $.kCGMouseEventClickState, 2); +$.CGEventSetIntegerValueField(up2, $.kCGMouseEventClickState, 2); +$.CGEventPost(tap, down2); +$.CGEventPost(tap, up2);`; +} + +/** Scroll deltas: positive wheel1 scrolls up, positive wheel2 scrolls left. */ +function _scrollScript( + direction: "up" | "down" | "left" | "right", + amount: number +): string { + const wheels = + direction === "left" + ? `2, 0, ${amount}` + : direction === "right" + ? `2, 0, ${-amount}` + : direction === "up" + ? `1, ${amount}` + : `1, ${-amount}`; + return ` +ObjC.import('CoreGraphics'); +const event = $.CGEventCreateScrollWheelEvent($(), $.kCGScrollEventUnitLine, ${wheels}); +$.CGEventPost($.kCGHIDEventTap, event);`; +} + +// -- keyboard ------------------------------------------------------------------ + +/** macOS key codes for named special keys (`key code N`). */ +const SPECIAL_KEY_CODES: Record = { + return: 36, + enter: 36, + tab: 48, + escape: 53, + delete: 51, + backspace: 51, + forwarddelete: 117, + space: 49, + up: 126, + down: 125, + left: 123, + right: 124, + home: 115, + end: 119, + pageup: 116, + pagedown: 121, + f1: 122, + f2: 120, + f3: 99, + f4: 118, + f5: 96, + f6: 97, + f7: 98, + f8: 100, + f9: 101, + f10: 109, + f11: 103, + f12: 111, +}; + +const MODIFIER_NAMES: Record = { + cmd: "command", + command: "command", + ctrl: "control", + control: "control", + alt: "option", + option: "option", + opt: "option", + shift: "shift", + fn: "fn", +}; + +// -- argument helpers ---------------------------------------------------------- + +function _requireNumber( + args: Record, + key: string, + min = 0 +): number { + const value = args[key]; + if ( + typeof value !== "number" || + !Number.isFinite(value) || + value < min || + (min === 0 && !Number.isInteger(value)) + ) { + throw new Error( + `${key} must be ${min > 0 ? `a number >= ${min}` : "a non-negative integer"}.` + ); + } + return value; +} + +// -- tools --------------------------------------------------------------------- + +function _computerScreenshotTool(deps: ComputerDependencies): ToolEntry { + const tool: BuiltinTool = { + type: "builtin", + name: "computer_screenshot", + icon: "camera", + description: + "Capture the screen and return it as an image so the model can see what is on screen. Can capture the full screen or a region. On macOS the app may need Screen Recording permission for window contents to appear.", + strict: true, + parameters: { + type: "object", + required: [], + properties: { + region: { + type: "object", + description: + "Optional pixel region to capture instead of the full screen.", + properties: { + x: { type: "number", description: "Left edge, in points." }, + y: { type: "number", description: "Top edge, in points." }, + width: { type: "number", description: "Width in points." }, + height: { type: "number", description: "Height in points." }, + }, + required: ["x", "y", "width", "height"], + additionalProperties: false, + }, + includeCursor: { + type: "boolean", + description: + "Whether to include the mouse cursor in the capture. Defaults to false.", + }, + }, + additionalProperties: false, + }, + }; + async function execute(args: Record): Promise { + const captureArgs = ["-x"]; + let note = "Captured the full screen."; + if (args.region !== undefined) { + const region = args.region as Record; + const x = _requireNumber(region, "x"); + const y = _requireNumber(region, "y"); + const width = _requireNumber(region, "width", 1); + const height = _requireNumber(region, "height", 1); + captureArgs.push(`-R${x},${y},${width},${height}`); + note = `Captured a ${width}x${height} region at (${x}, ${y}).`; + } + if (args.includeCursor === true) { + captureArgs.push("-C"); + } + const filePath = deps.temporaryPath(".png"); + captureArgs.push(filePath); + await deps.run(SCREENCAPURE, captureArgs); + try { + const png = await deps.readFile(filePath); + return createToolCallResponse([ + { + type: "image", + mimeType: "image/png", + data: Buffer.from(png).toString("base64"), + }, + { type: "text", text: note }, + ]); + } finally { + await deps.removeFile(filePath).catch(() => undefined); + } + } + return { tool, execute }; +} + +function _computerClickTool(deps: ComputerDependencies): ToolEntry { + const tool: BuiltinTool = { + type: "builtin", + name: "computer_click", + icon: "mouse-pointer", + description: + "Click the mouse at a screen coordinate. Supports left, right, and double clicks. Take a screenshot first to pick coordinates.", + strict: true, + parameters: { + type: "object", + required: ["x", "y"], + properties: { + x: { + type: "number", + description: "Horizontal position, in points from the left edge.", + }, + y: { + type: "number", + description: "Vertical position, in points from the top edge.", + }, + button: { + type: "string", + description: 'Which button to press. Defaults to "left".', + }, + clickCount: { + type: "number", + description: "1 for a single click (default), 2 for a double click.", + }, + }, + additionalProperties: false, + }, + }; + async function execute(args: Record): Promise { + const x = _requireNumber(args, "x"); + const y = _requireNumber(args, "y"); + const button = + args.button === undefined + ? "left" + : args.button === "left" || args.button === "right" + ? args.button + : undefined; + if (!button) { + throw new Error('button must be "left" or "right".'); + } + const clickCount = + args.clickCount === undefined + ? 1 + : args.clickCount === 1 || args.clickCount === 2 + ? args.clickCount + : undefined; + if (!clickCount) { + throw new Error("clickCount must be 1 or 2."); + } + await deps.run(OSASCRIPT, [ + "-l", + "JavaScript", + "-e", + _clickScript(x, y, button, clickCount), + ]); + return `Clicked ${button}${clickCount === 2 ? " (double)" : ""} at (${x}, ${y}).`; + } + return { tool, execute }; +} + +function _computerScrollTool(deps: ComputerDependencies): ToolEntry { + const tool: BuiltinTool = { + type: "builtin", + name: "computer_scroll", + icon: "mouse", + description: + "Scroll the view under the mouse cursor by a number of lines. Sends scroll-wheel events, so it works in lists, web pages, and editors.", + strict: true, + parameters: { + type: "object", + required: ["direction"], + properties: { + direction: { + type: "string", + description: "Which way to scroll the content.", + }, + amount: { + type: "number", + description: "Lines to scroll. Defaults to 3.", + }, + }, + additionalProperties: false, + }, + }; + async function execute(args: Record): Promise { + const direction = + args.direction === "up" || + args.direction === "down" || + args.direction === "left" || + args.direction === "right" + ? args.direction + : undefined; + if (!direction) { + throw new Error('direction must be "up", "down", "left", or "right".'); + } + const amount = + args.amount === undefined + ? 3 + : typeof args.amount === "number" && + Number.isFinite(args.amount) && + args.amount >= 1 && + args.amount <= 50 + ? Math.round(args.amount) + : undefined; + if (!amount) { + throw new Error("amount must be a number between 1 and 50."); + } + await deps.run(OSASCRIPT, [ + "-l", + "JavaScript", + "-e", + _scrollScript(direction, amount), + ]); + return `Scrolled ${direction} by ${amount} line${amount === 1 ? "" : "s"}.`; + } + return { tool, execute }; +} + +function _computerTypeTool(deps: ComputerDependencies): ToolEntry { + const tool: BuiltinTool = { + type: "builtin", + name: "computer_type", + icon: "keyboard", + description: + "Type text into the focused control as if on the keyboard. Click the target field first. ASCII text is typed directly; text with non-ASCII characters is typed by temporarily placing it on the clipboard and pasting.", + strict: true, + parameters: { + type: "object", + required: ["text"], + properties: { + text: { + type: "string", + description: "The text to type, including newlines if needed.", + }, + }, + additionalProperties: false, + }, + }; + async function execute(args: Record): Promise { + const text = args.text; + if (typeof text !== "string" || text.length < 1 || text.length > 5000) { + throw new Error("text must contain 1-5000 characters."); + } + const escaped = _escapeAppleScriptString(text); + if (/^[\x20-\x7E\r\n\t]*$/.test(text)) { + await deps.run(OSASCRIPT, [ + "-e", + `tell application "System Events" to keystroke "${escaped}"`, + ]); + return `Typed ${text.length} character${text.length === 1 ? "" : "s"}.`; + } + // System Events keystroke cannot type non-ASCII text; paste it instead. + await deps.run(OSASCRIPT, [ + "-e", + `set the clipboard to "${escaped}"\n` + + "delay 0.2\n" + + 'tell application "System Events" to keystroke "v" using command down', + ]); + return `Typed ${text.length} character${text.length === 1 ? "" : "s"} via the clipboard (the previous clipboard contents were replaced).`; + } + return { tool, execute }; +} + +function _computerKeyTool(deps: ComputerDependencies): ToolEntry { + const tool: BuiltinTool = { + type: "builtin", + name: "computer_key", + icon: "keyboard", + description: + 'Press a key or key combination, e.g. "return", "escape", "cmd+c", or "cmd+shift+t". Named keys: ' + + Object.keys(SPECIAL_KEY_CODES).join(", ") + + ".", + strict: true, + parameters: { + type: "object", + required: ["key"], + properties: { + key: { + type: "string", + description: + 'A single character (e.g. "c") or a named special key (e.g. "return"), optionally combined with "+"-separated modifiers in the same string (e.g. "cmd+shift+t").', + }, + }, + additionalProperties: false, + }, + }; + async function execute(args: Record): Promise { + const raw = args.key; + if (typeof raw !== "string" || !raw.trim()) { + throw new Error('key is required, e.g. "return" or "cmd+c".'); + } + const parts = raw + .trim() + .toLowerCase() + .split("+") + .map((part) => part.trim()) + .filter(Boolean); + if (parts.length === 0) { + throw new Error('key is required, e.g. "return" or "cmd+c".'); + } + const key = parts[parts.length - 1]; + const unknownModifiers = parts + .slice(0, -1) + .filter((part) => !(part in MODIFIER_NAMES)); + if (unknownModifiers.length > 0) { + throw new Error( + `Unknown modifier${unknownModifiers.length === 1 ? "" : "s"}: ${unknownModifiers.join(", ")}.` + ); + } + const appleModifiers = [ + ...new Set( + parts.slice(0, -1).map((part) => MODIFIER_NAMES[part]) + ), + ]; + const usingClause = + appleModifiers.length > 0 + ? ` using {${appleModifiers.map((name) => `${name} down`).join(", ")}}` + : ""; + const script = + key in SPECIAL_KEY_CODES + ? `tell application "System Events" to key code ${SPECIAL_KEY_CODES[key]}${usingClause}` + : key.length === 1 + ? `tell application "System Events" to keystroke "${_escapeAppleScriptString(key)}"${usingClause}` + : undefined; + if (!script) { + throw new Error( + `Unknown key "${key}". Use a single character or one of: ${Object.keys(SPECIAL_KEY_CODES).join(", ")}.` + ); + } + await deps.run(OSASCRIPT, ["-e", script]); + return `Pressed ${raw.trim()}.`; + } + return { tool, execute }; +} + +/** + * The computer-use tool set: see the screen, then act on it. Requires macOS's + * Screen Recording permission (screenshots) and Accessibility permission + * (clicks, typing, scrolling) for the app; macOS prompts for each on first use. + */ +export function createComputerBuiltInTools( + dependencies: ComputerDependencies = defaultComputerDependencies +): ToolEntry[] { + return [ + _computerScreenshotTool(dependencies), + _computerClickTool(dependencies), + _computerScrollTool(dependencies), + _computerTypeTool(dependencies), + _computerKeyTool(dependencies), + ]; +} diff --git a/packages/runtime/tests/tools/built-in/built-in-tools-module.test.ts b/packages/runtime/tests/tools/built-in/built-in-tools-module.test.ts index fa8994ca..ab202a24 100644 --- a/packages/runtime/tests/tools/built-in/built-in-tools-module.test.ts +++ b/packages/runtime/tests/tools/built-in/built-in-tools-module.test.ts @@ -100,7 +100,16 @@ describe("built-in tools module", () => { "present_files", "generate_image", ...(process.platform === "darwin" - ? ["list_voices", "speak", "stop_speaking"] + ? [ + "list_voices", + "speak", + "stop_speaking", + "computer_screenshot", + "computer_click", + "computer_scroll", + "computer_type", + "computer_key", + ] : []), "spawn_agent", "todo_write", diff --git a/packages/runtime/tests/tools/built-in/computer.test.ts b/packages/runtime/tests/tools/built-in/computer.test.ts new file mode 100644 index 00000000..3efd695e --- /dev/null +++ b/packages/runtime/tests/tools/built-in/computer.test.ts @@ -0,0 +1,363 @@ +import { describe, expect, test } from "bun:test"; +import os from "node:os"; +import path from "node:path"; + +import { + createComputerBuiltInTools, + defaultComputerDependencies, + type ComputerDependencies, +} from "../../../src/tools/built-in/computer"; + +interface RecordedInvocation { + command: string; + args: string[]; +} + +interface FakeComputerOptions { + /** Written to the temp path `screencapture` "created". */ + screenshotBytes?: Uint8Array; +} + +function _fakeDependencies(options: FakeComputerOptions = {}): { + deps: ComputerDependencies; + invocations: RecordedInvocation[]; + tempPaths: string[]; + removed: string[]; +} { + const invocations: RecordedInvocation[] = []; + const tempPaths: string[] = []; + const removed: string[] = []; + let counter = 0; + const deps: ComputerDependencies = { + async run(command, args) { + invocations.push({ command, args }); + return ""; + }, + async readFile(filePath) { + if (!tempPaths.includes(filePath)) { + throw new Error(`Unexpected read: ${filePath}`); + } + return options.screenshotBytes ?? new Uint8Array([1, 2, 3]); + }, + async removeFile(filePath) { + removed.push(filePath); + }, + temporaryPath(extension) { + counter += 1; + const filePath = path.join("/tmp", `fake-${counter}${extension}`); + tempPaths.push(filePath); + return filePath; + }, + }; + return { deps, invocations, tempPaths, removed }; +} + +function _tool( + name: string, + deps: ComputerDependencies +): + | { execute(args: Record): Promise } + | undefined { + return createComputerBuiltInTools(deps).find( + (entry) => entry.tool.name === name + ); +} + +describe("computer built-in tools", () => { + test("exposes the five tools with model-facing names", () => { + const { deps } = _fakeDependencies(); + const names = createComputerBuiltInTools(deps).map( + (entry) => entry.tool.name + ); + expect(names).toEqual([ + "computer_screenshot", + "computer_click", + "computer_scroll", + "computer_type", + "computer_key", + ]); + for (const entry of createComputerBuiltInTools(deps)) { + expect(entry.tool.type).toBe("builtin"); + expect(entry.tool.description.length).toBeGreaterThan(0); + expect(entry.tool.parameters).toBeTruthy(); + } + }); + + describe("computer_screenshot", () => { + test("captures the full screen and returns image content", async () => { + const { deps, invocations, tempPaths, removed } = _fakeDependencies({ + screenshotBytes: new Uint8Array([9, 9, 9]), + }); + const result = (await _tool("computer_screenshot", deps)!.execute({})) as { + content: { type: string; mimeType?: string; data?: string; text?: string }[]; + }; + + expect(invocations).toHaveLength(1); + expect(invocations[0].command).toBe("/usr/sbin/screencapture"); + expect(invocations[0].args).toEqual(["-x", tempPaths[0]]); + expect(result.content).toHaveLength(2); + expect(result.content[0]).toEqual({ + type: "image", + mimeType: "image/png", + data: Buffer.from([9, 9, 9]).toString("base64"), + }); + expect(result.content[1]).toMatchObject({ + type: "text", + text: "Captured the full screen.", + }); + // The temporary capture file is always cleaned up. + expect(removed).toEqual(tempPaths); + }); + + test("forwards a region and the cursor flag", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_screenshot", deps)!.execute({ + region: { x: 10, y: 20, width: 300, height: 200 }, + includeCursor: true, + }); + expect(invocations[0].args.slice(0, 3)).toEqual([ + "-x", + "-R10,20,300,200", + "-C", + ]); + expect(typeof invocations[0].args[3]).toBe("string"); + }); + + test("rejects invalid regions", async () => { + const { deps } = _fakeDependencies(); + let rejection: unknown; + try { + await _tool("computer_screenshot", deps)!.execute({ + region: { x: 0, y: 0, width: -5, height: 200 }, + }); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + expect((rejection as Error).message).toContain("width"); + }); + + test("cleans up the temp file even when reading fails", async () => { + const { deps, removed } = _fakeDependencies(); + const failing: ComputerDependencies = { + ...deps, + readFile: async () => { + throw new Error("disk gone"); + }, + }; + let rejection: unknown; + try { + await _tool("computer_screenshot", failing)!.execute({}); + } catch (error) { + rejection = error; + } + expect((rejection as Error).message).toBe("disk gone"); + expect(removed).toHaveLength(1); + }); + }); + + describe("computer_click", () => { + test("builds a single-click JXA script through CoreGraphics", async () => { + const { deps, invocations } = _fakeDependencies(); + const result = await _tool("computer_click", deps)!.execute({ + x: 120, + y: 80, + }); + expect(invocations).toHaveLength(1); + expect(invocations[0].command).toBe("/usr/bin/osascript"); + expect(invocations[0].args.slice(0, 2)).toEqual([ + "-l", + "JavaScript", + ]); + expect(typeof invocations[0].args[2]).toBe("string"); + const script = invocations[0].args[3]; + expect(script).toContain("ObjC.import('CoreGraphics')"); + expect(script).toContain("$.CGPointMake(120, 80)"); + expect(script).toContain("$.kCGEventLeftMouseDown"); + expect(script).toContain("$.kCGEventLeftMouseUp"); + expect(script).not.toContain("ClickState"); + expect(result).toBe("Clicked left at (120, 80)."); + }); + + test("builds a right double-click with ClickState 2", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_click", deps)!.execute({ + x: 1, + y: 2, + button: "right", + clickCount: 2, + }); + const script = invocations[0].args[3]; + expect(script).toContain("$.kCGEventRightMouseDown"); + expect(script).toContain("$.kCGMouseButtonRight"); + expect(script).toContain("$.kCGMouseEventClickState, 2"); + }); + + test("rejects bad buttons, click counts, and coordinates", async () => { + const { deps } = _fakeDependencies(); + for (const args of [ + { x: 1, y: 2, button: "middle" }, + { x: 1, y: 2, clickCount: 3 }, + { y: 2 }, + { x: 1.5, y: 2 }, + ]) { + let rejection: unknown; + try { + await _tool("computer_click", deps)!.execute(args); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + } + }); + }); + + describe("computer_scroll", () => { + test("positive wheel1 scrolls up, negative scrolls down", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_scroll", deps)!.execute({ + direction: "up", + amount: 5, + }); + expect(invocations[0].args[3]).toContain( + "CGEventCreateScrollWheelEvent($(), $.kCGScrollEventUnitLine, 1, 5)" + ); + invocations.length = 0; + await _tool("computer_scroll", deps)!.execute({ direction: "down" }); + expect(invocations[0].args[3]).toContain( + "CGEventCreateScrollWheelEvent($(), $.kCGScrollEventUnitLine, 1, -3)" + ); + }); + + test("horizontal scrolling uses the second wheel axis", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_scroll", deps)!.execute({ + direction: "right", + amount: 2, + }); + expect(invocations[0].args[3]).toContain( + "CGEventCreateScrollWheelEvent($(), $.kCGScrollEventUnitLine, 2, 0, -2)" + ); + }); + + test("rejects unknown directions and out-of-range amounts", async () => { + const { deps } = _fakeDependencies(); + for (const args of [ + { direction: "sideways" }, + { direction: "up", amount: 0 }, + { direction: "up", amount: 99 }, + ]) { + let rejection: unknown; + try { + await _tool("computer_scroll", deps)!.execute(args); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + } + }); + }); + + describe("computer_type", () => { + test("types ASCII directly through System Events keystroke", async () => { + const { deps, invocations } = _fakeDependencies(); + const result = await _tool("computer_type", deps)!.execute({ + text: 'Hello "world"\nline 2', + }); + expect(invocations).toEqual([ + { + command: "/usr/bin/osascript", + args: [ + "-e", + 'tell application "System Events" to keystroke "Hello \\"world\\"\\nline 2"', + ], + }, + ]); + expect(result).toBe("Typed 20 characters."); + }); + + test("routes non-ASCII text through the clipboard with a disclosure", async () => { + const { deps, invocations } = _fakeDependencies(); + const result = await _tool("computer_type", deps)!.execute({ + text: "你好", + }); + const script = invocations[0].args[1]; + expect(script).toContain('set the clipboard to "你好"'); + expect(script).toContain('keystroke "v" using command down'); + expect(result).toBe( + "Typed 2 characters via the clipboard (the previous clipboard contents were replaced)." + ); + }); + + test("rejects empty and oversized text", async () => { + const { deps } = _fakeDependencies(); + for (const text of ["", "x".repeat(5001)]) { + let rejection: unknown; + try { + await _tool("computer_type", deps)!.execute({ text }); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + } + }); + }); + + describe("computer_key", () => { + test("maps named keys to macOS key codes", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_key", deps)!.execute({ key: "return" }); + expect(invocations[0].args).toEqual([ + "-e", + 'tell application "System Events" to key code 36', + ]); + }); + + test("combines modifiers with special keys and characters", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_key", deps)!.execute({ key: "cmd+shift+t" }); + expect(invocations[0].args).toEqual([ + "-e", + 'tell application "System Events" to keystroke "t" using {command down, shift down}', + ]); + invocations.length = 0; + await _tool("computer_key", deps)!.execute({ key: "ctrl+up" }); + expect(invocations[0].args).toEqual([ + "-e", + 'tell application "System Events" to key code 126 using {control down}', + ]); + }); + + test("rejects unknown modifiers and unknown named keys", async () => { + const { deps } = _fakeDependencies(); + for (const key of ["hyper+c", "capslock", "cmd+hyperlock"]) { + let rejection: unknown; + try { + await _tool("computer_key", deps)!.execute({ key }); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + } + }); + + test("accepts modifier aliases", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_key", deps)!.execute({ key: "option+left" }); + expect(invocations[0].args).toEqual([ + "-e", + 'tell application "System Events" to key code 123 using {option down}', + ]); + }); + }); + + describe("default dependencies", () => { + test("produce unique temp paths with the requested extension", () => { + const first = defaultComputerDependencies.temporaryPath(".png"); + const second = defaultComputerDependencies.temporaryPath(".png"); + expect(first).not.toBe(second); + expect(first.endsWith(".png")).toBe(true); + expect(path.dirname(first)).toBe(os.tmpdir()); + }); + }); +}); diff --git a/packages/ui/src/components/thread-playground/tool/built-in-tool-icon.ts b/packages/ui/src/components/thread-playground/tool/built-in-tool-icon.ts index fe1fca13..ec44074c 100644 --- a/packages/ui/src/components/thread-playground/tool/built-in-tool-icon.ts +++ b/packages/ui/src/components/thread-playground/tool/built-in-tool-icon.ts @@ -2,6 +2,7 @@ import { BotIcon, CalculatorIcon, CalendarClockIcon, + CameraIcon, Code2Icon, CircleHelpIcon, CloudSunIcon, @@ -15,7 +16,10 @@ import { FolderTreeIcon, GlobeIcon, ImageIcon, + KeyboardIcon, ListTodoIcon, + MouseIcon, + MousePointer2Icon, ListTreeIcon, PackageCheckIcon, SearchIcon, @@ -29,6 +33,7 @@ const ICON_BY_KEY: Record = { bot: BotIcon, calculator: CalculatorIcon, "calendar-clock": CalendarClockIcon, + camera: CameraIcon, "code-2": Code2Icon, "file-text": FileTextIcon, "file-output": FileOutputIcon, @@ -47,6 +52,9 @@ const ICON_BY_KEY: Record = { files: FilesIcon, "list-todo": ListTodoIcon, image: ImageIcon, + keyboard: KeyboardIcon, + mouse: MouseIcon, + "mouse-pointer": MousePointer2Icon, }; /** Fallback for tools persisted before the `icon` field existed. */ @@ -72,6 +80,11 @@ const ICON_KEY_BY_NAME: Record = { todo_write: "list-todo", ask_user_question: "circle-help", generate_image: "image", + computer_screenshot: "camera", + computer_click: "mouse-pointer", + computer_scroll: "mouse", + computer_type: "keyboard", + computer_key: "keyboard", }; /** From 05fe02a77595186f016b5974136379699be726c9 Mon Sep 17 00:00:00 2001 From: zcai7675-bot <233973883+.zcai7675-bot@users.noreply.github.com> Date: Sun, 13 Sep 2026 11:38:39 +0800 Subject: [PATCH 2/2] fix(tools): correct computer-use coordinates on Retina displays Four fixes from a self-review against real hardware: - Screenshots now report the pixel-to-point scale in their text note (e.g. "Image: 3420x2214 px (2x points) ... divide image pixel coordinates by 2") by parsing the PNG IHDR for pixel size and probing the main display's point size through CoreGraphics (cached). Region captures derive the scale from their own requested width, so they work even when the screen probe fails. Without this, clicks aimed at coordinates read off a Retina capture landed at 2x the intended spot. - Removed the "fn" modifier: System Events rejects {fn down} at AppleScript compile time (verified with osacompile), so any fn combo errored. - Click and region coordinates accept fractional points now -- models dividing pixel coordinates naturally produce x.5 values. - Fixed the screen-size probe to destructure execFileAsync's result; calling .trim() on the {stdout, stderr} object threw and silently degraded every capture to the no-scale note (caught by a live end-to-end run, not by the fake-backed unit tests). Test fakes now emit a real PNG header and a configurable screen size, covering the scale note, the degraded notes, fractional coordinates, and the fn rejection. --- .../runtime/src/tools/built-in/computer.ts | 124 ++++++++++++++++-- .../tests/tools/built-in/computer.test.ts | 70 ++++++++-- 2 files changed, 175 insertions(+), 19 deletions(-) diff --git a/packages/runtime/src/tools/built-in/computer.ts b/packages/runtime/src/tools/built-in/computer.ts index e3baa2ea..5f3573ff 100644 --- a/packages/runtime/src/tools/built-in/computer.ts +++ b/packages/runtime/src/tools/built-in/computer.ts @@ -26,10 +26,19 @@ export interface ComputerDependencies { readFile(filePath: string): Promise; removeFile(filePath: string): Promise; temporaryPath(extension: string): string; + /** + * The main display's logical size in points, or null when it cannot be + * determined. Paired with the captured image's pixel size this yields the + * pixel-to-point scale the model must divide by before clicking. + */ + screenPoints(): Promise<{ width: number; height: number } | null>; } let screenshotCounter = 0; +/** Cached main-display size; the resolution rarely changes between captures. */ +let cachedScreenPoints: { width: number; height: number } | null = null; + export const defaultComputerDependencies: ComputerDependencies = { async run(command, args) { const { stdout } = await execFileAsync(command, args, { @@ -38,6 +47,33 @@ export const defaultComputerDependencies: ComputerDependencies = { }); return stdout; }, + async screenPoints() { + if (cachedScreenPoints) { + return cachedScreenPoints; + } + try { + const { stdout } = await execFileAsync( + OSASCRIPT, + [ + "-l", + "JavaScript", + "-e", + "ObjC.import('CoreGraphics');" + + "const d = $.CGMainDisplayID();" + + "$.CGDisplayBounds(d).size.width + 'x' + $.CGDisplayBounds(d).size.height", + ], + { timeout: COMMAND_TIMEOUT_MS, maxBuffer: 1024 } + ); + const parsed = _parseScreenPoints(stdout); + if (parsed) { + cachedScreenPoints = parsed; + return cachedScreenPoints; + } + } catch { + // Fall through: the capture still works without scale information. + } + return null; + }, readFile: (filePath) => readFile(filePath), removeFile: (filePath) => unlink(filePath), temporaryPath(extension) { @@ -133,6 +169,71 @@ const event = $.CGEventCreateScrollWheelEvent($(), $.kCGScrollEventUnitLine, ${w $.CGEventPost($.kCGHIDEventTap, event);`; } +// -- image helpers -------------------------------------------------------------- + +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; + +/** + * Read a PNG's pixel dimensions straight from the IHDR chunk (bytes 16-23), + * without spawning an image tool or pulling in a decoder. + */ +function _pngSize( + bytes: Uint8Array +): { width: number; height: number } | null { + if (bytes.length < 24) { + return null; + } + for (let index = 0; index < PNG_SIGNATURE.length; index += 1) { + if (bytes[index] !== PNG_SIGNATURE[index]) { + return null; + } + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const width = view.getUint32(16); + const height = view.getUint32(20); + if (width === 0 || height === 0) { + return null; + } + return { width, height }; +} + +/** + * Describe how image pixels map to the point space the input tools use, so the + * model can convert coordinates it reads off a Retina (2x) capture. + * `logicalWidth` is the capture's width in points: the screen width for a + * full capture, or the requested region width. + */ +function _scaleNote( + imageSize: { width: number; height: number } | null, + logicalWidth: number | null +): string { + if (!imageSize) { + return ""; + } + if (logicalWidth && logicalWidth > 0 && imageSize.width >= logicalWidth) { + const scale = Math.round((imageSize.width / logicalWidth) * 100) / 100; + return ( + ` Image: ${imageSize.width}x${imageSize.height} px (${scale}x points).` + + ` computer_click and computer_screenshot regions use points: divide image pixel coordinates by ${scale}.` + ); + } + return ( + ` Image: ${imageSize.width}x${imageSize.height} px. Input tools use points,` + + " which may differ from image pixels on Retina displays." + ); +} + +/** Parse the JXA probe's "WxH" stdout into a point size. */ +function _parseScreenPoints( + stdout: string +): { width: number; height: number } | null { + const match = /^(\d+(?:\.\d+)?)x(\d+(?:\.\d+)?)$/.exec(stdout.trim()); + if (!match) { + return null; + } + return { width: Number(match[1]), height: Number(match[2]) }; +} + // -- keyboard ------------------------------------------------------------------ /** macOS key codes for named special keys (`key code N`). */ @@ -176,7 +277,6 @@ const MODIFIER_NAMES: Record = { option: "option", opt: "option", shift: "shift", - fn: "fn", }; // -- argument helpers ---------------------------------------------------------- @@ -190,12 +290,9 @@ function _requireNumber( if ( typeof value !== "number" || !Number.isFinite(value) || - value < min || - (min === 0 && !Number.isInteger(value)) + value < min ) { - throw new Error( - `${key} must be ${min > 0 ? `a number >= ${min}` : "a non-negative integer"}.` - ); + throw new Error(`${key} must be a number >= ${min}.`); } return value; } @@ -239,12 +336,14 @@ function _computerScreenshotTool(deps: ComputerDependencies): ToolEntry { async function execute(args: Record): Promise { const captureArgs = ["-x"]; let note = "Captured the full screen."; + let regionWidth: number | null = null; if (args.region !== undefined) { const region = args.region as Record; const x = _requireNumber(region, "x"); const y = _requireNumber(region, "y"); const width = _requireNumber(region, "width", 1); const height = _requireNumber(region, "height", 1); + regionWidth = width; captureArgs.push(`-R${x},${y},${width},${height}`); note = `Captured a ${width}x${height} region at (${x}, ${y}).`; } @@ -256,13 +355,16 @@ function _computerScreenshotTool(deps: ComputerDependencies): ToolEntry { await deps.run(SCREENCAPURE, captureArgs); try { const png = await deps.readFile(filePath); + const logicalWidth = + regionWidth ?? ((await deps.screenPoints())?.width ?? null); + const scaleNote = _scaleNote(_pngSize(png), logicalWidth); return createToolCallResponse([ { type: "image", mimeType: "image/png", data: Buffer.from(png).toString("base64"), }, - { type: "text", text: note }, + { type: "text", text: note + scaleNote }, ]); } finally { await deps.removeFile(filePath).catch(() => undefined); @@ -277,7 +379,7 @@ function _computerClickTool(deps: ComputerDependencies): ToolEntry { name: "computer_click", icon: "mouse-pointer", description: - "Click the mouse at a screen coordinate. Supports left, right, and double clicks. Take a screenshot first to pick coordinates.", + "Click the mouse at a screen coordinate in points. Supports left, right, and double clicks. Take a screenshot first: it reports the pixel-to-point scale to divide its pixel coordinates by.", strict: true, parameters: { type: "object", @@ -285,11 +387,13 @@ function _computerClickTool(deps: ComputerDependencies): ToolEntry { properties: { x: { type: "number", - description: "Horizontal position, in points from the left edge.", + description: + "Horizontal position in points from the left edge of the main display.", }, y: { type: "number", - description: "Vertical position, in points from the top edge.", + description: + "Vertical position in points from the top edge of the main display.", }, button: { type: "string", diff --git a/packages/runtime/tests/tools/built-in/computer.test.ts b/packages/runtime/tests/tools/built-in/computer.test.ts index 3efd695e..140d7950 100644 --- a/packages/runtime/tests/tools/built-in/computer.test.ts +++ b/packages/runtime/tests/tools/built-in/computer.test.ts @@ -16,6 +16,20 @@ interface RecordedInvocation { interface FakeComputerOptions { /** Written to the temp path `screencapture` "created". */ screenshotBytes?: Uint8Array; + /** Logical main-display size reported to the scale note; null to disable. */ + screenPoints?: { width: number; height: number } | null; +} + +/** A minimal PNG whose IHDR advertises the given pixel dimensions. */ +function _pngBytes(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(32); + for (let index = 0; index < 8; index += 1) { + bytes[index] = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a][index]!; + } + const view = new DataView(bytes.buffer); + view.setUint32(16, width); + view.setUint32(20, height); + return bytes; } function _fakeDependencies(options: FakeComputerOptions = {}): { @@ -48,6 +62,11 @@ function _fakeDependencies(options: FakeComputerOptions = {}): { tempPaths.push(filePath); return filePath; }, + async screenPoints() { + return options.screenPoints === undefined + ? { width: 1710, height: 1107 } + : options.screenPoints; + }, }; return { deps, invocations, tempPaths, removed }; } @@ -84,9 +103,10 @@ describe("computer built-in tools", () => { }); describe("computer_screenshot", () => { - test("captures the full screen and returns image content", async () => { + test("captures the full screen and reports the pixel-to-point scale", async () => { const { deps, invocations, tempPaths, removed } = _fakeDependencies({ - screenshotBytes: new Uint8Array([9, 9, 9]), + screenshotBytes: _pngBytes(3420, 2214), + screenPoints: { width: 1710, height: 1107 }, }); const result = (await _tool("computer_screenshot", deps)!.execute({})) as { content: { type: string; mimeType?: string; data?: string; text?: string }[]; @@ -99,16 +119,40 @@ describe("computer built-in tools", () => { expect(result.content[0]).toEqual({ type: "image", mimeType: "image/png", - data: Buffer.from([9, 9, 9]).toString("base64"), - }); - expect(result.content[1]).toMatchObject({ - type: "text", - text: "Captured the full screen.", + data: Buffer.from(_pngBytes(3420, 2214)).toString("base64"), }); + const note = result.content[1].text!; + expect(note).toContain("Captured the full screen."); + // Retina math: 3420px over 1710pt -> 2x, and the note must say so. + expect(note).toContain("3420x2214 px"); + expect(note).toContain("(2x points)"); + expect(note).toContain("divide image pixel coordinates by 2"); // The temporary capture file is always cleaned up. expect(removed).toEqual(tempPaths); }); + test("still returns the image when the PNG is unparseable or the screen size is unknown", async () => { + const unparseable = _fakeDependencies({ + screenshotBytes: new Uint8Array([9, 9, 9]), + }); + const resultA = (await _tool("computer_screenshot", unparseable.deps)! + .execute({})) as { + content: { type: string; text?: string }[]; + }; + expect(resultA.content[1].text).toBe("Captured the full screen."); + + const noScreen = _fakeDependencies({ + screenshotBytes: _pngBytes(3420, 2214), + screenPoints: null, + }); + const resultB = (await _tool("computer_screenshot", noScreen.deps)! + .execute({})) as { + content: { type: string; text?: string }[]; + }; + expect(resultB.content[1].text).toContain("3420x2214 px"); + expect(resultB.content[1].text).toContain("may differ from image pixels"); + }); + test("forwards a region and the cursor flag", async () => { const { deps, invocations } = _fakeDependencies(); await _tool("computer_screenshot", deps)!.execute({ @@ -199,7 +243,7 @@ describe("computer built-in tools", () => { { x: 1, y: 2, button: "middle" }, { x: 1, y: 2, clickCount: 3 }, { y: 2 }, - { x: 1.5, y: 2 }, + { x: -1, y: 2 }, ]) { let rejection: unknown; try { @@ -210,6 +254,14 @@ describe("computer built-in tools", () => { expect(rejection).toBeInstanceOf(Error); } }); + + test("accepts fractional point coordinates from pixel conversion", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_click", deps)!.execute({ x: 855.5, y: 1106.25 }); + expect(invocations).toHaveLength(1); + expect(invocations[0].command).toBe("/usr/bin/osascript"); + expect(invocations[0].args[3]).toContain("$.CGPointMake(855.5, 1106.25)"); + }); }); describe("computer_scroll", () => { @@ -330,7 +382,7 @@ describe("computer built-in tools", () => { test("rejects unknown modifiers and unknown named keys", async () => { const { deps } = _fakeDependencies(); - for (const key of ["hyper+c", "capslock", "cmd+hyperlock"]) { + for (const key of ["hyper+c", "capslock", "cmd+hyperlock", "fn+c"]) { let rejection: unknown; try { await _tool("computer_key", deps)!.execute({ key });