diff --git a/packages/runtime/src/tools/built-in/built-in-tools-module.ts b/packages/runtime/src/tools/built-in/built-in-tools-module.ts index f36f6c88..2440c50c 100644 --- a/packages/runtime/src/tools/built-in/built-in-tools-module.ts +++ b/packages/runtime/src/tools/built-in/built-in-tools-module.ts @@ -1,6 +1,7 @@ import type { RuntimeModule } from "../runtime-module"; import { calculatorBuiltInTools } from "./calculator"; +import { createComputerBuiltInTools } from "./computer"; import { dateDifferenceBuiltInTools } from "./date-difference"; import { createExecCodeBuiltInTools, @@ -47,6 +48,10 @@ export function createBuiltInToolsModule( id: "llm-space.built-in-tools.speech", entries: createSpeechBuiltInTools(speech), }); + tools.register({ + id: "llm-space.built-in-tools.computer", + entries: createComputerBuiltInTools(), + }); } tools.register({ id: "llm-space.built-in-tools.misc", diff --git a/packages/runtime/src/tools/built-in/computer.ts b/packages/runtime/src/tools/built-in/computer.ts new file mode 100644 index 00000000..5f3573ff --- /dev/null +++ b/packages/runtime/src/tools/built-in/computer.ts @@ -0,0 +1,632 @@ +import { execFile } from "node:child_process"; +import { readFile, unlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +import type { BuiltinTool } from "@llm-space/core"; + +import { createToolCallResponse } from "../tool-registry"; +import type { ToolEntry } from "../tool-registry"; + +const SCREENCAPURE = "/usr/sbin/screencapture"; +const OSASCRIPT = "/usr/bin/osascript"; +const COMMAND_TIMEOUT_MS = 15_000; + +const execFileAsync = promisify(execFile); + +/** + * Injectable OS bridge. The defaults shell out to macOS's built-in + * `screencapture` and `osascript`; tests substitute fakes so nothing touches + * the real screen or clipboard. + */ +export interface ComputerDependencies { + /** Run a helper binary and resolve with its stdout. */ + run(command: string, args: string[]): Promise; + readFile(filePath: string): Promise; + removeFile(filePath: string): Promise; + temporaryPath(extension: string): string; + /** + * The main display's logical size in points, or null when it cannot be + * determined. Paired with the captured image's pixel size this yields the + * pixel-to-point scale the model must divide by before clicking. + */ + screenPoints(): Promise<{ width: number; height: number } | null>; +} + +let screenshotCounter = 0; + +/** Cached main-display size; the resolution rarely changes between captures. */ +let cachedScreenPoints: { width: number; height: number } | null = null; + +export const defaultComputerDependencies: ComputerDependencies = { + async run(command, args) { + const { stdout } = await execFileAsync(command, args, { + timeout: COMMAND_TIMEOUT_MS, + maxBuffer: 1024 * 1024, + }); + return stdout; + }, + async screenPoints() { + if (cachedScreenPoints) { + return cachedScreenPoints; + } + try { + const { stdout } = await execFileAsync( + OSASCRIPT, + [ + "-l", + "JavaScript", + "-e", + "ObjC.import('CoreGraphics');" + + "const d = $.CGMainDisplayID();" + + "$.CGDisplayBounds(d).size.width + 'x' + $.CGDisplayBounds(d).size.height", + ], + { timeout: COMMAND_TIMEOUT_MS, maxBuffer: 1024 } + ); + const parsed = _parseScreenPoints(stdout); + if (parsed) { + cachedScreenPoints = parsed; + return cachedScreenPoints; + } + } catch { + // Fall through: the capture still works without scale information. + } + return null; + }, + readFile: (filePath) => readFile(filePath), + removeFile: (filePath) => unlink(filePath), + temporaryPath(extension) { + screenshotCounter += 1; + return path.join( + os.tmpdir(), + `llm-space-computer-${Date.now()}-${screenshotCounter}${extension}` + ); + }, +}; + +// -- AppleScript / JXA helpers ------------------------------------------------- + +/** Escape a string for a double-quoted AppleScript string literal. */ +function _escapeAppleScriptString(text: string): string { + return text + .replace(/\\/g, "\\\\") + .replace(/"/g, '\\"') + .replace(/\r/g, "\\r") + .replace(/\n/g, "\\n") + .replace(/\t/g, "\\t"); +} + +const MOUSE_BUTTONS = { + left: { + down: "$.kCGEventLeftMouseDown", + up: "$.kCGEventLeftMouseUp", + cg: "$.kCGMouseButtonLeft", + }, + right: { + down: "$.kCGEventRightMouseDown", + up: "$.kCGEventRightMouseUp", + cg: "$.kCGMouseButtonRight", + }, +} as const; + +/** + * Post mouse events through CoreGraphics via the JXA ObjC bridge. Posting a + * move first lets hover states settle before the click lands, mirroring what + * a physical pointer does. + */ +function _clickScript( + x: number, + y: number, + button: keyof typeof MOUSE_BUTTONS, + clickCount: number +): string { + const types = MOUSE_BUTTONS[button]; + const head = ` +ObjC.import('CoreGraphics'); +const point = $.CGPointMake(${x}, ${y}); +const tap = $.kCGHIDEventTap; +const moved = $.CGEventCreateMouseEvent($(), $.kCGEventMouseMoved, point, ${types.cg}); +$.CGEventPost(tap, moved);`; + if (clickCount === 1) { + return `${head} +const down = $.CGEventCreateMouseEvent($(), ${types.down}, point, ${types.cg}); +const up = $.CGEventCreateMouseEvent($(), ${types.up}, point, ${types.cg}); +$.CGEventPost(tap, down); +$.CGEventPost(tap, up);`; + } + // A real double-click posts a second down/up pair with ClickState 2, so the + // target app recognizes it as a double rather than two singles. + return `${head} +const down1 = $.CGEventCreateMouseEvent($(), ${types.down}, point, ${types.cg}); +const up1 = $.CGEventCreateMouseEvent($(), ${types.up}, point, ${types.cg}); +$.CGEventPost(tap, down1); +$.CGEventPost(tap, up1); +const down2 = $.CGEventCreateMouseEvent($(), ${types.down}, point, ${types.cg}); +const up2 = $.CGEventCreateMouseEvent($(), ${types.up}, point, ${types.cg}); +$.CGEventSetIntegerValueField(down2, $.kCGMouseEventClickState, 2); +$.CGEventSetIntegerValueField(up2, $.kCGMouseEventClickState, 2); +$.CGEventPost(tap, down2); +$.CGEventPost(tap, up2);`; +} + +/** Scroll deltas: positive wheel1 scrolls up, positive wheel2 scrolls left. */ +function _scrollScript( + direction: "up" | "down" | "left" | "right", + amount: number +): string { + const wheels = + direction === "left" + ? `2, 0, ${amount}` + : direction === "right" + ? `2, 0, ${-amount}` + : direction === "up" + ? `1, ${amount}` + : `1, ${-amount}`; + return ` +ObjC.import('CoreGraphics'); +const event = $.CGEventCreateScrollWheelEvent($(), $.kCGScrollEventUnitLine, ${wheels}); +$.CGEventPost($.kCGHIDEventTap, event);`; +} + +// -- image helpers -------------------------------------------------------------- + +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; + +/** + * Read a PNG's pixel dimensions straight from the IHDR chunk (bytes 16-23), + * without spawning an image tool or pulling in a decoder. + */ +function _pngSize( + bytes: Uint8Array +): { width: number; height: number } | null { + if (bytes.length < 24) { + return null; + } + for (let index = 0; index < PNG_SIGNATURE.length; index += 1) { + if (bytes[index] !== PNG_SIGNATURE[index]) { + return null; + } + } + const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); + const width = view.getUint32(16); + const height = view.getUint32(20); + if (width === 0 || height === 0) { + return null; + } + return { width, height }; +} + +/** + * Describe how image pixels map to the point space the input tools use, so the + * model can convert coordinates it reads off a Retina (2x) capture. + * `logicalWidth` is the capture's width in points: the screen width for a + * full capture, or the requested region width. + */ +function _scaleNote( + imageSize: { width: number; height: number } | null, + logicalWidth: number | null +): string { + if (!imageSize) { + return ""; + } + if (logicalWidth && logicalWidth > 0 && imageSize.width >= logicalWidth) { + const scale = Math.round((imageSize.width / logicalWidth) * 100) / 100; + return ( + ` Image: ${imageSize.width}x${imageSize.height} px (${scale}x points).` + + ` computer_click and computer_screenshot regions use points: divide image pixel coordinates by ${scale}.` + ); + } + return ( + ` Image: ${imageSize.width}x${imageSize.height} px. Input tools use points,` + + " which may differ from image pixels on Retina displays." + ); +} + +/** Parse the JXA probe's "WxH" stdout into a point size. */ +function _parseScreenPoints( + stdout: string +): { width: number; height: number } | null { + const match = /^(\d+(?:\.\d+)?)x(\d+(?:\.\d+)?)$/.exec(stdout.trim()); + if (!match) { + return null; + } + return { width: Number(match[1]), height: Number(match[2]) }; +} + +// -- keyboard ------------------------------------------------------------------ + +/** macOS key codes for named special keys (`key code N`). */ +const SPECIAL_KEY_CODES: Record = { + return: 36, + enter: 36, + tab: 48, + escape: 53, + delete: 51, + backspace: 51, + forwarddelete: 117, + space: 49, + up: 126, + down: 125, + left: 123, + right: 124, + home: 115, + end: 119, + pageup: 116, + pagedown: 121, + f1: 122, + f2: 120, + f3: 99, + f4: 118, + f5: 96, + f6: 97, + f7: 98, + f8: 100, + f9: 101, + f10: 109, + f11: 103, + f12: 111, +}; + +const MODIFIER_NAMES: Record = { + cmd: "command", + command: "command", + ctrl: "control", + control: "control", + alt: "option", + option: "option", + opt: "option", + shift: "shift", +}; + +// -- argument helpers ---------------------------------------------------------- + +function _requireNumber( + args: Record, + key: string, + min = 0 +): number { + const value = args[key]; + if ( + typeof value !== "number" || + !Number.isFinite(value) || + value < min + ) { + throw new Error(`${key} must be a number >= ${min}.`); + } + return value; +} + +// -- tools --------------------------------------------------------------------- + +function _computerScreenshotTool(deps: ComputerDependencies): ToolEntry { + const tool: BuiltinTool = { + type: "builtin", + name: "computer_screenshot", + icon: "camera", + description: + "Capture the screen and return it as an image so the model can see what is on screen. Can capture the full screen or a region. On macOS the app may need Screen Recording permission for window contents to appear.", + strict: true, + parameters: { + type: "object", + required: [], + properties: { + region: { + type: "object", + description: + "Optional pixel region to capture instead of the full screen.", + properties: { + x: { type: "number", description: "Left edge, in points." }, + y: { type: "number", description: "Top edge, in points." }, + width: { type: "number", description: "Width in points." }, + height: { type: "number", description: "Height in points." }, + }, + required: ["x", "y", "width", "height"], + additionalProperties: false, + }, + includeCursor: { + type: "boolean", + description: + "Whether to include the mouse cursor in the capture. Defaults to false.", + }, + }, + additionalProperties: false, + }, + }; + async function execute(args: Record): Promise { + const captureArgs = ["-x"]; + let note = "Captured the full screen."; + let regionWidth: number | null = null; + if (args.region !== undefined) { + const region = args.region as Record; + const x = _requireNumber(region, "x"); + const y = _requireNumber(region, "y"); + const width = _requireNumber(region, "width", 1); + const height = _requireNumber(region, "height", 1); + regionWidth = width; + captureArgs.push(`-R${x},${y},${width},${height}`); + note = `Captured a ${width}x${height} region at (${x}, ${y}).`; + } + if (args.includeCursor === true) { + captureArgs.push("-C"); + } + const filePath = deps.temporaryPath(".png"); + captureArgs.push(filePath); + await deps.run(SCREENCAPURE, captureArgs); + try { + const png = await deps.readFile(filePath); + const logicalWidth = + regionWidth ?? ((await deps.screenPoints())?.width ?? null); + const scaleNote = _scaleNote(_pngSize(png), logicalWidth); + return createToolCallResponse([ + { + type: "image", + mimeType: "image/png", + data: Buffer.from(png).toString("base64"), + }, + { type: "text", text: note + scaleNote }, + ]); + } finally { + await deps.removeFile(filePath).catch(() => undefined); + } + } + return { tool, execute }; +} + +function _computerClickTool(deps: ComputerDependencies): ToolEntry { + const tool: BuiltinTool = { + type: "builtin", + name: "computer_click", + icon: "mouse-pointer", + description: + "Click the mouse at a screen coordinate in points. Supports left, right, and double clicks. Take a screenshot first: it reports the pixel-to-point scale to divide its pixel coordinates by.", + strict: true, + parameters: { + type: "object", + required: ["x", "y"], + properties: { + x: { + type: "number", + description: + "Horizontal position in points from the left edge of the main display.", + }, + y: { + type: "number", + description: + "Vertical position in points from the top edge of the main display.", + }, + button: { + type: "string", + description: 'Which button to press. Defaults to "left".', + }, + clickCount: { + type: "number", + description: "1 for a single click (default), 2 for a double click.", + }, + }, + additionalProperties: false, + }, + }; + async function execute(args: Record): Promise { + const x = _requireNumber(args, "x"); + const y = _requireNumber(args, "y"); + const button = + args.button === undefined + ? "left" + : args.button === "left" || args.button === "right" + ? args.button + : undefined; + if (!button) { + throw new Error('button must be "left" or "right".'); + } + const clickCount = + args.clickCount === undefined + ? 1 + : args.clickCount === 1 || args.clickCount === 2 + ? args.clickCount + : undefined; + if (!clickCount) { + throw new Error("clickCount must be 1 or 2."); + } + await deps.run(OSASCRIPT, [ + "-l", + "JavaScript", + "-e", + _clickScript(x, y, button, clickCount), + ]); + return `Clicked ${button}${clickCount === 2 ? " (double)" : ""} at (${x}, ${y}).`; + } + return { tool, execute }; +} + +function _computerScrollTool(deps: ComputerDependencies): ToolEntry { + const tool: BuiltinTool = { + type: "builtin", + name: "computer_scroll", + icon: "mouse", + description: + "Scroll the view under the mouse cursor by a number of lines. Sends scroll-wheel events, so it works in lists, web pages, and editors.", + strict: true, + parameters: { + type: "object", + required: ["direction"], + properties: { + direction: { + type: "string", + description: "Which way to scroll the content.", + }, + amount: { + type: "number", + description: "Lines to scroll. Defaults to 3.", + }, + }, + additionalProperties: false, + }, + }; + async function execute(args: Record): Promise { + const direction = + args.direction === "up" || + args.direction === "down" || + args.direction === "left" || + args.direction === "right" + ? args.direction + : undefined; + if (!direction) { + throw new Error('direction must be "up", "down", "left", or "right".'); + } + const amount = + args.amount === undefined + ? 3 + : typeof args.amount === "number" && + Number.isFinite(args.amount) && + args.amount >= 1 && + args.amount <= 50 + ? Math.round(args.amount) + : undefined; + if (!amount) { + throw new Error("amount must be a number between 1 and 50."); + } + await deps.run(OSASCRIPT, [ + "-l", + "JavaScript", + "-e", + _scrollScript(direction, amount), + ]); + return `Scrolled ${direction} by ${amount} line${amount === 1 ? "" : "s"}.`; + } + return { tool, execute }; +} + +function _computerTypeTool(deps: ComputerDependencies): ToolEntry { + const tool: BuiltinTool = { + type: "builtin", + name: "computer_type", + icon: "keyboard", + description: + "Type text into the focused control as if on the keyboard. Click the target field first. ASCII text is typed directly; text with non-ASCII characters is typed by temporarily placing it on the clipboard and pasting.", + strict: true, + parameters: { + type: "object", + required: ["text"], + properties: { + text: { + type: "string", + description: "The text to type, including newlines if needed.", + }, + }, + additionalProperties: false, + }, + }; + async function execute(args: Record): Promise { + const text = args.text; + if (typeof text !== "string" || text.length < 1 || text.length > 5000) { + throw new Error("text must contain 1-5000 characters."); + } + const escaped = _escapeAppleScriptString(text); + if (/^[\x20-\x7E\r\n\t]*$/.test(text)) { + await deps.run(OSASCRIPT, [ + "-e", + `tell application "System Events" to keystroke "${escaped}"`, + ]); + return `Typed ${text.length} character${text.length === 1 ? "" : "s"}.`; + } + // System Events keystroke cannot type non-ASCII text; paste it instead. + await deps.run(OSASCRIPT, [ + "-e", + `set the clipboard to "${escaped}"\n` + + "delay 0.2\n" + + 'tell application "System Events" to keystroke "v" using command down', + ]); + return `Typed ${text.length} character${text.length === 1 ? "" : "s"} via the clipboard (the previous clipboard contents were replaced).`; + } + return { tool, execute }; +} + +function _computerKeyTool(deps: ComputerDependencies): ToolEntry { + const tool: BuiltinTool = { + type: "builtin", + name: "computer_key", + icon: "keyboard", + description: + 'Press a key or key combination, e.g. "return", "escape", "cmd+c", or "cmd+shift+t". Named keys: ' + + Object.keys(SPECIAL_KEY_CODES).join(", ") + + ".", + strict: true, + parameters: { + type: "object", + required: ["key"], + properties: { + key: { + type: "string", + description: + 'A single character (e.g. "c") or a named special key (e.g. "return"), optionally combined with "+"-separated modifiers in the same string (e.g. "cmd+shift+t").', + }, + }, + additionalProperties: false, + }, + }; + async function execute(args: Record): Promise { + const raw = args.key; + if (typeof raw !== "string" || !raw.trim()) { + throw new Error('key is required, e.g. "return" or "cmd+c".'); + } + const parts = raw + .trim() + .toLowerCase() + .split("+") + .map((part) => part.trim()) + .filter(Boolean); + if (parts.length === 0) { + throw new Error('key is required, e.g. "return" or "cmd+c".'); + } + const key = parts[parts.length - 1]; + const unknownModifiers = parts + .slice(0, -1) + .filter((part) => !(part in MODIFIER_NAMES)); + if (unknownModifiers.length > 0) { + throw new Error( + `Unknown modifier${unknownModifiers.length === 1 ? "" : "s"}: ${unknownModifiers.join(", ")}.` + ); + } + const appleModifiers = [ + ...new Set( + parts.slice(0, -1).map((part) => MODIFIER_NAMES[part]) + ), + ]; + const usingClause = + appleModifiers.length > 0 + ? ` using {${appleModifiers.map((name) => `${name} down`).join(", ")}}` + : ""; + const script = + key in SPECIAL_KEY_CODES + ? `tell application "System Events" to key code ${SPECIAL_KEY_CODES[key]}${usingClause}` + : key.length === 1 + ? `tell application "System Events" to keystroke "${_escapeAppleScriptString(key)}"${usingClause}` + : undefined; + if (!script) { + throw new Error( + `Unknown key "${key}". Use a single character or one of: ${Object.keys(SPECIAL_KEY_CODES).join(", ")}.` + ); + } + await deps.run(OSASCRIPT, ["-e", script]); + return `Pressed ${raw.trim()}.`; + } + return { tool, execute }; +} + +/** + * The computer-use tool set: see the screen, then act on it. Requires macOS's + * Screen Recording permission (screenshots) and Accessibility permission + * (clicks, typing, scrolling) for the app; macOS prompts for each on first use. + */ +export function createComputerBuiltInTools( + dependencies: ComputerDependencies = defaultComputerDependencies +): ToolEntry[] { + return [ + _computerScreenshotTool(dependencies), + _computerClickTool(dependencies), + _computerScrollTool(dependencies), + _computerTypeTool(dependencies), + _computerKeyTool(dependencies), + ]; +} diff --git a/packages/runtime/tests/tools/built-in/built-in-tools-module.test.ts b/packages/runtime/tests/tools/built-in/built-in-tools-module.test.ts index fa8994ca..ab202a24 100644 --- a/packages/runtime/tests/tools/built-in/built-in-tools-module.test.ts +++ b/packages/runtime/tests/tools/built-in/built-in-tools-module.test.ts @@ -100,7 +100,16 @@ describe("built-in tools module", () => { "present_files", "generate_image", ...(process.platform === "darwin" - ? ["list_voices", "speak", "stop_speaking"] + ? [ + "list_voices", + "speak", + "stop_speaking", + "computer_screenshot", + "computer_click", + "computer_scroll", + "computer_type", + "computer_key", + ] : []), "spawn_agent", "todo_write", diff --git a/packages/runtime/tests/tools/built-in/computer.test.ts b/packages/runtime/tests/tools/built-in/computer.test.ts new file mode 100644 index 00000000..140d7950 --- /dev/null +++ b/packages/runtime/tests/tools/built-in/computer.test.ts @@ -0,0 +1,415 @@ +import { describe, expect, test } from "bun:test"; +import os from "node:os"; +import path from "node:path"; + +import { + createComputerBuiltInTools, + defaultComputerDependencies, + type ComputerDependencies, +} from "../../../src/tools/built-in/computer"; + +interface RecordedInvocation { + command: string; + args: string[]; +} + +interface FakeComputerOptions { + /** Written to the temp path `screencapture` "created". */ + screenshotBytes?: Uint8Array; + /** Logical main-display size reported to the scale note; null to disable. */ + screenPoints?: { width: number; height: number } | null; +} + +/** A minimal PNG whose IHDR advertises the given pixel dimensions. */ +function _pngBytes(width: number, height: number): Uint8Array { + const bytes = new Uint8Array(32); + for (let index = 0; index < 8; index += 1) { + bytes[index] = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a][index]!; + } + const view = new DataView(bytes.buffer); + view.setUint32(16, width); + view.setUint32(20, height); + return bytes; +} + +function _fakeDependencies(options: FakeComputerOptions = {}): { + deps: ComputerDependencies; + invocations: RecordedInvocation[]; + tempPaths: string[]; + removed: string[]; +} { + const invocations: RecordedInvocation[] = []; + const tempPaths: string[] = []; + const removed: string[] = []; + let counter = 0; + const deps: ComputerDependencies = { + async run(command, args) { + invocations.push({ command, args }); + return ""; + }, + async readFile(filePath) { + if (!tempPaths.includes(filePath)) { + throw new Error(`Unexpected read: ${filePath}`); + } + return options.screenshotBytes ?? new Uint8Array([1, 2, 3]); + }, + async removeFile(filePath) { + removed.push(filePath); + }, + temporaryPath(extension) { + counter += 1; + const filePath = path.join("/tmp", `fake-${counter}${extension}`); + tempPaths.push(filePath); + return filePath; + }, + async screenPoints() { + return options.screenPoints === undefined + ? { width: 1710, height: 1107 } + : options.screenPoints; + }, + }; + return { deps, invocations, tempPaths, removed }; +} + +function _tool( + name: string, + deps: ComputerDependencies +): + | { execute(args: Record): Promise } + | undefined { + return createComputerBuiltInTools(deps).find( + (entry) => entry.tool.name === name + ); +} + +describe("computer built-in tools", () => { + test("exposes the five tools with model-facing names", () => { + const { deps } = _fakeDependencies(); + const names = createComputerBuiltInTools(deps).map( + (entry) => entry.tool.name + ); + expect(names).toEqual([ + "computer_screenshot", + "computer_click", + "computer_scroll", + "computer_type", + "computer_key", + ]); + for (const entry of createComputerBuiltInTools(deps)) { + expect(entry.tool.type).toBe("builtin"); + expect(entry.tool.description.length).toBeGreaterThan(0); + expect(entry.tool.parameters).toBeTruthy(); + } + }); + + describe("computer_screenshot", () => { + test("captures the full screen and reports the pixel-to-point scale", async () => { + const { deps, invocations, tempPaths, removed } = _fakeDependencies({ + screenshotBytes: _pngBytes(3420, 2214), + screenPoints: { width: 1710, height: 1107 }, + }); + const result = (await _tool("computer_screenshot", deps)!.execute({})) as { + content: { type: string; mimeType?: string; data?: string; text?: string }[]; + }; + + expect(invocations).toHaveLength(1); + expect(invocations[0].command).toBe("/usr/sbin/screencapture"); + expect(invocations[0].args).toEqual(["-x", tempPaths[0]]); + expect(result.content).toHaveLength(2); + expect(result.content[0]).toEqual({ + type: "image", + mimeType: "image/png", + data: Buffer.from(_pngBytes(3420, 2214)).toString("base64"), + }); + const note = result.content[1].text!; + expect(note).toContain("Captured the full screen."); + // Retina math: 3420px over 1710pt -> 2x, and the note must say so. + expect(note).toContain("3420x2214 px"); + expect(note).toContain("(2x points)"); + expect(note).toContain("divide image pixel coordinates by 2"); + // The temporary capture file is always cleaned up. + expect(removed).toEqual(tempPaths); + }); + + test("still returns the image when the PNG is unparseable or the screen size is unknown", async () => { + const unparseable = _fakeDependencies({ + screenshotBytes: new Uint8Array([9, 9, 9]), + }); + const resultA = (await _tool("computer_screenshot", unparseable.deps)! + .execute({})) as { + content: { type: string; text?: string }[]; + }; + expect(resultA.content[1].text).toBe("Captured the full screen."); + + const noScreen = _fakeDependencies({ + screenshotBytes: _pngBytes(3420, 2214), + screenPoints: null, + }); + const resultB = (await _tool("computer_screenshot", noScreen.deps)! + .execute({})) as { + content: { type: string; text?: string }[]; + }; + expect(resultB.content[1].text).toContain("3420x2214 px"); + expect(resultB.content[1].text).toContain("may differ from image pixels"); + }); + + test("forwards a region and the cursor flag", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_screenshot", deps)!.execute({ + region: { x: 10, y: 20, width: 300, height: 200 }, + includeCursor: true, + }); + expect(invocations[0].args.slice(0, 3)).toEqual([ + "-x", + "-R10,20,300,200", + "-C", + ]); + expect(typeof invocations[0].args[3]).toBe("string"); + }); + + test("rejects invalid regions", async () => { + const { deps } = _fakeDependencies(); + let rejection: unknown; + try { + await _tool("computer_screenshot", deps)!.execute({ + region: { x: 0, y: 0, width: -5, height: 200 }, + }); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + expect((rejection as Error).message).toContain("width"); + }); + + test("cleans up the temp file even when reading fails", async () => { + const { deps, removed } = _fakeDependencies(); + const failing: ComputerDependencies = { + ...deps, + readFile: async () => { + throw new Error("disk gone"); + }, + }; + let rejection: unknown; + try { + await _tool("computer_screenshot", failing)!.execute({}); + } catch (error) { + rejection = error; + } + expect((rejection as Error).message).toBe("disk gone"); + expect(removed).toHaveLength(1); + }); + }); + + describe("computer_click", () => { + test("builds a single-click JXA script through CoreGraphics", async () => { + const { deps, invocations } = _fakeDependencies(); + const result = await _tool("computer_click", deps)!.execute({ + x: 120, + y: 80, + }); + expect(invocations).toHaveLength(1); + expect(invocations[0].command).toBe("/usr/bin/osascript"); + expect(invocations[0].args.slice(0, 2)).toEqual([ + "-l", + "JavaScript", + ]); + expect(typeof invocations[0].args[2]).toBe("string"); + const script = invocations[0].args[3]; + expect(script).toContain("ObjC.import('CoreGraphics')"); + expect(script).toContain("$.CGPointMake(120, 80)"); + expect(script).toContain("$.kCGEventLeftMouseDown"); + expect(script).toContain("$.kCGEventLeftMouseUp"); + expect(script).not.toContain("ClickState"); + expect(result).toBe("Clicked left at (120, 80)."); + }); + + test("builds a right double-click with ClickState 2", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_click", deps)!.execute({ + x: 1, + y: 2, + button: "right", + clickCount: 2, + }); + const script = invocations[0].args[3]; + expect(script).toContain("$.kCGEventRightMouseDown"); + expect(script).toContain("$.kCGMouseButtonRight"); + expect(script).toContain("$.kCGMouseEventClickState, 2"); + }); + + test("rejects bad buttons, click counts, and coordinates", async () => { + const { deps } = _fakeDependencies(); + for (const args of [ + { x: 1, y: 2, button: "middle" }, + { x: 1, y: 2, clickCount: 3 }, + { y: 2 }, + { x: -1, y: 2 }, + ]) { + let rejection: unknown; + try { + await _tool("computer_click", deps)!.execute(args); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + } + }); + + test("accepts fractional point coordinates from pixel conversion", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_click", deps)!.execute({ x: 855.5, y: 1106.25 }); + expect(invocations).toHaveLength(1); + expect(invocations[0].command).toBe("/usr/bin/osascript"); + expect(invocations[0].args[3]).toContain("$.CGPointMake(855.5, 1106.25)"); + }); + }); + + describe("computer_scroll", () => { + test("positive wheel1 scrolls up, negative scrolls down", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_scroll", deps)!.execute({ + direction: "up", + amount: 5, + }); + expect(invocations[0].args[3]).toContain( + "CGEventCreateScrollWheelEvent($(), $.kCGScrollEventUnitLine, 1, 5)" + ); + invocations.length = 0; + await _tool("computer_scroll", deps)!.execute({ direction: "down" }); + expect(invocations[0].args[3]).toContain( + "CGEventCreateScrollWheelEvent($(), $.kCGScrollEventUnitLine, 1, -3)" + ); + }); + + test("horizontal scrolling uses the second wheel axis", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_scroll", deps)!.execute({ + direction: "right", + amount: 2, + }); + expect(invocations[0].args[3]).toContain( + "CGEventCreateScrollWheelEvent($(), $.kCGScrollEventUnitLine, 2, 0, -2)" + ); + }); + + test("rejects unknown directions and out-of-range amounts", async () => { + const { deps } = _fakeDependencies(); + for (const args of [ + { direction: "sideways" }, + { direction: "up", amount: 0 }, + { direction: "up", amount: 99 }, + ]) { + let rejection: unknown; + try { + await _tool("computer_scroll", deps)!.execute(args); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + } + }); + }); + + describe("computer_type", () => { + test("types ASCII directly through System Events keystroke", async () => { + const { deps, invocations } = _fakeDependencies(); + const result = await _tool("computer_type", deps)!.execute({ + text: 'Hello "world"\nline 2', + }); + expect(invocations).toEqual([ + { + command: "/usr/bin/osascript", + args: [ + "-e", + 'tell application "System Events" to keystroke "Hello \\"world\\"\\nline 2"', + ], + }, + ]); + expect(result).toBe("Typed 20 characters."); + }); + + test("routes non-ASCII text through the clipboard with a disclosure", async () => { + const { deps, invocations } = _fakeDependencies(); + const result = await _tool("computer_type", deps)!.execute({ + text: "你好", + }); + const script = invocations[0].args[1]; + expect(script).toContain('set the clipboard to "你好"'); + expect(script).toContain('keystroke "v" using command down'); + expect(result).toBe( + "Typed 2 characters via the clipboard (the previous clipboard contents were replaced)." + ); + }); + + test("rejects empty and oversized text", async () => { + const { deps } = _fakeDependencies(); + for (const text of ["", "x".repeat(5001)]) { + let rejection: unknown; + try { + await _tool("computer_type", deps)!.execute({ text }); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + } + }); + }); + + describe("computer_key", () => { + test("maps named keys to macOS key codes", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_key", deps)!.execute({ key: "return" }); + expect(invocations[0].args).toEqual([ + "-e", + 'tell application "System Events" to key code 36', + ]); + }); + + test("combines modifiers with special keys and characters", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_key", deps)!.execute({ key: "cmd+shift+t" }); + expect(invocations[0].args).toEqual([ + "-e", + 'tell application "System Events" to keystroke "t" using {command down, shift down}', + ]); + invocations.length = 0; + await _tool("computer_key", deps)!.execute({ key: "ctrl+up" }); + expect(invocations[0].args).toEqual([ + "-e", + 'tell application "System Events" to key code 126 using {control down}', + ]); + }); + + test("rejects unknown modifiers and unknown named keys", async () => { + const { deps } = _fakeDependencies(); + for (const key of ["hyper+c", "capslock", "cmd+hyperlock", "fn+c"]) { + let rejection: unknown; + try { + await _tool("computer_key", deps)!.execute({ key }); + } catch (error) { + rejection = error; + } + expect(rejection).toBeInstanceOf(Error); + } + }); + + test("accepts modifier aliases", async () => { + const { deps, invocations } = _fakeDependencies(); + await _tool("computer_key", deps)!.execute({ key: "option+left" }); + expect(invocations[0].args).toEqual([ + "-e", + 'tell application "System Events" to key code 123 using {option down}', + ]); + }); + }); + + describe("default dependencies", () => { + test("produce unique temp paths with the requested extension", () => { + const first = defaultComputerDependencies.temporaryPath(".png"); + const second = defaultComputerDependencies.temporaryPath(".png"); + expect(first).not.toBe(second); + expect(first.endsWith(".png")).toBe(true); + expect(path.dirname(first)).toBe(os.tmpdir()); + }); + }); +}); diff --git a/packages/ui/src/components/thread-playground/tool/built-in-tool-icon.ts b/packages/ui/src/components/thread-playground/tool/built-in-tool-icon.ts index fe1fca13..ec44074c 100644 --- a/packages/ui/src/components/thread-playground/tool/built-in-tool-icon.ts +++ b/packages/ui/src/components/thread-playground/tool/built-in-tool-icon.ts @@ -2,6 +2,7 @@ import { BotIcon, CalculatorIcon, CalendarClockIcon, + CameraIcon, Code2Icon, CircleHelpIcon, CloudSunIcon, @@ -15,7 +16,10 @@ import { FolderTreeIcon, GlobeIcon, ImageIcon, + KeyboardIcon, ListTodoIcon, + MouseIcon, + MousePointer2Icon, ListTreeIcon, PackageCheckIcon, SearchIcon, @@ -29,6 +33,7 @@ const ICON_BY_KEY: Record = { bot: BotIcon, calculator: CalculatorIcon, "calendar-clock": CalendarClockIcon, + camera: CameraIcon, "code-2": Code2Icon, "file-text": FileTextIcon, "file-output": FileOutputIcon, @@ -47,6 +52,9 @@ const ICON_BY_KEY: Record = { files: FilesIcon, "list-todo": ListTodoIcon, image: ImageIcon, + keyboard: KeyboardIcon, + mouse: MouseIcon, + "mouse-pointer": MousePointer2Icon, }; /** Fallback for tools persisted before the `icon` field existed. */ @@ -72,6 +80,11 @@ const ICON_KEY_BY_NAME: Record = { todo_write: "list-todo", ask_user_question: "circle-help", generate_image: "image", + computer_screenshot: "camera", + computer_click: "mouse-pointer", + computer_scroll: "mouse", + computer_type: "keyboard", + computer_key: "keyboard", }; /**